Tree states v3.4.0 alpha.0

Squashed commit of the following:

commit 1ba4f80cc0
Author: Michael Sproul <michael@sigmaprime.io>
Date:   Tue Jan 17 11:43:18 2023 +1100

    Bye 1.0.0 beta, hello 0.5.x

commit a862b234b2
Author: Michael Sproul <michael@sigmaprime.io>
Date:   Tue Jan 17 10:54:46 2023 +1100

    Cargo fmt

commit e29f358a9e
Author: Michael Sproul <michael@sigmaprime.io>
Date:   Mon Jan 16 18:21:42 2023 +1100

    It compiles :O

commit 1ee4514b7d
Author: Michael Sproul <michael@sigmaprime.io>
Date:   Mon Jan 16 17:27:10 2023 +1100

    Ethereum hashing

commit 69bdd1d61f
Author: Michael Sproul <michael@sigmaprime.io>
Date:   Mon Jan 16 17:24:58 2023 +1100

    Tree hash et al

commit 7cae5d99d7
Author: Michael Sproul <michael@sigmaprime.io>
Date:   Mon Jan 16 17:21:03 2023 +1100

    Delete crates!

commit dd9ee38084
Author: Michael Sproul <michael@sigmaprime.io>
Date:   Mon Jan 16 17:19:19 2023 +1100

    Delete overrides

commit 0d54534eb4
Author: Michael Sproul <michael@sigmaprime.io>
Date:   Mon Jan 16 17:19:04 2023 +1100

    Crate renames

.cargo/config.toml

@@ -0,0 +1,4 @@
+[env]
+# Set the number of arenas to 16 when using jemalloc.
+JEMALLOC_SYS_WITH_MALLOC_CONF = "abort_conf:true,narenas:16"
+

.dockerignore

@@ -1,4 +1,5 @@
 testing/ef_tests/consensus-spec-tests
+testing/execution_engine_integration/execution_clients
 target/
 *.data
 *.tar.gz

.github/custom/clippy.toml

@@ -0,0 +1,23 @@
+disallowed-from-async-methods = [
+    "tokio::runtime::Handle::block_on",
+    "tokio::runtime::Runtime::block_on",
+    "tokio::task::LocalSet::block_on",
+    "tokio::sync::Mutex::blocking_lock",
+    "tokio::sync::RwLock::blocking_read",
+    "tokio::sync::mpsc::Receiver::blocking_recv",
+    "tokio::sync::mpsc::UnboundedReceiver::blocking_recv",
+    "tokio::sync::oneshot::Receiver::blocking_recv",
+    "tokio::sync::mpsc::Sender::blocking_send",
+    "tokio::sync::RwLock::blocking_write",
+]
+async-wrapper-methods = [
+    "tokio::runtime::Handle::spawn_blocking",
+    "task_executor::TaskExecutor::spawn_blocking",
+    "task_executor::TaskExecutor::spawn_blocking_handle",
+    "warp_utils::task::blocking_task",
+    "warp_utils::task::blocking_json_task",
+    "beacon_chain::beacon_chain::BeaconChain::spawn_blocking_handle",
+    "validator_client::http_api::blocking_signed_json_task",
+    "execution_layer::test_utils::MockServer::new",
+    "execution_layer::test_utils::MockServer::new_with_config",
+]

.github/workflows/book.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   build-and-upload-to-s3:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@master
 

.github/workflows/docker-antithesis.yml

@@ -15,9 +15,9 @@ env:
 
 jobs:
     build-docker:
-        runs-on: ubuntu-18.04
+        runs-on: ubuntu-22.04
         steps:
-            - uses: actions/checkout@v2
+            - uses: actions/checkout@v3
             - name: Update Rust
               run: rustup update stable
             - name: Dockerhub login

.github/workflows/docker.yml

@@ -22,7 +22,7 @@ jobs:
     # `unstable`, but for now we keep the two parts of the version separate for backwards
     # compatibility.
     extract-version:
-        runs-on: ubuntu-18.04
+        runs-on: ubuntu-22.04
         steps:
             - name: Extract version (if stable)
               if: github.event.ref == 'refs/heads/stable'
@@ -44,13 +44,16 @@ jobs:
             VERSION_SUFFIX: ${{ env.VERSION_SUFFIX }}
     build-docker-single-arch:
         name: build-docker-${{ matrix.binary }}
-        runs-on: ubuntu-18.04
+        runs-on: ubuntu-22.04
         strategy:
             matrix:
                 binary: [aarch64,
                          aarch64-portable,
                          x86_64,
                          x86_64-portable]
+                include:
+                    - profile: maxperf
+
         needs: [extract-version]
         env:
             # We need to enable experimental docker features in order to use `docker buildx`
@@ -58,7 +61,7 @@ jobs:
             VERSION: ${{ needs.extract-version.outputs.VERSION }}
             VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }}
         steps:
-            - uses: actions/checkout@v2
+            - uses: actions/checkout@v3
             - name: Update Rust
               run: rustup update stable
             - name: Dockerhub login
@@ -67,17 +70,17 @@ jobs:
             - name: Cross build Lighthouse binary
               run: |
                   cargo install cross
-                  make build-${{ matrix.binary }}
+                  env CROSS_PROFILE=${{ matrix.profile }} make build-${{ matrix.binary }}
             - name: Move cross-built binary into Docker scope (if ARM)
               if: startsWith(matrix.binary, 'aarch64')
               run: |
                   mkdir ./bin;
-                  mv ./target/aarch64-unknown-linux-gnu/release/lighthouse ./bin;
+                  mv ./target/aarch64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ./bin;
             - name: Move cross-built binary into Docker scope (if x86_64)
               if: startsWith(matrix.binary, 'x86_64')
               run: |
                   mkdir ./bin;
-                  mv ./target/x86_64-unknown-linux-gnu/release/lighthouse ./bin;
+                  mv ./target/x86_64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ./bin;
             - name: Map aarch64 to arm64 short arch
               if: startsWith(matrix.binary, 'aarch64')
               run: echo "SHORT_ARCH=arm64" >> $GITHUB_ENV
@@ -99,7 +102,7 @@ jobs:
                       --push
     build-docker-multiarch:
         name: build-docker-multiarch${{ matrix.modernity }}
-        runs-on: ubuntu-18.04
+        runs-on: ubuntu-22.04
         needs: [build-docker-single-arch, extract-version]
         strategy:
             matrix:
@@ -120,13 +123,13 @@ jobs:
                       --amend ${IMAGE_NAME}:${VERSION}-amd64${VERSION_SUFFIX}${{ matrix.modernity }};
                   docker manifest push ${IMAGE_NAME}:${VERSION}${VERSION_SUFFIX}${{ matrix.modernity }}
     build-docker-lcli:
-        runs-on: ubuntu-18.04
+        runs-on: ubuntu-22.04
         needs: [extract-version]
         env:
             VERSION: ${{ needs.extract-version.outputs.VERSION }}
             VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }}
         steps:
-            - uses: actions/checkout@v2
+            - uses: actions/checkout@v3
             - name: Dockerhub login
               run: |
                   echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin

.github/workflows/linkcheck.yml

@@ -15,13 +15,13 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Create docker network
         run: docker network create book
 
       - name: Run mdbook server
-        run: docker run -v ${{ github.workspace }}/book:/book --network book --name book -p 3000:3000 -d peaceiris/mdbook:latest serve --hostname 0.0.0.0
+        run: docker run -v ${{ github.workspace }}/book:/book --network book --name book -p 3000:3000 -d peaceiris/mdbook:v0.4.20-rust serve --hostname 0.0.0.0
 
       - name: Print logs
         run: docker logs book

.github/workflows/local-testnet.yml

@@ -12,17 +12,23 @@ jobs:
     strategy:
       matrix:
         os:
-          - ubuntu-18.04
-          - macos-latest
+          - ubuntu-22.04
+          - macos-12
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v3
 
+      - name: Get latest version of stable Rust
+        run: rustup update stable
+      - name: Install Protoc
+        uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
+        with:
+            repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Install ganache
-        run: npm install ganache-cli@latest --global
+        run: npm install ganache@latest --global
 
       # https://github.com/actions/cache/blob/main/examples.md#rust---cargo
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         id: cache-cargo
         with:
           path: |
@@ -34,17 +40,32 @@ jobs:
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
 
       - name: Install lighthouse
-        if: steps.cache-cargo.outputs.cache-hit != 'true'
         run: make && make install-lcli
 
       - name: Start local testnet
-        run: ./start_local_testnet.sh
+        run: ./start_local_testnet.sh && sleep 60
         working-directory: scripts/local_testnet
 
       - name: Print logs
-        run: ./print_logs.sh
+        run: ./dump_logs.sh
         working-directory: scripts/local_testnet
 
       - name: Stop local testnet
         run: ./stop_local_testnet.sh
         working-directory: scripts/local_testnet
+
+      - name: Clean-up testnet
+        run: ./clean.sh
+        working-directory: scripts/local_testnet
+
+      - name: Start local testnet with blinded block production
+        run: ./start_local_testnet.sh -p && sleep 60
+        working-directory: scripts/local_testnet
+
+      - name: Print logs for blinded block testnet
+        run: ./dump_logs.sh
+        working-directory: scripts/local_testnet
+
+      - name: Stop local testnet with blinded block production
+        run: ./stop_local_testnet.sh
+        working-directory: scripts/local_testnet

.github/workflows/publish-crate.yml

@@ -19,7 +19,7 @@ jobs:
         runs-on: ubuntu-latest
         steps:
             - name: Extract tag
-              run: echo "::set-output name=TAG::$(echo ${GITHUB_REF#refs/tags/})"
+              run: echo "TAG=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_OUTPUT
               id: extract_tag
         outputs:
             TAG: ${{ steps.extract_tag.outputs.TAG }}
@@ -30,7 +30,7 @@ jobs:
         env:
             TAG: ${{ needs.extract-tag.outputs.TAG }}
         steps:
-            - uses: actions/checkout@v2
+            - uses: actions/checkout@v3
             - name: Update Rust
               run: rustup update stable
             - name: Cargo login

.github/workflows/release.yml

@@ -8,15 +8,15 @@ on:
 env:
     DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
     DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-    REPO_NAME: sigp/lighthouse
-    IMAGE_NAME: sigp/lighthouse
+    REPO_NAME: ${{ github.repository_owner }}/lighthouse
+    IMAGE_NAME: ${{ github.repository_owner }}/lighthouse
 
 jobs:
     extract-version:
         runs-on: ubuntu-latest
         steps:
             - name: Extract version
-              run: echo "::set-output name=VERSION::$(echo ${GITHUB_REF#refs/tags/})"
+              run: echo "VERSION=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_OUTPUT
               id: extract_version
         outputs:
             VERSION: ${{ steps.extract_version.outputs.VERSION }}
@@ -35,32 +35,36 @@ jobs:
                 include:
                     -   arch: aarch64-unknown-linux-gnu
                         platform: ubuntu-latest
+                        profile: maxperf
                     -   arch: aarch64-unknown-linux-gnu-portable
                         platform: ubuntu-latest
+                        profile: maxperf
                     -   arch: x86_64-unknown-linux-gnu
                         platform: ubuntu-latest
+                        profile: maxperf
                     -   arch: x86_64-unknown-linux-gnu-portable
                         platform: ubuntu-latest
+                        profile: maxperf
                     -   arch: x86_64-apple-darwin
                         platform: macos-latest
+                        profile: maxperf
                     -   arch: x86_64-apple-darwin-portable
                         platform: macos-latest
+                        profile: maxperf
                     -   arch: x86_64-windows
-                        platform: windows-latest
+                        platform: windows-2019
+                        profile: maxperf
                     -   arch: x86_64-windows-portable
-                        platform: windows-latest
+                        platform: windows-2019
+                        profile: maxperf
 
         runs-on:    ${{ matrix.platform }}
         needs: extract-version
         steps:
             - name: Checkout sources
-              uses: actions/checkout@v2
-            - name: Build toolchain
-              uses: actions-rs/toolchain@v1
-              with:
-                  toolchain: stable
-                  profile:  minimal
-                  override: true
+              uses: actions/checkout@v3
+            - name: Get latest version of stable Rust
+              run: rustup update stable
 
             # ==============================
             # Windows dependencies
@@ -75,6 +79,15 @@ jobs:
               if: startsWith(matrix.arch, 'x86_64-windows')
               run: echo "LIBCLANG_PATH=$((gcm clang).source -replace "clang.exe")" >> $env:GITHUB_ENV
 
+            # ==============================
+            # Windows & Mac dependencies
+            # ==============================
+            - name: Install Protoc
+              if: contains(matrix.arch, 'darwin') || contains(matrix.arch, 'windows')
+              uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
+              with:
+                repo-token: ${{ secrets.GITHUB_TOKEN }}
+
             # ==============================
             #       Builds
             # ==============================
@@ -83,49 +96,49 @@ jobs:
               if:   matrix.arch == 'aarch64-unknown-linux-gnu-portable'
               run:  |
                 cargo install cross
-                make build-aarch64-portable
+                env CROSS_PROFILE=${{ matrix.profile }} make build-aarch64-portable
 
             - name: Build Lighthouse for aarch64-unknown-linux-gnu
               if:   matrix.arch == 'aarch64-unknown-linux-gnu'
               run:  |
                 cargo install cross
-                make build-aarch64
+                env CROSS_PROFILE=${{ matrix.profile }} make build-aarch64
 
             - name: Build Lighthouse for x86_64-unknown-linux-gnu-portable
               if:   matrix.arch == 'x86_64-unknown-linux-gnu-portable'
               run:  |
                 cargo install cross
-                make build-x86_64-portable
+                env CROSS_PROFILE=${{ matrix.profile }} make build-x86_64-portable
 
             - name: Build Lighthouse for x86_64-unknown-linux-gnu
               if:   matrix.arch == 'x86_64-unknown-linux-gnu'
               run:  |
                 cargo install cross
-                make build-x86_64
+                env CROSS_PROFILE=${{ matrix.profile }} make build-x86_64
 
             - name: Move cross-compiled binary
               if:  startsWith(matrix.arch, 'aarch64')
-              run: mv target/aarch64-unknown-linux-gnu/release/lighthouse ~/.cargo/bin/lighthouse
+              run: mv target/aarch64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ~/.cargo/bin/lighthouse
 
             - name: Move cross-compiled binary
               if:  startsWith(matrix.arch, 'x86_64-unknown-linux-gnu')
-              run: mv target/x86_64-unknown-linux-gnu/release/lighthouse ~/.cargo/bin/lighthouse
+              run: mv target/x86_64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ~/.cargo/bin/lighthouse
 
             - name: Build Lighthouse for x86_64-apple-darwin portable
               if:   matrix.arch == 'x86_64-apple-darwin-portable'
-              run:  cargo install --path lighthouse --force --locked --features portable,gnosis
+              run:  cargo install --path lighthouse --force --locked --features portable,gnosis --profile ${{ matrix.profile }}
 
             - name: Build Lighthouse for x86_64-apple-darwin modern
               if:   matrix.arch == 'x86_64-apple-darwin'
-              run:  cargo install --path lighthouse --force --locked --features modern,gnosis
+              run:  cargo install --path lighthouse --force --locked --features modern,gnosis --profile ${{ matrix.profile }}
 
             - name: Build Lighthouse for Windows portable
               if:   matrix.arch == 'x86_64-windows-portable'
-              run:  cargo install --path lighthouse --force --locked --features portable,gnosis
+              run:  cargo install --path lighthouse --force --locked --features portable,gnosis --profile ${{ matrix.profile }}
 
             - name: Build Lighthouse for Windows modern
               if:   matrix.arch == 'x86_64-windows'
-              run:  cargo install --path lighthouse --force --locked --features modern,gnosis
+              run:  cargo install --path lighthouse --force --locked --features modern,gnosis --profile ${{ matrix.profile }}
 
             - name: Configure GPG and create artifacts
               if: startsWith(matrix.arch, 'x86_64-windows') != true
@@ -162,13 +175,13 @@ jobs:
             # =======================================================================
 
             - name:  Upload artifact
-              uses:  actions/upload-artifact@v2
+              uses:  actions/upload-artifact@v3
               with:
                   name: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz
                   path: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz
 
             - name: Upload signature
-              uses: actions/upload-artifact@v2
+              uses: actions/upload-artifact@v3
               with:
                   name: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc
                   path: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc
@@ -182,7 +195,7 @@ jobs:
         steps:
             # This is necessary for generating the changelog. It has to come before "Download Artifacts" or else it deletes the artifacts.
             - name: Checkout sources
-              uses: actions/checkout@v2
+              uses: actions/checkout@v3
               with:
                   fetch-depth: 0
 
@@ -191,7 +204,7 @@ jobs:
             # ==============================
 
             - name: Download artifacts
-              uses: actions/download-artifact@v2
+              uses: actions/download-artifact@v3
 
             # ==============================
             #       Create release draft
@@ -199,11 +212,14 @@ jobs:
 
             - name: Generate Full Changelog
               id: changelog
-              run: echo "::set-output name=CHANGELOG::$(git log --pretty=format:"- %s" $(git describe --tags --abbrev=0 ${{ env.VERSION }}^)..${{ env.VERSION }})"
+              run: |
+                echo "CHANGELOG<<EOF" >> $GITHUB_OUTPUT
+                echo "$(git log --pretty=format:"- %s" $(git describe --tags --abbrev=0 ${{ env.VERSION }}^)..${{ env.VERSION }})" >> $GITHUB_OUTPUT
+                echo "EOF" >> $GITHUB_OUTPUT
 
             - name: Create Release Draft
               env:
-                  GITHUB_USER: sigp
+                  GITHUB_USER: ${{ github.repository_owner }}
                   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
               # The formatting here is borrowed from OpenEthereum: https://github.com/openethereum/openethereum/blob/main/.github/workflows/build.yml
@@ -212,7 +228,7 @@ jobs:
                   <Rick and Morty character>
 
                   ## Testing Checklist (DELETE ME)
-                  
+
                   - [ ] Run on synced Prater Sigma Prime nodes.
                   - [ ] Run on synced Canary (mainnet) Sigma Prime nodes.
                   - [ ] Resync a Prater node.

.github/workflows/test-suite.yml

@@ -12,20 +12,34 @@ env:
   # Deny warnings in CI
   RUSTFLAGS: "-D warnings"
   # The Nightly version used for cargo-udeps, might need updating from time to time.
-  PINNED_NIGHTLY: nightly-2021-12-01
+  PINNED_NIGHTLY: nightly-2022-12-15
+  # Prevent Github API rate limiting.
+  LIGHTHOUSE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 jobs:
   target-branch-check:
     name: target-branch-check
     runs-on: ubuntu-latest
     if: github.event_name == 'pull_request'
     steps:
-        - name: Check that pull request is targeting unstable branch
-          run: test ${{ github.base_ref }} = "unstable"
+        - name: Check that the pull request is not targeting the stable branch
+          run: test ${{ github.base_ref }} != "stable"
+  extract-msrv:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Extract Minimum Supported Rust Version (MSRV)
+      run: |
+        metadata=$(cargo metadata --no-deps --format-version 1)
+        msrv=$(echo $metadata | jq -r '.packages | map(select(.name == "lighthouse")) | .[0].rust_version')
+        echo "MSRV=$msrv" >> $GITHUB_OUTPUT
+      id: extract_msrv
+    outputs:
+      MSRV: ${{ steps.extract_msrv.outputs.MSRV }}
   cargo-fmt:
     name: cargo-fmt
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
     - name: Get latest version of stable Rust
       run: rustup update stable
     - name: Check formatting with cargo fmt
@@ -35,23 +49,35 @@ jobs:
     runs-on: ubuntu-latest
     needs: cargo-fmt
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
     - name: Get latest version of stable Rust
       run: rustup update stable
-    - name: Install ganache-cli
-      run: sudo npm install -g ganache-cli
+    - name: Install Protoc
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Install ganache
+      run: sudo npm install -g ganache
     - name: Run tests in release
       run: make test-release
   release-tests-windows:
     name: release-tests-windows
-    runs-on: windows-latest
+    runs-on: windows-2019
     needs: cargo-fmt
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
     - name: Get latest version of stable Rust
       run: rustup update stable
-    - name: Install ganache-cli
-      run: npm install -g ganache-cli
+    - name: Use Node.js
+      uses: actions/setup-node@v2
+      with:
+        node-version: '14'
+    - name: Install windows build tools
+      run: |
+        choco install python protoc visualstudio2019-workload-vctools -y
+        npm config set msvs_version 2019
+    - name: Install ganache
+      run: npm install -g ganache --loglevel verbose
     - name: Install make
       run: choco install -y make
     - uses: KyleMayes/install-llvm-action@v1
@@ -67,9 +93,13 @@ jobs:
     runs-on: ubuntu-latest
     needs: cargo-fmt
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
     - name: Get latest version of stable Rust
       run: rustup update stable
+    - name: Install Protoc
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Run beacon_chain tests for all known forks
       run: make test-beacon-chain
   op-pool-tests:
@@ -77,21 +107,39 @@ jobs:
     runs-on: ubuntu-latest
     needs: cargo-fmt
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
     - name: Get latest version of stable Rust
       run: rustup update stable
+    - name: Install Protoc
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Run operation_pool tests for all known forks
       run: make test-op-pool
-  debug-tests-ubuntu:
-    name: debug-tests-ubuntu
+  slasher-tests:
+    name: slasher-tests
     runs-on: ubuntu-latest
     needs: cargo-fmt
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
     - name: Get latest version of stable Rust
       run: rustup update stable
-    - name: Install ganache-cli
-      run: sudo npm install -g ganache-cli
+    - name: Run slasher tests for all supported backends
+      run: make test-slasher
+  debug-tests-ubuntu:
+    name: debug-tests-ubuntu
+    runs-on: ubuntu-22.04
+    needs: cargo-fmt
+    steps:
+    - uses: actions/checkout@v3
+    - name: Get latest version of stable Rust
+      run: rustup update stable
+    - name: Install Protoc
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Install ganache
+      run: sudo npm install -g ganache
     - name: Run tests in debug
       run: make test-debug
   state-transition-vectors-ubuntu:
@@ -99,9 +147,13 @@ jobs:
     runs-on: ubuntu-latest
     needs: cargo-fmt
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
     - name: Get latest version of stable Rust
       run: rustup update stable
+    - name: Install Protoc
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Run state_transition_vectors in release.
       run: make run-state-transition-tests
   ef-tests-ubuntu:
@@ -109,9 +161,13 @@ jobs:
     runs-on: ubuntu-latest
     needs: cargo-fmt
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
     - name: Get latest version of stable Rust
       run: rustup update stable
+    - name: Install Protoc
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Run consensus-spec-tests with blst, milagro and fake_crypto
       run: make test-ef
   dockerfile-ubuntu:
@@ -119,7 +175,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: cargo-fmt
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
     - name: Get latest version of stable Rust
       run: rustup update stable
     - name: Build the root Dockerfile
@@ -131,23 +187,47 @@ jobs:
     runs-on: ubuntu-latest
     needs: cargo-fmt
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
     - name: Get latest version of stable Rust
       run: rustup update stable
-    - name: Install ganache-cli
-      run: sudo npm install -g ganache-cli
+    - name: Install Protoc
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Install ganache
+      run: sudo npm install -g ganache
     - name: Run the beacon chain sim that starts from an eth1 contract
       run: cargo run --release --bin simulator eth1-sim
+  merge-transition-ubuntu:
+    name: merge-transition-ubuntu
+    runs-on: ubuntu-latest
+    needs: cargo-fmt
+    steps:
+    - uses: actions/checkout@v3
+    - name: Get latest version of stable Rust
+      run: rustup update stable
+    - name: Install Protoc
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Install ganache
+      run: sudo npm install -g ganache
+    - name: Run the beacon chain sim and go through the merge transition
+      run: cargo run --release --bin simulator eth1-sim --post-merge
   no-eth1-simulator-ubuntu:
     name: no-eth1-simulator-ubuntu
     runs-on: ubuntu-latest
     needs: cargo-fmt
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
     - name: Get latest version of stable Rust
       run: rustup update stable
-    - name: Install ganache-cli
-      run: sudo npm install -g ganache-cli
+    - name: Install Protoc
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Install ganache
+      run: sudo npm install -g ganache
     - name: Run the beacon chain sim without an eth1 connection
       run: cargo run --release --bin simulator no-eth1-sim
   syncing-simulator-ubuntu:
@@ -155,43 +235,75 @@ jobs:
     runs-on: ubuntu-latest
     needs: cargo-fmt
     steps:
-      - uses: actions/checkout@v1
-      - name: Get latest version of stable Rust
-        run: rustup update stable
-      - name: Install ganache-cli
-        run: sudo npm install -g ganache-cli
-      - name: Run the syncing simulator
-        run: cargo run --release --bin simulator syncing-sim
+    - uses: actions/checkout@v3
+    - name: Get latest version of stable Rust
+      run: rustup update stable
+    - name: Install Protoc
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Install ganache
+      run: sudo npm install -g ganache
+    - name: Run the syncing simulator
+      run: cargo run --release --bin simulator syncing-sim
   doppelganger-protection-test:
-      name: doppelganger-protection-test
-      runs-on: ubuntu-latest
-      needs: cargo-fmt
-      steps:
-          - uses: actions/checkout@v1
-          - name: Get latest version of stable Rust
-            run: rustup update stable
-          - name: Install ganache-cli
-            run: sudo npm install -g ganache-cli
-          - name: Install lighthouse and lcli
-            run: |
-                make
-                make install-lcli
-          - name: Run the doppelganger protection success test script
-            run: |
-                cd scripts/tests
-                ./doppelganger_protection.sh success
-          - name: Run the doppelganger protection failure test script
-            run: |
-                cd scripts/tests
-                ./doppelganger_protection.sh failure
+    name: doppelganger-protection-test
+    runs-on: ubuntu-latest
+    needs: cargo-fmt
+    steps:
+    - uses: actions/checkout@v3
+    - name: Get latest version of stable Rust
+      run: rustup update stable
+    - name: Install Protoc
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Install ganache
+      run: sudo npm install -g ganache
+    - name: Install lighthouse and lcli
+      run: |
+          make
+          make install-lcli
+    - name: Run the doppelganger protection success test script
+      run: |
+          cd scripts/tests
+          ./doppelganger_protection.sh success
+    - name: Run the doppelganger protection failure test script
+      run: |
+          cd scripts/tests
+          ./doppelganger_protection.sh failure
+  execution-engine-integration-ubuntu:
+    name: execution-engine-integration-ubuntu
+    runs-on: ubuntu-latest
+    needs: cargo-fmt
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-go@v3
+      with:
+        go-version: '1.17'
+    - uses: actions/setup-dotnet@v3
+      with:
+        dotnet-version: '6.0.201'
+    - name: Get latest version of stable Rust
+      run: rustup update stable
+    - name: Install Protoc
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Run exec engine integration tests in release
+      run: make test-exec-engine
   check-benchmarks:
     name: check-benchmarks
     runs-on: ubuntu-latest
     needs: cargo-fmt
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
     - name: Get latest version of stable Rust
       run: rustup update stable
+    - name: Install Protoc
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Typecheck benchmark code without running it
       run: make check-benches
   check-consensus:
@@ -199,7 +311,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: cargo-fmt
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
     - name: Get latest version of stable Rust
       run: rustup update stable
     - name: Typecheck consensus code in strict mode
@@ -209,19 +321,37 @@ jobs:
     runs-on: ubuntu-latest
     needs: cargo-fmt
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
     - name: Get latest version of stable Rust
       run: rustup update stable
+    - name: Install Protoc
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Lint code for quality and style with Clippy
       run: make lint
     - name: Certify Cargo.lock freshness
       run: git diff --exit-code Cargo.lock
+  check-msrv:
+    name: check-msrv
+    runs-on: ubuntu-latest
+    needs: [cargo-fmt, extract-msrv]
+    steps:
+    - uses: actions/checkout@v3
+    - name: Install Rust @ MSRV (${{ needs.extract-msrv.outputs.MSRV }})
+      run: rustup override set ${{ needs.extract-msrv.outputs.MSRV }}
+    - name: Install Protoc
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Run cargo check
+      run: cargo check --workspace
   arbitrary-check:
     name: arbitrary-check
     runs-on: ubuntu-latest
     needs: cargo-fmt
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
     - name: Get latest version of stable Rust
       run: rustup update stable
     - name: Validate state_processing feature arbitrary-fuzz
@@ -231,7 +361,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: cargo-fmt
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
     - name: Get latest version of stable Rust
       run: rustup update stable
     - name: Run cargo audit to identify known security vulnerabilities reported to the RustSec Advisory Database
@@ -241,7 +371,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: cargo-fmt
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
     - name: Run cargo vendor to make sure dependencies can be vendored for packaging, reproducibility and archival purpose
       run:  CARGO_HOME=$(readlink -f $HOME) make vendor
   cargo-udeps:
@@ -249,11 +379,17 @@ jobs:
     runs-on: ubuntu-latest
     needs: cargo-fmt
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
     - name: Install Rust (${{ env.PINNED_NIGHTLY }})
       run: rustup toolchain install $PINNED_NIGHTLY
+    # NOTE: cargo-udeps version is pinned until this issue is resolved:
+    # https://github.com/est31/cargo-udeps/issues/135
+    - name: Install Protoc
+      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Install cargo-udeps
-      run: cargo install cargo-udeps --locked
+      run: cargo install cargo-udeps --locked --force --version 0.1.30
     - name: Create Cargo config dir
       run: mkdir -p .cargo
     - name: Install custom Cargo config
@@ -263,3 +399,14 @@ jobs:
     env:
       # Allow warnings on Nightly
       RUSTFLAGS: ""
+  compile-with-beta-compiler:
+    name: compile-with-beta-compiler
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Install dependencies
+      run: sudo apt install -y git gcc g++ make cmake pkg-config llvm-dev libclang-dev clang protobuf-compiler
+    - name: Use Rust beta
+      run: rustup override set beta
+    - name: Run make
+      run: make

.gitignore

@@ -8,3 +8,7 @@ perf.data*
 *.tar.gz
 /bin
 genesis.ssz
+/clippy.toml
+
+# IntelliJ
+/*.iml

CONTRIBUTING.md

@@ -1,4 +1,5 @@
 # Contributors Guide
+[![GitPOAP badge](https://public-api.gitpoap.io/v1/repo/sigp/lighthouse/badge)](https://www.gitpoap.io/gh/sigp/lighthouse)
 
 Lighthouse is an open-source Ethereum 2.0 client. We're community driven and
 welcome all contribution. We aim to provide a constructive, respectful and fun
@@ -45,7 +46,7 @@ questions.
    (github.com/YOUR_NAME/lighthouse) of the main repository
    (github.com/sigp/lighthouse).
 3. Once you feel you have addressed the issue, **create a pull-request** to merge
-   your changes in to the main repository.
+   your changes into the main repository.
 4. Wait for the repository maintainers to **review your changes** to ensure the
    issue is addressed satisfactorily. Optionally, mention your PR on
 [discord](https://discord.gg/cyAszAh).

Cargo.toml

@@ -4,6 +4,7 @@ members = [
 
     "beacon_node",
     "beacon_node/beacon_chain",
+    "beacon_node/builder_client",
     "beacon_node/client",
     "beacon_node/eth1",
     "beacon_node/lighthouse_network",
@@ -14,7 +15,7 @@ members = [
     "beacon_node/store",
     "beacon_node/timer",
 
-	"boot_node",
+    "boot_node",
 
     "common/account_utils",
     "common/clap_utils",
@@ -27,39 +28,35 @@ members = [
     "common/eth2_interop_keypairs",
     "common/eth2_network_config",
     "common/eth2_wallet_manager",
-    "common/hashset_delay",
     "common/lighthouse_metrics",
     "common/lighthouse_version",
     "common/lockfile",
     "common/logging",
     "common/lru_cache",
     "common/malloc_utils",
+    "common/oneshot_broadcast",
     "common/sensitive_url",
     "common/slot_clock",
+    "common/system_health",
     "common/task_executor",
     "common/target_check",
     "common/test_random_derive",
+    "common/unused_port",
     "common/validator_dir",
     "common/warp_utils",
-    "common/fallback",
     "common/monitoring_api",
 
+    "database_manager",
+
     "consensus/cached_tree_hash",
     "consensus/int_to_bytes",
     "consensus/fork_choice",
     "consensus/proto_array",
     "consensus/safe_arith",
-    "consensus/ssz",
-    "consensus/ssz_derive",
-    "consensus/ssz_types",
-    "consensus/serde_utils",
     "consensus/state_processing",
     "consensus/swap_or_not_shuffle",
-    "consensus/tree_hash",
-    "consensus/tree_hash_derive",
 
     "crypto/bls",
-    "crypto/eth2_hashing",
     "crypto/eth2_key_derivation",
     "crypto/eth2_keystore",
     "crypto/eth2_wallet",
@@ -74,6 +71,7 @@ members = [
 
     "testing/ef_tests",
     "testing/eth1_test_rig",
+    "testing/execution_engine_integration",
     "testing/node_test_rig",
     "testing/simulator",
     "testing/test-test_logger",
@@ -87,8 +85,10 @@ members = [
 [patch]
 [patch.crates-io]
 fixed-hash = { git = "https://github.com/paritytech/parity-common", rev="df638ab0885293d21d656dc300d39236b69ce57d" }
-warp = { git = "https://github.com/macladson/warp", rev ="7e75acc" }
-eth2_ssz = { path = "consensus/ssz" }
-eth2_ssz_types = { path = "consensus/ssz_types" }
-tree_hash = { path = "consensus/tree_hash" }
-eth2_serde_utils = { path = "consensus/serde_utils" }
+warp = { git = "https://github.com/macladson/warp", rev="7e75acc368229a46a236a8c991bf251fe7fe50ef" }
+
+[profile.maxperf]
+inherits = "release"
+lto = "fat"
+codegen-units = 1
+incremental = false

Cross.toml

@@ -1,15 +1,5 @@
-[build.env]
-passthrough = [
-    "RUSTFLAGS",
-]
-
-# These custom images are required to work around the lack of Clang in the default `cross` images.
-# We need Clang to run `bindgen` for MDBX, and the `BINDGEN_EXTRA_CLANG_ARGS` flags must also be set
-# while cross-compiling for ARM to prevent bindgen from attempting to include headers from the host.
-#
-# For more information see https://github.com/rust-embedded/cross/pull/608
 [target.x86_64-unknown-linux-gnu]
-image = "michaelsproul/cross-clang:x86_64-latest"
+dockerfile = './scripts/cross/Dockerfile'
 
 [target.aarch64-unknown-linux-gnu]
-image = "michaelsproul/cross-clang:aarch64-latest"
+dockerfile = './scripts/cross/Dockerfile'

Dockerfile

@@ -1,11 +1,11 @@
-FROM rust:1.58.1-bullseye AS builder
-RUN apt-get update && apt-get -y upgrade && apt-get install -y cmake libclang-dev
+FROM rust:1.62.1-bullseye AS builder
+RUN apt-get update && apt-get -y upgrade && apt-get install -y cmake libclang-dev protobuf-compiler
 COPY . lighthouse
 ARG FEATURES
 ENV FEATURES $FEATURES
 RUN cd lighthouse && make
 
-FROM ubuntu:latest
+FROM ubuntu:22.04
 RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends \
   libssl-dev \
   ca-certificates \

Dockerfile.cross

@@ -1,7 +1,7 @@
 # This image is meant to enable cross-architecture builds.
 # It assumes the lighthouse binary has already been
 # compiled for `$TARGETPLATFORM` and moved to `./bin`.
-FROM --platform=$TARGETPLATFORM ubuntu:latest
+FROM --platform=$TARGETPLATFORM ubuntu:22.04
 RUN apt-get update && apt-get install -y --no-install-recommends \
   libssl-dev \
   ca-certificates \

Makefile

@@ -2,6 +2,7 @@
 
 EF_TESTS = "testing/ef_tests"
 STATE_TRANSITION_VECTORS = "testing/state_transition_vectors"
+EXECUTION_ENGINE_INTEGRATION = "testing/execution_engine_integration"
 GIT_TAG := $(shell git describe --tags --candidates 1)
 BIN_DIR = "bin"
 
@@ -11,20 +12,30 @@ AARCH64_TAG = "aarch64-unknown-linux-gnu"
 BUILD_PATH_AARCH64 = "target/$(AARCH64_TAG)/release"
 
 PINNED_NIGHTLY ?= nightly
+CLIPPY_PINNED_NIGHTLY=nightly-2022-05-19
+
+# List of features to use when cross-compiling. Can be overridden via the environment.
+CROSS_FEATURES ?= gnosis,slasher-lmdb,slasher-mdbx
+
+# Cargo profile for Cross builds. Default is for local builds, CI uses an override.
+CROSS_PROFILE ?= release
+
+# Cargo profile for regular builds.
+PROFILE ?= release
 
 # List of all hard forks. This list is used to set env variables for several tests so that
 # they run for different forks.
-FORKS=phase0 altair
+FORKS=phase0 altair merge
 
 # Builds the Lighthouse binary in release (optimized).
 #
 # Binaries will most likely be found in `./target/release`
 install:
-	cargo install --path lighthouse --force --locked --features "$(FEATURES)"
+	cargo install --path lighthouse --force --locked --features "$(FEATURES)" --profile "$(PROFILE)"
 
 # Builds the lcli binary in release (optimized).
 install-lcli:
-	cargo install --path lcli --force --locked --features "$(FEATURES)"
+	cargo install --path lcli --force --locked --features "$(FEATURES)" --profile "$(PROFILE)"
 
 # The following commands use `cross` to build a cross-compile.
 #
@@ -40,13 +51,13 @@ install-lcli:
 # optimized CPU functions that may not be available on some systems. This
 # results in a more portable binary with ~20% slower BLS verification.
 build-x86_64:
-	cross build --release --bin lighthouse --target x86_64-unknown-linux-gnu --features modern,gnosis
+	cross build --bin lighthouse --target x86_64-unknown-linux-gnu --features "modern,$(CROSS_FEATURES)" --profile "$(CROSS_PROFILE)"
 build-x86_64-portable:
-	cross build --release --bin lighthouse --target x86_64-unknown-linux-gnu --features portable,gnosis
+	cross build --bin lighthouse --target x86_64-unknown-linux-gnu --features "portable,$(CROSS_FEATURES)" --profile "$(CROSS_PROFILE)"
 build-aarch64:
-	cross build --release --bin lighthouse --target aarch64-unknown-linux-gnu --features gnosis
+	cross build --bin lighthouse --target aarch64-unknown-linux-gnu --features "$(CROSS_FEATURES)" --profile "$(CROSS_PROFILE)"
 build-aarch64-portable:
-	cross build --release --bin lighthouse --target aarch64-unknown-linux-gnu --features portable,gnosis
+	cross build --bin lighthouse --target aarch64-unknown-linux-gnu --features "portable,$(CROSS_FEATURES)" --profile "$(CROSS_PROFILE)"
 
 # Create a `.tar.gz` containing a binary for a specific target.
 define tarball_release_binary
@@ -75,7 +86,7 @@ build-release-tarballs:
 # Runs the full workspace tests in **release**, without downloading any additional
 # test vectors.
 test-release:
-	cargo test --workspace --release --exclude ef_tests --exclude beacon_chain
+	cargo test --workspace --release --exclude ef_tests --exclude beacon_chain --exclude slasher
 
 # Runs the full workspace tests in **debug**, without downloading any additional test
 # vectors.
@@ -116,6 +127,11 @@ test-op-pool-%:
 		--features 'beacon_chain/fork_from_env'\
 		-p operation_pool
 
+# Run the tests in the `slasher` crate for all supported database backends.
+test-slasher:
+	cargo test --release -p slasher --features mdbx
+	cargo test --release -p slasher --no-default-features --features lmdb
+
 # Runs only the tests/state_transition_vectors tests.
 run-state-transition-tests:
 	make -C $(STATE_TRANSITION_VECTORS) test
@@ -123,12 +139,16 @@ run-state-transition-tests:
 # Downloads and runs the EF test vectors.
 test-ef: make-ef-tests run-ef-tests
 
+# Runs tests checking interop between Lighthouse and execution clients.
+test-exec-engine:
+	make -C $(EXECUTION_ENGINE_INTEGRATION) test
+
 # Runs the full workspace tests in release, without downloading any additional
 # test vectors.
 test: test-release
 
 # Runs the entire test suite, downloading test vectors if required.
-test-full: cargo-fmt test-release test-debug test-ef
+test-full: cargo-fmt test-release test-debug test-ef test-exec-engine
 
 # Lints the code for bad style and potentially unsafe arithmetic using Clippy.
 # Clippy lints are opt-in per-crate for now. By default, everything is allowed except for performance and correctness lints.
@@ -136,9 +156,18 @@ lint:
 	cargo clippy --workspace --tests -- \
 		-D clippy::fn_to_numeric_cast_any \
 		-D warnings \
+		-A clippy::derive_partial_eq_without_eq \
 		-A clippy::from-over-into \
 		-A clippy::upper-case-acronyms \
-		-A clippy::vec-init-then-push
+		-A clippy::vec-init-then-push \
+	    -A clippy::question-mark
+
+nightly-lint:
+	cp .github/custom/clippy.toml .
+	cargo +$(CLIPPY_PINNED_NIGHTLY) clippy --workspace --tests --release -- \
+		-A clippy::all \
+		-D clippy::disallowed_from_async
+	rm clippy.toml
 
 # Runs the makefile in the `ef_tests` repo.
 #
@@ -156,7 +185,7 @@ arbitrary-fuzz:
 # Runs cargo audit (Audit Cargo.lock files for crates with security vulnerabilities reported to the RustSec Advisory Database)
 audit:
 	cargo install --force cargo-audit
-	cargo audit --ignore RUSTSEC-2020-0071 --ignore RUSTSEC-2020-0159 --ignore RUSTSEC-2022-0009
+	cargo audit --ignore RUSTSEC-2020-0071 --ignore RUSTSEC-2020-0159
 
 # Runs `cargo vendor` to make sure dependencies can be vendored for packaging, reproducibility and archival purpose.
 vendor:

README.md

@@ -1,11 +1,9 @@
-# Lighthouse: Ethereum 2.0
+# Lighthouse: Ethereum consensus client
 
-An open-source Ethereum 2.0 client, written in Rust and maintained by Sigma Prime.
+An open-source Ethereum consensus client, written in Rust and maintained by Sigma Prime.
 
-[![Build Status]][Build Link] [![Book Status]][Book Link] [![Chat Badge]][Chat Link]
+[![Book Status]][Book Link] [![Chat Badge]][Chat Link]
 
-[Build Status]: https://github.com/sigp/lighthouse/workflows/test-suite/badge.svg?branch=stable
-[Build Link]: https://github.com/sigp/lighthouse/actions
 [Chat Badge]: https://img.shields.io/badge/chat-discord-%237289da
 [Chat Link]: https://discord.gg/cyAszAh
 [Book Status]:https://img.shields.io/badge/user--docs-unstable-informational
@@ -22,7 +20,7 @@ An open-source Ethereum 2.0 client, written in Rust and maintained by Sigma Prim
 
 Lighthouse is:
 
-- Ready for use on Eth2 mainnet.
+- Ready for use on Ethereum consensus mainnet.
 - Fully open-source, licensed under Apache 2.0.
 - Security-focused. Fuzzing techniques have been continuously applied and several external security reviews have been performed.
 - Built in [Rust](https://www.rust-lang.org), a modern language providing unique safety guarantees and
@@ -30,20 +28,20 @@ Lighthouse is:
 - Funded by various organisations, including Sigma Prime, the
 	Ethereum Foundation, ConsenSys, the Decentralization Foundation and private individuals.
 - Actively involved in the specification and security analysis of the
-	Ethereum 2.0 specification.
+	Ethereum proof-of-stake consensus specification.
 
-## Eth2 Deposit Contract
+## Staking Deposit Contract
 
 The Lighthouse team acknowledges
 [`0x00000000219ab540356cBB839Cbe05303d7705Fa`](https://etherscan.io/address/0x00000000219ab540356cbb839cbe05303d7705fa)
-as the canonical Eth2 deposit contract address.
+as the canonical staking deposit contract address.
 
 ## Documentation
 
 The [Lighthouse Book](https://lighthouse-book.sigmaprime.io) contains information for users and
 developers.
 
-The Lighthouse team maintains a blog at [lighthouse.sigmaprime.io][blog] which contains periodical
+The Lighthouse team maintains a blog at [lighthouse-blog.sigmaprime.io][blog] which contains periodical
 progress updates, roadmap insights and interesting findings.
 
 ## Branches
@@ -66,9 +64,9 @@ of the Lighthouse book.
 ## Contact
 
 The best place for discussion is the [Lighthouse Discord
-server](https://discord.gg/cyAszAh). 
+server](https://discord.gg/cyAszAh).
 
-Sign up to the [Lighthouse Development Updates](http://eepurl.com/dh9Lvb) mailing list for email
+Sign up to the [Lighthouse Development Updates](https://eepurl.com/dh9Lvb/) mailing list for email
 notifications about releases, network status and other important information.
 
 Encrypt sensitive messages using our [PGP

account_manager/Cargo.toml

@@ -2,7 +2,7 @@
 name = "account_manager"
 version = "0.3.5"
 authors = ["Paul Hauner <paul@paulhauner.com>", "Luke Anderson <luke@sigmaprime.io>"]
-edition = "2018"
+edition = "2021"
 
 [dependencies]
 bls = { path = "../crypto/bls" }

account_manager/src/lib.rs

@@ -10,6 +10,7 @@ use types::EthSpec;
 pub const CMD: &str = "account_manager";
 pub const SECRETS_DIR_FLAG: &str = "secrets-dir";
 pub const VALIDATOR_DIR_FLAG: &str = "validator-dir";
+pub const VALIDATOR_DIR_FLAG_ALIAS: &str = "validators-dir";
 pub const WALLETS_DIR_FLAG: &str = "wallets-dir";
 
 pub fn cli_app<'a, 'b>() -> App<'a, 'b> {

account_manager/src/validator/create.rs

@@ -114,7 +114,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
 
 pub fn cli_run<T: EthSpec>(
     matches: &ArgMatches,
-    mut env: Environment<T>,
+    env: Environment<T>,
     validator_dir: PathBuf,
 ) -> Result<(), String> {
     let spec = env.core_context().eth2_config.spec;

account_manager/src/validator/exit.rs

@@ -212,8 +212,8 @@ async fn publish_voluntary_exit<E: EthSpec>(
         let validator_data = get_validator_data(client, &keypair.pk).await?;
         match validator_data.status {
             ValidatorStatus::ActiveExiting => {
-                let exit_epoch = validator_data.validator.exit_epoch;
-                let withdrawal_epoch = validator_data.validator.withdrawable_epoch;
+                let exit_epoch = validator_data.validator.exit_epoch();
+                let withdrawal_epoch = validator_data.validator.withdrawable_epoch();
                 let current_epoch = get_current_epoch::<E>(genesis_data.genesis_time, spec)
                     .ok_or("Failed to get current epoch. Please check your system time")?;
                 eprintln!("Voluntary exit has been accepted into the beacon chain, but not yet finalized. \
@@ -233,7 +233,7 @@ async fn publish_voluntary_exit<E: EthSpec>(
             ValidatorStatus::ExitedSlashed | ValidatorStatus::ExitedUnslashed => {
                 eprintln!(
                     "Validator has exited on epoch: {}",
-                    validator_data.validator.exit_epoch
+                    validator_data.validator.exit_epoch()
                 );
                 break;
             }
@@ -259,7 +259,7 @@ async fn get_validator_index_for_exit(
         ValidatorStatus::ActiveOngoing => {
             let eligible_epoch = validator_data
                 .validator
-                .activation_epoch
+                .activation_epoch()
                 .safe_add(spec.shard_committee_period)
                 .map_err(|e| format!("Failed to calculate eligible epoch, validator activation epoch too high: {:?}", e))?;
 
@@ -349,7 +349,7 @@ fn load_voting_keypair(
     password_file_path: Option<&PathBuf>,
     stdin_inputs: bool,
 ) -> Result<Keypair, String> {
-    let keystore = Keystore::from_json_file(&voting_keystore_path).map_err(|e| {
+    let keystore = Keystore::from_json_file(voting_keystore_path).map_err(|e| {
         format!(
             "Unable to read keystore JSON {:?}: {:?}",
             voting_keystore_path, e

account_manager/src/validator/import.rs

@@ -176,7 +176,7 @@ pub fn cli_run(matches: &ArgMatches, validator_dir: PathBuf) -> Result<(), Strin
 
             let password = match keystore_password_path.as_ref() {
                 Some(path) => {
-                    let password_from_file: ZeroizeString = fs::read_to_string(&path)
+                    let password_from_file: ZeroizeString = fs::read_to_string(path)
                         .map_err(|e| format!("Unable to read {:?}: {:?}", path, e))?
                         .into();
                     password_from_file.without_newlines()
@@ -256,7 +256,7 @@ pub fn cli_run(matches: &ArgMatches, validator_dir: PathBuf) -> Result<(), Strin
             .ok_or_else(|| format!("Badly formatted file name: {:?}", src_keystore))?;
 
         // Copy the keystore to the new location.
-        fs::copy(&src_keystore, &dest_keystore)
+        fs::copy(src_keystore, &dest_keystore)
             .map_err(|e| format!("Unable to copy keystore: {:?}", e))?;
 
         // Register with slashing protection.
@@ -280,6 +280,8 @@ pub fn cli_run(matches: &ArgMatches, validator_dir: PathBuf) -> Result<(), Strin
             password_opt,
             graffiti,
             suggested_fee_recipient,
+            None,
+            None,
         )
         .map_err(|e| format!("Unable to create new validator definition: {:?}", e))?;
 

account_manager/src/validator/mod.rs

@@ -6,7 +6,7 @@ pub mod modify;
 pub mod recover;
 pub mod slashing_protection;
 
-use crate::VALIDATOR_DIR_FLAG;
+use crate::{VALIDATOR_DIR_FLAG, VALIDATOR_DIR_FLAG_ALIAS};
 use clap::{App, Arg, ArgMatches};
 use directory::{parse_path_or_default_with_flag, DEFAULT_VALIDATOR_DIR};
 use environment::Environment;
@@ -21,6 +21,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
         .arg(
             Arg::with_name(VALIDATOR_DIR_FLAG)
                 .long(VALIDATOR_DIR_FLAG)
+                .alias(VALIDATOR_DIR_FLAG_ALIAS)
                 .value_name("VALIDATOR_DIRECTORY")
                 .help(
                     "The path to search for validator directories. \

account_manager/src/validator/slashing_protection.rs

@@ -158,7 +158,7 @@ pub fn cli_run<T: EthSpec>(
                             InterchangeImportOutcome::Success { pubkey, summary } => {
                                 eprintln!("- {:?}", pubkey);
                                 eprintln!(
-                                    "    - latest block: {}",
+                                    "    - latest proposed block: {}",
                                     display_slot(summary.max_block_slot)
                                 );
                                 eprintln!(

account_manager/src/wallet/create.rs

@@ -159,7 +159,7 @@ pub fn create_wallet_from_mnemonic(
         unknown => return Err(format!("--{} {} is not supported", TYPE_FLAG, unknown)),
     };
 
-    let mgr = WalletManager::open(&wallet_base_dir)
+    let mgr = WalletManager::open(wallet_base_dir)
         .map_err(|e| format!("Unable to open --{}: {:?}", WALLETS_DIR_FLAG, e))?;
 
     let wallet_password: PlainText = match wallet_password_path {

account_manager/src/wallet/list.rs

@@ -10,7 +10,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
 }
 
 pub fn cli_run(wallet_base_dir: PathBuf) -> Result<(), String> {
-    let mgr = WalletManager::open(&wallet_base_dir)
+    let mgr = WalletManager::open(wallet_base_dir)
         .map_err(|e| format!("Unable to open --{}: {:?}", WALLETS_DIR_FLAG, e))?;
 
     for (name, _uuid) in mgr

beacon_node/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
 name = "beacon_node"
-version = "2.1.3"
+version = "3.4.0-tree.0"
 authors = ["Paul Hauner <paul@paulhauner.com>", "Age Manning <Age@AgeManning.com"]
-edition = "2018"
+edition = "2021"
 
 [lib]
 name = "beacon_node"
@@ -29,6 +29,7 @@ environment = { path = "../lighthouse/environment" }
 task_executor = { path = "../common/task_executor" }
 genesis = { path = "genesis" }
 eth2_network_config = { path = "../common/eth2_network_config" }
+execution_layer = { path = "execution_layer" }
 lighthouse_network = { path = "./lighthouse_network" }
 serde = "1.0.116"
 clap_utils = { path = "../common/clap_utils" }
@@ -39,3 +40,5 @@ slasher = { path = "../slasher" }
 monitoring_api = { path = "../common/monitoring_api" }
 sensitive_url = { path = "../common/sensitive_url" }
 http_api = { path = "http_api" }
+unused_port = { path = "../common/unused_port" }
+strum = "0.24.1"

beacon_node/beacon_chain/Cargo.toml

@@ -2,7 +2,7 @@
 name = "beacon_chain"
 version = "0.2.0"
 authors = ["Paul Hauner <paul@paulhauner.com>", "Age Manning <Age@AgeManning.com>"]
-edition = "2018"
+edition = "2021"
 autotests = false # using a single test binary compiles faster
 
 [features]
@@ -14,11 +14,12 @@ fork_from_env = [] # Initialise the harness chain spec from the FORK_NAME env va
 [dev-dependencies]
 maplit = "1.0.2"
 environment = { path = "../../lighthouse/environment" }
+serde_json = "1.0.58"
 
 [dependencies]
 merkle_proof = { path = "../../consensus/merkle_proof" }
 store = { path = "../store" }
-parking_lot = "0.11.0"
+parking_lot = "0.12.0"
 lazy_static = "1.4.0"
 smallvec = "1.6.1"
 lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
@@ -29,23 +30,23 @@ serde_derive = "1.0.116"
 slog = { version = "2.5.2", features = ["max_level_trace"] }
 sloggers = { version = "2.1.1", features = ["json"] }
 slot_clock = { path = "../../common/slot_clock" }
-eth2_hashing = "0.2.0"
-eth2_ssz = "0.4.1"
-eth2_ssz_types = "0.2.2"
-eth2_ssz_derive = "0.3.0"
+ethereum_hashing = "1.0.0-beta.2"
+ethereum_ssz = "0.5.0"
+ssz_types = "0.5.0"
+ethereum_ssz_derive = "0.5.0"
 state_processing = { path = "../../consensus/state_processing" }
-tree_hash = "0.4.1"
+tree_hash = "0.5.0"
 types = { path = "../../consensus/types" }
 tokio = "1.14.0"
 eth1 = { path = "../eth1" }
 futures = "0.3.7"
 genesis = { path = "../genesis" }
 int_to_bytes = { path = "../../consensus/int_to_bytes" }
-rand = "0.7.3"
+rand = "0.8.5"
 proto_array = { path = "../../consensus/proto_array" }
 lru = "0.7.1"
 tempfile = "3.1.0"
-bitvec = "0.19.3"
+bitvec = "0.20.4"
 bls = { path = "../../crypto/bls" }
 safe_arith = { path = "../../consensus/safe_arith" }
 fork_choice = { path = "../../consensus/fork_choice" }
@@ -54,11 +55,15 @@ derivative = "2.1.1"
 itertools = "0.10.0"
 slasher = { path = "../../slasher" }
 eth2 = { path = "../../common/eth2" }
-strum = { version = "0.21.0", features = ["derive"] }
+strum = { version = "0.24.0", features = ["derive"] }
 logging = { path = "../../common/logging" }
 execution_layer = { path = "../execution_layer" }
 sensitive_url = { path = "../../common/sensitive_url" }
-superstruct = "0.4.0"
+superstruct = "0.7.0"
+hex = "0.4.2"
+exit-future = "0.2.0"
+unused_port = {path = "../../common/unused_port"}
+oneshot_broadcast = { path = "../../common/oneshot_broadcast" }
 
 [[test]]
 name = "beacon_chain_tests"

beacon_node/beacon_chain/src/attestation_verification.rs

@@ -318,10 +318,17 @@ impl<'a, T: BeaconChainTypes> Clone for IndexedUnaggregatedAttestation<'a, T> {
 
 /// A helper trait implemented on wrapper types that can be progressed to a state where they can be
 /// verified for application to fork choice.
-pub trait VerifiedAttestation<T: BeaconChainTypes> {
+pub trait VerifiedAttestation<T: BeaconChainTypes>: Sized {
     fn attestation(&self) -> &Attestation<T::EthSpec>;
 
     fn indexed_attestation(&self) -> &IndexedAttestation<T::EthSpec>;
+
+    // Inefficient default implementation. This is overridden for gossip verified attestations.
+    fn into_attestation_and_indices(self) -> (Attestation<T::EthSpec>, Vec<u64>) {
+        let attestation = self.attestation().clone();
+        let attesting_indices = self.indexed_attestation().attesting_indices.clone().into();
+        (attestation, attesting_indices)
+    }
 }
 
 impl<'a, T: BeaconChainTypes> VerifiedAttestation<T> for VerifiedAggregatedAttestation<'a, T> {
@@ -961,7 +968,6 @@ impl<'a, T: BeaconChainTypes> VerifiedUnaggregatedAttestation<'a, T> {
 }
 
 /// Returns `Ok(())` if the `attestation.data.beacon_block_root` is known to this chain.
-/// You can use this `shuffling_id` to read from the shuffling cache.
 ///
 /// The block root may not be known for two reasons:
 ///
@@ -977,8 +983,8 @@ fn verify_head_block_is_known<T: BeaconChainTypes>(
     max_skip_slots: Option<u64>,
 ) -> Result<ProtoBlock, Error> {
     let block_opt = chain
-        .fork_choice
-        .read()
+        .canonical_head
+        .fork_choice_read_lock()
         .get_block(&attestation.data.beacon_block_root)
         .or_else(|| {
             chain
@@ -1246,7 +1252,10 @@ where
     // processing an attestation that does not include our latest finalized block in its chain.
     //
     // We do not delay consideration for later, we simply drop the attestation.
-    if !chain.fork_choice.read().contains_block(&target.root)
+    if !chain
+        .canonical_head
+        .fork_choice_read_lock()
+        .contains_block(&target.root)
         && !chain.early_attester_cache.contains_block(target.root)
     {
         return Err(Error::UnknownTargetRoot(target.root));

beacon_node/beacon_chain/src/attestation_verification/batch.rs

@@ -65,7 +65,7 @@ where
             .try_read_for(VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT)
             .ok_or(BeaconChainError::ValidatorPubkeyCacheLockTimeout)?;
 
-        let fork = chain.with_head(|head| Ok::<_, BeaconChainError>(head.beacon_state.fork()))?;
+        let fork = chain.canonical_head.cached_head().head_fork();
 
         let mut signature_sets = Vec::with_capacity(num_indexed * 3);
 
@@ -169,13 +169,13 @@ where
             &metrics::ATTESTATION_PROCESSING_BATCH_UNAGG_SIGNATURE_SETUP_TIMES,
         );
 
+        let fork = chain.canonical_head.cached_head().head_fork();
+
         let pubkey_cache = chain
             .validator_pubkey_cache
             .try_read_for(VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT)
             .ok_or(BeaconChainError::ValidatorPubkeyCacheLockTimeout)?;
 
-        let fork = chain.with_head(|head| Ok::<_, BeaconChainError>(head.beacon_state.fork()))?;
-
         let mut signature_sets = Vec::with_capacity(num_partially_verified);
 
         // Iterate, flattening to get only the `Ok` values.

beacon_node/beacon_chain/src/beacon_fork_choice_store.rs

@@ -7,13 +7,17 @@
 use crate::{metrics, BeaconSnapshot};
 use derivative::Derivative;
 use fork_choice::ForkChoiceStore;
+use proto_array::JustifiedBalances;
+use safe_arith::ArithError;
 use ssz_derive::{Decode, Encode};
+use std::collections::BTreeSet;
 use std::marker::PhantomData;
 use std::sync::Arc;
 use store::{Error as StoreError, HotColdDB, ItemStore};
 use superstruct::superstruct;
 use types::{
-    BeaconBlock, BeaconState, BeaconStateError, Checkpoint, Epoch, EthSpec, Hash256, Slot,
+    BeaconBlockRef, BeaconState, BeaconStateError, Checkpoint, Epoch, EthSpec, ExecPayload,
+    Hash256, Slot,
 };
 
 #[derive(Debug)]
@@ -29,6 +33,7 @@ pub enum Error {
     MissingState(Hash256),
     InvalidPersistedBytes(ssz::DecodeError),
     BeaconStateError(BeaconStateError),
+    Arith(ArithError),
 }
 
 impl From<BeaconStateError> for Error {
@@ -37,29 +42,17 @@ impl From<BeaconStateError> for Error {
     }
 }
 
+impl From<ArithError> for Error {
+    fn from(e: ArithError) -> Self {
+        Error::Arith(e)
+    }
+}
+
 /// The number of validator balance sets that are cached within `BalancesCache`.
 const MAX_BALANCE_CACHE_SIZE: usize = 4;
 
-/// Returns the effective balances for every validator in the given `state`.
-///
-/// Any validator who is not active in the epoch of the given `state` is assigned a balance of
-/// zero.
-pub fn get_effective_balances<T: EthSpec>(state: &BeaconState<T>) -> Vec<u64> {
-    state
-        .validators()
-        .iter()
-        .map(|validator| {
-            if validator.is_active_at(state.current_epoch()) {
-                validator.effective_balance
-            } else {
-                0
-            }
-        })
-        .collect()
-}
-
 #[superstruct(
-    variants(V1, V8),
+    variants(V8),
     variant_attributes(derive(PartialEq, Clone, Debug, Encode, Decode)),
     no_enum
 )]
@@ -73,13 +66,11 @@ pub(crate) struct CacheItem {
 pub(crate) type CacheItem = CacheItemV8;
 
 #[superstruct(
-    variants(V1, V8),
+    variants(V8),
     variant_attributes(derive(PartialEq, Clone, Default, Debug, Encode, Decode)),
     no_enum
 )]
 pub struct BalancesCache {
-    #[superstruct(only(V1))]
-    pub(crate) items: Vec<CacheItemV1>,
     #[superstruct(only(V8))]
     pub(crate) items: Vec<CacheItemV8>,
 }
@@ -113,7 +104,7 @@ impl BalancesCache {
             let item = CacheItem {
                 block_root: epoch_boundary_root,
                 epoch,
-                balances: get_effective_balances(state),
+                balances: JustifiedBalances::from_justified_state(state)?.effective_balances,
             };
 
             if self.items.len() == MAX_BALANCE_CACHE_SIZE {
@@ -152,9 +143,12 @@ pub struct BeaconForkChoiceStore<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<
     time: Slot,
     finalized_checkpoint: Checkpoint,
     justified_checkpoint: Checkpoint,
-    justified_balances: Vec<u64>,
+    justified_balances: JustifiedBalances,
     best_justified_checkpoint: Checkpoint,
+    unrealized_justified_checkpoint: Checkpoint,
+    unrealized_finalized_checkpoint: Checkpoint,
     proposer_boost_root: Hash256,
+    equivocating_indices: BTreeSet<u64>,
     _phantom: PhantomData<E>,
 }
 
@@ -178,7 +172,7 @@ where
     pub fn get_forkchoice_store(
         store: Arc<HotColdDB<E, Hot, Cold>>,
         anchor: &BeaconSnapshot<E>,
-    ) -> Self {
+    ) -> Result<Self, Error> {
         let anchor_state = &anchor.beacon_state;
         let mut anchor_block_header = anchor_state.latest_block_header().clone();
         if anchor_block_header.state_root == Hash256::zero() {
@@ -191,18 +185,22 @@ where
             root: anchor_root,
         };
         let finalized_checkpoint = justified_checkpoint;
+        let justified_balances = JustifiedBalances::from_justified_state(anchor_state)?;
 
-        Self {
+        Ok(Self {
             store,
             balances_cache: <_>::default(),
             time: anchor_state.slot(),
             justified_checkpoint,
-            justified_balances: anchor_state.balances().clone().into(),
+            justified_balances,
             finalized_checkpoint,
             best_justified_checkpoint: justified_checkpoint,
+            unrealized_justified_checkpoint: justified_checkpoint,
+            unrealized_finalized_checkpoint: finalized_checkpoint,
             proposer_boost_root: Hash256::zero(),
+            equivocating_indices: BTreeSet::new(),
             _phantom: PhantomData,
-        }
+        })
     }
 
     /// Save the current state of `Self` to a `PersistedForkChoiceStore` which can be stored to the
@@ -213,9 +211,12 @@ where
             time: self.time,
             finalized_checkpoint: self.finalized_checkpoint,
             justified_checkpoint: self.justified_checkpoint,
-            justified_balances: self.justified_balances.clone(),
+            justified_balances: self.justified_balances.effective_balances.clone(),
             best_justified_checkpoint: self.best_justified_checkpoint,
+            unrealized_justified_checkpoint: self.unrealized_justified_checkpoint,
+            unrealized_finalized_checkpoint: self.unrealized_finalized_checkpoint,
             proposer_boost_root: self.proposer_boost_root,
+            equivocating_indices: self.equivocating_indices.clone(),
         }
     }
 
@@ -224,15 +225,20 @@ where
         persisted: PersistedForkChoiceStore,
         store: Arc<HotColdDB<E, Hot, Cold>>,
     ) -> Result<Self, Error> {
+        let justified_balances =
+            JustifiedBalances::from_effective_balances(persisted.justified_balances)?;
         Ok(Self {
             store,
             balances_cache: persisted.balances_cache,
             time: persisted.time,
             finalized_checkpoint: persisted.finalized_checkpoint,
             justified_checkpoint: persisted.justified_checkpoint,
-            justified_balances: persisted.justified_balances,
+            justified_balances,
             best_justified_checkpoint: persisted.best_justified_checkpoint,
+            unrealized_justified_checkpoint: persisted.unrealized_justified_checkpoint,
+            unrealized_finalized_checkpoint: persisted.unrealized_finalized_checkpoint,
             proposer_boost_root: persisted.proposer_boost_root,
+            equivocating_indices: persisted.equivocating_indices,
             _phantom: PhantomData,
         })
     }
@@ -254,9 +260,9 @@ where
         self.time = slot
     }
 
-    fn on_verified_block(
+    fn on_verified_block<Payload: ExecPayload<E>>(
         &mut self,
-        _block: &BeaconBlock<E>,
+        _block: BeaconBlockRef<E, Payload>,
         block_root: Hash256,
         state: &BeaconState<E>,
     ) -> Result<(), Self::Error> {
@@ -267,7 +273,7 @@ where
         &self.justified_checkpoint
     }
 
-    fn justified_balances(&self) -> &[u64] {
+    fn justified_balances(&self) -> &JustifiedBalances {
         &self.justified_balances
     }
 
@@ -279,6 +285,14 @@ where
         &self.finalized_checkpoint
     }
 
+    fn unrealized_justified_checkpoint(&self) -> &Checkpoint {
+        &self.unrealized_justified_checkpoint
+    }
+
+    fn unrealized_finalized_checkpoint(&self) -> &Checkpoint {
+        &self.unrealized_finalized_checkpoint
+    }
+
     fn proposer_boost_root(&self) -> Hash256 {
         self.proposer_boost_root
     }
@@ -294,13 +308,14 @@ where
             self.justified_checkpoint.root,
             self.justified_checkpoint.epoch,
         ) {
+            // NOTE: could avoid this re-calculation by introducing a `PersistedCacheItem`.
             metrics::inc_counter(&metrics::BALANCES_CACHE_HITS);
-            self.justified_balances = balances;
+            self.justified_balances = JustifiedBalances::from_effective_balances(balances)?;
         } else {
             metrics::inc_counter(&metrics::BALANCES_CACHE_MISSES);
             let justified_block = self
                 .store
-                .get_block(&self.justified_checkpoint.root)
+                .get_blinded_block(&self.justified_checkpoint.root, None)
                 .map_err(Error::FailedToReadBlock)?
                 .ok_or(Error::MissingBlock(self.justified_checkpoint.root))?
                 .deconstruct()
@@ -312,7 +327,7 @@ where
                 .map_err(Error::FailedToReadState)?
                 .ok_or_else(|| Error::MissingState(justified_block.state_root()))?;
 
-            self.justified_balances = get_effective_balances(&state);
+            self.justified_balances = JustifiedBalances::from_justified_state(&state)?;
         }
 
         Ok(())
@@ -322,29 +337,45 @@ where
         self.best_justified_checkpoint = checkpoint
     }
 
+    fn set_unrealized_justified_checkpoint(&mut self, checkpoint: Checkpoint) {
+        self.unrealized_justified_checkpoint = checkpoint;
+    }
+
+    fn set_unrealized_finalized_checkpoint(&mut self, checkpoint: Checkpoint) {
+        self.unrealized_finalized_checkpoint = checkpoint;
+    }
+
     fn set_proposer_boost_root(&mut self, proposer_boost_root: Hash256) {
         self.proposer_boost_root = proposer_boost_root;
     }
+
+    fn equivocating_indices(&self) -> &BTreeSet<u64> {
+        &self.equivocating_indices
+    }
+
+    fn extend_equivocating_indices(&mut self, indices: impl IntoIterator<Item = u64>) {
+        self.equivocating_indices.extend(indices);
+    }
 }
 
 /// A container which allows persisting the `BeaconForkChoiceStore` to the on-disk database.
-#[superstruct(
-    variants(V1, V7, V8),
-    variant_attributes(derive(Encode, Decode)),
-    no_enum
-)]
+#[superstruct(variants(V11), variant_attributes(derive(Encode, Decode)), no_enum)]
 pub struct PersistedForkChoiceStore {
-    #[superstruct(only(V1, V7))]
-    pub balances_cache: BalancesCacheV1,
-    #[superstruct(only(V8))]
+    #[superstruct(only(V11))]
     pub balances_cache: BalancesCacheV8,
     pub time: Slot,
     pub finalized_checkpoint: Checkpoint,
     pub justified_checkpoint: Checkpoint,
     pub justified_balances: Vec<u64>,
     pub best_justified_checkpoint: Checkpoint,
-    #[superstruct(only(V7, V8))]
+    #[superstruct(only(V11))]
+    pub unrealized_justified_checkpoint: Checkpoint,
+    #[superstruct(only(V11))]
+    pub unrealized_finalized_checkpoint: Checkpoint,
+    #[superstruct(only(V11))]
     pub proposer_boost_root: Hash256,
+    #[superstruct(only(V11))]
+    pub equivocating_indices: BTreeSet<u64>,
 }
 
-pub type PersistedForkChoiceStore = PersistedForkChoiceStoreV8;
+pub type PersistedForkChoiceStore = PersistedForkChoiceStoreV11;

beacon_node/beacon_chain/src/beacon_proposer_cache.rs

@@ -8,9 +8,15 @@
 //! very simple to reason about, but it might store values that are useless due to finalization. The
 //! values it stores are very small, so this should not be an issue.
 
+use crate::{BeaconChain, BeaconChainError, BeaconChainTypes};
+use fork_choice::ExecutionStatus;
 use lru::LruCache;
 use smallvec::SmallVec;
-use types::{BeaconStateError, Epoch, EthSpec, Fork, Hash256, Slot, Unsigned};
+use state_processing::state_advance::partial_state_advance;
+use std::cmp::Ordering;
+use types::{
+    BeaconState, BeaconStateError, ChainSpec, Epoch, EthSpec, Fork, Hash256, Slot, Unsigned,
+};
 
 /// The number of sets of proposer indices that should be cached.
 const CACHE_SIZE: usize = 16;
@@ -125,3 +131,68 @@ impl BeaconProposerCache {
         Ok(())
     }
 }
+
+/// Compute the proposer duties using the head state without cache.
+pub fn compute_proposer_duties_from_head<T: BeaconChainTypes>(
+    current_epoch: Epoch,
+    chain: &BeaconChain<T>,
+) -> Result<(Vec<usize>, Hash256, ExecutionStatus, Fork), BeaconChainError> {
+    // Atomically collect information about the head whilst holding the canonical head `Arc` as
+    // short as possible.
+    let (mut state, head_state_root, head_block_root) = {
+        let head = chain.canonical_head.cached_head();
+        // Take a copy of the head state.
+        let head_state = head.snapshot.beacon_state.clone();
+        let head_state_root = head.head_state_root();
+        let head_block_root = head.head_block_root();
+        (head_state, head_state_root, head_block_root)
+    };
+
+    let execution_status = chain
+        .canonical_head
+        .fork_choice_read_lock()
+        .get_block_execution_status(&head_block_root)
+        .ok_or(BeaconChainError::HeadMissingFromForkChoice(head_block_root))?;
+
+    // Advance the state into the requested epoch.
+    ensure_state_is_in_epoch(&mut state, head_state_root, current_epoch, &chain.spec)?;
+
+    let indices = state
+        .get_beacon_proposer_indices(&chain.spec)
+        .map_err(BeaconChainError::from)?;
+
+    let dependent_root = state
+        // The only block which decides its own shuffling is the genesis block.
+        .proposer_shuffling_decision_root(chain.genesis_block_root)
+        .map_err(BeaconChainError::from)?;
+
+    Ok((indices, dependent_root, execution_status, state.fork()))
+}
+
+/// If required, advance `state` to `target_epoch`.
+///
+/// ## Details
+///
+/// - Returns an error if `state.current_epoch() > target_epoch`.
+/// - No-op if `state.current_epoch() == target_epoch`.
+/// - It must be the case that `state.canonical_root() == state_root`, but this function will not
+///     check that.
+pub fn ensure_state_is_in_epoch<E: EthSpec>(
+    state: &mut BeaconState<E>,
+    state_root: Hash256,
+    target_epoch: Epoch,
+    spec: &ChainSpec,
+) -> Result<(), BeaconChainError> {
+    match state.current_epoch().cmp(&target_epoch) {
+        // Protects against an inconsistent slot clock.
+        Ordering::Greater => Err(BeaconStateError::SlotOutOfBounds.into()),
+        // The state needs to be advanced.
+        Ordering::Less => {
+            let target_slot = target_epoch.start_slot(E::slots_per_epoch());
+            partial_state_advance(state, Some(state_root), target_slot, spec)
+                .map_err(BeaconChainError::from)
+        }
+        // The state is suitable, nothing to do.
+        Ordering::Equal => Ok(()),
+    }
+}

beacon_node/beacon_chain/src/beacon_snapshot.rs

@@ -1,19 +1,36 @@
 use serde_derive::Serialize;
-use types::{beacon_state::CloneConfig, BeaconState, EthSpec, Hash256, SignedBeaconBlock};
+use std::sync::Arc;
+use types::{
+    BeaconState, EthSpec, ExecPayload, FullPayload, Hash256, SignedBeaconBlock,
+    SignedBlindedBeaconBlock,
+};
 
 /// Represents some block and its associated state. Generally, this will be used for tracking the
 /// head, justified head and finalized head.
 #[derive(Clone, Serialize, PartialEq, Debug)]
-pub struct BeaconSnapshot<E: EthSpec> {
-    pub beacon_block: SignedBeaconBlock<E>,
+pub struct BeaconSnapshot<E: EthSpec, Payload: ExecPayload<E> = FullPayload<E>> {
+    pub beacon_block: Arc<SignedBeaconBlock<E, Payload>>,
     pub beacon_block_root: Hash256,
     pub beacon_state: BeaconState<E>,
 }
 
-impl<E: EthSpec> BeaconSnapshot<E> {
+/// This snapshot is to be used for verifying a child of `self.beacon_block`.
+#[derive(Debug)]
+pub struct PreProcessingSnapshot<T: EthSpec> {
+    /// This state is equivalent to the `self.beacon_block.state_root()` state that has been
+    /// advanced forward one slot using `per_slot_processing`. This state is "primed and ready" for
+    /// the application of another block.
+    pub pre_state: BeaconState<T>,
+    /// This value is only set to `Some` if the `pre_state` was *not* advanced forward.
+    pub beacon_state_root: Option<Hash256>,
+    pub beacon_block: SignedBlindedBeaconBlock<T>,
+    pub beacon_block_root: Hash256,
+}
+
+impl<E: EthSpec, Payload: ExecPayload<E>> BeaconSnapshot<E, Payload> {
     /// Create a new checkpoint.
     pub fn new(
-        beacon_block: SignedBeaconBlock<E>,
+        beacon_block: Arc<SignedBeaconBlock<E, Payload>>,
         beacon_block_root: Hash256,
         beacon_state: BeaconState<E>,
     ) -> Self {
@@ -36,7 +53,7 @@ impl<E: EthSpec> BeaconSnapshot<E> {
     /// Update all fields of the checkpoint.
     pub fn update(
         &mut self,
-        beacon_block: SignedBeaconBlock<E>,
+        beacon_block: Arc<SignedBeaconBlock<E, Payload>>,
         beacon_block_root: Hash256,
         beacon_state: BeaconState<E>,
     ) {
@@ -44,12 +61,4 @@ impl<E: EthSpec> BeaconSnapshot<E> {
         self.beacon_block_root = beacon_block_root;
         self.beacon_state = beacon_state;
     }
-
-    pub fn clone_with(&self, clone_config: CloneConfig) -> Self {
-        Self {
-            beacon_block: self.beacon_block.clone(),
-            beacon_block_root: self.beacon_block_root,
-            beacon_state: self.beacon_state.clone_with(clone_config),
-        }
-    }
 }

beacon_node/beacon_chain/src/block_reward.rs

@@ -1,29 +1,50 @@
 use crate::{BeaconChain, BeaconChainError, BeaconChainTypes};
 use eth2::lighthouse::{AttestationRewards, BlockReward, BlockRewardMeta};
-use operation_pool::{AttMaxCover, MaxCover};
-use state_processing::per_block_processing::altair::sync_committee::compute_sync_aggregate_rewards;
-use types::{BeaconBlockRef, BeaconState, EthSpec, Hash256, RelativeEpoch};
+use operation_pool::{AttMaxCover, MaxCover, RewardCache, SplitAttestation};
+use state_processing::{
+    common::get_attesting_indices_from_state,
+    per_block_processing::altair::sync_committee::compute_sync_aggregate_rewards,
+};
+use types::{BeaconBlockRef, BeaconState, EthSpec, ExecPayload, Hash256};
 
 impl<T: BeaconChainTypes> BeaconChain<T> {
-    pub fn compute_block_reward(
+    pub fn compute_block_reward<Payload: ExecPayload<T::EthSpec>>(
         &self,
-        block: BeaconBlockRef<'_, T::EthSpec>,
+        block: BeaconBlockRef<'_, T::EthSpec, Payload>,
         block_root: Hash256,
         state: &BeaconState<T::EthSpec>,
+        reward_cache: &mut RewardCache,
+        include_attestations: bool,
     ) -> Result<BlockReward, BeaconChainError> {
         if block.slot() != state.slot() {
             return Err(BeaconChainError::BlockRewardSlotError);
         }
 
-        let active_indices = state.get_cached_active_validator_indices(RelativeEpoch::Current)?;
-        let total_active_balance = state.get_total_balance(active_indices, &self.spec)?;
-        let mut per_attestation_rewards = block
+        reward_cache.update(state)?;
+
+        let total_active_balance = state.get_total_active_balance()?;
+
+        let split_attestations = block
             .body()
             .attestations()
             .iter()
             .map(|att| {
-                AttMaxCover::new(att, state, total_active_balance, &self.spec)
-                    .ok_or(BeaconChainError::BlockRewardAttestationError)
+                let attesting_indices = get_attesting_indices_from_state(state, att)?;
+                Ok(SplitAttestation::new(att.clone(), attesting_indices))
+            })
+            .collect::<Result<Vec<_>, BeaconChainError>>()?;
+
+        let mut per_attestation_rewards = split_attestations
+            .iter()
+            .map(|att| {
+                AttMaxCover::new(
+                    att.as_ref(),
+                    state,
+                    reward_cache,
+                    total_active_balance,
+                    &self.spec,
+                )
+                .ok_or(BeaconChainError::BlockRewardAttestationError)
             })
             .collect::<Result<Vec<_>, _>>()?;
 
@@ -34,7 +55,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             let latest_att = &updated[i];
 
             for att in to_update {
-                att.update_covering_set(latest_att.object(), latest_att.covering_set());
+                att.update_covering_set(latest_att.intermediate(), latest_att.covering_set());
             }
         }
 
@@ -60,11 +81,24 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             .map(|cover| cover.fresh_validators_rewards)
             .collect();
 
+        // Add the attestation data if desired.
+        let attestations = if include_attestations {
+            block
+                .body()
+                .attestations()
+                .iter()
+                .map(|a| a.data.clone())
+                .collect()
+        } else {
+            vec![]
+        };
+
         let attestation_rewards = AttestationRewards {
             total: attestation_total,
             prev_epoch_total,
             curr_epoch_total,
             per_attestation_rewards,
+            attestations,
         };
 
         // Sync committee rewards.

beacon_node/beacon_chain/src/block_times_cache.rs

@@ -18,6 +18,7 @@ type BlockRoot = Hash256;
 #[derive(Clone, Default)]
 pub struct Timestamps {
     pub observed: Option<Duration>,
+    pub attestable: Option<Duration>,
     pub imported: Option<Duration>,
     pub set_as_head: Option<Duration>,
 }
@@ -26,6 +27,7 @@ pub struct Timestamps {
 #[derive(Default)]
 pub struct BlockDelays {
     pub observed: Option<Duration>,
+    pub attestable: Option<Duration>,
     pub imported: Option<Duration>,
     pub set_as_head: Option<Duration>,
 }
@@ -35,6 +37,9 @@ impl BlockDelays {
         let observed = times
             .observed
             .and_then(|observed_time| observed_time.checked_sub(slot_start_time));
+        let attestable = times
+            .attestable
+            .and_then(|attestable_time| attestable_time.checked_sub(slot_start_time));
         let imported = times
             .imported
             .and_then(|imported_time| imported_time.checked_sub(times.observed?));
@@ -43,6 +48,7 @@ impl BlockDelays {
             .and_then(|set_as_head_time| set_as_head_time.checked_sub(times.imported?));
         BlockDelays {
             observed,
+            attestable,
             imported,
             set_as_head,
         }
@@ -99,6 +105,14 @@ impl BlockTimesCache {
         };
     }
 
+    pub fn set_time_attestable(&mut self, block_root: BlockRoot, slot: Slot, timestamp: Duration) {
+        let block_times = self
+            .cache
+            .entry(block_root)
+            .or_insert_with(|| BlockTimesCacheValue::new(slot));
+        block_times.timestamps.attestable = Some(timestamp);
+    }
+
     pub fn set_time_imported(&mut self, block_root: BlockRoot, slot: Slot, timestamp: Duration) {
         let block_times = self
             .cache

beacon_node/beacon_chain/src/builder.rs

@@ -1,14 +1,14 @@
-use crate::beacon_chain::{BEACON_CHAIN_DB_KEY, ETH1_CACHE_DB_KEY, OP_POOL_DB_KEY};
+use crate::beacon_chain::{CanonicalHead, BEACON_CHAIN_DB_KEY, ETH1_CACHE_DB_KEY, OP_POOL_DB_KEY};
 use crate::eth1_chain::{CachingEth1Backend, SszEth1};
+use crate::eth1_finalization_cache::Eth1FinalizationCache;
+use crate::fork_choice_signal::ForkChoiceSignalTx;
 use crate::fork_revert::{reset_fork_choice_to_finalization, revert_to_fork_boundary};
 use crate::head_tracker::HeadTracker;
 use crate::migrate::{BackgroundMigrator, MigratorConfig};
 use crate::persisted_beacon_chain::PersistedBeaconChain;
 use crate::shuffling_cache::ShufflingCache;
-use crate::snapshot_cache::{SnapshotCache, DEFAULT_SNAPSHOT_CACHE_SIZE};
 use crate::timeout_rw_lock::TimeoutRwLock;
 use crate::validator_monitor::ValidatorMonitor;
-use crate::validator_pubkey_cache::ValidatorPubkeyCache;
 use crate::ChainConfig;
 use crate::{
     BeaconChain, BeaconChainTypes, BeaconForkChoiceStore, BeaconSnapshot, Eth1Chain,
@@ -16,10 +16,11 @@ use crate::{
 };
 use eth1::Config as Eth1Config;
 use execution_layer::ExecutionLayer;
-use fork_choice::ForkChoice;
+use fork_choice::{ForkChoice, ResetPayloadStatuses};
 use futures::channel::mpsc::Sender;
 use operation_pool::{OperationPool, PersistedOperationPool};
 use parking_lot::RwLock;
+use proto_array::ReOrgThreshold;
 use slasher::Slasher;
 use slog::{crit, error, info, Logger};
 use slot_clock::{SlotClock, TestingSlotClock};
@@ -27,10 +28,10 @@ use std::marker::PhantomData;
 use std::sync::Arc;
 use std::time::Duration;
 use store::{Error as StoreError, HotColdDB, ItemStore, KeyValueStoreOp};
-use task_executor::ShutdownReason;
+use task_executor::{ShutdownReason, TaskExecutor};
 use types::{
-    BeaconBlock, BeaconState, ChainSpec, Checkpoint, EthSpec, Graffiti, Hash256, PublicKeyBytes,
-    Signature, SignedBeaconBlock, Slot,
+    BeaconBlock, BeaconState, ChainSpec, Checkpoint, Epoch, EthSpec, Graffiti, Hash256,
+    PublicKeyBytes, Signature, SignedBeaconBlock, Slot,
 };
 
 /// An empty struct used to "witness" all the `BeaconChainTypes` traits. It has no user-facing
@@ -76,12 +77,11 @@ pub struct BeaconChainBuilder<T: BeaconChainTypes> {
     >,
     op_pool: Option<OperationPool<T::EthSpec>>,
     eth1_chain: Option<Eth1Chain<T::Eth1Chain, T::EthSpec>>,
-    execution_layer: Option<ExecutionLayer>,
+    execution_layer: Option<ExecutionLayer<T::EthSpec>>,
     event_handler: Option<ServerSentEventHandler<T::EthSpec>>,
     slot_clock: Option<T::SlotClock>,
     shutdown_sender: Option<Sender<ShutdownReason>>,
     head_tracker: Option<HeadTracker>,
-    validator_pubkey_cache: Option<ValidatorPubkeyCache<T>>,
     spec: ChainSpec,
     chain_config: ChainConfig,
     log: Option<Logger>,
@@ -91,6 +91,7 @@ pub struct BeaconChainBuilder<T: BeaconChainTypes> {
     // Pending I/O batch that is constructed during building and should be executed atomically
     // alongside `PersistedBeaconChain` storage when `BeaconChainBuilder::build` is called.
     pending_io_batch: Vec<KeyValueStoreOp>,
+    task_executor: Option<TaskExecutor>,
 }
 
 impl<TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>
@@ -121,7 +122,6 @@ where
             slot_clock: None,
             shutdown_sender: None,
             head_tracker: None,
-            validator_pubkey_cache: None,
             spec: TEthSpec::default_spec(),
             chain_config: ChainConfig::default(),
             log: None,
@@ -129,6 +129,7 @@ where
             slasher: None,
             validator_monitor: None,
             pending_io_batch: vec![],
+            task_executor: None,
         }
     }
 
@@ -155,6 +156,21 @@ where
         self
     }
 
+    /// Sets the proposer re-org threshold.
+    pub fn proposer_re_org_threshold(mut self, threshold: Option<ReOrgThreshold>) -> Self {
+        self.chain_config.re_org_threshold = threshold;
+        self
+    }
+
+    /// Sets the proposer re-org max epochs since finalization.
+    pub fn proposer_re_org_max_epochs_since_finalization(
+        mut self,
+        epochs_since_finalization: Epoch,
+    ) -> Self {
+        self.chain_config.re_org_max_epochs_since_finalization = epochs_since_finalization;
+        self
+    }
+
     /// Sets the store (database).
     ///
     /// Should generally be called early in the build chain.
@@ -182,6 +198,13 @@ where
         self.log = Some(log);
         self
     }
+
+    /// Sets the task executor.
+    pub fn task_executor(mut self, task_executor: TaskExecutor) -> Self {
+        self.task_executor = Some(task_executor);
+        self
+    }
+
     /// Attempt to load an existing eth1 cache from the builder's `Store`.
     pub fn get_persisted_eth1_backend(&self) -> Result<Option<SszEth1>, String> {
         let store = self
@@ -235,12 +258,18 @@ where
         let fork_choice =
             BeaconChain::<Witness<TSlotClock, TEth1Backend, _, _, _>>::load_fork_choice(
                 store.clone(),
+                ResetPayloadStatuses::always_reset_conditionally(
+                    self.chain_config.always_reset_payload_statuses,
+                ),
+                self.chain_config.count_unrealized_full,
+                &self.spec,
+                log,
             )
             .map_err(|e| format!("Unable to load fork choice from disk: {:?}", e))?
             .ok_or("Fork choice not found in store")?;
 
         let genesis_block = store
-            .get_block(&chain.genesis_block_root)
+            .get_blinded_block(&chain.genesis_block_root, Some(Slot::new(0)))
             .map_err(|e| descriptive_db_error("genesis block", &e))?
             .ok_or("Genesis block not found in store")?;
         let genesis_state = store
@@ -265,16 +294,12 @@ where
                 .unwrap_or_else(OperationPool::new),
         );
 
-        let pubkey_cache = ValidatorPubkeyCache::load_from_store(store)
-            .map_err(|e| format!("Unable to open persisted pubkey cache: {:?}", e))?;
-
         self.genesis_block_root = Some(chain.genesis_block_root);
         self.genesis_state_root = Some(genesis_block.state_root());
         self.head_tracker = Some(
             HeadTracker::from_ssz_container(&chain.ssz_head_tracker)
                 .map_err(|e| format!("Failed to decode head tracker for database: {:?}", e))?,
         );
-        self.validator_pubkey_cache = Some(pubkey_cache);
         self.fork_choice = Some(fork_choice);
 
         Ok(self)
@@ -295,6 +320,7 @@ where
             .ok_or("set_genesis_state requires a store")?;
 
         let beacon_block = genesis_block(&mut beacon_state, &self.spec)?;
+        let blinded_block = beacon_block.clone_as_blinded();
 
         beacon_state
             .build_all_caches(&self.spec)
@@ -303,16 +329,20 @@ where
         let beacon_state_root = beacon_block.message().state_root();
         let beacon_block_root = beacon_block.canonical_root();
 
+        store
+            .update_finalized_state(beacon_state_root, beacon_block_root, beacon_state.clone())
+            .map_err(|e| format!("Failed to set genesis state as finalized state: {:?}", e))?;
+
         store
             .put_state(&beacon_state_root, &beacon_state)
             .map_err(|e| format!("Failed to store genesis state: {:?}", e))?;
         store
-            .put_block(&beacon_block_root, beacon_block.clone())
+            .put_cold_blinded_block(&beacon_block_root, &blinded_block)
             .map_err(|e| format!("Failed to store genesis block: {:?}", e))?;
 
         // Store the genesis block under the `ZERO_HASH` key.
         store
-            .put_block(&Hash256::zero(), beacon_block.clone())
+            .put_cold_blinded_block(&Hash256::zero(), &blinded_block)
             .map_err(|e| {
                 format!(
                     "Failed to store genesis block under 0x00..00 alias: {:?}",
@@ -327,7 +357,7 @@ where
         Ok((
             BeaconSnapshot {
                 beacon_block_root,
-                beacon_block,
+                beacon_block: Arc::new(beacon_block),
                 beacon_state,
             },
             self,
@@ -341,13 +371,18 @@ where
         let (genesis, updated_builder) = self.set_genesis_state(beacon_state)?;
         self = updated_builder;
 
-        let fc_store = BeaconForkChoiceStore::get_forkchoice_store(store, &genesis);
+        let fc_store = BeaconForkChoiceStore::get_forkchoice_store(store, &genesis)
+            .map_err(|e| format!("Unable to initialize fork choice store: {e:?}"))?;
+        let current_slot = None;
 
         let fork_choice = ForkChoice::from_anchor(
             fc_store,
             genesis.beacon_block_root,
             &genesis.beacon_block,
             &genesis.beacon_state,
+            current_slot,
+            self.chain_config.count_unrealized_full,
+            &self.spec,
         )
         .map_err(|e| format!("Unable to initialize ForkChoice: {:?}", e))?;
 
@@ -389,6 +424,12 @@ where
             ));
         }
 
+        // Prime all caches before storing the state in the database and computing the tree hash
+        // root.
+        weak_subj_state
+            .build_all_caches(&self.spec)
+            .map_err(|e| format!("Error building caches on checkpoint state: {e:?}"))?;
+
         let computed_state_root = weak_subj_state
             .update_tree_hash_cache()
             .map_err(|e| format!("Error computing checkpoint state root: {:?}", e))?;
@@ -417,8 +458,21 @@ where
         let (_, updated_builder) = self.set_genesis_state(genesis_state)?;
         self = updated_builder;
 
+        // Build the committee caches before storing. The database assumes that states have
+        // committee caches built before storing.
+        weak_subj_state
+            .build_all_committee_caches(&self.spec)
+            .map_err(|e| format!("Error building caches on checkpoint state: {:?}", e))?;
+
         // Write the state and block non-atomically, it doesn't matter if they're forgotten
         // about on a crash restart.
+        store
+            .update_finalized_state(
+                weak_subj_state_root,
+                weak_subj_block_root,
+                weak_subj_state.clone(),
+            )
+            .map_err(|e| format!("Failed to set checkpoint state as finalized state: {:?}", e))?;
         store
             .put_state(&weak_subj_state_root, &weak_subj_state)
             .map_err(|e| format!("Failed to store weak subjectivity state: {:?}", e))?;
@@ -429,7 +483,11 @@ where
         // Stage the database's metadata fields for atomic storage when `build` is called.
         // This prevents the database from restarting in an inconsistent state if the anchor
         // info or split point is written before the `PersistedBeaconChain`.
-        self.pending_io_batch.push(store.store_split_in_batch());
+        self.pending_io_batch.push(
+            store
+                .store_split_in_batch()
+                .map_err(|e| format!("Failed to store split: {:?}", e))?,
+        );
         self.pending_io_batch.push(
             store
                 .init_anchor_info(weak_subj_block.message())
@@ -437,25 +495,33 @@ where
         );
 
         // Store pruning checkpoint to prevent attempting to prune before the anchor state.
-        self.pending_io_batch
-            .push(store.pruning_checkpoint_store_op(Checkpoint {
-                root: weak_subj_block_root,
-                epoch: weak_subj_state.slot().epoch(TEthSpec::slots_per_epoch()),
-            }));
+        self.pending_io_batch.push(
+            store
+                .pruning_checkpoint_store_op(Checkpoint {
+                    root: weak_subj_block_root,
+                    epoch: weak_subj_state.slot().epoch(TEthSpec::slots_per_epoch()),
+                })
+                .map_err(|e| format!("{:?}", e))?,
+        );
 
         let snapshot = BeaconSnapshot {
             beacon_block_root: weak_subj_block_root,
-            beacon_block: weak_subj_block,
+            beacon_block: Arc::new(weak_subj_block),
             beacon_state: weak_subj_state,
         };
 
-        let fc_store = BeaconForkChoiceStore::get_forkchoice_store(store, &snapshot);
+        let fc_store = BeaconForkChoiceStore::get_forkchoice_store(store, &snapshot)
+            .map_err(|e| format!("Unable to initialize fork choice store: {e:?}"))?;
 
+        let current_slot = Some(snapshot.beacon_block.slot());
         let fork_choice = ForkChoice::from_anchor(
             fc_store,
             snapshot.beacon_block_root,
             &snapshot.beacon_block,
             &snapshot.beacon_state,
+            current_slot,
+            self.chain_config.count_unrealized_full,
+            &self.spec,
         )
         .map_err(|e| format!("Unable to initialize ForkChoice: {:?}", e))?;
 
@@ -471,7 +537,7 @@ where
     }
 
     /// Sets the `BeaconChain` execution layer.
-    pub fn execution_layer(mut self, execution_layer: Option<ExecutionLayer>) -> Self {
+    pub fn execution_layer(mut self, execution_layer: Option<ExecutionLayer<TEthSpec>>) -> Self {
         self.execution_layer = execution_layer;
         self
     }
@@ -530,11 +596,13 @@ where
         mut self,
         auto_register: bool,
         validators: Vec<PublicKeyBytes>,
+        individual_metrics_threshold: usize,
         log: Logger,
     ) -> Self {
         self.validator_monitor = Some(ValidatorMonitor::new(
             validators,
             auto_register,
+            individual_metrics_threshold,
             log.clone(),
         ));
         self
@@ -588,7 +656,7 @@ where
         // Try to decode the head block according to the current fork, if that fails, try
         // to backtrack to before the most recent fork.
         let (head_block_root, head_block, head_reverted) =
-            match store.get_block(&initial_head_block_root) {
+            match store.get_full_block(&initial_head_block_root, None) {
                 Ok(Some(block)) => (initial_head_block_root, block, false),
                 Ok(None) => return Err("Head block not found in store".into()),
                 Err(StoreError::SszDecodeError(_)) => {
@@ -628,17 +696,20 @@ where
                 head_block_root,
                 &head_state,
                 store.clone(),
+                Some(current_slot),
                 &self.spec,
+                self.chain_config.count_unrealized.into(),
+                self.chain_config.count_unrealized_full,
             )?;
         }
 
-        let mut canonical_head = BeaconSnapshot {
+        let mut head_snapshot = BeaconSnapshot {
             beacon_block_root: head_block_root,
-            beacon_block: head_block,
+            beacon_block: Arc::new(head_block),
             beacon_state: head_state,
         };
 
-        canonical_head
+        head_snapshot
             .beacon_state
             .build_all_caches(&self.spec)
             .map_err(|e| format!("Failed to build state caches: {:?}", e))?;
@@ -648,27 +719,26 @@ where
         //
         // This is a sanity check to detect database corruption.
         let fc_finalized = fork_choice.finalized_checkpoint();
-        let head_finalized = canonical_head.beacon_state.finalized_checkpoint();
-        if fc_finalized != head_finalized {
-            let is_genesis = head_finalized.root.is_zero()
-                && head_finalized.epoch == fc_finalized.epoch
-                && fc_finalized.root == genesis_block_root;
-            let is_wss = store.get_anchor_slot().map_or(false, |anchor_slot| {
-                fc_finalized.epoch == anchor_slot.epoch(TEthSpec::slots_per_epoch())
-            });
-            if !is_genesis && !is_wss {
-                return Err(format!(
-                    "Database corrupt: fork choice is finalized at {:?} whilst head is finalized at \
+        let head_finalized = head_snapshot.beacon_state.finalized_checkpoint();
+        if fc_finalized.epoch < head_finalized.epoch {
+            return Err(format!(
+                "Database corrupt: fork choice is finalized at {:?} whilst head is finalized at \
                     {:?}",
-                    fc_finalized, head_finalized
-                ));
-            }
+                fc_finalized, head_finalized
+            ));
         }
 
-        let validator_pubkey_cache = self.validator_pubkey_cache.map(Ok).unwrap_or_else(|| {
-            ValidatorPubkeyCache::new(&canonical_head.beacon_state, store.clone())
-                .map_err(|e| format!("Unable to init validator pubkey cache: {:?}", e))
-        })?;
+        let validator_pubkey_cache = store.immutable_validators.clone();
+
+        // Update pubkey cache on first start in case we have started from genesis.
+        let kv_store_ops = validator_pubkey_cache
+            .write()
+            .import_new_pubkeys(&head_snapshot.beacon_state)
+            .map_err(|e| format!("error initializing pubkey cache: {e:?}"))?;
+        store
+            .hot_db
+            .do_atomically(kv_store_ops)
+            .map_err(|e| format!("error writing validator store: {e:?}"))?;
 
         let migrator_config = self.store_migrator_config.unwrap_or_default();
         let store_migrator = BackgroundMigrator::new(
@@ -681,10 +751,20 @@ where
         if let Some(slot) = slot_clock.now() {
             validator_monitor.process_valid_state(
                 slot.epoch(TEthSpec::slots_per_epoch()),
-                &canonical_head.beacon_state,
+                &head_snapshot.beacon_state,
             );
         }
 
+        // If enabled, set up the fork choice signaller.
+        let (fork_choice_signal_tx, fork_choice_signal_rx) =
+            if self.chain_config.fork_choice_before_proposal_timeout_ms != 0 {
+                let tx = ForkChoiceSignalTx::new();
+                let rx = tx.get_receiver();
+                (Some(tx), Some(rx))
+            } else {
+                (None, None)
+            };
+
         // Store the `PersistedBeaconChain` in the database atomically with the metadata so that on
         // restart we can correctly detect the presence of an initialized database.
         //
@@ -694,21 +774,28 @@ where
             Witness<TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>,
         >::persist_head_in_batch_standalone(
             genesis_block_root, &head_tracker
-        ));
+        ).map_err(|e| format!("{:?}", e))?);
         self.pending_io_batch.push(BeaconChain::<
             Witness<TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>,
         >::persist_fork_choice_in_batch_standalone(
             &fork_choice
-        ));
+        ).map_err(|e| format!("{:?}", e))?);
         store
             .hot_db
             .do_atomically(self.pending_io_batch)
             .map_err(|e| format!("Error writing chain & metadata to disk: {:?}", e))?;
 
+        let genesis_validators_root = head_snapshot.beacon_state.genesis_validators_root();
+        let genesis_time = head_snapshot.beacon_state.genesis_time();
+        let canonical_head = CanonicalHead::new(fork_choice, Arc::new(head_snapshot));
+
         let beacon_chain = BeaconChain {
             spec: self.spec,
             config: self.chain_config,
             store,
+            task_executor: self
+                .task_executor
+                .ok_or("Cannot build without task executor")?,
             store_migrator,
             slot_clock,
             op_pool: self.op_pool.ok_or("Cannot build without op pool")?,
@@ -736,24 +823,25 @@ where
             observed_voluntary_exits: <_>::default(),
             observed_proposer_slashings: <_>::default(),
             observed_attester_slashings: <_>::default(),
+            latest_seen_finality_update: <_>::default(),
+            latest_seen_optimistic_update: <_>::default(),
             eth1_chain: self.eth1_chain,
             execution_layer: self.execution_layer,
-            genesis_validators_root: canonical_head.beacon_state.genesis_validators_root(),
-            canonical_head: TimeoutRwLock::new(canonical_head.clone()),
+            genesis_validators_root,
+            genesis_time,
+            canonical_head,
             genesis_block_root,
             genesis_state_root,
-            fork_choice: RwLock::new(fork_choice),
+            fork_choice_signal_tx,
+            fork_choice_signal_rx,
             event_handler: self.event_handler,
             head_tracker,
-            snapshot_cache: TimeoutRwLock::new(SnapshotCache::new(
-                DEFAULT_SNAPSHOT_CACHE_SIZE,
-                canonical_head,
-            )),
             shuffling_cache: TimeoutRwLock::new(ShufflingCache::new()),
+            eth1_finalization_cache: TimeoutRwLock::new(Eth1FinalizationCache::new(log.clone())),
             beacon_proposer_cache: <_>::default(),
             block_times_cache: <_>::default(),
             pre_finalization_block_cache: <_>::default(),
-            validator_pubkey_cache: TimeoutRwLock::new(validator_pubkey_cache),
+            validator_pubkey_cache,
             attester_cache: <_>::default(),
             early_attester_cache: <_>::default(),
             shutdown_sender: self
@@ -765,9 +853,7 @@ where
             validator_monitor: RwLock::new(validator_monitor),
         };
 
-        let head = beacon_chain
-            .head()
-            .map_err(|e| format!("Failed to get head: {:?}", e))?;
+        let head = beacon_chain.head_snapshot();
 
         // Prime the attester cache with the head state.
         beacon_chain
@@ -813,6 +899,20 @@ where
             beacon_chain.store_migrator.process_reconstruction();
         }
 
+        // Prune finalized execution payloads in the background.
+        if beacon_chain.store.get_config().prune_payloads {
+            let store = beacon_chain.store.clone();
+            let log = log.clone();
+            beacon_chain.task_executor.spawn_blocking(
+                move || {
+                    if let Err(e) = store.try_prune_execution_payloads(false) {
+                        error!(log, "Error pruning payloads in background"; "error" => ?e);
+                    }
+                },
+                "prune_payloads_background",
+            );
+        }
+
         Ok(beacon_chain)
     }
 }
@@ -840,7 +940,7 @@ where
             .ok_or("dummy_eth1_backend requires a log")?;
 
         let backend =
-            CachingEth1Backend::new(Eth1Config::default(), log.clone(), self.spec.clone());
+            CachingEth1Backend::new(Eth1Config::default(), log.clone(), self.spec.clone())?;
 
         self.eth1_chain = Some(Eth1Chain::new_dummy(backend));
 
@@ -910,7 +1010,8 @@ fn descriptive_db_error(item: &str, error: &StoreError) -> String {
 #[cfg(test)]
 mod test {
     use super::*;
-    use eth2_hashing::hash;
+    use crate::validator_monitor::DEFAULT_INDIVIDUAL_TRACKING_THRESHOLD;
+    use ethereum_hashing::hash;
     use genesis::{
         generate_deterministic_keypairs, interop_genesis_state, DEFAULT_ETH1_BLOCK_HASH,
     };
@@ -919,6 +1020,7 @@ mod test {
     use std::time::Duration;
     use store::config::StoreConfig;
     use store::{HotColdDB, MemoryStore};
+    use task_executor::test_utils::TestRuntime;
     use types::{EthSpec, MinimalEthSpec, Slot};
 
     type TestEthSpec = MinimalEthSpec;
@@ -952,10 +1054,12 @@ mod test {
         .expect("should create interop genesis state");
 
         let (shutdown_tx, _) = futures::channel::mpsc::channel(1);
+        let runtime = TestRuntime::default();
 
         let chain = BeaconChainBuilder::new(MinimalEthSpec)
             .logger(log.clone())
             .store(Arc::new(store))
+            .task_executor(runtime.task_executor.clone())
             .genesis_state(genesis_state)
             .expect("should build state using recent genesis")
             .dummy_eth1_backend()
@@ -963,14 +1067,19 @@ mod test {
             .testing_slot_clock(Duration::from_secs(1))
             .expect("should configure testing slot clock")
             .shutdown_sender(shutdown_tx)
-            .monitor_validators(true, vec![], log.clone())
+            .monitor_validators(
+                true,
+                vec![],
+                DEFAULT_INDIVIDUAL_TRACKING_THRESHOLD,
+                log.clone(),
+            )
             .build()
             .expect("should build");
 
-        let head = chain.head().expect("should get head");
+        let head = chain.head_snapshot();
 
-        let state = head.beacon_state;
-        let block = head.beacon_block;
+        let state = &head.beacon_state;
+        let block = &head.beacon_block;
 
         assert_eq!(state.slot(), Slot::new(0), "should start from genesis");
         assert_eq!(
@@ -986,10 +1095,10 @@ mod test {
         assert_eq!(
             chain
                 .store
-                .get_block(&Hash256::zero())
+                .get_blinded_block(&Hash256::zero(), None)
                 .expect("should read db")
                 .expect("should find genesis block"),
-            block,
+            block.clone_as_blinded(),
             "should store genesis block under zero hash alias"
         );
         assert_eq!(
@@ -1041,14 +1150,15 @@ mod test {
         }
 
         for v in state.validators() {
-            let creds = v.withdrawal_credentials.as_bytes();
+            let creds = v.withdrawal_credentials();
+            let creds = creds.as_bytes();
             assert_eq!(
                 creds[0], spec.bls_withdrawal_prefix_byte,
                 "first byte of withdrawal creds should be bls prefix"
             );
             assert_eq!(
                 &creds[1..],
-                &hash(&v.pubkey.as_ssz_bytes())[1..],
+                &hash(&v.pubkey().as_ssz_bytes())[1..],
                 "rest of withdrawal creds should be pubkey hash"
             )
         }

beacon_node/beacon_chain/src/chain_config.rs

@@ -1,5 +1,17 @@
+pub use proto_array::{CountUnrealizedFull, ReOrgThreshold};
 use serde_derive::{Deserialize, Serialize};
-use types::Checkpoint;
+use std::time::Duration;
+use types::{Checkpoint, Epoch};
+
+pub const DEFAULT_RE_ORG_THRESHOLD: ReOrgThreshold = ReOrgThreshold(20);
+pub const DEFAULT_RE_ORG_MAX_EPOCHS_SINCE_FINALIZATION: Epoch = Epoch::new(2);
+pub const DEFAULT_FORK_CHOICE_BEFORE_PROPOSAL_TIMEOUT: u64 = 250;
+
+/// Default fraction of a slot lookahead for payload preparation (12/3 = 4 seconds on mainnet).
+pub const DEFAULT_PREPARE_PAYLOAD_LOOKAHEAD_FACTOR: u32 = 3;
+
+/// Fraction of a slot lookahead for fork choice in the state advance timer (500ms on mainnet).
+pub const FORK_CHOICE_LOOKAHEAD_FACTOR: u32 = 24;
 
 #[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]
 pub struct ChainConfig {
@@ -18,6 +30,43 @@ pub struct ChainConfig {
     pub enable_lock_timeouts: bool,
     /// The max size of a message that can be sent over the network.
     pub max_network_size: usize,
+    /// Maximum percentage of committee weight at which to attempt re-orging the canonical head.
+    pub re_org_threshold: Option<ReOrgThreshold>,
+    /// Maximum number of epochs since finalization for attempting a proposer re-org.
+    pub re_org_max_epochs_since_finalization: Epoch,
+    /// Number of milliseconds to wait for fork choice before proposing a block.
+    ///
+    /// If set to 0 then block proposal will not wait for fork choice at all.
+    pub fork_choice_before_proposal_timeout_ms: u64,
+    /// Number of skip slots in a row before the BN refuses to use connected builders during payload construction.
+    pub builder_fallback_skips: usize,
+    /// Number of skip slots in the past `SLOTS_PER_EPOCH` before the BN refuses to use connected
+    /// builders during payload construction.
+    pub builder_fallback_skips_per_epoch: usize,
+    /// Number of epochs since finalization before the BN refuses to use connected builders during
+    /// payload construction.
+    pub builder_fallback_epochs_since_finalization: usize,
+    /// Whether any chain health checks should be considered when deciding whether to use the builder API.
+    pub builder_fallback_disable_checks: bool,
+    /// When set to `true`, weigh the "unrealized" FFG progression when choosing a head in fork
+    /// choice.
+    pub count_unrealized: bool,
+    /// When set to `true`, forget any valid/invalid/optimistic statuses in fork choice during start
+    /// up.
+    pub always_reset_payload_statuses: bool,
+    /// Whether to apply paranoid checks to blocks proposed by this beacon node.
+    pub paranoid_block_proposal: bool,
+    /// Whether to strictly count unrealized justified votes.
+    pub count_unrealized_full: CountUnrealizedFull,
+    /// Optionally set timeout for calls to checkpoint sync endpoint.
+    pub checkpoint_sync_url_timeout: u64,
+    /// The offset before the start of a proposal slot at which payload attributes should be sent.
+    ///
+    /// Low values are useful for execution engines which don't improve their payload after the
+    /// first call, and high values are useful for ensuring the EL is given ample notice.
+    pub prepare_payload_lookahead: Duration,
+    /// Use EL-free optimistic sync for the finalized part of the chain.
+    pub optimistic_finalized_sync: bool,
 }
 
 impl Default for ChainConfig {
@@ -28,6 +77,21 @@ impl Default for ChainConfig {
             reconstruct_historic_states: false,
             enable_lock_timeouts: true,
             max_network_size: 10 * 1_048_576, // 10M
+            re_org_threshold: Some(DEFAULT_RE_ORG_THRESHOLD),
+            re_org_max_epochs_since_finalization: DEFAULT_RE_ORG_MAX_EPOCHS_SINCE_FINALIZATION,
+            fork_choice_before_proposal_timeout_ms: DEFAULT_FORK_CHOICE_BEFORE_PROPOSAL_TIMEOUT,
+            // Builder fallback configs that are set in `clap` will override these.
+            builder_fallback_skips: 3,
+            builder_fallback_skips_per_epoch: 8,
+            builder_fallback_epochs_since_finalization: 3,
+            builder_fallback_disable_checks: false,
+            count_unrealized: true,
+            always_reset_payload_statuses: false,
+            paranoid_block_proposal: false,
+            count_unrealized_full: CountUnrealizedFull::default(),
+            checkpoint_sync_url_timeout: 60,
+            prepare_payload_lookahead: Duration::from_secs(4),
+            optimistic_finalized_sync: true,
         }
     }
 }

beacon_node/beacon_chain/src/early_attester_cache.rs

@@ -4,6 +4,7 @@ use crate::{
 };
 use parking_lot::RwLock;
 use proto_array::Block as ProtoBlock;
+use std::sync::Arc;
 use types::*;
 
 pub struct CacheItem<E: EthSpec> {
@@ -18,7 +19,7 @@ pub struct CacheItem<E: EthSpec> {
     /*
      * Values used to make the block available.
      */
-    block: SignedBeaconBlock<E>,
+    block: Arc<SignedBeaconBlock<E>>,
     proto_block: ProtoBlock,
 }
 
@@ -48,7 +49,7 @@ impl<E: EthSpec> EarlyAttesterCache<E> {
     pub fn add_head_block(
         &self,
         beacon_block_root: Hash256,
-        block: SignedBeaconBlock<E>,
+        block: Arc<SignedBeaconBlock<E>>,
         proto_block: ProtoBlock,
         state: &BeaconState<E>,
         spec: &ChainSpec,
@@ -85,7 +86,7 @@ impl<E: EthSpec> EarlyAttesterCache<E> {
     ///
     /// - There is a cache `item` present.
     /// - If `request_slot` is in the same epoch as `item.epoch`.
-    /// - If `request_index` does not exceed `item.comittee_count`.
+    /// - If `request_index` does not exceed `item.committee_count`.
     pub fn try_attest(
         &self,
         request_slot: Slot,
@@ -104,6 +105,10 @@ impl<E: EthSpec> EarlyAttesterCache<E> {
             return Ok(None);
         }
 
+        if request_slot < item.block.slot() {
+            return Ok(None);
+        }
+
         let committee_count = item
             .committee_lengths
             .get_committee_count_per_slot::<E>(spec)?;
@@ -142,7 +147,7 @@ impl<E: EthSpec> EarlyAttesterCache<E> {
     }
 
     /// Returns the block, if `block_root` matches the cached item.
-    pub fn get_block(&self, block_root: Hash256) -> Option<SignedBeaconBlock<E>> {
+    pub fn get_block(&self, block_root: Hash256) -> Option<Arc<SignedBeaconBlock<E>>> {
         self.item
             .read()
             .as_ref()

beacon_node/beacon_chain/src/errors.rs

@@ -8,6 +8,8 @@ use crate::naive_aggregation_pool::Error as NaiveAggregationError;
 use crate::observed_aggregates::Error as ObservedAttestationsError;
 use crate::observed_attesters::Error as ObservedAttestersError;
 use crate::observed_block_producers::Error as ObservedBlockProducersError;
+use execution_layer::PayloadStatus;
+use fork_choice::ExecutionStatus;
 use futures::channel::mpsc::TrySendError;
 use operation_pool::OpPoolError;
 use safe_arith::ArithError;
@@ -24,6 +26,7 @@ use state_processing::{
 };
 use std::time::Duration;
 use task_executor::ShutdownReason;
+use tokio::task::JoinError;
 use types::*;
 
 macro_rules! easy_from_to {
@@ -42,8 +45,8 @@ pub enum BeaconChainError {
     UnableToReadSlot,
     UnableToComputeTimeAtSlot,
     RevertedFinalizedEpoch {
-        previous_epoch: Epoch,
-        new_epoch: Epoch,
+        old: Checkpoint,
+        new: Checkpoint,
     },
     SlotClockDidNotStart,
     NoStateForSlot(Slot),
@@ -82,13 +85,10 @@ pub enum BeaconChainError {
     ValidatorPubkeyCacheLockTimeout,
     SnapshotCacheLockTimeout,
     IncorrectStateForAttestation(RelativeEpochError),
-    InvalidValidatorPubkeyBytes(bls::Error),
     ValidatorPubkeyCacheIncomplete(usize),
     SignatureSetError(SignatureSetError),
     BlockSignatureVerifierError(state_processing::block_signature_verifier::Error),
     BlockReplayError(BlockReplayError),
-    DuplicateValidatorPublicKey,
-    ValidatorPubkeyCacheFileError(String),
     ValidatorIndexUnknown(usize),
     ValidatorPubkeyUnknown(PublicKeyBytes),
     OpPoolError(OpPoolError),
@@ -135,14 +135,73 @@ pub enum BeaconChainError {
         new_slot: Slot,
     },
     AltairForkDisabled,
+    BuilderMissing,
     ExecutionLayerMissing,
+    BlockVariantLacksExecutionPayload(Hash256),
+    ExecutionLayerErrorPayloadReconstruction(ExecutionBlockHash, execution_layer::Error),
+    BlockHashMissingFromExecutionLayer(ExecutionBlockHash),
+    InconsistentPayloadReconstructed {
+        slot: Slot,
+        exec_block_hash: ExecutionBlockHash,
+        canonical_payload_root: Hash256,
+        reconstructed_payload_root: Hash256,
+        canonical_transactions_root: Hash256,
+        reconstructed_transactions_root: Hash256,
+    },
+    AddPayloadLogicError,
     ExecutionForkChoiceUpdateFailed(execution_layer::Error),
+    PrepareProposerBlockingFailed(execution_layer::Error),
+    ExecutionForkChoiceUpdateInvalid {
+        status: PayloadStatus,
+    },
     BlockRewardSlotError,
     BlockRewardAttestationError,
     BlockRewardSyncError,
     HeadMissingFromForkChoice(Hash256),
     FinalizedBlockMissingFromForkChoice(Hash256),
+    HeadBlockMissingFromForkChoice(Hash256),
+    InvalidFinalizedPayload {
+        finalized_root: Hash256,
+        execution_block_hash: ExecutionBlockHash,
+    },
     InvalidFinalizedPayloadShutdownError(TrySendError<ShutdownReason>),
+    JustifiedPayloadInvalid {
+        justified_root: Hash256,
+        execution_block_hash: Option<ExecutionBlockHash>,
+    },
+    ForkchoiceUpdate(execution_layer::Error),
+    FinalizedCheckpointMismatch {
+        head_state: Checkpoint,
+        fork_choice: Hash256,
+    },
+    InvalidSlot(Slot),
+    HeadBlockNotFullyVerified {
+        beacon_block_root: Hash256,
+        execution_status: ExecutionStatus,
+    },
+    CannotAttestToFinalizedBlock {
+        beacon_block_root: Hash256,
+    },
+    SyncContributionDataReferencesFinalizedBlock {
+        beacon_block_root: Hash256,
+    },
+    RuntimeShutdown,
+    TokioJoin(tokio::task::JoinError),
+    ProcessInvalidExecutionPayload(JoinError),
+    ForkChoiceSignalOutOfOrder {
+        current: Slot,
+        latest: Slot,
+    },
+    ForkchoiceUpdateParamsMissing,
+    HeadHasInvalidPayload {
+        block_root: Hash256,
+        execution_status: ExecutionStatus,
+    },
+    AttestationHeadNotInForkChoice(Hash256),
+    MissingPersistedForkChoice,
+    CommitteePromiseFailed(oneshot_broadcast::Error),
+    MaxCommitteePromises(usize),
+    ProposerHeadForkChoiceError(fork_choice::Error<proto_array::Error>),
 }
 
 easy_from_to!(SlotProcessingError, BeaconChainError);
@@ -168,12 +227,12 @@ easy_from_to!(BlockReplayError, BeaconChainError);
 
 #[derive(Debug)]
 pub enum BlockProductionError {
-    UnableToGetHeadInfo(BeaconChainError),
     UnableToGetBlockRootFromState,
     UnableToReadSlot,
     UnableToProduceAtSlot(Slot),
     SlotProcessingError(SlotProcessingError),
     BlockProcessingError(BlockProcessingError),
+    ForkChoiceError(ForkChoiceError),
     Eth1ChainError(Eth1ChainError),
     BeaconStateError(BeaconStateError),
     StateAdvanceError(StateAdvanceError),
@@ -190,8 +249,14 @@ pub enum BlockProductionError {
     TerminalPoWBlockLookupFailed(execution_layer::Error),
     GetPayloadFailed(execution_layer::Error),
     FailedToReadFinalizedBlock(store::Error),
+    FailedToLoadState(store::Error),
     MissingFinalizedBlock(Hash256),
     BlockTooLarge(usize),
+    ShuttingDown,
+    MissingSyncAggregate,
+    MissingExecutionPayload,
+    TokioJoin(tokio::task::JoinError),
+    BeaconChain(BeaconChainError),
 }
 
 easy_from_to!(BlockProcessingError, BlockProductionError);
@@ -199,3 +264,4 @@ easy_from_to!(BeaconStateError, BlockProductionError);
 easy_from_to!(SlotProcessingError, BlockProductionError);
 easy_from_to!(Eth1ChainError, BlockProductionError);
 easy_from_to!(StateAdvanceError, BlockProductionError);
+easy_from_to!(ForkChoiceError, BlockProductionError);

beacon_node/beacon_chain/src/eth1_chain.rs

@@ -1,7 +1,7 @@
 use crate::metrics;
 use eth1::{Config as Eth1Config, Eth1Block, Service as HttpService};
 use eth2::lighthouse::Eth1SyncStatusData;
-use eth2_hashing::hash;
+use ethereum_hashing::hash;
 use int_to_bytes::int_to_bytes32;
 use slog::{debug, error, trace, Logger};
 use ssz::{Decode, Encode};
@@ -16,7 +16,6 @@ use store::{DBColumn, Error as StoreError, StoreItem};
 use task_executor::TaskExecutor;
 use types::{
     BeaconState, BeaconStateError, ChainSpec, Deposit, Eth1Data, EthSpec, Hash256, Slot, Unsigned,
-    DEPOSIT_TREE_DEPTH,
 };
 
 type BlockNumber = u64;
@@ -170,8 +169,8 @@ fn get_sync_status<T: EthSpec>(
 
 #[derive(Encode, Decode, Clone)]
 pub struct SszEth1 {
-    use_dummy_backend: bool,
-    backend_bytes: Vec<u8>,
+    pub use_dummy_backend: bool,
+    pub backend_bytes: Vec<u8>,
 }
 
 impl StoreItem for SszEth1 {
@@ -179,8 +178,8 @@ impl StoreItem for SszEth1 {
         DBColumn::Eth1Cache
     }
 
-    fn as_store_bytes(&self) -> Vec<u8> {
-        self.as_ssz_bytes()
+    fn as_store_bytes(&self) -> Result<Vec<u8>, StoreError> {
+        Ok(self.as_ssz_bytes())
     }
 
     fn from_store_bytes(bytes: &[u8]) -> Result<Self, StoreError> {
@@ -305,6 +304,12 @@ where
         }
     }
 
+    /// Set in motion the finalization of `Eth1Data`. This method is called during block import
+    /// so it should be fast.
+    pub fn finalize_eth1_data(&self, eth1_data: Eth1Data) {
+        self.backend.finalize_eth1_data(eth1_data);
+    }
+
     /// Consumes `self`, returning the backend.
     pub fn into_backend(self) -> T {
         self.backend
@@ -335,6 +340,10 @@ pub trait Eth1ChainBackend<T: EthSpec>: Sized + Send + Sync {
     /// beacon node eth1 cache is.
     fn latest_cached_block(&self) -> Option<Eth1Block>;
 
+    /// Set in motion the finalization of `Eth1Data`. This method is called during block import
+    /// so it should be fast.
+    fn finalize_eth1_data(&self, eth1_data: Eth1Data);
+
     /// Returns the block at the head of the chain (ignoring follow distance, etc). Used to obtain
     /// an idea of how up-to-date the remote eth1 node is.
     fn head_block(&self) -> Option<Eth1Block>;
@@ -389,6 +398,8 @@ impl<T: EthSpec> Eth1ChainBackend<T> for DummyEth1ChainBackend<T> {
         None
     }
 
+    fn finalize_eth1_data(&self, _eth1_data: Eth1Data) {}
+
     fn head_block(&self) -> Option<Eth1Block> {
         None
     }
@@ -431,12 +442,13 @@ impl<T: EthSpec> CachingEth1Backend<T> {
     /// Instantiates `self` with empty caches.
     ///
     /// Does not connect to the eth1 node or start any tasks to keep the cache updated.
-    pub fn new(config: Eth1Config, log: Logger, spec: ChainSpec) -> Self {
-        Self {
-            core: HttpService::new(config, log.clone(), spec),
+    pub fn new(config: Eth1Config, log: Logger, spec: ChainSpec) -> Result<Self, String> {
+        Ok(Self {
+            core: HttpService::new(config, log.clone(), spec)
+                .map_err(|e| format!("Failed to create eth1 http service: {:?}", e))?,
             log,
             _phantom: PhantomData,
-        }
+        })
     }
 
     /// Starts the routine which connects to the external eth1 node and updates the caches.
@@ -546,7 +558,7 @@ impl<T: EthSpec> Eth1ChainBackend<T> for CachingEth1Backend<T> {
                     .deposits()
                     .read()
                     .cache
-                    .get_deposits(next, last, deposit_count, DEPOSIT_TREE_DEPTH)
+                    .get_deposits(next, last, deposit_count)
                     .map_err(|e| Error::BackendError(format!("Failed to get deposits: {:?}", e)))
                     .map(|(_deposit_root, deposits)| deposits)
             }
@@ -557,6 +569,12 @@ impl<T: EthSpec> Eth1ChainBackend<T> for CachingEth1Backend<T> {
         self.core.latest_cached_block()
     }
 
+    /// This only writes the eth1_data to a temporary cache so that the service
+    /// thread can later do the actual finalizing of the deposit tree.
+    fn finalize_eth1_data(&self, eth1_data: Eth1Data) {
+        self.core.set_to_finalize(Some(eth1_data));
+    }
+
     fn head_block(&self) -> Option<Eth1Block> {
         self.core.head_block()
     }
@@ -730,11 +748,9 @@ mod test {
             };
 
             let log = null_logger().unwrap();
-            Eth1Chain::new(CachingEth1Backend::new(
-                eth1_config,
-                log,
-                MainnetEthSpec::default_spec(),
-            ))
+            Eth1Chain::new(
+                CachingEth1Backend::new(eth1_config, log, MainnetEthSpec::default_spec()).unwrap(),
+            )
         }
 
         fn get_deposit_log(i: u64, spec: &ChainSpec) -> DepositLog {
@@ -1009,6 +1025,7 @@ mod test {
 
     mod collect_valid_votes {
         use super::*;
+        use types::VList;
 
         fn get_eth1_data_vec(n: u64, block_number_offset: u64) -> Vec<(Eth1Data, BlockNumber)> {
             (0..n)
@@ -1056,12 +1073,14 @@ mod test {
 
             let votes_to_consider = get_eth1_data_vec(slots, 0);
 
-            *state.eth1_data_votes_mut() = votes_to_consider[0..slots as usize / 4]
-                .iter()
-                .map(|(eth1_data, _)| eth1_data)
-                .cloned()
-                .collect::<Vec<_>>()
-                .into();
+            *state.eth1_data_votes_mut() = VList::new(
+                votes_to_consider[0..slots as usize / 4]
+                    .iter()
+                    .map(|(eth1_data, _)| eth1_data)
+                    .cloned()
+                    .collect::<Vec<_>>(),
+            )
+            .unwrap();
 
             let votes =
                 collect_valid_votes(&state, &votes_to_consider.clone().into_iter().collect());
@@ -1085,12 +1104,14 @@ mod test {
                 .expect("should have some eth1 data")
                 .clone();
 
-            *state.eth1_data_votes_mut() = vec![duplicate_eth1_data.clone(); 4]
-                .iter()
-                .map(|(eth1_data, _)| eth1_data)
-                .cloned()
-                .collect::<Vec<_>>()
-                .into();
+            *state.eth1_data_votes_mut() = VList::new(
+                vec![duplicate_eth1_data.clone(); 4]
+                    .iter()
+                    .map(|(eth1_data, _)| eth1_data)
+                    .cloned()
+                    .collect::<Vec<_>>(),
+            )
+            .unwrap();
 
             let votes = collect_valid_votes(&state, &votes_to_consider.into_iter().collect());
             assert_votes!(

beacon_node/beacon_chain/src/eth1_finalization_cache.rs

@@ -0,0 +1,498 @@
+use slog::{debug, Logger};
+use std::cmp;
+use std::collections::BTreeMap;
+use types::{Checkpoint, Epoch, Eth1Data, Hash256 as Root};
+
+/// The default size of the cache.
+/// The beacon chain only looks at the last 4 epochs for finalization.
+/// Add 1 for current epoch and 4 earlier epochs.
+pub const DEFAULT_ETH1_CACHE_SIZE: usize = 5;
+
+/// These fields are named the same as the corresponding fields in the `BeaconState`
+/// as this structure stores these values from the `BeaconState` at a `Checkpoint`
+#[derive(Clone)]
+pub struct Eth1FinalizationData {
+    pub eth1_data: Eth1Data,
+    pub eth1_deposit_index: u64,
+}
+
+impl Eth1FinalizationData {
+    /// Ensures the deposit finalization conditions have been met. See:
+    /// https://eips.ethereum.org/EIPS/eip-4881#deposit-finalization-conditions
+    fn fully_imported(&self) -> bool {
+        self.eth1_deposit_index >= self.eth1_data.deposit_count
+    }
+}
+
+/// Implements map from Checkpoint -> Eth1CacheData
+pub struct CheckpointMap {
+    capacity: usize,
+    // There shouldn't be more than a couple of potential checkpoints at the same
+    // epoch. Searching through a vector for the matching Root should be faster
+    // than using another map from Root->Eth1CacheData
+    store: BTreeMap<Epoch, Vec<(Root, Eth1FinalizationData)>>,
+}
+
+impl Default for CheckpointMap {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+/// Provides a map of `Eth1CacheData` referenced by `Checkpoint`
+///
+/// ## Cache Queuing
+///
+/// The cache keeps a maximum number of (`capacity`) epochs. Because there may be
+/// forks at the epoch boundary, it's possible that there exists more than one
+/// `Checkpoint` for the same `Epoch`. This cache will store all checkpoints for
+/// a given `Epoch`. When adding data for a new `Checkpoint` would cause the number
+/// of `Epoch`s stored to exceed `capacity`, the data for oldest `Epoch` is dropped
+impl CheckpointMap {
+    pub fn new() -> Self {
+        CheckpointMap {
+            capacity: DEFAULT_ETH1_CACHE_SIZE,
+            store: BTreeMap::new(),
+        }
+    }
+
+    pub fn with_capacity(capacity: usize) -> Self {
+        CheckpointMap {
+            capacity: cmp::max(1, capacity),
+            store: BTreeMap::new(),
+        }
+    }
+
+    pub fn insert(&mut self, checkpoint: Checkpoint, eth1_finalization_data: Eth1FinalizationData) {
+        self.store
+            .entry(checkpoint.epoch)
+            .or_insert_with(Vec::new)
+            .push((checkpoint.root, eth1_finalization_data));
+
+        // faster to reduce size after the fact than do pre-checking to see
+        // if the current data would increase the size of the BTreeMap
+        while self.store.len() > self.capacity {
+            let oldest_stored_epoch = self.store.keys().next().cloned().unwrap();
+            self.store.remove(&oldest_stored_epoch);
+        }
+    }
+
+    pub fn get(&self, checkpoint: &Checkpoint) -> Option<&Eth1FinalizationData> {
+        match self.store.get(&checkpoint.epoch) {
+            Some(vec) => {
+                for (root, data) in vec {
+                    if *root == checkpoint.root {
+                        return Some(data);
+                    }
+                }
+                None
+            }
+            None => None,
+        }
+    }
+
+    #[cfg(test)]
+    pub fn len(&self) -> usize {
+        self.store.len()
+    }
+}
+
+/// This cache stores `Eth1CacheData` that could potentially be finalized within 4
+/// future epochs.
+pub struct Eth1FinalizationCache {
+    by_checkpoint: CheckpointMap,
+    pending_eth1: BTreeMap<u64, Eth1Data>,
+    last_finalized: Option<Eth1Data>,
+    log: Logger,
+}
+
+/// Provides a cache of `Eth1CacheData` at epoch boundaries. This is used to
+/// finalize deposits when a new epoch is finalized.
+///
+impl Eth1FinalizationCache {
+    pub fn new(log: Logger) -> Self {
+        Eth1FinalizationCache {
+            by_checkpoint: CheckpointMap::new(),
+            pending_eth1: BTreeMap::new(),
+            last_finalized: None,
+            log,
+        }
+    }
+
+    pub fn with_capacity(log: Logger, capacity: usize) -> Self {
+        Eth1FinalizationCache {
+            by_checkpoint: CheckpointMap::with_capacity(capacity),
+            pending_eth1: BTreeMap::new(),
+            last_finalized: None,
+            log,
+        }
+    }
+
+    pub fn insert(&mut self, checkpoint: Checkpoint, eth1_finalization_data: Eth1FinalizationData) {
+        if !eth1_finalization_data.fully_imported() {
+            self.pending_eth1.insert(
+                eth1_finalization_data.eth1_data.deposit_count,
+                eth1_finalization_data.eth1_data.clone(),
+            );
+            debug!(
+                self.log,
+                "Eth1Cache: inserted pending eth1";
+                "eth1_data.deposit_count" => eth1_finalization_data.eth1_data.deposit_count,
+                "eth1_deposit_index" => eth1_finalization_data.eth1_deposit_index,
+            );
+        }
+        self.by_checkpoint
+            .insert(checkpoint, eth1_finalization_data);
+    }
+
+    pub fn finalize(&mut self, checkpoint: &Checkpoint) -> Option<Eth1Data> {
+        if let Some(eth1_finalized_data) = self.by_checkpoint.get(checkpoint) {
+            let finalized_deposit_index = eth1_finalized_data.eth1_deposit_index;
+            let mut result = None;
+            while let Some(pending_count) = self.pending_eth1.keys().next().cloned() {
+                if finalized_deposit_index >= pending_count {
+                    result = self.pending_eth1.remove(&pending_count);
+                    debug!(
+                        self.log,
+                        "Eth1Cache: dropped pending eth1";
+                        "pending_count" => pending_count,
+                        "finalized_deposit_index" => finalized_deposit_index,
+                    );
+                } else {
+                    break;
+                }
+            }
+            if eth1_finalized_data.fully_imported() {
+                result = Some(eth1_finalized_data.eth1_data.clone())
+            }
+            if result.is_some() {
+                self.last_finalized = result;
+            }
+            self.last_finalized.clone()
+        } else {
+            debug!(
+                self.log,
+                "Eth1Cache: cache miss";
+                "epoch" => checkpoint.epoch,
+            );
+            None
+        }
+    }
+
+    #[cfg(test)]
+    pub fn by_checkpoint(&self) -> &CheckpointMap {
+        &self.by_checkpoint
+    }
+
+    #[cfg(test)]
+    pub fn pending_eth1(&self) -> &BTreeMap<u64, Eth1Data> {
+        &self.pending_eth1
+    }
+}
+
+#[cfg(test)]
+pub mod tests {
+    use super::*;
+    use sloggers::null::NullLoggerBuilder;
+    use sloggers::Build;
+    use std::collections::HashMap;
+
+    const SLOTS_PER_EPOCH: u64 = 32;
+    const MAX_DEPOSITS: u64 = 16;
+    const EPOCHS_PER_ETH1_VOTING_PERIOD: u64 = 64;
+
+    fn eth1cache() -> Eth1FinalizationCache {
+        let log_builder = NullLoggerBuilder;
+        Eth1FinalizationCache::new(log_builder.build().expect("should build log"))
+    }
+
+    fn random_eth1_data(deposit_count: u64) -> Eth1Data {
+        Eth1Data {
+            deposit_root: Root::random(),
+            deposit_count,
+            block_hash: Root::random(),
+        }
+    }
+
+    fn random_checkpoint(epoch: u64) -> Checkpoint {
+        Checkpoint {
+            epoch: epoch.into(),
+            root: Root::random(),
+        }
+    }
+
+    fn random_checkpoints(n: usize) -> Vec<Checkpoint> {
+        let mut result = Vec::with_capacity(n);
+        for epoch in 0..n {
+            result.push(random_checkpoint(epoch as u64))
+        }
+        result
+    }
+
+    #[test]
+    fn fully_imported_deposits() {
+        let epochs = 16;
+        let deposits_imported = 128;
+
+        let eth1data = random_eth1_data(deposits_imported);
+        let checkpoints = random_checkpoints(epochs as usize);
+        let mut eth1cache = eth1cache();
+
+        for epoch in 4..epochs {
+            assert_eq!(
+                eth1cache.by_checkpoint().len(),
+                cmp::min((epoch - 4) as usize, DEFAULT_ETH1_CACHE_SIZE),
+                "Unexpected cache size"
+            );
+
+            let checkpoint = checkpoints
+                .get(epoch as usize)
+                .expect("should get checkpoint");
+            eth1cache.insert(
+                *checkpoint,
+                Eth1FinalizationData {
+                    eth1_data: eth1data.clone(),
+                    eth1_deposit_index: deposits_imported,
+                },
+            );
+
+            let finalized_checkpoint = checkpoints
+                .get((epoch - 4) as usize)
+                .expect("should get finalized checkpoint");
+            assert!(
+                eth1cache.pending_eth1().is_empty(),
+                "Deposits are fully imported so pending cache should be empty"
+            );
+            if epoch < 8 {
+                assert_eq!(
+                    eth1cache.finalize(finalized_checkpoint),
+                    None,
+                    "Should have cache miss"
+                );
+            } else {
+                assert_eq!(
+                    eth1cache.finalize(finalized_checkpoint),
+                    Some(eth1data.clone()),
+                    "Should have cache hit"
+                )
+            }
+        }
+    }
+
+    #[test]
+    fn partially_imported_deposits() {
+        let epochs = 16;
+        let initial_deposits_imported = 1024;
+        let deposits_imported_per_epoch = MAX_DEPOSITS * SLOTS_PER_EPOCH;
+        let full_import_epoch = 13;
+        let total_deposits =
+            initial_deposits_imported + deposits_imported_per_epoch * full_import_epoch;
+
+        let eth1data = random_eth1_data(total_deposits);
+        let checkpoints = random_checkpoints(epochs as usize);
+        let mut eth1cache = eth1cache();
+
+        for epoch in 0..epochs {
+            assert_eq!(
+                eth1cache.by_checkpoint().len(),
+                cmp::min(epoch as usize, DEFAULT_ETH1_CACHE_SIZE),
+                "Unexpected cache size"
+            );
+
+            let checkpoint = checkpoints
+                .get(epoch as usize)
+                .expect("should get checkpoint");
+            let deposits_imported = cmp::min(
+                total_deposits,
+                initial_deposits_imported + deposits_imported_per_epoch * epoch,
+            );
+            eth1cache.insert(
+                *checkpoint,
+                Eth1FinalizationData {
+                    eth1_data: eth1data.clone(),
+                    eth1_deposit_index: deposits_imported,
+                },
+            );
+
+            if epoch >= 4 {
+                let finalized_epoch = epoch - 4;
+                let finalized_checkpoint = checkpoints
+                    .get(finalized_epoch as usize)
+                    .expect("should get finalized checkpoint");
+                if finalized_epoch < full_import_epoch {
+                    assert_eq!(
+                        eth1cache.finalize(finalized_checkpoint),
+                        None,
+                        "Deposits not fully finalized so cache should return no Eth1Data",
+                    );
+                    assert_eq!(
+                        eth1cache.pending_eth1().len(),
+                        1,
+                        "Deposits not fully finalized. Pending eth1 cache should have 1 entry"
+                    );
+                } else {
+                    assert_eq!(
+                        eth1cache.finalize(finalized_checkpoint),
+                        Some(eth1data.clone()),
+                        "Deposits fully imported and finalized. Cache should return Eth1Data. finalized_deposits[{}]",
+                        (initial_deposits_imported + deposits_imported_per_epoch * finalized_epoch),
+                    );
+                    assert!(
+                        eth1cache.pending_eth1().is_empty(),
+                        "Deposits fully imported and finalized. Pending cache should be empty"
+                    );
+                }
+            }
+        }
+    }
+
+    #[test]
+    fn fork_at_epoch_boundary() {
+        let epochs = 12;
+        let deposits_imported = 128;
+
+        let eth1data = random_eth1_data(deposits_imported);
+        let checkpoints = random_checkpoints(epochs as usize);
+        let mut forks = HashMap::new();
+        let mut eth1cache = eth1cache();
+
+        for epoch in 0..epochs {
+            assert_eq!(
+                eth1cache.by_checkpoint().len(),
+                cmp::min(epoch as usize, DEFAULT_ETH1_CACHE_SIZE),
+                "Unexpected cache size"
+            );
+
+            let checkpoint = checkpoints
+                .get(epoch as usize)
+                .expect("should get checkpoint");
+            eth1cache.insert(
+                *checkpoint,
+                Eth1FinalizationData {
+                    eth1_data: eth1data.clone(),
+                    eth1_deposit_index: deposits_imported,
+                },
+            );
+            // lets put a fork at every third epoch
+            if epoch % 3 == 0 {
+                let fork = random_checkpoint(epoch);
+                eth1cache.insert(
+                    fork,
+                    Eth1FinalizationData {
+                        eth1_data: eth1data.clone(),
+                        eth1_deposit_index: deposits_imported,
+                    },
+                );
+                forks.insert(epoch as usize, fork);
+            }
+
+            assert!(
+                eth1cache.pending_eth1().is_empty(),
+                "Deposits are fully imported so pending cache should be empty"
+            );
+            if epoch >= 4 {
+                let finalized_epoch = (epoch - 4) as usize;
+                let finalized_checkpoint = if finalized_epoch % 3 == 0 {
+                    forks.get(&finalized_epoch).expect("should get fork")
+                } else {
+                    checkpoints
+                        .get(finalized_epoch)
+                        .expect("should get checkpoint")
+                };
+                assert_eq!(
+                    eth1cache.finalize(finalized_checkpoint),
+                    Some(eth1data.clone()),
+                    "Should have cache hit"
+                );
+                if finalized_epoch >= 3 {
+                    let dropped_epoch = finalized_epoch - 3;
+                    if let Some(dropped_checkpoint) = forks.get(&dropped_epoch) {
+                        // got checkpoint for an old fork that should no longer
+                        // be in the cache because it is from too long ago
+                        assert_eq!(
+                            eth1cache.finalize(dropped_checkpoint),
+                            None,
+                            "Should have cache miss"
+                        );
+                    }
+                }
+            }
+        }
+    }
+
+    #[test]
+    fn massive_deposit_queue() {
+        // Simulating a situation where deposits don't get imported within an eth1 voting period
+        let eth1_voting_periods = 8;
+        let initial_deposits_imported = 1024;
+        let deposits_imported_per_epoch = MAX_DEPOSITS * SLOTS_PER_EPOCH;
+        let initial_deposit_queue =
+            deposits_imported_per_epoch * EPOCHS_PER_ETH1_VOTING_PERIOD * 2 + 32;
+        let new_deposits_per_voting_period =
+            EPOCHS_PER_ETH1_VOTING_PERIOD * deposits_imported_per_epoch / 2;
+
+        let mut epoch_data = BTreeMap::new();
+        let mut eth1s_by_count = BTreeMap::new();
+        let mut eth1cache = eth1cache();
+        let mut last_period_deposits = initial_deposits_imported;
+        for period in 0..eth1_voting_periods {
+            let period_deposits = initial_deposits_imported
+                + initial_deposit_queue
+                + period * new_deposits_per_voting_period;
+            let period_eth1_data = random_eth1_data(period_deposits);
+            eth1s_by_count.insert(period_eth1_data.deposit_count, period_eth1_data.clone());
+
+            for epoch_mod_period in 0..EPOCHS_PER_ETH1_VOTING_PERIOD {
+                let epoch = period * EPOCHS_PER_ETH1_VOTING_PERIOD + epoch_mod_period;
+                let checkpoint = random_checkpoint(epoch);
+                let deposits_imported = cmp::min(
+                    period_deposits,
+                    last_period_deposits + deposits_imported_per_epoch * epoch_mod_period,
+                );
+                eth1cache.insert(
+                    checkpoint,
+                    Eth1FinalizationData {
+                        eth1_data: period_eth1_data.clone(),
+                        eth1_deposit_index: deposits_imported,
+                    },
+                );
+                epoch_data.insert(epoch, (checkpoint, deposits_imported));
+
+                if epoch >= 4 {
+                    let finalized_epoch = epoch - 4;
+                    let (finalized_checkpoint, finalized_deposits) = epoch_data
+                        .get(&finalized_epoch)
+                        .expect("should get epoch data");
+
+                    let pending_eth1s = eth1s_by_count.range((finalized_deposits + 1)..).count();
+                    let last_finalized_eth1 = eth1s_by_count
+                        .range(0..(finalized_deposits + 1))
+                        .map(|(_, eth1)| eth1)
+                        .last()
+                        .cloned();
+                    assert_eq!(
+                        eth1cache.finalize(finalized_checkpoint),
+                        last_finalized_eth1,
+                        "finalized checkpoint mismatch",
+                    );
+                    assert_eq!(
+                        eth1cache.pending_eth1().len(),
+                        pending_eth1s,
+                        "pending eth1 mismatch"
+                    );
+                }
+            }
+
+            // remove unneeded stuff from old epochs
+            while epoch_data.len() > DEFAULT_ETH1_CACHE_SIZE {
+                let oldest_stored_epoch = epoch_data
+                    .keys()
+                    .next()
+                    .cloned()
+                    .expect("should get oldest epoch");
+                epoch_data.remove(&oldest_stored_epoch);
+            }
+            last_period_deposits = period_deposits;
+        }
+    }
+}

beacon_node/beacon_chain/src/execution_payload.rs

@@ -7,65 +7,205 @@
 //! So, this module contains functions that one might expect to find in other crates, but they live
 //! here for good reason.
 
+use crate::otb_verification_service::OptimisticTransitionBlock;
 use crate::{
     BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, BlockProductionError,
     ExecutionPayloadError,
 };
-use execution_layer::ExecutePayloadResponseStatus;
-use fork_choice::PayloadVerificationStatus;
+use execution_layer::{BuilderParams, PayloadStatus};
+use fork_choice::{InvalidationOperation, PayloadVerificationStatus};
 use proto_array::{Block as ProtoBlock, ExecutionStatus};
-use slog::debug;
+use slog::{debug, warn};
 use slot_clock::SlotClock;
 use state_processing::per_block_processing::{
     compute_timestamp_at_slot, is_execution_enabled, is_merge_transition_complete,
     partially_verify_execution_payload,
 };
+use std::sync::Arc;
+use tokio::task::JoinHandle;
+use tree_hash::TreeHash;
 use types::*;
 
+pub type PreparePayloadResult<Payload> = Result<Payload, BlockProductionError>;
+pub type PreparePayloadHandle<Payload> = JoinHandle<Option<PreparePayloadResult<Payload>>>;
+
+#[derive(PartialEq)]
+pub enum AllowOptimisticImport {
+    Yes,
+    No,
+}
+
+/// Signal whether the execution payloads of new blocks should be
+/// immediately verified with the EL or imported optimistically without
+/// any EL communication.
+#[derive(Default, Clone, Copy)]
+pub enum NotifyExecutionLayer {
+    #[default]
+    Yes,
+    No,
+}
+
+/// Used to await the result of executing payload with a remote EE.
+pub struct PayloadNotifier<T: BeaconChainTypes> {
+    pub chain: Arc<BeaconChain<T>>,
+    pub block: Arc<SignedBeaconBlock<T::EthSpec>>,
+    payload_verification_status: Option<PayloadVerificationStatus>,
+}
+
+impl<T: BeaconChainTypes> PayloadNotifier<T> {
+    pub fn new(
+        chain: Arc<BeaconChain<T>>,
+        block: Arc<SignedBeaconBlock<T::EthSpec>>,
+        state: &BeaconState<T::EthSpec>,
+        notify_execution_layer: NotifyExecutionLayer,
+    ) -> Result<Self, BlockError<T::EthSpec>> {
+        let payload_verification_status = if is_execution_enabled(state, block.message().body()) {
+            // Perform the initial stages of payload verification.
+            //
+            // We will duplicate these checks again during `per_block_processing`, however these
+            // checks are cheap and doing them here ensures we have verified them before marking
+            // the block as optimistically imported. This is particularly relevant in the case
+            // where we do not send the block to the EL at all.
+            let block_message = block.message();
+            let payload = block_message.execution_payload()?;
+            partially_verify_execution_payload(state, block.slot(), payload, &chain.spec)
+                .map_err(BlockError::PerBlockProcessingError)?;
+
+            match notify_execution_layer {
+                NotifyExecutionLayer::No if chain.config.optimistic_finalized_sync => {
+                    // Verify the block hash here in Lighthouse and immediately mark the block as
+                    // optimistically imported. This saves a lot of roundtrips to the EL.
+                    let execution_layer = chain
+                        .execution_layer
+                        .as_ref()
+                        .ok_or(ExecutionPayloadError::NoExecutionConnection)?;
+
+                    if let Err(e) =
+                        execution_layer.verify_payload_block_hash(&payload.execution_payload)
+                    {
+                        warn!(
+                            chain.log,
+                            "Falling back to slow block hash verification";
+                            "block_number" => payload.block_number(),
+                            "info" => "you can silence this warning with --disable-optimistic-finalized-sync",
+                            "error" => ?e,
+                        );
+                        None
+                    } else {
+                        Some(PayloadVerificationStatus::Optimistic)
+                    }
+                }
+                _ => None,
+            }
+        } else {
+            Some(PayloadVerificationStatus::Irrelevant)
+        };
+
+        Ok(Self {
+            chain,
+            block,
+            payload_verification_status,
+        })
+    }
+
+    pub async fn notify_new_payload(
+        self,
+    ) -> Result<PayloadVerificationStatus, BlockError<T::EthSpec>> {
+        if let Some(precomputed_status) = self.payload_verification_status {
+            Ok(precomputed_status)
+        } else {
+            notify_new_payload(&self.chain, self.block.message()).await
+        }
+    }
+}
+
 /// Verify that `execution_payload` contained by `block` is considered valid by an execution
 /// engine.
 ///
 /// ## Specification
 ///
-/// Equivalent to the `execute_payload` function in the merge Beacon Chain Changes, although it
+/// Equivalent to the `notify_new_payload` function in the merge Beacon Chain Changes, although it
 /// contains a few extra checks by running `partially_verify_execution_payload` first:
 ///
-/// https://github.com/ethereum/consensus-specs/blob/v1.1.5/specs/merge/beacon-chain.md#execute_payload
-pub fn execute_payload<T: BeaconChainTypes>(
-    chain: &BeaconChain<T>,
-    state: &BeaconState<T::EthSpec>,
-    block: BeaconBlockRef<T::EthSpec>,
+/// https://github.com/ethereum/consensus-specs/blob/v1.1.9/specs/bellatrix/beacon-chain.md#notify_new_payload
+async fn notify_new_payload<'a, T: BeaconChainTypes>(
+    chain: &Arc<BeaconChain<T>>,
+    block: BeaconBlockRef<'a, T::EthSpec>,
 ) -> Result<PayloadVerificationStatus, BlockError<T::EthSpec>> {
-    if !is_execution_enabled(state, block.body()) {
-        return Ok(PayloadVerificationStatus::Irrelevant);
-    }
-
     let execution_payload = block.execution_payload()?;
 
-    // Perform the initial stages of payload verification.
-    //
-    // We will duplicate these checks again during `per_block_processing`, however these checks
-    // are cheap and doing them here ensures we protect the execution payload from junk.
-    partially_verify_execution_payload(state, execution_payload, &chain.spec)
-        .map_err(BlockError::PerBlockProcessingError)?;
-
     let execution_layer = chain
         .execution_layer
         .as_ref()
         .ok_or(ExecutionPayloadError::NoExecutionConnection)?;
-    let execute_payload_response = execution_layer
-        .block_on(|execution_layer| execution_layer.execute_payload(execution_payload));
 
-    match execute_payload_response {
-        Ok((status, _latest_valid_hash)) => match status {
-            ExecutePayloadResponseStatus::Valid => Ok(PayloadVerificationStatus::Verified),
-            // TODO(merge): invalidate any invalid ancestors of this block in fork choice.
-            ExecutePayloadResponseStatus::Invalid => {
-                Err(ExecutionPayloadError::RejectedByExecutionEngine.into())
+    let new_payload_response = execution_layer
+        .notify_new_payload(&execution_payload.execution_payload)
+        .await;
+
+    match new_payload_response {
+        Ok(status) => match status {
+            PayloadStatus::Valid => Ok(PayloadVerificationStatus::Verified),
+            PayloadStatus::Syncing | PayloadStatus::Accepted => {
+                Ok(PayloadVerificationStatus::Optimistic)
+            }
+            PayloadStatus::Invalid {
+                latest_valid_hash,
+                ref validation_error,
+            } => {
+                debug!(
+                    chain.log,
+                    "Invalid execution payload";
+                    "validation_error" => ?validation_error,
+                    "latest_valid_hash" => ?latest_valid_hash,
+                    "execution_block_hash" => ?execution_payload.execution_payload.block_hash,
+                    "root" => ?block.tree_hash_root(),
+                    "graffiti" => block.body().graffiti().as_utf8_lossy(),
+                    "proposer_index" => block.proposer_index(),
+                    "slot" => block.slot(),
+                    "method" => "new_payload",
+                );
+
+                // latest_valid_hash == 0 implies that this was the terminal block
+                // Hence, we don't need to run `BeaconChain::process_invalid_execution_payload`.
+                if latest_valid_hash == ExecutionBlockHash::zero() {
+                    return Err(ExecutionPayloadError::RejectedByExecutionEngine { status }.into());
+                }
+                // This block has not yet been applied to fork choice, so the latest block that was
+                // imported to fork choice was the parent.
+                let latest_root = block.parent_root();
+                chain
+                    .process_invalid_execution_payload(&InvalidationOperation::InvalidateMany {
+                        head_block_root: latest_root,
+                        always_invalidate_head: false,
+                        latest_valid_ancestor: latest_valid_hash,
+                    })
+                    .await?;
+
+                Err(ExecutionPayloadError::RejectedByExecutionEngine { status }.into())
+            }
+            PayloadStatus::InvalidBlockHash {
+                ref validation_error,
+            } => {
+                debug!(
+                    chain.log,
+                    "Invalid execution payload block hash";
+                    "validation_error" => ?validation_error,
+                    "execution_block_hash" => ?execution_payload.execution_payload.block_hash,
+                    "root" => ?block.tree_hash_root(),
+                    "graffiti" => block.body().graffiti().as_utf8_lossy(),
+                    "proposer_index" => block.proposer_index(),
+                    "slot" => block.slot(),
+                    "method" => "new_payload",
+                );
+
+                // Returning an error here should be sufficient to invalidate the block. We have no
+                // information to indicate its parent is invalid, so no need to run
+                // `BeaconChain::process_invalid_execution_payload`.
+                Err(ExecutionPayloadError::RejectedByExecutionEngine { status }.into())
             }
-            ExecutePayloadResponseStatus::Syncing => Ok(PayloadVerificationStatus::NotVerified),
         },
-        Err(_) => Err(ExecutionPayloadError::RejectedByExecutionEngine.into()),
+        Err(e) => Err(ExecutionPayloadError::RequestFailed(e).into()),
     }
 }
 
@@ -81,15 +221,16 @@ pub fn execute_payload<T: BeaconChainTypes>(
 /// Equivalent to the `validate_merge_block` function in the merge Fork Choice Changes:
 ///
 /// https://github.com/ethereum/consensus-specs/blob/v1.1.5/specs/merge/fork-choice.md#validate_merge_block
-pub fn validate_merge_block<T: BeaconChainTypes>(
-    chain: &BeaconChain<T>,
-    block: BeaconBlockRef<T::EthSpec>,
+pub async fn validate_merge_block<'a, T: BeaconChainTypes>(
+    chain: &Arc<BeaconChain<T>>,
+    block: BeaconBlockRef<'a, T::EthSpec>,
+    allow_optimistic_import: AllowOptimisticImport,
 ) -> Result<(), BlockError<T::EthSpec>> {
     let spec = &chain.spec;
     let block_epoch = block.slot().epoch(T::EthSpec::slots_per_epoch());
     let execution_payload = block.execution_payload()?;
 
-    if spec.terminal_block_hash != Hash256::zero() {
+    if spec.terminal_block_hash != ExecutionBlockHash::zero() {
         if block_epoch < spec.terminal_block_hash_activation_epoch {
             return Err(ExecutionPayloadError::InvalidActivationEpoch {
                 activation_epoch: spec.terminal_block_hash_activation_epoch,
@@ -98,10 +239,10 @@ pub fn validate_merge_block<T: BeaconChainTypes>(
             .into());
         }
 
-        if execution_payload.parent_hash != spec.terminal_block_hash {
+        if execution_payload.parent_hash() != spec.terminal_block_hash {
             return Err(ExecutionPayloadError::InvalidTerminalBlockHash {
                 terminal_block_hash: spec.terminal_block_hash,
-                payload_parent_hash: execution_payload.parent_hash,
+                payload_parent_hash: execution_payload.parent_hash(),
             }
             .into());
         }
@@ -115,29 +256,67 @@ pub fn validate_merge_block<T: BeaconChainTypes>(
         .ok_or(ExecutionPayloadError::NoExecutionConnection)?;
 
     let is_valid_terminal_pow_block = execution_layer
-        .block_on(|execution_layer| {
-            execution_layer.is_valid_terminal_pow_block_hash(execution_payload.parent_hash, spec)
-        })
+        .is_valid_terminal_pow_block_hash(execution_payload.parent_hash(), spec)
+        .await
         .map_err(ExecutionPayloadError::from)?;
 
     match is_valid_terminal_pow_block {
         Some(true) => Ok(()),
         Some(false) => Err(ExecutionPayloadError::InvalidTerminalPoWBlock {
-            parent_hash: execution_payload.parent_hash,
+            parent_hash: execution_payload.parent_hash(),
         }
         .into()),
         None => {
-            debug!(
-                chain.log,
-                "Optimistically accepting terminal block";
-                "block_hash" => ?execution_payload.parent_hash,
-                "msg" => "the terminal block/parent was unavailable"
-            );
-            Ok(())
+            if allow_optimistic_import == AllowOptimisticImport::Yes
+                && is_optimistic_candidate_block(chain, block.slot(), block.parent_root()).await?
+            {
+                debug!(
+                    chain.log,
+                    "Optimistically importing merge transition block";
+                    "block_hash" => ?execution_payload.parent_hash(),
+                    "msg" => "the terminal block/parent was unavailable"
+                );
+                // Store Optimistic Transition Block in Database for later Verification
+                OptimisticTransitionBlock::from_block(block)
+                    .persist_in_store::<T, _>(&chain.store)?;
+                Ok(())
+            } else {
+                Err(ExecutionPayloadError::UnverifiedNonOptimisticCandidate.into())
+            }
         }
     }
 }
 
+/// Check to see if a block with the given parameters is valid to be imported optimistically.
+pub async fn is_optimistic_candidate_block<T: BeaconChainTypes>(
+    chain: &Arc<BeaconChain<T>>,
+    block_slot: Slot,
+    block_parent_root: Hash256,
+) -> Result<bool, BeaconChainError> {
+    let current_slot = chain.slot()?;
+    let inner_chain = chain.clone();
+
+    // Use a blocking task to check if the block is an optimistic candidate. Interacting
+    // with the `fork_choice` lock in an async task can block the core executor.
+    chain
+        .spawn_blocking_handle(
+            move || {
+                inner_chain
+                    .canonical_head
+                    .fork_choice_read_lock()
+                    .is_optimistic_candidate_block(
+                        current_slot,
+                        block_slot,
+                        &block_parent_root,
+                        &inner_chain.spec,
+                    )
+            },
+            "validate_merge_block_optimistic_candidate",
+        )
+        .await?
+        .map_err(BeaconChainError::from)
+}
+
 /// Validate the gossip block's execution_payload according to the checks described here:
 /// https://github.com/ethereum/consensus-specs/blob/dev/specs/merge/p2p-interface.md#beacon_block
 pub fn validate_execution_payload_for_gossip<T: BeaconChainTypes>(
@@ -152,7 +331,7 @@ pub fn validate_execution_payload_for_gossip<T: BeaconChainTypes>(
 
         let is_merge_transition_complete = match parent_block.execution_status {
             // Optimistically declare that an "unknown" status block has completed the merge.
-            ExecutionStatus::Valid(_) | ExecutionStatus::Unknown(_) => true,
+            ExecutionStatus::Valid(_) | ExecutionStatus::Optimistic(_) => true,
             // It's impossible for an irrelevant block to have completed the merge. It is pre-merge
             // by definition.
             ExecutionStatus::Irrelevant(_) => false,
@@ -175,11 +354,11 @@ pub fn validate_execution_payload_for_gossip<T: BeaconChainTypes>(
                 ))?;
 
             // The block's execution payload timestamp is correct with respect to the slot
-            if execution_payload.timestamp != expected_timestamp {
+            if execution_payload.timestamp() != expected_timestamp {
                 return Err(BlockError::ExecutionPayloadError(
                     ExecutionPayloadError::InvalidPayloadTimestamp {
                         expected: expected_timestamp,
-                        found: execution_payload.timestamp,
+                        found: execution_payload.timestamp(),
                     },
                 ));
             }
@@ -201,30 +380,49 @@ pub fn validate_execution_payload_for_gossip<T: BeaconChainTypes>(
 /// Equivalent to the `get_execution_payload` function in the Validator Guide:
 ///
 /// https://github.com/ethereum/consensus-specs/blob/v1.1.5/specs/merge/validator.md#block-proposal
-pub fn get_execution_payload<T: BeaconChainTypes>(
-    chain: &BeaconChain<T>,
+pub fn get_execution_payload<
+    T: BeaconChainTypes,
+    Payload: ExecPayload<T::EthSpec> + Default + Send + 'static,
+>(
+    chain: Arc<BeaconChain<T>>,
     state: &BeaconState<T::EthSpec>,
     proposer_index: u64,
-) -> Result<ExecutionPayload<T::EthSpec>, BlockProductionError> {
-    Ok(prepare_execution_payload_blocking(chain, state, proposer_index)?.unwrap_or_default())
-}
+    builder_params: BuilderParams,
+) -> Result<PreparePayloadHandle<Payload>, BlockProductionError> {
+    // Compute all required values from the `state` now to avoid needing to pass it into a spawned
+    // task.
+    let spec = &chain.spec;
+    let current_epoch = state.current_epoch();
+    let is_merge_transition_complete = is_merge_transition_complete(state);
+    let timestamp =
+        compute_timestamp_at_slot(state, state.slot(), spec).map_err(BeaconStateError::from)?;
+    let random = *state.get_randao_mix(current_epoch)?;
+    let latest_execution_payload_header_block_hash =
+        state.latest_execution_payload_header()?.block_hash;
 
-/// Wraps the async `prepare_execution_payload` function as a blocking task.
-pub fn prepare_execution_payload_blocking<T: BeaconChainTypes>(
-    chain: &BeaconChain<T>,
-    state: &BeaconState<T::EthSpec>,
-    proposer_index: u64,
-) -> Result<Option<ExecutionPayload<T::EthSpec>>, BlockProductionError> {
-    let execution_layer = chain
-        .execution_layer
-        .as_ref()
-        .ok_or(BlockProductionError::ExecutionLayerMissing)?;
+    // Spawn a task to obtain the execution payload from the EL via a series of async calls. The
+    // `join_handle` can be used to await the result of the function.
+    let join_handle = chain
+        .task_executor
+        .clone()
+        .spawn_handle(
+            async move {
+                prepare_execution_payload::<T, Payload>(
+                    &chain,
+                    is_merge_transition_complete,
+                    timestamp,
+                    random,
+                    proposer_index,
+                    latest_execution_payload_header_block_hash,
+                    builder_params,
+                )
+                .await
+            },
+            "get_execution_payload",
+        )
+        .ok_or(BlockProductionError::ShuttingDown)?;
 
-    execution_layer
-        .block_on_generic(|_| async {
-            prepare_execution_payload(chain, state, proposer_index).await
-        })
-        .map_err(BlockProductionError::BlockingFailed)?
+    Ok(join_handle)
 }
 
 /// Prepares an execution payload for inclusion in a block.
@@ -241,74 +439,87 @@ pub fn prepare_execution_payload_blocking<T: BeaconChainTypes>(
 /// Equivalent to the `prepare_execution_payload` function in the Validator Guide:
 ///
 /// https://github.com/ethereum/consensus-specs/blob/v1.1.5/specs/merge/validator.md#block-proposal
-pub async fn prepare_execution_payload<T: BeaconChainTypes>(
-    chain: &BeaconChain<T>,
-    state: &BeaconState<T::EthSpec>,
+#[allow(clippy::too_many_arguments)]
+pub async fn prepare_execution_payload<T, Payload>(
+    chain: &Arc<BeaconChain<T>>,
+    is_merge_transition_complete: bool,
+    timestamp: u64,
+    random: Hash256,
     proposer_index: u64,
-) -> Result<Option<ExecutionPayload<T::EthSpec>>, BlockProductionError> {
+    latest_execution_payload_header_block_hash: ExecutionBlockHash,
+    builder_params: BuilderParams,
+) -> Result<Payload, BlockProductionError>
+where
+    T: BeaconChainTypes,
+    Payload: ExecPayload<T::EthSpec> + Default,
+{
+    let current_epoch = builder_params.slot.epoch(T::EthSpec::slots_per_epoch());
     let spec = &chain.spec;
     let execution_layer = chain
         .execution_layer
         .as_ref()
         .ok_or(BlockProductionError::ExecutionLayerMissing)?;
 
-    let parent_hash = if !is_merge_transition_complete(state) {
-        let is_terminal_block_hash_set = spec.terminal_block_hash != Hash256::zero();
+    let parent_hash = if !is_merge_transition_complete {
+        let is_terminal_block_hash_set = spec.terminal_block_hash != ExecutionBlockHash::zero();
         let is_activation_epoch_reached =
-            state.current_epoch() >= spec.terminal_block_hash_activation_epoch;
+            current_epoch >= spec.terminal_block_hash_activation_epoch;
 
         if is_terminal_block_hash_set && !is_activation_epoch_reached {
-            return Ok(None);
+            // Use the "empty" payload if there's a terminal block hash, but we haven't reached the
+            // terminal block epoch yet.
+            return Ok(<_>::default());
         }
 
         let terminal_pow_block_hash = execution_layer
-            .get_terminal_pow_block_hash(spec)
+            .get_terminal_pow_block_hash(spec, timestamp)
             .await
             .map_err(BlockProductionError::TerminalPoWBlockLookupFailed)?;
 
         if let Some(terminal_pow_block_hash) = terminal_pow_block_hash {
             terminal_pow_block_hash
         } else {
-            return Ok(None);
+            // If the merge transition hasn't occurred yet and the EL hasn't found the terminal
+            // block, return an "empty" payload.
+            return Ok(<_>::default());
         }
     } else {
-        state.latest_execution_payload_header()?.block_hash
+        latest_execution_payload_header_block_hash
     };
 
-    let timestamp = compute_timestamp_at_slot(state, spec).map_err(BeaconStateError::from)?;
-    let random = *state.get_randao_mix(state.current_epoch())?;
-    let finalized_root = state.finalized_checkpoint().root;
-
-    // The finalized block hash is not included in the specification, however we provide this
-    // parameter so that the execution layer can produce a payload id if one is not already known
-    // (e.g., due to a recent reorg).
-    let finalized_block_hash =
-        if let Some(block) = chain.fork_choice.read().get_block(&finalized_root) {
-            block.execution_status.block_hash()
-        } else {
-            chain
-                .store
-                .get_block(&finalized_root)
-                .map_err(BlockProductionError::FailedToReadFinalizedBlock)?
-                .ok_or(BlockProductionError::MissingFinalizedBlock(finalized_root))?
-                .message()
-                .body()
-                .execution_payload()
-                .ok()
-                .map(|ep| ep.block_hash)
-        };
+    // Try to obtain the fork choice update parameters from the cached head.
+    //
+    // Use a blocking task to interact with the `canonical_head` lock otherwise we risk blocking the
+    // core `tokio` executor.
+    let inner_chain = chain.clone();
+    let forkchoice_update_params = chain
+        .spawn_blocking_handle(
+            move || {
+                inner_chain
+                    .canonical_head
+                    .cached_head()
+                    .forkchoice_update_parameters()
+            },
+            "prepare_execution_payload_forkchoice_update_params",
+        )
+        .await
+        .map_err(BlockProductionError::BeaconChain)?;
 
     // Note: the suggested_fee_recipient is stored in the `execution_layer`, it will add this parameter.
+    //
+    // This future is not executed here, it's up to the caller to await it.
     let execution_payload = execution_layer
-        .get_payload(
+        .get_payload::<Payload>(
             parent_hash,
             timestamp,
             random,
-            finalized_block_hash.unwrap_or_else(Hash256::zero),
             proposer_index,
+            forkchoice_update_params,
+            builder_params,
+            &chain.spec,
         )
         .await
         .map_err(BlockProductionError::GetPayloadFailed)?;
 
-    Ok(Some(execution_payload))
+    Ok(execution_payload)
 }

beacon_node/beacon_chain/src/fork_choice_signal.rs

@@ -0,0 +1,97 @@
+//! Concurrency helpers for synchronising block proposal with fork choice.
+//!
+//! The transmitter provides a way for a thread runnning fork choice on a schedule to signal
+//! to the receiver that fork choice has been updated for a given slot.
+use crate::BeaconChainError;
+use parking_lot::{Condvar, Mutex};
+use std::sync::Arc;
+use std::time::Duration;
+use types::Slot;
+
+/// Sender, for use by the per-slot task timer.
+pub struct ForkChoiceSignalTx {
+    pair: Arc<(Mutex<Slot>, Condvar)>,
+}
+
+/// Receiver, for use by the beacon chain waiting on fork choice to complete.
+pub struct ForkChoiceSignalRx {
+    pair: Arc<(Mutex<Slot>, Condvar)>,
+}
+
+pub enum ForkChoiceWaitResult {
+    /// Successfully reached a slot greater than or equal to the awaited slot.
+    Success(Slot),
+    /// Fork choice was updated to a lower slot, indicative of lag or processing delays.
+    Behind(Slot),
+    /// Timed out waiting for the fork choice update from the sender.
+    TimeOut,
+}
+
+impl ForkChoiceSignalTx {
+    pub fn new() -> Self {
+        let pair = Arc::new((Mutex::new(Slot::new(0)), Condvar::new()));
+        Self { pair }
+    }
+
+    pub fn get_receiver(&self) -> ForkChoiceSignalRx {
+        ForkChoiceSignalRx {
+            pair: self.pair.clone(),
+        }
+    }
+
+    /// Signal to the receiver that fork choice has been updated to `slot`.
+    ///
+    /// Return an error if the provided `slot` is strictly less than any previously provided slot.
+    pub fn notify_fork_choice_complete(&self, slot: Slot) -> Result<(), BeaconChainError> {
+        let &(ref lock, ref condvar) = &*self.pair;
+
+        let mut current_slot = lock.lock();
+
+        if slot < *current_slot {
+            return Err(BeaconChainError::ForkChoiceSignalOutOfOrder {
+                current: *current_slot,
+                latest: slot,
+            });
+        } else {
+            *current_slot = slot;
+        }
+
+        // We use `notify_all` because there may be multiple block proposals waiting simultaneously.
+        // Usually there'll be 0-1.
+        condvar.notify_all();
+
+        Ok(())
+    }
+}
+
+impl Default for ForkChoiceSignalTx {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl ForkChoiceSignalRx {
+    pub fn wait_for_fork_choice(&self, slot: Slot, timeout: Duration) -> ForkChoiceWaitResult {
+        let &(ref lock, ref condvar) = &*self.pair;
+
+        let mut current_slot = lock.lock();
+
+        // Wait for `current_slot >= slot`.
+        //
+        // Do not loop and wait, if we receive an update for the wrong slot then something is
+        // quite out of whack and we shouldn't waste more time waiting.
+        if *current_slot < slot {
+            let timeout_result = condvar.wait_for(&mut current_slot, timeout);
+
+            if timeout_result.timed_out() {
+                return ForkChoiceWaitResult::TimeOut;
+            }
+        }
+
+        if *current_slot >= slot {
+            ForkChoiceWaitResult::Success(*current_slot)
+        } else {
+            ForkChoiceWaitResult::Behind(*current_slot)
+        }
+    }
+}

beacon_node/beacon_chain/src/fork_revert.rs

@@ -1,10 +1,12 @@
 use crate::{BeaconForkChoiceStore, BeaconSnapshot};
-use fork_choice::{ForkChoice, PayloadVerificationStatus};
+use fork_choice::{CountUnrealized, ForkChoice, PayloadVerificationStatus};
 use itertools::process_results;
+use proto_array::CountUnrealizedFull;
 use slog::{info, warn, Logger};
 use state_processing::state_advance::complete_state_advance;
 use state_processing::{
-    per_block_processing, per_block_processing::BlockSignatureStrategy, VerifyBlockRoot,
+    per_block_processing, per_block_processing::BlockSignatureStrategy, ConsensusContext,
+    VerifyBlockRoot,
 };
 use std::sync::Arc;
 use std::time::Duration;
@@ -48,7 +50,7 @@ pub fn revert_to_fork_boundary<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>
     );
     let block_iter = ParentRootBlockIterator::fork_tolerant(&store, head_block_root);
 
-    process_results(block_iter, |mut iter| {
+    let (block_root, blinded_block) = process_results(block_iter, |mut iter| {
         iter.find_map(|(block_root, block)| {
             if block.slot() < fork_epoch.start_slot(E::slots_per_epoch()) {
                 Some((block_root, block))
@@ -69,7 +71,13 @@ pub fn revert_to_fork_boundary<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>
             e, CORRUPT_DB_MESSAGE
         )
     })?
-    .ok_or_else(|| format!("No pre-fork blocks found. {}", CORRUPT_DB_MESSAGE))
+    .ok_or_else(|| format!("No pre-fork blocks found. {}", CORRUPT_DB_MESSAGE))?;
+
+    let block = store
+        .make_full_block(&block_root, blinded_block)
+        .map_err(|e| format!("Unable to add payload to new head block: {:?}", e))?;
+
+    Ok((block_root, block))
 }
 
 /// Reset fork choice to the finalized checkpoint of the supplied head state.
@@ -91,13 +99,16 @@ pub fn reset_fork_choice_to_finalization<E: EthSpec, Hot: ItemStore<E>, Cold: It
     head_block_root: Hash256,
     head_state: &BeaconState<E>,
     store: Arc<HotColdDB<E, Hot, Cold>>,
+    current_slot: Option<Slot>,
     spec: &ChainSpec,
+    count_unrealized_config: CountUnrealized,
+    count_unrealized_full_config: CountUnrealizedFull,
 ) -> Result<ForkChoice<BeaconForkChoiceStore<E, Hot, Cold>, E>, String> {
     // Fetch finalized block.
     let finalized_checkpoint = head_state.finalized_checkpoint();
     let finalized_block_root = finalized_checkpoint.root;
     let finalized_block = store
-        .get_block(&finalized_block_root)
+        .get_full_block(&finalized_block_root, None)
         .map_err(|e| format!("Error loading finalized block: {:?}", e))?
         .ok_or_else(|| {
             format!(
@@ -132,17 +143,21 @@ pub fn reset_fork_choice_to_finalization<E: EthSpec, Hot: ItemStore<E>, Cold: It
     })?;
     let finalized_snapshot = BeaconSnapshot {
         beacon_block_root: finalized_block_root,
-        beacon_block: finalized_block,
+        beacon_block: Arc::new(finalized_block),
         beacon_state: finalized_state,
     };
 
-    let fc_store = BeaconForkChoiceStore::get_forkchoice_store(store.clone(), &finalized_snapshot);
+    let fc_store = BeaconForkChoiceStore::get_forkchoice_store(store.clone(), &finalized_snapshot)
+        .map_err(|e| format!("Unable to reset fork choice store for revert: {e:?}"))?;
 
     let mut fork_choice = ForkChoice::from_anchor(
         fc_store,
         finalized_block_root,
         &finalized_snapshot.beacon_block,
         &finalized_snapshot.beacon_state,
+        current_slot,
+        count_unrealized_full_config,
+        spec,
     )
     .map_err(|e| format!("Unable to reset fork choice for revert: {:?}", e))?;
 
@@ -154,16 +169,19 @@ pub fn reset_fork_choice_to_finalization<E: EthSpec, Hot: ItemStore<E>, Cold: It
         .map_err(|e| format!("Error loading blocks to replay for fork choice: {:?}", e))?;
 
     let mut state = finalized_snapshot.beacon_state;
-    for block in blocks {
+    let blocks_len = blocks.len();
+    for (i, block) in blocks.into_iter().enumerate() {
         complete_state_advance(&mut state, None, block.slot(), spec)
             .map_err(|e| format!("State advance failed: {:?}", e))?;
 
+        let mut ctxt = ConsensusContext::new(block.slot())
+            .set_proposer_index(block.message().proposer_index());
         per_block_processing(
             &mut state,
             &block,
-            None,
             BlockSignatureStrategy::NoVerification,
             VerifyBlockRoot::True,
+            &mut ctxt,
             spec,
         )
         .map_err(|e| format!("Error replaying block: {:?}", e))?;
@@ -172,19 +190,28 @@ pub fn reset_fork_choice_to_finalization<E: EthSpec, Hot: ItemStore<E>, Cold: It
         // retro-actively determine if they were valid or not.
         //
         // This scenario is so rare that it seems OK to double-verify some blocks.
-        let payload_verification_status = PayloadVerificationStatus::NotVerified;
+        let payload_verification_status = PayloadVerificationStatus::Optimistic;
+
+        // Because we are replaying a single chain of blocks, we only need to calculate unrealized
+        // justification for the last block in the chain.
+        let is_last_block = i + 1 == blocks_len;
+        let count_unrealized = if is_last_block {
+            count_unrealized_config
+        } else {
+            CountUnrealized::False
+        };
 
-        let (block, _) = block.deconstruct();
         fork_choice
             .on_block(
                 block.slot(),
-                &block,
+                block.message(),
                 block.canonical_root(),
                 // Reward proposer boost. We are reinforcing the canonical chain.
                 Duration::from_secs(0),
                 &state,
                 payload_verification_status,
                 spec,
+                count_unrealized,
             )
             .map_err(|e| format!("Error applying replayed block to fork choice: {:?}", e))?;
     }

beacon_node/beacon_chain/src/head_tracker.rs

@@ -45,7 +45,7 @@ impl HeadTracker {
     /// Returns a `SszHeadTracker`, which contains all necessary information to restore the state
     /// of `Self` at some later point.
     pub fn to_ssz_container(&self) -> SszHeadTracker {
-        SszHeadTracker::from_map(&*self.0.read())
+        SszHeadTracker::from_map(&self.0.read())
     }
 
     /// Creates a new `Self` from the given `SszHeadTracker`, restoring `Self` to the same state of
@@ -83,8 +83,8 @@ impl PartialEq<HeadTracker> for HeadTracker {
 /// This is used when persisting the state of the `BeaconChain` to disk.
 #[derive(Encode, Decode, Clone)]
 pub struct SszHeadTracker {
-    roots: Vec<Hash256>,
-    slots: Vec<Slot>,
+    pub roots: Vec<Hash256>,
+    pub slots: Vec<Slot>,
 }
 
 impl SszHeadTracker {

beacon_node/beacon_chain/src/historical_blocks.rs

@@ -7,9 +7,10 @@ use state_processing::{
 };
 use std::borrow::Cow;
 use std::iter;
+use std::sync::Arc;
 use std::time::Duration;
 use store::{chunked_vector::BlockRoots, AnchorInfo, ChunkWriter, KeyValueStore};
-use types::{Hash256, SignedBeaconBlock, Slot};
+use types::{Hash256, SignedBlindedBeaconBlock, Slot};
 
 /// Use a longer timeout on the pubkey cache.
 ///
@@ -58,7 +59,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     /// Return the number of blocks successfully imported.
     pub fn import_historical_block_batch(
         &self,
-        blocks: &[SignedBeaconBlock<T::EthSpec>],
+        blocks: Vec<Arc<SignedBlindedBeaconBlock<T::EthSpec>>>,
     ) -> Result<usize, Error> {
         let anchor_info = self
             .store
@@ -92,7 +93,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             ChunkWriter::<BlockRoots, _, _>::new(&self.store.cold_db, prev_block_slot.as_usize())?;
 
         let mut cold_batch = Vec::with_capacity(blocks.len());
-        let mut hot_batch = Vec::with_capacity(blocks.len());
 
         for block in blocks_to_import.iter().rev() {
             // Check chain integrity.
@@ -106,8 +106,9 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                 .into());
             }
 
-            // Store block in the hot database.
-            hot_batch.push(self.store.block_as_kv_store_op(&block_root, block));
+            // Store block in the hot database without payload.
+            self.store
+                .blinded_block_as_cold_kv_store_ops(&block_root, block, &mut cold_batch)?;
 
             // Store block roots, including at all skip slots in the freezer DB.
             for slot in (block.slot().as_usize()..prev_block_slot.as_usize()).rev() {
@@ -175,10 +176,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         drop(verify_timer);
         drop(sig_timer);
 
-        // Write the I/O batches to disk, writing the blocks themselves first, as it's better
-        // for the hot DB to contain extra blocks than for the cold DB to point to blocks that
-        // do not exist.
-        self.store.hot_db.do_atomically(hot_batch)?;
+        // Write the I/O batch to disk.
         self.store.cold_db.do_atomically(cold_batch)?;
 
         // Update the anchor.

beacon_node/beacon_chain/src/lib.rs

@@ -3,55 +3,68 @@ pub mod attestation_verification;
 mod attester_cache;
 mod beacon_chain;
 mod beacon_fork_choice_store;
-mod beacon_proposer_cache;
+pub mod beacon_proposer_cache;
 mod beacon_snapshot;
 pub mod block_reward;
 mod block_times_cache;
 mod block_verification;
 pub mod builder;
+pub mod canonical_head;
 pub mod chain_config;
 mod early_attester_cache;
 mod errors;
 pub mod eth1_chain;
+mod eth1_finalization_cache;
 pub mod events;
-mod execution_payload;
+pub mod execution_payload;
+pub mod fork_choice_signal;
 pub mod fork_revert;
 mod head_tracker;
 pub mod historical_blocks;
-mod metrics;
+pub mod light_client_finality_update_verification;
+pub mod light_client_optimistic_update_verification;
+pub mod merge_readiness;
+pub mod metrics;
 pub mod migrate;
 mod naive_aggregation_pool;
 mod observed_aggregates;
 mod observed_attesters;
 mod observed_block_producers;
 pub mod observed_operations;
+pub mod otb_verification_service;
 mod persisted_beacon_chain;
 mod persisted_fork_choice;
 mod pre_finalization_cache;
+pub mod proposer_prep_service;
 pub mod schema_change;
 mod shuffling_cache;
-mod snapshot_cache;
 pub mod state_advance_timer;
 pub mod sync_committee_verification;
 pub mod test_utils;
 mod timeout_rw_lock;
 pub mod validator_monitor;
-mod validator_pubkey_cache;
 
 pub use self::beacon_chain::{
     AttestationProcessingOutcome, BeaconChain, BeaconChainTypes, BeaconStore, ChainSegmentResult,
-    ForkChoiceError, HeadInfo, HeadSafetyStatus, StateSkipConfig, WhenSlotSkipped,
-    MAXIMUM_GOSSIP_CLOCK_DISPARITY,
+    CountUnrealized, ForkChoiceError, OverrideForkchoiceUpdate, ProduceBlockVerification,
+    StateSkipConfig, WhenSlotSkipped, INVALID_FINALIZED_MERGE_TRANSITION_BLOCK_SHUTDOWN_REASON,
+    INVALID_JUSTIFIED_PAYLOAD_SHUTDOWN_REASON, MAXIMUM_GOSSIP_CLOCK_DISPARITY,
 };
 pub use self::beacon_snapshot::BeaconSnapshot;
-pub use self::chain_config::ChainConfig;
+pub use self::chain_config::{ChainConfig, CountUnrealizedFull};
 pub use self::errors::{BeaconChainError, BlockProductionError};
 pub use self::historical_blocks::HistoricalBlockError;
 pub use attestation_verification::Error as AttestationError;
 pub use beacon_fork_choice_store::{BeaconForkChoiceStore, Error as ForkChoiceStoreError};
-pub use block_verification::{BlockError, ExecutionPayloadError, GossipVerifiedBlock};
+pub use block_verification::{
+    get_block_root, BlockError, ExecutionPayloadError, GossipVerifiedBlock,
+};
+pub use canonical_head::{CachedHead, CanonicalHead, CanonicalHeadRwLock};
 pub use eth1_chain::{Eth1Chain, Eth1ChainBackend};
 pub use events::ServerSentEventHandler;
+pub use execution_layer::EngineState;
+pub use execution_payload::NotifyExecutionLayer;
+pub use fork_choice::{ExecutionStatus, ForkchoiceUpdateParameters};
 pub use metrics::scrape_for_metrics;
 pub use parking_lot;
 pub use slot_clock;
@@ -62,3 +75,13 @@ pub use state_processing::per_block_processing::errors::{
 pub use store;
 pub use timeout_rw_lock::TimeoutRwLock;
 pub use types;
+
+pub mod validator_pubkey_cache {
+    use crate::BeaconChainTypes;
+
+    pub type ValidatorPubkeyCache<T> = store::ValidatorPubkeyCache<
+        <T as BeaconChainTypes>::EthSpec,
+        <T as BeaconChainTypes>::HotStore,
+        <T as BeaconChainTypes>::ColdStore,
+    >;
+}

beacon_node/beacon_chain/src/light_client_finality_update_verification.rs

@@ -0,0 +1,135 @@
+use crate::{
+    beacon_chain::MAXIMUM_GOSSIP_CLOCK_DISPARITY, BeaconChain, BeaconChainError, BeaconChainTypes,
+};
+use derivative::Derivative;
+use slot_clock::SlotClock;
+use std::time::Duration;
+use strum::AsRefStr;
+use types::{
+    light_client_update::Error as LightClientUpdateError, LightClientFinalityUpdate, Slot,
+};
+
+/// Returned when a light client finality update was not successfully verified. It might not have been verified for
+/// two reasons:
+///
+/// - The light client finality message is malformed or inappropriate for the context (indicated by all variants
+///   other than `BeaconChainError`).
+/// - The application encountered an internal error whilst attempting to determine validity
+///   (the `BeaconChainError` variant)
+#[derive(Debug, AsRefStr)]
+pub enum Error {
+    /// Light client finality update message with a lower or equal finalized_header slot already forwarded.
+    FinalityUpdateAlreadySeen,
+    /// The light client finality message was received is prior to one-third of slot duration passage. (with
+    /// respect to the gossip clock disparity and slot clock duration).
+    ///
+    /// ## Peer scoring
+    ///
+    /// Assuming the local clock is correct, the peer has sent an invalid message.
+    TooEarly,
+    /// Light client finality update message does not match the locally constructed one.
+    ///
+    /// ## Peer Scoring
+    ///
+    InvalidLightClientFinalityUpdate,
+    /// Signature slot start time is none.
+    SigSlotStartIsNone,
+    /// Failed to construct a LightClientFinalityUpdate from state.
+    FailedConstructingUpdate,
+    /// Beacon chain error occured.
+    BeaconChainError(BeaconChainError),
+    LightClientUpdateError(LightClientUpdateError),
+}
+
+impl From<BeaconChainError> for Error {
+    fn from(e: BeaconChainError) -> Self {
+        Error::BeaconChainError(e)
+    }
+}
+
+impl From<LightClientUpdateError> for Error {
+    fn from(e: LightClientUpdateError) -> Self {
+        Error::LightClientUpdateError(e)
+    }
+}
+
+/// Wraps a `LightClientFinalityUpdate` that has been verified for propagation on the gossip network.
+#[derive(Derivative)]
+#[derivative(Clone(bound = "T: BeaconChainTypes"))]
+pub struct VerifiedLightClientFinalityUpdate<T: BeaconChainTypes> {
+    light_client_finality_update: LightClientFinalityUpdate<T::EthSpec>,
+    seen_timestamp: Duration,
+}
+
+impl<T: BeaconChainTypes> VerifiedLightClientFinalityUpdate<T> {
+    /// Returns `Ok(Self)` if the `light_client_finality_update` is valid to be (re)published on the gossip
+    /// network.
+    pub fn verify(
+        light_client_finality_update: LightClientFinalityUpdate<T::EthSpec>,
+        chain: &BeaconChain<T>,
+        seen_timestamp: Duration,
+    ) -> Result<Self, Error> {
+        let gossiped_finality_slot = light_client_finality_update.finalized_header.slot;
+        let one_third_slot_duration = Duration::new(chain.spec.seconds_per_slot / 3, 0);
+        let signature_slot = light_client_finality_update.signature_slot;
+        let start_time = chain.slot_clock.start_of(signature_slot);
+        let mut latest_seen_finality_update = chain.latest_seen_finality_update.lock();
+
+        let head = chain.canonical_head.cached_head();
+        let head_block = &head.snapshot.beacon_block;
+        let attested_block_root = head_block.message().parent_root();
+        let attested_block = chain
+            .get_blinded_block(&attested_block_root)?
+            .ok_or(Error::FailedConstructingUpdate)?;
+        let mut attested_state = chain
+            .get_state(&attested_block.state_root(), Some(attested_block.slot()))?
+            .ok_or(Error::FailedConstructingUpdate)?;
+
+        let finalized_block_root = attested_state.finalized_checkpoint().root;
+        let finalized_block = chain
+            .get_blinded_block(&finalized_block_root)?
+            .ok_or(Error::FailedConstructingUpdate)?;
+        let latest_seen_finality_update_slot = match latest_seen_finality_update.as_ref() {
+            Some(update) => update.finalized_header.slot,
+            None => Slot::new(0),
+        };
+
+        // verify that no other finality_update with a lower or equal
+        // finalized_header.slot was already forwarded on the network
+        if gossiped_finality_slot <= latest_seen_finality_update_slot {
+            return Err(Error::FinalityUpdateAlreadySeen);
+        }
+
+        // verify that enough time has passed for the block to have been propagated
+        match start_time {
+            Some(time) => {
+                if seen_timestamp + MAXIMUM_GOSSIP_CLOCK_DISPARITY < time + one_third_slot_duration
+                {
+                    return Err(Error::TooEarly);
+                }
+            }
+            None => return Err(Error::SigSlotStartIsNone),
+        }
+
+        let head_state = &head.snapshot.beacon_state;
+        let finality_update = LightClientFinalityUpdate::new(
+            &chain.spec,
+            head_state,
+            head_block,
+            &mut attested_state,
+            &finalized_block,
+        )?;
+
+        // verify that the gossiped finality update is the same as the locally constructed one.
+        if finality_update != light_client_finality_update {
+            return Err(Error::InvalidLightClientFinalityUpdate);
+        }
+
+        *latest_seen_finality_update = Some(light_client_finality_update.clone());
+
+        Ok(Self {
+            light_client_finality_update,
+            seen_timestamp,
+        })
+    }
+}

beacon_node/beacon_chain/src/light_client_optimistic_update_verification.rs

@@ -0,0 +1,125 @@
+use crate::{
+    beacon_chain::MAXIMUM_GOSSIP_CLOCK_DISPARITY, BeaconChain, BeaconChainError, BeaconChainTypes,
+};
+use derivative::Derivative;
+use slot_clock::SlotClock;
+use std::time::Duration;
+use strum::AsRefStr;
+use types::{
+    light_client_update::Error as LightClientUpdateError, LightClientOptimisticUpdate, Slot,
+};
+
+/// Returned when a light client optimistic update was not successfully verified. It might not have been verified for
+/// two reasons:
+///
+/// - The light client optimistic message is malformed or inappropriate for the context (indicated by all variants
+///   other than `BeaconChainError`).
+/// - The application encountered an internal error whilst attempting to determine validity
+///   (the `BeaconChainError` variant)
+#[derive(Debug, AsRefStr)]
+pub enum Error {
+    /// Light client optimistic update message with a lower or equal optimistic_header slot already forwarded.
+    OptimisticUpdateAlreadySeen,
+    /// The light client optimistic message was received is prior to one-third of slot duration passage. (with
+    /// respect to the gossip clock disparity and slot clock duration).
+    ///
+    /// ## Peer scoring
+    ///
+    /// Assuming the local clock is correct, the peer has sent an invalid message.
+    TooEarly,
+    /// Light client optimistic update message does not match the locally constructed one.
+    ///
+    /// ## Peer Scoring
+    ///
+    InvalidLightClientOptimisticUpdate,
+    /// Signature slot start time is none.
+    SigSlotStartIsNone,
+    /// Failed to construct a LightClientOptimisticUpdate from state.
+    FailedConstructingUpdate,
+    /// Beacon chain error occured.
+    BeaconChainError(BeaconChainError),
+    LightClientUpdateError(LightClientUpdateError),
+}
+
+impl From<BeaconChainError> for Error {
+    fn from(e: BeaconChainError) -> Self {
+        Error::BeaconChainError(e)
+    }
+}
+
+impl From<LightClientUpdateError> for Error {
+    fn from(e: LightClientUpdateError) -> Self {
+        Error::LightClientUpdateError(e)
+    }
+}
+
+/// Wraps a `LightClientOptimisticUpdate` that has been verified for propagation on the gossip network.
+#[derive(Derivative)]
+#[derivative(Clone(bound = "T: BeaconChainTypes"))]
+pub struct VerifiedLightClientOptimisticUpdate<T: BeaconChainTypes> {
+    light_client_optimistic_update: LightClientOptimisticUpdate<T::EthSpec>,
+    seen_timestamp: Duration,
+}
+
+impl<T: BeaconChainTypes> VerifiedLightClientOptimisticUpdate<T> {
+    /// Returns `Ok(Self)` if the `light_client_optimistic_update` is valid to be (re)published on the gossip
+    /// network.
+    pub fn verify(
+        light_client_optimistic_update: LightClientOptimisticUpdate<T::EthSpec>,
+        chain: &BeaconChain<T>,
+        seen_timestamp: Duration,
+    ) -> Result<Self, Error> {
+        let gossiped_optimistic_slot = light_client_optimistic_update.attested_header.slot;
+        let one_third_slot_duration = Duration::new(chain.spec.seconds_per_slot / 3, 0);
+        let signature_slot = light_client_optimistic_update.signature_slot;
+        let start_time = chain.slot_clock.start_of(signature_slot);
+        let mut latest_seen_optimistic_update = chain.latest_seen_optimistic_update.lock();
+
+        let head = chain.canonical_head.cached_head();
+        let head_block = &head.snapshot.beacon_block;
+        let attested_block_root = head_block.message().parent_root();
+        let attested_block = chain
+            .get_blinded_block(&attested_block_root)?
+            .ok_or(Error::FailedConstructingUpdate)?;
+
+        let attested_state = chain
+            .get_state(&attested_block.state_root(), Some(attested_block.slot()))?
+            .ok_or(Error::FailedConstructingUpdate)?;
+        let latest_seen_optimistic_update_slot = match latest_seen_optimistic_update.as_ref() {
+            Some(update) => update.attested_header.slot,
+            None => Slot::new(0),
+        };
+
+        // verify that no other optimistic_update with a lower or equal
+        // optimistic_header.slot was already forwarded on the network
+        if gossiped_optimistic_slot <= latest_seen_optimistic_update_slot {
+            return Err(Error::OptimisticUpdateAlreadySeen);
+        }
+
+        // verify that enough time has passed for the block to have been propagated
+        match start_time {
+            Some(time) => {
+                if seen_timestamp + MAXIMUM_GOSSIP_CLOCK_DISPARITY < time + one_third_slot_duration
+                {
+                    return Err(Error::TooEarly);
+                }
+            }
+            None => return Err(Error::SigSlotStartIsNone),
+        }
+
+        let optimistic_update =
+            LightClientOptimisticUpdate::new(&chain.spec, head_block, &attested_state)?;
+
+        // verify that the gossiped optimistic update is the same as the locally constructed one.
+        if optimistic_update != light_client_optimistic_update {
+            return Err(Error::InvalidLightClientOptimisticUpdate);
+        }
+
+        *latest_seen_optimistic_update = Some(light_client_optimistic_update.clone());
+
+        Ok(Self {
+            light_client_optimistic_update,
+            seen_timestamp,
+        })
+    }
+}

beacon_node/beacon_chain/src/merge_readiness.rs

@@ -0,0 +1,192 @@
+//! Provides tools for checking if a node is ready for the Bellatrix upgrade and following merge
+//! transition.
+
+use crate::{BeaconChain, BeaconChainTypes};
+use serde::{Deserialize, Serialize, Serializer};
+use std::fmt;
+use std::fmt::Write;
+use types::*;
+
+/// The time before the Bellatrix fork when we will start issuing warnings about preparation.
+const SECONDS_IN_A_WEEK: u64 = 604800;
+pub const MERGE_READINESS_PREPARATION_SECONDS: u64 = SECONDS_IN_A_WEEK * 2;
+
+#[derive(Default, Debug, Serialize, Deserialize)]
+pub struct MergeConfig {
+    #[serde(serialize_with = "serialize_uint256")]
+    pub terminal_total_difficulty: Option<Uint256>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub terminal_block_hash: Option<ExecutionBlockHash>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub terminal_block_hash_epoch: Option<Epoch>,
+}
+
+impl fmt::Display for MergeConfig {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        if self.terminal_block_hash.is_none()
+            && self.terminal_block_hash_epoch.is_none()
+            && self.terminal_total_difficulty.is_none()
+        {
+            return write!(
+                f,
+                "Merge terminal difficulty parameters not configured, check your config"
+            );
+        }
+        let mut display_string = String::new();
+        if let Some(terminal_total_difficulty) = self.terminal_total_difficulty {
+            write!(
+                display_string,
+                "terminal_total_difficulty: {},",
+                terminal_total_difficulty
+            )?;
+        }
+        if let Some(terminal_block_hash) = self.terminal_block_hash {
+            write!(
+                display_string,
+                "terminal_block_hash: {},",
+                terminal_block_hash
+            )?;
+        }
+        if let Some(terminal_block_hash_epoch) = self.terminal_block_hash_epoch {
+            write!(
+                display_string,
+                "terminal_block_hash_epoch: {},",
+                terminal_block_hash_epoch
+            )?;
+        }
+        write!(f, "{}", display_string.trim_end_matches(','))?;
+        Ok(())
+    }
+}
+impl MergeConfig {
+    /// Instantiate `self` from the values in a `ChainSpec`.
+    pub fn from_chainspec(spec: &ChainSpec) -> Self {
+        let mut params = MergeConfig::default();
+        if spec.terminal_total_difficulty != Uint256::max_value() {
+            params.terminal_total_difficulty = Some(spec.terminal_total_difficulty);
+        }
+        if spec.terminal_block_hash != ExecutionBlockHash::zero() {
+            params.terminal_block_hash = Some(spec.terminal_block_hash);
+        }
+        if spec.terminal_block_hash_activation_epoch != Epoch::max_value() {
+            params.terminal_block_hash_epoch = Some(spec.terminal_block_hash_activation_epoch);
+        }
+        params
+    }
+}
+
+/// Indicates if a node is ready for the Bellatrix upgrade and subsequent merge transition.
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+#[serde(tag = "type")]
+pub enum MergeReadiness {
+    /// The node is ready, as far as we can tell.
+    Ready {
+        config: MergeConfig,
+        #[serde(serialize_with = "serialize_uint256")]
+        current_difficulty: Option<Uint256>,
+    },
+    /// The transition configuration with the EL failed, there might be a problem with
+    /// connectivity, authentication or a difference in configuration.
+    ExchangeTransitionConfigurationFailed { error: String },
+    /// The EL can be reached and has the correct configuration, however it's not yet synced.
+    NotSynced,
+    /// The user has not configured this node to use an execution endpoint.
+    NoExecutionEndpoint,
+}
+
+impl fmt::Display for MergeReadiness {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            MergeReadiness::Ready {
+                config: params,
+                current_difficulty,
+            } => {
+                write!(
+                    f,
+                    "This node appears ready for the merge. \
+                        Params: {}, current_difficulty: {:?}",
+                    params, current_difficulty
+                )
+            }
+            MergeReadiness::ExchangeTransitionConfigurationFailed { error } => write!(
+                f,
+                "Could not confirm the transition configuration with the \
+                    execution endpoint: {:?}",
+                error
+            ),
+            MergeReadiness::NotSynced => write!(
+                f,
+                "The execution endpoint is connected and configured, \
+                    however it is not yet synced"
+            ),
+            MergeReadiness::NoExecutionEndpoint => write!(
+                f,
+                "The --execution-endpoint flag is not specified, this is a \
+                    requirement for the merge"
+            ),
+        }
+    }
+}
+
+impl<T: BeaconChainTypes> BeaconChain<T> {
+    /// Returns `true` if user has an EL configured, or if the Bellatrix fork has occurred or will
+    /// occur within `MERGE_READINESS_PREPARATION_SECONDS`.
+    pub fn is_time_to_prepare_for_bellatrix(&self, current_slot: Slot) -> bool {
+        if let Some(bellatrix_epoch) = self.spec.bellatrix_fork_epoch {
+            let bellatrix_slot = bellatrix_epoch.start_slot(T::EthSpec::slots_per_epoch());
+            let merge_readiness_preparation_slots =
+                MERGE_READINESS_PREPARATION_SECONDS / self.spec.seconds_per_slot;
+
+            if self.execution_layer.is_some() {
+                // The user has already configured an execution layer, start checking for readiness
+                // right away.
+                true
+            } else {
+                // Return `true` if Bellatrix has happened or is within the preparation time.
+                current_slot + merge_readiness_preparation_slots > bellatrix_slot
+            }
+        } else {
+            // The Bellatrix fork epoch has not been defined yet, no need to prepare.
+            false
+        }
+    }
+
+    /// Attempts to connect to the EL and confirm that it is ready for the merge.
+    pub async fn check_merge_readiness(&self) -> MergeReadiness {
+        if let Some(el) = self.execution_layer.as_ref() {
+            if let Err(e) = el.exchange_transition_configuration(&self.spec).await {
+                // The EL was either unreachable, responded with an error or has a different
+                // configuration.
+                return MergeReadiness::ExchangeTransitionConfigurationFailed {
+                    error: format!("{:?}", e),
+                };
+            }
+
+            if !el.is_synced_for_notifier().await {
+                // The EL is not synced.
+                return MergeReadiness::NotSynced;
+            }
+            let params = MergeConfig::from_chainspec(&self.spec);
+            let current_difficulty = el.get_current_difficulty().await.ok();
+            MergeReadiness::Ready {
+                config: params,
+                current_difficulty,
+            }
+        } else {
+            // There is no EL configured.
+            MergeReadiness::NoExecutionEndpoint
+        }
+    }
+}
+
+/// Utility function to serialize a Uint256 as a decimal string.
+fn serialize_uint256<S>(val: &Option<Uint256>, s: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    match val {
+        Some(v) => v.to_string().serialize(s),
+        None => s.serialize_none(),
+    }
+}

beacon_node/beacon_chain/src/metrics.rs

@@ -4,12 +4,8 @@ use crate::{BeaconChain, BeaconChainError, BeaconChainTypes};
 use lazy_static::lazy_static;
 pub use lighthouse_metrics::*;
 use slot_clock::SlotClock;
-use std::time::Duration;
 use types::{BeaconState, Epoch, EthSpec, Hash256, Slot};
 
-/// The maximum time to wait for the snapshot cache lock during a metrics scrape.
-const SNAPSHOT_CACHE_TIMEOUT: Duration = Duration::from_millis(100);
-
 lazy_static! {
     /*
      * Block Processing
@@ -64,6 +60,11 @@ lazy_static! {
         "beacon_block_processing_state_root_seconds",
         "Time spent calculating the state root when processing a block."
     );
+    pub static ref BLOCK_PROCESSING_POST_EXEC_PROCESSING: Result<Histogram> = try_create_histogram_with_buckets(
+        "beacon_block_processing_post_exec_pre_attestable_seconds",
+        "Time between finishing execution processing and the block becoming attestable",
+        linear_buckets(5e-3, 5e-3, 10)
+    );
     pub static ref BLOCK_PROCESSING_DB_WRITE: Result<Histogram> = try_create_histogram(
         "beacon_block_processing_db_write_seconds",
         "Time spent writing a newly processed block and state to DB"
@@ -72,6 +73,11 @@ lazy_static! {
         "beacon_block_processing_attestation_observation_seconds",
         "Time spent hashing and remembering all the attestations in the block"
     );
+    pub static ref BLOCK_PROCESSING_FORK_CHOICE: Result<Histogram> = try_create_histogram_with_buckets(
+        "beacon_block_processing_fork_choice_seconds",
+        "Time spent running fork choice's `get_head` during block import",
+        exponential_buckets(1e-3, 2.0, 8)
+    );
     pub static ref BLOCK_SYNC_AGGREGATE_SET_BITS: Result<IntGauge> = try_create_int_gauge(
         "block_sync_aggregate_set_bits",
         "The number of true bits in the last sync aggregate in a block"
@@ -90,6 +96,15 @@ lazy_static! {
     );
     pub static ref BLOCK_PRODUCTION_TIMES: Result<Histogram> =
         try_create_histogram("beacon_block_production_seconds", "Full runtime of block production");
+    pub static ref BLOCK_PRODUCTION_FORK_CHOICE_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_block_production_fork_choice_seconds",
+        "Time taken to run fork choice before block production"
+    );
+    pub static ref BLOCK_PRODUCTION_GET_PROPOSER_HEAD_TIMES: Result<Histogram> = try_create_histogram_with_buckets(
+        "beacon_block_production_get_proposer_head_times",
+        "Time taken for fork choice to compute the proposer head before block production",
+        exponential_buckets(1e-3, 2.0, 8)
+    );
     pub static ref BLOCK_PRODUCTION_STATE_LOAD_TIMES: Result<Histogram> = try_create_histogram(
         "beacon_block_production_state_load_seconds",
         "Time taken to load the base state for block production"
@@ -118,14 +133,17 @@ lazy_static! {
     /*
      * Block Statistics
      */
-    pub static ref OPERATIONS_PER_BLOCK_ATTESTATION: Result<Histogram> = try_create_histogram(
+    pub static ref OPERATIONS_PER_BLOCK_ATTESTATION: Result<Histogram> = try_create_histogram_with_buckets(
         "beacon_operations_per_block_attestation_total",
-        "Number of attestations in a block"
+        "Number of attestations in a block",
+        // Full block is 128.
+        Ok(vec![0_f64, 1_f64, 3_f64, 15_f64, 31_f64, 63_f64, 127_f64, 255_f64])
     );
 
-    pub static ref BLOCK_SIZE: Result<Histogram> = try_create_histogram(
+    pub static ref BLOCK_SIZE: Result<Histogram> = try_create_histogram_with_buckets(
         "beacon_block_total_size",
-        "Size of a signed beacon block"
+        "Size of a signed beacon block",
+        linear_buckets(5120_f64,5120_f64,10)
     );
 
     /*
@@ -247,6 +265,10 @@ lazy_static! {
         try_create_int_counter("beacon_shuffling_cache_hits_total", "Count of times shuffling cache fulfils request");
     pub static ref SHUFFLING_CACHE_MISSES: Result<IntCounter> =
         try_create_int_counter("beacon_shuffling_cache_misses_total", "Count of times shuffling cache fulfils request");
+    pub static ref SHUFFLING_CACHE_PROMISE_HITS: Result<IntCounter> =
+        try_create_int_counter("beacon_shuffling_cache_promise_hits_total", "Count of times shuffling cache returns a promise to future shuffling");
+    pub static ref SHUFFLING_CACHE_PROMISE_FAILS: Result<IntCounter> =
+        try_create_int_counter("beacon_shuffling_cache_promise_fails_total", "Count of times shuffling cache detects a failed promise");
 
     /*
      * Early attester cache
@@ -298,14 +320,34 @@ lazy_static! {
         "beacon_fork_choice_reorg_total",
         "Count of occasions fork choice has switched to a different chain"
     );
+    pub static ref FORK_CHOICE_REORG_DISTANCE: Result<IntGauge> = try_create_int_gauge(
+        "beacon_fork_choice_reorg_distance",
+        "The distance of each re-org of the fork choice algorithm"
+    );
     pub static ref FORK_CHOICE_REORG_COUNT_INTEROP: Result<IntCounter> = try_create_int_counter(
         "beacon_reorgs_total",
         "Count of occasions fork choice has switched to a different chain"
     );
-    pub static ref FORK_CHOICE_TIMES: Result<Histogram> =
-        try_create_histogram("beacon_fork_choice_seconds", "Full runtime of fork choice");
-    pub static ref FORK_CHOICE_FIND_HEAD_TIMES: Result<Histogram> =
-        try_create_histogram("beacon_fork_choice_find_head_seconds", "Full runtime of fork choice find_head function");
+    pub static ref FORK_CHOICE_TIMES: Result<Histogram> = try_create_histogram_with_buckets(
+        "beacon_fork_choice_seconds",
+        "Full runtime of fork choice",
+        linear_buckets(10e-3, 20e-3, 10)
+    );
+    pub static ref FORK_CHOICE_OVERRIDE_FCU_TIMES: Result<Histogram> = try_create_histogram_with_buckets(
+        "beacon_fork_choice_override_fcu_seconds",
+        "Time taken to compute the optional forkchoiceUpdated override",
+        exponential_buckets(1e-3, 2.0, 8)
+    );
+    pub static ref FORK_CHOICE_AFTER_NEW_HEAD_TIMES: Result<Histogram> = try_create_histogram_with_buckets(
+        "beacon_fork_choice_after_new_head_seconds",
+        "Time taken to run `after_new_head`",
+        exponential_buckets(1e-3, 2.0, 10)
+    );
+    pub static ref FORK_CHOICE_AFTER_FINALIZATION_TIMES: Result<Histogram> = try_create_histogram_with_buckets(
+        "beacon_fork_choice_after_finalization_seconds",
+        "Time taken to run `after_finalization`",
+        exponential_buckets(1e-3, 2.0, 10)
+    );
     pub static ref FORK_CHOICE_PROCESS_BLOCK_TIMES: Result<Histogram> = try_create_histogram(
         "beacon_fork_choice_process_block_seconds",
         "Time taken to add a block and all attestations to fork choice"
@@ -321,7 +363,7 @@ lazy_static! {
     pub static ref BALANCES_CACHE_HITS: Result<IntCounter> =
         try_create_int_counter("beacon_balances_cache_hits_total", "Count of times balances cache fulfils request");
     pub static ref BALANCES_CACHE_MISSES: Result<IntCounter> =
-        try_create_int_counter("beacon_balances_cache_misses_total", "Count of times balances cache fulfils request");
+        try_create_int_counter("beacon_balances_cache_misses_total", "Count of times balances cache misses request");
 
     /*
      * Persisting BeaconChain components to disk
@@ -767,31 +809,62 @@ lazy_static! {
         "Number of attester slashings seen",
         &["src", "validator"]
     );
+}
 
+// Fourth lazy-static block is used to account for macro recursion limit.
+lazy_static! {
     /*
      * Block Delay Metrics
      */
-    pub static ref BEACON_BLOCK_OBSERVED_SLOT_START_DELAY_TIME: Result<Histogram> = try_create_histogram(
+    pub static ref BEACON_BLOCK_OBSERVED_SLOT_START_DELAY_TIME: Result<Histogram> = try_create_histogram_with_buckets(
         "beacon_block_observed_slot_start_delay_time",
         "Duration between the start of the block's slot and the time the block was observed.",
+        // [0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50]
+        decimal_buckets(-1,2)
     );
-    pub static ref BEACON_BLOCK_IMPORTED_OBSERVED_DELAY_TIME: Result<Histogram> = try_create_histogram(
+    pub static ref BEACON_BLOCK_IMPORTED_OBSERVED_DELAY_TIME: Result<Histogram> = try_create_histogram_with_buckets(
         "beacon_block_imported_observed_delay_time",
         "Duration between the time the block was observed and the time when it was imported.",
+        // [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5]
+        decimal_buckets(-2,0)
     );
-    pub static ref BEACON_BLOCK_HEAD_IMPORTED_DELAY_TIME: Result<Histogram> = try_create_histogram(
+    pub static ref BEACON_BLOCK_HEAD_IMPORTED_DELAY_TIME: Result<Histogram> = try_create_histogram_with_buckets(
         "beacon_block_head_imported_delay_time",
         "Duration between the time the block was imported and the time when it was set as head.",
+        // [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5]
+        decimal_buckets(-2,-1)
     );
-    pub static ref BEACON_BLOCK_HEAD_SLOT_START_DELAY_TIME: Result<Histogram> = try_create_histogram(
+    pub static ref BEACON_BLOCK_HEAD_ATTESTABLE_DELAY_TIME: Result<Histogram> = try_create_histogram(
+        "beacon_block_head_attestable_delay_time",
+        "Duration between the start of the slot and the time at which the block could be attested to.",
+    );
+    pub static ref BEACON_BLOCK_HEAD_SLOT_START_DELAY_TIME: Result<Histogram> = try_create_histogram_with_buckets(
         "beacon_block_head_slot_start_delay_time",
         "Duration between the start of the block's slot and the time when it was set as head.",
+        // [0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50]
+        decimal_buckets(-1,2)
     );
     pub static ref BEACON_BLOCK_HEAD_SLOT_START_DELAY_EXCEEDED_TOTAL: Result<IntCounter> = try_create_int_counter(
         "beacon_block_head_slot_start_delay_exceeded_total",
         "Triggered when the duration between the start of the block's slot and the current time \
         will result in failed attestations.",
     );
+    pub static ref BEACON_BLOCK_HEAD_MISSED_ATT_DEADLINE_LATE: Result<IntCounter> = try_create_int_counter(
+        "beacon_block_head_missed_att_deadline_late",
+        "Total number of delayed head blocks that arrived late"
+    );
+    pub static ref BEACON_BLOCK_HEAD_MISSED_ATT_DEADLINE_BORDERLINE: Result<IntCounter> = try_create_int_counter(
+        "beacon_block_head_missed_att_deadline_borderline",
+        "Total number of delayed head blocks that arrived very close to the deadline"
+    );
+    pub static ref BEACON_BLOCK_HEAD_MISSED_ATT_DEADLINE_SLOW: Result<IntCounter> = try_create_int_counter(
+        "beacon_block_head_missed_att_deadline_slow",
+        "Total number of delayed head blocks that arrived on time but not processed in time"
+    );
+    pub static ref BEACON_BLOCK_HEAD_MISSED_ATT_DEADLINE_OTHER: Result<IntCounter> = try_create_int_counter(
+        "beacon_block_head_missed_att_deadline_other",
+        "Total number of delayed head blocks that were not late and not slow to process"
+    );
 
     /*
      * General block metrics
@@ -801,10 +874,7 @@ lazy_static! {
             "gossip_beacon_block_skipped_slots",
             "For each gossip blocks, the number of skip slots between it and its parent"
         );
-}
 
-// Fourth lazy-static block is used to account for macro recursion limit.
-lazy_static! {
     /*
      * Sync Committee Message Verification
      */
@@ -920,6 +990,24 @@ lazy_static! {
         );
 }
 
+// Fifth lazy-static block is used to account for macro recursion limit.
+lazy_static! {
+    /*
+    * Light server message verification
+    */
+    pub static ref FINALITY_UPDATE_PROCESSING_SUCCESSES: Result<IntCounter> = try_create_int_counter(
+        "light_client_finality_update_verification_success_total",
+        "Number of light client finality updates verified for gossip"
+    );
+    /*
+    * Light server message verification
+    */
+    pub static ref OPTIMISTIC_UPDATE_PROCESSING_SUCCESSES: Result<IntCounter> = try_create_int_counter(
+        "light_client_optimistic_update_verification_success_total",
+        "Number of light client optimistic updates verified for gossip"
+    );
+}
+
 /// Scrape the `beacon_chain` for metrics that are not constantly updated (e.g., the present slot,
 /// head state info, etc) and update the Prometheus `DEFAULT_REGISTRY`.
 pub fn scrape_for_metrics<T: BeaconChainTypes>(beacon_chain: &BeaconChain<T>) {
@@ -935,15 +1023,10 @@ pub fn scrape_for_metrics<T: BeaconChainTypes>(beacon_chain: &BeaconChain<T>) {
 
     let attestation_stats = beacon_chain.op_pool.attestation_stats();
 
-    if let Some(snapshot_cache) = beacon_chain
-        .snapshot_cache
-        .try_write_for(SNAPSHOT_CACHE_TIMEOUT)
-    {
-        set_gauge(
-            &BLOCK_PROCESSING_SNAPSHOT_CACHE_SIZE,
-            snapshot_cache.len() as i64,
-        )
-    }
+    set_gauge_by_usize(
+        &BLOCK_PROCESSING_SNAPSHOT_CACHE_SIZE,
+        beacon_chain.store.state_cache_len(),
+    );
 
     if let Some((size, num_lookups)) = beacon_chain.pre_finalization_block_cache.metrics() {
         set_gauge_by_usize(&PRE_FINALIZATION_BLOCK_CACHE_SIZE, size);
@@ -1055,7 +1138,7 @@ fn scrape_head_state<T: EthSpec>(state: &BeaconState<T>, state_root: Hash256) {
             num_active += 1;
         }
 
-        if v.slashed {
+        if v.slashed() {
             num_slashed += 1;
         }
 

beacon_node/beacon_chain/src/migrate.rs

@@ -3,7 +3,8 @@ use crate::errors::BeaconChainError;
 use crate::head_tracker::{HeadTracker, SszHeadTracker};
 use crate::persisted_beacon_chain::{PersistedBeaconChain, DUMMY_CANONICAL_HEAD_BLOCK_ROOT};
 use parking_lot::Mutex;
-use slog::{debug, error, info, warn, Logger};
+use serde::{Deserialize, Serialize};
+use slog::{debug, error, info, trace, warn, Logger};
 use std::collections::{HashMap, HashSet};
 use std::mem;
 use std::sync::{mpsc, Arc};
@@ -25,20 +26,43 @@ const MIN_COMPACTION_PERIOD_SECONDS: u64 = 7200;
 /// Compact after a large finality gap, if we respect `MIN_COMPACTION_PERIOD_SECONDS`.
 const COMPACTION_FINALITY_DISTANCE: u64 = 1024;
 
+/// Default number of epochs to wait between finalization migrations.
+pub const DEFAULT_EPOCHS_PER_RUN: u64 = 4;
+
 /// The background migrator runs a thread to perform pruning and migrate state from the hot
 /// to the cold database.
 pub struct BackgroundMigrator<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> {
     db: Arc<HotColdDB<E, Hot, Cold>>,
     #[allow(clippy::type_complexity)]
     tx_thread: Option<Mutex<(mpsc::Sender<Notification>, thread::JoinHandle<()>)>>,
+    /// Record of when the last migration ran, for enforcing `epochs_per_run`.
+    prev_migration: Arc<Mutex<PrevMigration>>,
     /// Genesis block root, for persisting the `PersistedBeaconChain`.
     genesis_block_root: Hash256,
     log: Logger,
 }
 
-#[derive(Debug, Default, Clone, PartialEq, Eq)]
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub struct MigratorConfig {
     pub blocking: bool,
+    /// Run migrations at most once per `epochs_per_run`.
+    ///
+    /// If set to 0, then run every finalization.
+    pub epochs_per_run: u64,
+}
+
+impl Default for MigratorConfig {
+    fn default() -> Self {
+        Self {
+            blocking: false,
+            epochs_per_run: DEFAULT_EPOCHS_PER_RUN,
+        }
+    }
+}
+
+pub struct PrevMigration {
+    epoch: Option<Epoch>,
+    epochs_per_run: u64,
 }
 
 impl MigratorConfig {
@@ -46,6 +70,11 @@ impl MigratorConfig {
         self.blocking = true;
         self
     }
+
+    pub fn epochs_per_run(mut self, epochs_per_run: u64) -> Self {
+        self.epochs_per_run = epochs_per_run;
+        self
+    }
 }
 
 /// Pruning can be successful, or in rare cases deferred to a later point.
@@ -55,7 +84,13 @@ pub enum PruningOutcome {
     Successful {
         old_finalized_checkpoint: Checkpoint,
     },
-    DeferredConcurrentMutation,
+    /// The run was aborted because the new finalized checkpoint is older than the previous one.
+    OutOfOrderFinalization {
+        old_finalized_checkpoint: Checkpoint,
+        new_finalized_checkpoint: Checkpoint,
+    },
+    /// The run was aborted due to a concurrent mutation of the head tracker.
+    DeferredConcurrentHeadTrackerMutation,
 }
 
 /// Logic errors that can occur during pruning, none of these should ever happen.
@@ -68,6 +103,10 @@ pub enum PruningError {
     MissingInfoForCanonicalChain {
         slot: Slot,
     },
+    FinalizedStateOutOfOrder {
+        old_finalized_checkpoint: Checkpoint,
+        new_finalized_checkpoint: Checkpoint,
+    },
     UnexpectedEqualStateRoots,
     UnexpectedUnequalStateRoots,
 }
@@ -85,6 +124,18 @@ pub struct FinalizationNotification {
     genesis_block_root: Hash256,
 }
 
+impl Notification {
+    pub fn epoch(&self) -> Option<Epoch> {
+        match self {
+            Notification::Finalization(FinalizationNotification {
+                finalized_checkpoint,
+                ..
+            }) => Some(finalized_checkpoint.epoch),
+            Notification::Reconstruction => None,
+        }
+    }
+}
+
 impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Hot, Cold> {
     /// Create a new `BackgroundMigrator` and spawn its thread if necessary.
     pub fn new(
@@ -93,14 +144,23 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
         genesis_block_root: Hash256,
         log: Logger,
     ) -> Self {
+        let prev_migration = Arc::new(Mutex::new(PrevMigration {
+            epoch: None,
+            epochs_per_run: config.epochs_per_run,
+        }));
         let tx_thread = if config.blocking {
             None
         } else {
-            Some(Mutex::new(Self::spawn_thread(db.clone(), log.clone())))
+            Some(Mutex::new(Self::spawn_thread(
+                db.clone(),
+                prev_migration.clone(),
+                log.clone(),
+            )))
         };
         Self {
             db,
             tx_thread,
+            prev_migration,
             genesis_block_root,
             log,
         }
@@ -163,7 +223,11 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
 
             // Restart the background thread if it has crashed.
             if let Err(tx_err) = tx.send(notif) {
-                let (new_tx, new_thread) = Self::spawn_thread(self.db.clone(), self.log.clone());
+                let (new_tx, new_thread) = Self::spawn_thread(
+                    self.db.clone(),
+                    self.prev_migration.clone(),
+                    self.log.clone(),
+                );
 
                 *tx = new_tx;
                 let old_thread = mem::replace(thread, new_thread);
@@ -197,6 +261,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
         debug!(log, "Database consolidation started");
 
         let finalized_state_root = notif.finalized_state_root;
+        let finalized_block_root = notif.finalized_checkpoint.root;
 
         let finalized_state = match db.get_state(&finalized_state_root.into(), None) {
             Ok(Some(state)) => state,
@@ -223,7 +288,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
             Ok(PruningOutcome::Successful {
                 old_finalized_checkpoint,
             }) => old_finalized_checkpoint,
-            Ok(PruningOutcome::DeferredConcurrentMutation) => {
+            Ok(PruningOutcome::DeferredConcurrentHeadTrackerMutation) => {
                 warn!(
                     log,
                     "Pruning deferred because of a concurrent mutation";
@@ -231,13 +296,31 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
                 );
                 return;
             }
+            Ok(PruningOutcome::OutOfOrderFinalization {
+                old_finalized_checkpoint,
+                new_finalized_checkpoint,
+            }) => {
+                warn!(
+                    log,
+                    "Ignoring out of order finalization request";
+                    "old_finalized_epoch" => old_finalized_checkpoint.epoch,
+                    "new_finalized_epoch" => new_finalized_checkpoint.epoch,
+                    "message" => "this is expected occasionally due to a (harmless) race condition"
+                );
+                return;
+            }
             Err(e) => {
-                warn!(log, "Block pruning failed"; "error" => format!("{:?}", e));
+                warn!(log, "Block pruning failed"; "error" => ?e);
                 return;
             }
         };
 
-        match migrate_database(db.clone(), finalized_state_root.into(), &finalized_state) {
+        match migrate_database(
+            db.clone(),
+            finalized_state_root.into(),
+            finalized_block_root,
+            &finalized_state,
+        ) {
             Ok(()) => {}
             Err(Error::HotColdDBError(HotColdDBError::FreezeSlotUnaligned(slot))) => {
                 debug!(
@@ -274,6 +357,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
     /// Return a channel handle for sending requests to the thread.
     fn spawn_thread(
         db: Arc<HotColdDB<E, Hot, Cold>>,
+        prev_migration: Arc<Mutex<PrevMigration>>,
         log: Logger,
     ) -> (mpsc::Sender<Notification>, thread::JoinHandle<()>) {
         let (tx, rx) = mpsc::channel();
@@ -299,6 +383,29 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
                             }
                         });
 
+                // Do not run too frequently.
+                if let Some(epoch) = notif.epoch() {
+                    let mut prev_migration = prev_migration.lock();
+
+                    if let Some(prev_epoch) = prev_migration.epoch {
+                        if epoch < prev_epoch + prev_migration.epochs_per_run {
+                            debug!(
+                                log,
+                                "Database consolidation deferred";
+                                "last_finalized_epoch" => prev_epoch,
+                                "new_finalized_epoch" => epoch,
+                                "epochs_per_run" => prev_migration.epochs_per_run,
+                            );
+                            continue;
+                        }
+                    }
+
+                    // We intend to run at this epoch, update the in-memory record of the last epoch
+                    // at which we ran. This value isn't tracked on disk so we will always migrate
+                    // on the first finalization after startup.
+                    prev_migration.epoch = Some(epoch);
+                }
+
                 match notif {
                     Notification::Reconstruction => Self::run_reconstruction(db.clone(), &log),
                     Notification::Finalization(fin) => Self::run_migration(db.clone(), fin, &log),
@@ -347,6 +454,16 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
             .into());
         }
 
+        // The new finalized state must be newer than the previous finalized state.
+        // I think this can happen sometimes currently due to `fork_choice` running in parallel
+        // with itself and sending us notifications out of order.
+        if old_finalized_slot > new_finalized_slot {
+            return Ok(PruningOutcome::OutOfOrderFinalization {
+                old_finalized_checkpoint,
+                new_finalized_checkpoint,
+            });
+        }
+
         debug!(
             log,
             "Starting database pruning";
@@ -374,15 +491,14 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
         // We don't know which blocks are shared among abandoned chains, so we buffer and delete
         // everything in one fell swoop.
         let mut abandoned_blocks: HashSet<SignedBeaconBlockHash> = HashSet::new();
-        let mut abandoned_states: HashSet<(Slot, BeaconStateHash)> = HashSet::new();
         let mut abandoned_heads: HashSet<Hash256> = HashSet::new();
 
         let heads = head_tracker.heads();
         debug!(
             log,
             "Extra pruning information";
-            "old_finalized_root" => format!("{:?}", old_finalized_checkpoint.root),
-            "new_finalized_root" => format!("{:?}", new_finalized_checkpoint.root),
+            "old_finalized_root" => ?old_finalized_checkpoint.root,
+            "new_finalized_root" => ?new_finalized_checkpoint.root,
             "head_count" => heads.len(),
         );
 
@@ -391,7 +507,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
             // so delete it from the head tracker but leave it and its states in the database
             // This is suboptimal as it wastes disk space, but it's difficult to fix. A re-sync
             // can be used to reclaim the space.
-            let head_state_root = match store.get_block(&head_hash) {
+            let head_state_root = match store.get_blinded_block(&head_hash, Some(head_slot)) {
                 Ok(Some(block)) => block.state_root(),
                 Ok(None) => {
                     return Err(BeaconStateError::MissingBeaconBlock(head_hash.into()).into())
@@ -410,9 +526,9 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
             };
 
             let mut potentially_abandoned_head = Some(head_hash);
-            let mut potentially_abandoned_blocks = vec![];
+            let mut potentially_abandoned_blocks = HashSet::new();
 
-            // Iterate backwards from this head, staging blocks and states for deletion.
+            // Iterate backwards from this head, staging blocks for deletion.
             let iter = std::iter::once(Ok((head_hash, head_state_root, head_slot)))
                 .chain(RootsIterator::from_block(&store, head_hash)?);
 
@@ -427,11 +543,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
                     // the fork's block and state for possible deletion.
                     None => {
                         if slot > new_finalized_slot {
-                            potentially_abandoned_blocks.push((
-                                slot,
-                                Some(block_root),
-                                Some(state_root),
-                            ));
+                            potentially_abandoned_blocks.insert(block_root);
                         } else if slot >= old_finalized_slot {
                             return Err(PruningError::MissingInfoForCanonicalChain { slot }.into());
                         } else {
@@ -441,7 +553,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
                             warn!(
                                 log,
                                 "Found a chain that should already have been pruned";
-                                "head_block_root" => format!("{:?}", head_hash),
+                                "head_block_root" => ?head_hash,
                                 "head_slot" => head_slot,
                             );
                             potentially_abandoned_head.take();
@@ -469,26 +581,14 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
                             // we will have just staged the common block for deletion.
                             // Unstage it.
                             else {
-                                for (_, block_root, _) in
-                                    potentially_abandoned_blocks.iter_mut().rev()
-                                {
-                                    if block_root.as_ref() == Some(finalized_block_root) {
-                                        *block_root = None;
-                                    } else {
-                                        break;
-                                    }
-                                }
+                                potentially_abandoned_blocks.remove(finalized_block_root);
                             }
                             break;
                         } else {
                             if state_root == *finalized_state_root {
                                 return Err(PruningError::UnexpectedEqualStateRoots.into());
                             }
-                            potentially_abandoned_blocks.push((
-                                slot,
-                                Some(block_root),
-                                Some(state_root),
-                            ));
+                            potentially_abandoned_blocks.insert(block_root);
                         }
                     }
                 }
@@ -498,18 +598,11 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
                 debug!(
                     log,
                     "Pruning head";
-                    "head_block_root" => format!("{:?}", abandoned_head),
+                    "head_block_root" => ?abandoned_head,
                     "head_slot" => head_slot,
                 );
                 abandoned_heads.insert(abandoned_head);
-                abandoned_blocks.extend(
-                    potentially_abandoned_blocks
-                        .iter()
-                        .filter_map(|(_, maybe_block_hash, _)| *maybe_block_hash),
-                );
-                abandoned_states.extend(potentially_abandoned_blocks.iter().filter_map(
-                    |(slot, _, maybe_state_hash)| maybe_state_hash.map(|sr| (*slot, sr)),
-                ));
+                abandoned_blocks.extend(potentially_abandoned_blocks);
             }
         }
 
@@ -523,7 +616,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
         // later.
         for head_hash in &abandoned_heads {
             if !head_tracker_lock.contains_key(head_hash) {
-                return Ok(PruningOutcome::DeferredConcurrentMutation);
+                return Ok(PruningOutcome::DeferredConcurrentHeadTrackerMutation);
             }
         }
 
@@ -532,34 +625,73 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
             head_tracker_lock.remove(&head_hash);
         }
 
-        let batch: Vec<StoreOp<E>> = abandoned_blocks
+        let num_deleted_blocks = abandoned_blocks.len();
+        let mut batch: Vec<StoreOp<E>> = abandoned_blocks
             .into_iter()
             .map(Into::into)
-            .map(StoreOp::DeleteBlock)
-            .chain(
-                abandoned_states
-                    .into_iter()
-                    .map(|(slot, state_hash)| StoreOp::DeleteState(state_hash.into(), Some(slot))),
-            )
+            .flat_map(|block_root: Hash256| {
+                [
+                    StoreOp::DeleteBlock(block_root),
+                    StoreOp::DeleteExecutionPayload(block_root),
+                ]
+            })
             .collect();
 
-        let mut kv_batch = store.convert_to_kv_batch(&batch)?;
-
         // Persist the head in case the process is killed or crashes here. This prevents
         // the head tracker reverting after our mutation above.
         let persisted_head = PersistedBeaconChain {
             _canonical_head_block_root: DUMMY_CANONICAL_HEAD_BLOCK_ROOT,
             genesis_block_root,
-            ssz_head_tracker: SszHeadTracker::from_map(&*head_tracker_lock),
+            ssz_head_tracker: SszHeadTracker::from_map(&head_tracker_lock),
         };
         drop(head_tracker_lock);
-        kv_batch.push(persisted_head.as_kv_store_op(BEACON_CHAIN_DB_KEY));
+        batch.push(StoreOp::KeyValueOp(
+            persisted_head.as_kv_store_op(BEACON_CHAIN_DB_KEY)?,
+        ));
 
         // Persist the new finalized checkpoint as the pruning checkpoint.
-        kv_batch.push(store.pruning_checkpoint_store_op(new_finalized_checkpoint));
+        batch.push(StoreOp::KeyValueOp(
+            store.pruning_checkpoint_store_op(new_finalized_checkpoint)?,
+        ));
 
-        store.hot_db.do_atomically(kv_batch)?;
-        debug!(log, "Database pruning complete");
+        store.do_atomically(batch)?;
+        debug!(
+            log,
+            "Database block pruning complete";
+            "num_deleted_blocks" => num_deleted_blocks,
+        );
+
+        // Do a separate pass to clean up irrelevant states.
+        let mut state_delete_batch = vec![];
+        for res in store.iter_hot_state_summaries() {
+            let (state_root, summary) = res?;
+
+            if summary.slot <= new_finalized_slot {
+                // If state root doesn't match state root from canonical chain, or this slot
+                // is not part of the recently finalized chain, then delete.
+                if newly_finalized_chain
+                    .get(&summary.slot)
+                    .map_or(true, |(_, canonical_state_root)| {
+                        state_root != Hash256::from(*canonical_state_root)
+                    })
+                {
+                    trace!(
+                        log,
+                        "Deleting state";
+                        "state_root" => ?state_root,
+                        "slot" => summary.slot,
+                    );
+                    state_delete_batch.push(StoreOp::DeleteState(state_root, Some(summary.slot)));
+                }
+            }
+        }
+        let num_deleted_states = state_delete_batch.len();
+        store.do_atomically(state_delete_batch)?;
+        debug!(
+            log,
+            "Database state pruning complete";
+            "num_deleted_states" => num_deleted_states,
+        );
 
         Ok(PruningOutcome::Successful {
             old_finalized_checkpoint,

beacon_node/beacon_chain/src/naive_aggregation_pool.rs

@@ -402,7 +402,7 @@ impl<T: AggregateMap> NaiveAggregationPool<T> {
 
     /// Returns the total number of items stored in `self`.
     pub fn num_items(&self) -> usize {
-        self.maps.iter().map(|(_, map)| map.len()).sum()
+        self.maps.values().map(T::len).sum()
     }
 
     /// Returns an aggregated `T::Value` with the given `T::Data`, if any.
@@ -421,10 +421,7 @@ impl<T: AggregateMap> NaiveAggregationPool<T> {
 
     /// Iterate all items in all slots of `self`.
     pub fn iter(&self) -> impl Iterator<Item = &T::Value> {
-        self.maps
-            .iter()
-            .map(|(_slot, map)| map.get_map().iter().map(|(_key, value)| value))
-            .flatten()
+        self.maps.values().flat_map(|map| map.get_map().values())
     }
 
     /// Removes any items with a slot lower than `current_slot` and bars any future
@@ -451,11 +448,7 @@ impl<T: AggregateMap> NaiveAggregationPool<T> {
         // If we have too many maps, remove the lowest amount to ensure we only have
         // `SLOTS_RETAINED` left.
         if self.maps.len() > SLOTS_RETAINED {
-            let mut slots = self
-                .maps
-                .iter()
-                .map(|(slot, _map)| *slot)
-                .collect::<Vec<_>>();
+            let mut slots = self.maps.keys().copied().collect::<Vec<_>>();
             // Sort is generally pretty slow, however `SLOTS_RETAINED` is quite low so it should be
             // negligible.
             slots.sort_unstable();

beacon_node/beacon_chain/src/observed_aggregates.rs

@@ -203,6 +203,7 @@ impl<T: TreeHash + SlotData + Consts, E: EthSpec> ObservedAggregates<T, E> {
     /// Check to see if the `root` of `item` is in self.
     ///
     /// `root` must equal `a.tree_hash_root()`.
+    #[allow(clippy::wrong_self_convention)]
     pub fn is_known(&mut self, item: &T, root: Hash256) -> Result<bool, Error> {
         let index = self.get_set_index(item.get_slot())?;
 

beacon_node/beacon_chain/src/observed_operations.rs

@@ -1,10 +1,12 @@
 use derivative::Derivative;
 use smallvec::SmallVec;
+use ssz::{Decode, Encode};
 use state_processing::{SigVerifiedOp, VerifyOperation};
 use std::collections::HashSet;
 use std::marker::PhantomData;
 use types::{
-    AttesterSlashing, BeaconState, ChainSpec, EthSpec, ProposerSlashing, SignedVoluntaryExit,
+    AttesterSlashing, BeaconState, ChainSpec, EthSpec, ForkName, ProposerSlashing,
+    SignedVoluntaryExit, Slot,
 };
 
 /// Number of validator indices to store on the stack in `observed_validators`.
@@ -24,13 +26,16 @@ pub struct ObservedOperations<T: ObservableOperation<E>, E: EthSpec> {
     /// previously seen attester slashings, i.e. those validators in the intersection of
     /// `attestation_1.attester_indices` and `attestation_2.attester_indices`.
     observed_validator_indices: HashSet<u64>,
+    /// The name of the current fork. The default will be overwritten on first use.
+    #[derivative(Default(value = "ForkName::Base"))]
+    current_fork: ForkName,
     _phantom: PhantomData<(T, E)>,
 }
 
 /// Was the observed operation new and valid for further processing, or a useless duplicate?
 #[derive(Debug, PartialEq, Eq, Clone)]
-pub enum ObservationOutcome<T> {
-    New(SigVerifiedOp<T>),
+pub enum ObservationOutcome<T: Encode + Decode, E: EthSpec> {
+    New(SigVerifiedOp<T, E>),
     AlreadyKnown,
 }
 
@@ -81,7 +86,9 @@ impl<T: ObservableOperation<E>, E: EthSpec> ObservedOperations<T, E> {
         op: T,
         head_state: &BeaconState<E>,
         spec: &ChainSpec,
-    ) -> Result<ObservationOutcome<T>, T::Error> {
+    ) -> Result<ObservationOutcome<T, E>, T::Error> {
+        self.reset_at_fork_boundary(head_state.slot(), spec);
+
         let observed_validator_indices = &mut self.observed_validator_indices;
         let new_validator_indices = op.observed_validators();
 
@@ -107,4 +114,23 @@ impl<T: ObservableOperation<E>, E: EthSpec> ObservedOperations<T, E> {
 
         Ok(ObservationOutcome::New(verified_op))
     }
+
+    /// Reset the cache when crossing a fork boundary.
+    ///
+    /// This prevents an attacker from crafting a self-slashing which is only valid before the fork
+    /// (e.g. using the Altair fork domain at a Bellatrix epoch), in order to prevent propagation of
+    /// all other slashings due to the duplicate check.
+    ///
+    /// It doesn't matter if this cache gets reset too often, as we reset it on restart anyway and a
+    /// false negative just results in propagation of messages which should have been ignored.
+    ///
+    /// In future we could check slashing relevance against the op pool itself, but that would
+    /// require indexing the attester slashings in the op pool by validator index.
+    fn reset_at_fork_boundary(&mut self, head_slot: Slot, spec: &ChainSpec) {
+        let head_fork = spec.fork_name_at_slot::<E>(head_slot);
+        if head_fork != self.current_fork {
+            self.observed_validator_indices.clear();
+            self.current_fork = head_fork;
+        }
+    }
 }

beacon_node/beacon_chain/src/otb_verification_service.rs

@@ -0,0 +1,381 @@
+use crate::execution_payload::{validate_merge_block, AllowOptimisticImport};
+use crate::{
+    BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, ExecutionPayloadError,
+    INVALID_FINALIZED_MERGE_TRANSITION_BLOCK_SHUTDOWN_REASON,
+};
+use itertools::process_results;
+use proto_array::InvalidationOperation;
+use slog::{crit, debug, error, info, warn};
+use slot_clock::SlotClock;
+use ssz::{Decode, Encode};
+use ssz_derive::{Decode, Encode};
+use state_processing::per_block_processing::is_merge_transition_complete;
+use std::sync::Arc;
+use store::{DBColumn, Error as StoreError, HotColdDB, KeyValueStore, StoreItem};
+use task_executor::{ShutdownReason, TaskExecutor};
+use tokio::time::sleep;
+use tree_hash::TreeHash;
+use types::{BeaconBlockRef, EthSpec, Hash256, Slot};
+use DBColumn::OptimisticTransitionBlock as OTBColumn;
+
+#[derive(Clone, Debug, Decode, Encode, PartialEq)]
+pub struct OptimisticTransitionBlock {
+    root: Hash256,
+    slot: Slot,
+}
+
+impl OptimisticTransitionBlock {
+    // types::BeaconBlockRef<'_, <T as BeaconChainTypes>::EthSpec>
+    pub fn from_block<E: EthSpec>(block: BeaconBlockRef<E>) -> Self {
+        Self {
+            root: block.tree_hash_root(),
+            slot: block.slot(),
+        }
+    }
+
+    pub fn root(&self) -> &Hash256 {
+        &self.root
+    }
+
+    pub fn slot(&self) -> &Slot {
+        &self.slot
+    }
+
+    pub fn persist_in_store<T, A>(&self, store: A) -> Result<(), StoreError>
+    where
+        T: BeaconChainTypes,
+        A: AsRef<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>,
+    {
+        if store
+            .as_ref()
+            .item_exists::<OptimisticTransitionBlock>(&self.root)?
+        {
+            Ok(())
+        } else {
+            store.as_ref().put_item(&self.root, self)
+        }
+    }
+
+    pub fn remove_from_store<T, A>(&self, store: A) -> Result<(), StoreError>
+    where
+        T: BeaconChainTypes,
+        A: AsRef<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>,
+    {
+        store
+            .as_ref()
+            .hot_db
+            .key_delete(OTBColumn.into(), self.root.as_bytes())
+    }
+
+    fn is_canonical<T: BeaconChainTypes>(
+        &self,
+        chain: &BeaconChain<T>,
+    ) -> Result<bool, BeaconChainError> {
+        Ok(chain
+            .forwards_iter_block_roots_until(self.slot, self.slot)?
+            .next()
+            .transpose()?
+            .map(|(root, _)| root)
+            == Some(self.root))
+    }
+}
+
+impl StoreItem for OptimisticTransitionBlock {
+    fn db_column() -> DBColumn {
+        OTBColumn
+    }
+
+    fn as_store_bytes(&self) -> Result<Vec<u8>, StoreError> {
+        Ok(self.as_ssz_bytes())
+    }
+
+    fn from_store_bytes(bytes: &[u8]) -> Result<Self, StoreError> {
+        Ok(Self::from_ssz_bytes(bytes)?)
+    }
+}
+
+/// The routine is expected to run once per epoch, 1/4th through the epoch.
+pub const EPOCH_DELAY_FACTOR: u32 = 4;
+
+/// Spawns a routine which checks the validity of any optimistically imported transition blocks
+///
+/// This routine will run once per epoch, at `epoch_duration / EPOCH_DELAY_FACTOR` after
+/// the start of each epoch.
+///
+/// The service will not be started if there is no `execution_layer` on the `chain`.
+pub fn start_otb_verification_service<T: BeaconChainTypes>(
+    executor: TaskExecutor,
+    chain: Arc<BeaconChain<T>>,
+) {
+    // Avoid spawning the service if there's no EL, it'll just error anyway.
+    if chain.execution_layer.is_some() {
+        executor.spawn(
+            async move { otb_verification_service(chain).await },
+            "otb_verification_service",
+        );
+    }
+}
+
+pub fn load_optimistic_transition_blocks<T: BeaconChainTypes>(
+    chain: &BeaconChain<T>,
+) -> Result<Vec<OptimisticTransitionBlock>, StoreError> {
+    process_results(
+        chain.store.hot_db.iter_column::<Hash256>(OTBColumn),
+        |iter| {
+            iter.map(|(_, bytes)| OptimisticTransitionBlock::from_store_bytes(&bytes))
+                .collect()
+        },
+    )?
+}
+
+#[derive(Debug)]
+pub enum Error {
+    ForkChoice(String),
+    BeaconChain(BeaconChainError),
+    StoreError(StoreError),
+    NoBlockFound(OptimisticTransitionBlock),
+}
+
+pub async fn validate_optimistic_transition_blocks<T: BeaconChainTypes>(
+    chain: &Arc<BeaconChain<T>>,
+    otbs: Vec<OptimisticTransitionBlock>,
+) -> Result<(), Error> {
+    let finalized_slot = chain
+        .canonical_head
+        .fork_choice_read_lock()
+        .get_finalized_block()
+        .map_err(|e| Error::ForkChoice(format!("{:?}", e)))?
+        .slot;
+
+    // separate otbs into
+    //     non-canonical
+    //     finalized canonical
+    //     unfinalized canonical
+    let mut non_canonical_otbs = vec![];
+    let (finalized_canonical_otbs, unfinalized_canonical_otbs) = process_results(
+        otbs.into_iter().map(|otb| {
+            otb.is_canonical(chain)
+                .map(|is_canonical| (otb, is_canonical))
+        }),
+        |pair_iter| {
+            pair_iter
+                .filter_map(|(otb, is_canonical)| {
+                    if is_canonical {
+                        Some(otb)
+                    } else {
+                        non_canonical_otbs.push(otb);
+                        None
+                    }
+                })
+                .partition::<Vec<_>, _>(|otb| *otb.slot() <= finalized_slot)
+        },
+    )
+    .map_err(Error::BeaconChain)?;
+
+    // remove non-canonical blocks that conflict with finalized checkpoint from the database
+    for otb in non_canonical_otbs {
+        if *otb.slot() <= finalized_slot {
+            otb.remove_from_store::<T, _>(&chain.store)
+                .map_err(Error::StoreError)?;
+        }
+    }
+
+    // ensure finalized canonical otb are valid, otherwise kill client
+    for otb in finalized_canonical_otbs {
+        match chain.get_block(otb.root()).await {
+            Ok(Some(block)) => {
+                match validate_merge_block(chain, block.message(), AllowOptimisticImport::No).await
+                {
+                    Ok(()) => {
+                        // merge transition block is valid, remove it from OTB
+                        otb.remove_from_store::<T, _>(&chain.store)
+                            .map_err(Error::StoreError)?;
+                        info!(
+                            chain.log,
+                            "Validated merge transition block";
+                            "block_root" => ?otb.root(),
+                            "type" => "finalized"
+                        );
+                    }
+                    // The block was not able to be verified by the EL. Leave the OTB in the
+                    // database since the EL is likely still syncing and may verify the block
+                    // later.
+                    Err(BlockError::ExecutionPayloadError(
+                        ExecutionPayloadError::UnverifiedNonOptimisticCandidate,
+                    )) => (),
+                    Err(BlockError::ExecutionPayloadError(
+                        ExecutionPayloadError::InvalidTerminalPoWBlock { .. },
+                    )) => {
+                        // Finalized Merge Transition Block is Invalid! Kill the Client!
+                        crit!(
+                            chain.log,
+                            "Finalized merge transition block is invalid!";
+                            "msg" => "You must use the `--purge-db` flag to clear the database and restart sync. \
+                            You may be on a hostile network.",
+                            "block_hash" => ?block.canonical_root()
+                        );
+                        let mut shutdown_sender = chain.shutdown_sender();
+                        if let Err(e) = shutdown_sender.try_send(ShutdownReason::Failure(
+                            INVALID_FINALIZED_MERGE_TRANSITION_BLOCK_SHUTDOWN_REASON,
+                        )) {
+                            crit!(
+                                chain.log,
+                                "Failed to shut down client";
+                                "error" => ?e,
+                                "shutdown_reason" => INVALID_FINALIZED_MERGE_TRANSITION_BLOCK_SHUTDOWN_REASON
+                            );
+                        }
+                    }
+                    _ => {}
+                }
+            }
+            Ok(None) => return Err(Error::NoBlockFound(otb)),
+            // Our database has pruned the payload and the payload was unavailable on the EL since
+            // the EL is still syncing or the payload is non-canonical.
+            Err(BeaconChainError::BlockHashMissingFromExecutionLayer(_)) => (),
+            Err(e) => return Err(Error::BeaconChain(e)),
+        }
+    }
+
+    // attempt to validate any non-finalized canonical otb blocks
+    for otb in unfinalized_canonical_otbs {
+        match chain.get_block(otb.root()).await {
+            Ok(Some(block)) => {
+                match validate_merge_block(chain, block.message(), AllowOptimisticImport::No).await
+                {
+                    Ok(()) => {
+                        // merge transition block is valid, remove it from OTB
+                        otb.remove_from_store::<T, _>(&chain.store)
+                            .map_err(Error::StoreError)?;
+                        info!(
+                            chain.log,
+                            "Validated merge transition block";
+                            "block_root" => ?otb.root(),
+                            "type" => "not finalized"
+                        );
+                    }
+                    // The block was not able to be verified by the EL. Leave the OTB in the
+                    // database since the EL is likely still syncing and may verify the block
+                    // later.
+                    Err(BlockError::ExecutionPayloadError(
+                        ExecutionPayloadError::UnverifiedNonOptimisticCandidate,
+                    )) => (),
+                    Err(BlockError::ExecutionPayloadError(
+                        ExecutionPayloadError::InvalidTerminalPoWBlock { .. },
+                    )) => {
+                        // Unfinalized Merge Transition Block is Invalid -> Run process_invalid_execution_payload
+                        warn!(
+                            chain.log,
+                            "Merge transition block invalid";
+                            "block_root" => ?otb.root()
+                        );
+                        chain
+                            .process_invalid_execution_payload(
+                                &InvalidationOperation::InvalidateOne {
+                                    block_root: *otb.root(),
+                                },
+                            )
+                            .await
+                            .map_err(|e| {
+                                warn!(
+                                    chain.log,
+                                    "Error checking merge transition block";
+                                    "error" => ?e,
+                                    "location" => "process_invalid_execution_payload"
+                                );
+                                Error::BeaconChain(e)
+                            })?;
+                    }
+                    _ => {}
+                }
+            }
+            Ok(None) => return Err(Error::NoBlockFound(otb)),
+            // Our database has pruned the payload and the payload was unavailable on the EL since
+            // the EL is still syncing or the payload is non-canonical.
+            Err(BeaconChainError::BlockHashMissingFromExecutionLayer(_)) => (),
+            Err(e) => return Err(Error::BeaconChain(e)),
+        }
+    }
+
+    Ok(())
+}
+
+/// Loop until any optimistically imported merge transition blocks have been verified and
+/// the merge has been finalized.
+async fn otb_verification_service<T: BeaconChainTypes>(chain: Arc<BeaconChain<T>>) {
+    let epoch_duration = chain.slot_clock.slot_duration() * T::EthSpec::slots_per_epoch() as u32;
+    loop {
+        match chain
+            .slot_clock
+            .duration_to_next_epoch(T::EthSpec::slots_per_epoch())
+        {
+            Some(duration) => {
+                let additional_delay = epoch_duration / EPOCH_DELAY_FACTOR;
+                sleep(duration + additional_delay).await;
+
+                debug!(
+                    chain.log,
+                    "OTB verification service firing";
+                );
+
+                if !is_merge_transition_complete(
+                    &chain.canonical_head.cached_head().snapshot.beacon_state,
+                ) {
+                    // We are pre-merge. Nothing to do yet.
+                    continue;
+                }
+
+                // load all optimistically imported transition blocks from the database
+                match load_optimistic_transition_blocks(chain.as_ref()) {
+                    Ok(otbs) => {
+                        if otbs.is_empty() {
+                            if chain
+                                .canonical_head
+                                .fork_choice_read_lock()
+                                .get_finalized_block()
+                                .map_or(false, |block| {
+                                    block.execution_status.is_execution_enabled()
+                                })
+                            {
+                                // there are no optimistic blocks in the database, we can exit
+                                // the service since the merge transition is finalized and we'll
+                                // never see another transition block
+                                break;
+                            } else {
+                                debug!(
+                                    chain.log,
+                                    "No optimistic transition blocks";
+                                    "info" => "waiting for the merge transition to finalize"
+                                )
+                            }
+                        }
+                        if let Err(e) = validate_optimistic_transition_blocks(&chain, otbs).await {
+                            warn!(
+                                chain.log,
+                                "Error while validating optimistic transition blocks";
+                                "error" => ?e
+                            );
+                        }
+                    }
+                    Err(e) => {
+                        error!(
+                            chain.log,
+                            "Error loading optimistic transition blocks";
+                            "error" => ?e
+                        );
+                    }
+                };
+            }
+            None => {
+                error!(chain.log, "Failed to read slot clock");
+                // If we can't read the slot clock, just wait another slot.
+                sleep(chain.slot_clock.slot_duration()).await;
+            }
+        };
+    }
+    debug!(
+        chain.log,
+        "No optimistic transition blocks in database";
+        "msg" => "shutting down OTB verification service"
+    );
+}

beacon_node/beacon_chain/src/persisted_beacon_chain.rs

@@ -26,8 +26,8 @@ impl StoreItem for PersistedBeaconChain {
         DBColumn::BeaconChain
     }
 
-    fn as_store_bytes(&self) -> Vec<u8> {
-        self.as_ssz_bytes()
+    fn as_store_bytes(&self) -> Result<Vec<u8>, StoreError> {
+        Ok(self.as_ssz_bytes())
     }
 
     fn from_store_bytes(bytes: &[u8]) -> Result<Self, StoreError> {

beacon_node/beacon_chain/src/persisted_fork_choice.rs

@@ -1,27 +1,17 @@
-use crate::beacon_fork_choice_store::{
-    PersistedForkChoiceStoreV1, PersistedForkChoiceStoreV7, PersistedForkChoiceStoreV8,
-};
+use crate::beacon_fork_choice_store::PersistedForkChoiceStoreV11;
 use ssz::{Decode, Encode};
 use ssz_derive::{Decode, Encode};
 use store::{DBColumn, Error, StoreItem};
 use superstruct::superstruct;
 
 // If adding a new version you should update this type alias and fix the breakages.
-pub type PersistedForkChoice = PersistedForkChoiceV8;
+pub type PersistedForkChoice = PersistedForkChoiceV11;
 
-#[superstruct(
-    variants(V1, V7, V8),
-    variant_attributes(derive(Encode, Decode)),
-    no_enum
-)]
+#[superstruct(variants(V11), variant_attributes(derive(Encode, Decode)), no_enum)]
 pub struct PersistedForkChoice {
     pub fork_choice: fork_choice::PersistedForkChoice,
-    #[superstruct(only(V1))]
-    pub fork_choice_store: PersistedForkChoiceStoreV1,
-    #[superstruct(only(V7))]
-    pub fork_choice_store: PersistedForkChoiceStoreV7,
-    #[superstruct(only(V8))]
-    pub fork_choice_store: PersistedForkChoiceStoreV8,
+    #[superstruct(only(V11))]
+    pub fork_choice_store: PersistedForkChoiceStoreV11,
 }
 
 macro_rules! impl_store_item {
@@ -31,17 +21,15 @@ macro_rules! impl_store_item {
                 DBColumn::ForkChoice
             }
 
-            fn as_store_bytes(&self) -> Vec<u8> {
-                self.as_ssz_bytes()
+            fn as_store_bytes(&self) -> Result<Vec<u8>, Error> {
+                Ok(self.as_ssz_bytes())
             }
 
-            fn from_store_bytes(bytes: &[u8]) -> std::result::Result<Self, Error> {
+            fn from_store_bytes(bytes: &[u8]) -> Result<Self, Error> {
                 Self::from_ssz_bytes(bytes).map_err(Into::into)
             }
         }
     };
 }
 
-impl_store_item!(PersistedForkChoiceV1);
-impl_store_item!(PersistedForkChoiceV7);
-impl_store_item!(PersistedForkChoiceV8);
+impl_store_item!(PersistedForkChoiceV11);

beacon_node/beacon_chain/src/pre_finalization_cache.rs

@@ -71,7 +71,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         }
 
         // 2. Check on disk.
-        if self.store.get_block(&block_root)?.is_some() {
+        if self.store.get_blinded_block(&block_root, None)?.is_some() {
             cache.block_roots.put(block_root, ());
             return Ok(true);
         }

beacon_node/beacon_chain/src/proposer_prep_service.rs

@@ -0,0 +1,72 @@
+use crate::{BeaconChain, BeaconChainTypes};
+use slog::{debug, error};
+use slot_clock::SlotClock;
+use std::sync::Arc;
+use task_executor::TaskExecutor;
+use tokio::time::sleep;
+
+/// Spawns a routine which ensures the EL is provided advance notice of any block producers.
+///
+/// This routine will run once per slot, at `chain.prepare_payload_lookahead()`
+/// before the start of each slot.
+///
+/// The service will not be started if there is no `execution_layer` on the `chain`.
+pub fn start_proposer_prep_service<T: BeaconChainTypes>(
+    executor: TaskExecutor,
+    chain: Arc<BeaconChain<T>>,
+) {
+    // Avoid spawning the service if there's no EL, it'll just error anyway.
+    if chain.execution_layer.is_some() {
+        executor.clone().spawn(
+            async move { proposer_prep_service(executor, chain).await },
+            "proposer_prep_service",
+        );
+    }
+}
+
+/// Loop indefinitely, calling `BeaconChain::prepare_beacon_proposer_async` at an interval.
+async fn proposer_prep_service<T: BeaconChainTypes>(
+    executor: TaskExecutor,
+    chain: Arc<BeaconChain<T>>,
+) {
+    let slot_duration = chain.slot_clock.slot_duration();
+
+    loop {
+        match chain.slot_clock.duration_to_next_slot() {
+            Some(duration) => {
+                let additional_delay =
+                    slot_duration.saturating_sub(chain.config.prepare_payload_lookahead);
+                sleep(duration + additional_delay).await;
+
+                debug!(
+                    chain.log,
+                    "Proposer prepare routine firing";
+                );
+
+                let inner_chain = chain.clone();
+                executor.spawn(
+                    async move {
+                        if let Ok(current_slot) = inner_chain.slot() {
+                            if let Err(e) = inner_chain.prepare_beacon_proposer(current_slot).await
+                            {
+                                error!(
+                                    inner_chain.log,
+                                    "Proposer prepare routine failed";
+                                    "error" => ?e
+                                );
+                            }
+                        } else {
+                            debug!(inner_chain.log, "No slot for proposer prepare routine");
+                        }
+                    },
+                    "proposer_prep_update",
+                );
+            }
+            None => {
+                error!(chain.log, "Failed to read slot clock");
+                // If we can't read the slot clock, just wait another slot.
+                sleep(slot_duration).await;
+            }
+        };
+    }
+}

beacon_node/beacon_chain/src/schema_change.rs

@@ -1,180 +1,122 @@
 //! Utilities for managing database schema changes.
-mod migration_schema_v6;
-mod migration_schema_v7;
-mod migration_schema_v8;
-mod types;
+mod migration_schema_v12;
+mod migration_schema_v13;
+mod migration_schema_v20;
 
-use crate::beacon_chain::{BeaconChainTypes, FORK_CHOICE_DB_KEY, OP_POOL_DB_KEY};
-use crate::persisted_fork_choice::{PersistedForkChoiceV1, PersistedForkChoiceV7};
-use crate::validator_pubkey_cache::ValidatorPubkeyCache;
-use operation_pool::{PersistedOperationPool, PersistedOperationPoolBase};
+use crate::beacon_chain::{BeaconChainTypes, ETH1_CACHE_DB_KEY};
+use crate::eth1_chain::SszEth1;
+use crate::types::ChainSpec;
 use slog::{warn, Logger};
-use ssz::{Decode, Encode};
-use ssz_derive::{Decode, Encode};
-use std::fs;
-use std::path::Path;
 use std::sync::Arc;
-use store::config::OnDiskStoreConfig;
 use store::hot_cold_store::{HotColdDB, HotColdDBError};
-use store::metadata::{SchemaVersion, CONFIG_KEY, CURRENT_SCHEMA_VERSION};
-use store::{DBColumn, Error as StoreError, ItemStore, StoreItem};
-
-const PUBKEY_CACHE_FILENAME: &str = "pubkey_cache.ssz";
+use store::metadata::{SchemaVersion, CURRENT_SCHEMA_VERSION};
+use store::{Error as StoreError, StoreItem};
 
 /// Migrate the database from one schema version to another, applying all requisite mutations.
+#[allow(clippy::only_used_in_recursion)] // spec is not used but likely to be used in future
 pub fn migrate_schema<T: BeaconChainTypes>(
     db: Arc<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>,
-    datadir: &Path,
+    deposit_contract_deploy_block: u64,
     from: SchemaVersion,
     to: SchemaVersion,
     log: Logger,
+    spec: &ChainSpec,
 ) -> Result<(), StoreError> {
     match (from, to) {
-        // Migrating from the current schema version to iself is always OK, a no-op.
+        // Migrating from the current schema version to itself is always OK, a no-op.
         (_, _) if from == to && to == CURRENT_SCHEMA_VERSION => Ok(()),
-        // Migrate across multiple versions by recursively migrating one step at a time.
+        // Upgrade for tree-states database changes.
+        (SchemaVersion(12), SchemaVersion(20)) => {
+            migration_schema_v20::upgrade_to_v20::<T>(db, log)
+        }
+        // Downgrade for tree-states database changes.
+        (SchemaVersion(20), SchemaVersion(12)) => {
+            migration_schema_v20::downgrade_from_v20::<T>(db, log)
+        }
+        // Upgrade across multiple versions by recursively migrating one step at a time.
         (_, _) if from.as_u64() + 1 < to.as_u64() => {
             let next = SchemaVersion(from.as_u64() + 1);
-            migrate_schema::<T>(db.clone(), datadir, from, next, log.clone())?;
-            migrate_schema::<T>(db, datadir, next, to, log)
+            migrate_schema::<T>(
+                db.clone(),
+                deposit_contract_deploy_block,
+                from,
+                next,
+                log.clone(),
+                spec,
+            )?;
+            migrate_schema::<T>(db, deposit_contract_deploy_block, next, to, log, spec)
         }
-        // Migration from v0.3.0 to v0.3.x, adding the temporary states column.
-        // Nothing actually needs to be done, but once a DB uses v2 it shouldn't go back.
-        (SchemaVersion(1), SchemaVersion(2)) => {
-            db.store_schema_version(to)?;
-            Ok(())
+        // Downgrade across multiple versions by recursively migrating one step at a time.
+        (_, _) if to.as_u64() + 1 < from.as_u64() => {
+            let next = SchemaVersion(from.as_u64() - 1);
+            migrate_schema::<T>(
+                db.clone(),
+                deposit_contract_deploy_block,
+                from,
+                next,
+                log.clone(),
+                spec,
+            )?;
+            migrate_schema::<T>(db, deposit_contract_deploy_block, next, to, log, spec)
         }
-        // Migration for removing the pubkey cache.
-        (SchemaVersion(2), SchemaVersion(3)) => {
-            let pk_cache_path = datadir.join(PUBKEY_CACHE_FILENAME);
 
-            // Load from file, store to DB.
-            ValidatorPubkeyCache::<T>::load_from_file(&pk_cache_path)
-                .and_then(|cache| ValidatorPubkeyCache::convert(cache, db.clone()))
-                .map_err(|e| StoreError::SchemaMigrationError(format!("{:?}", e)))?;
+        //
+        // Migrations from before SchemaVersion(11) are deprecated.
+        //
 
-            db.store_schema_version(to)?;
-
-            // Delete cache file now that keys are stored in the DB.
-            fs::remove_file(&pk_cache_path).map_err(|e| {
-                StoreError::SchemaMigrationError(format!(
-                    "unable to delete {}: {:?}",
-                    pk_cache_path.display(),
-                    e
-                ))
-            })?;
-
-            Ok(())
+        // Upgrade from v11 to v12 to store richer metadata in the attestation op pool.
+        (SchemaVersion(11), SchemaVersion(12)) => {
+            let ops = migration_schema_v12::upgrade_to_v12::<T>(db.clone(), log)?;
+            db.store_schema_version_atomically(to, ops)
         }
-        // Migration for adding sync committee contributions to the persisted op pool.
-        (SchemaVersion(3), SchemaVersion(4)) => {
-            // Deserialize from what exists in the database using the `PersistedOperationPoolBase`
-            // variant and convert it to the Altair variant.
-            let pool_opt = db
-                .get_item::<PersistedOperationPoolBase<T::EthSpec>>(&OP_POOL_DB_KEY)?
-                .map(PersistedOperationPool::Base)
-                .map(PersistedOperationPool::base_to_altair);
-
-            if let Some(pool) = pool_opt {
-                // Store the converted pool under the same key.
-                db.put_item::<PersistedOperationPool<T::EthSpec>>(&OP_POOL_DB_KEY, &pool)?;
+        // Downgrade from v12 to v11 to drop richer metadata from the attestation op pool.
+        (SchemaVersion(12), SchemaVersion(11)) => {
+            let ops = migration_schema_v12::downgrade_from_v12::<T>(db.clone(), log)?;
+            db.store_schema_version_atomically(to, ops)
+        }
+        (SchemaVersion(12), SchemaVersion(13)) => {
+            let mut ops = vec![];
+            if let Some(persisted_eth1_v1) = db.get_item::<SszEth1>(&ETH1_CACHE_DB_KEY)? {
+                let upgraded_eth1_cache =
+                    match migration_schema_v13::update_eth1_cache(persisted_eth1_v1) {
+                        Ok(upgraded_eth1) => upgraded_eth1,
+                        Err(e) => {
+                            warn!(log, "Failed to deserialize SszEth1CacheV1"; "error" => ?e);
+                            warn!(log, "Reinitializing eth1 cache");
+                            migration_schema_v13::reinitialized_eth1_cache_v13(
+                                deposit_contract_deploy_block,
+                            )
+                        }
+                    };
+                ops.push(upgraded_eth1_cache.as_kv_store_op(ETH1_CACHE_DB_KEY)?);
             }
 
-            db.store_schema_version(to)?;
+            db.store_schema_version_atomically(to, ops)?;
 
             Ok(())
         }
-        // Migration for weak subjectivity sync support and clean up of `OnDiskStoreConfig` (#1784).
-        (SchemaVersion(4), SchemaVersion(5)) => {
-            if let Some(OnDiskStoreConfigV4 {
-                slots_per_restore_point,
-                ..
-            }) = db.hot_db.get(&CONFIG_KEY)?
-            {
-                let new_config = OnDiskStoreConfig {
-                    slots_per_restore_point,
+        (SchemaVersion(13), SchemaVersion(12)) => {
+            let mut ops = vec![];
+            if let Some(persisted_eth1_v13) = db.get_item::<SszEth1>(&ETH1_CACHE_DB_KEY)? {
+                let downgraded_eth1_cache = match migration_schema_v13::downgrade_eth1_cache(
+                    persisted_eth1_v13,
+                ) {
+                    Ok(Some(downgraded_eth1)) => downgraded_eth1,
+                    Ok(None) => {
+                        warn!(log, "Unable to downgrade eth1 cache from newer version: reinitializing eth1 cache");
+                        migration_schema_v13::reinitialized_eth1_cache_v1(
+                            deposit_contract_deploy_block,
+                        )
+                    }
+                    Err(e) => {
+                        warn!(log, "Unable to downgrade eth1 cache from newer version: failed to deserialize SszEth1CacheV13"; "error" => ?e);
+                        warn!(log, "Reinitializing eth1 cache");
+                        migration_schema_v13::reinitialized_eth1_cache_v1(
+                            deposit_contract_deploy_block,
+                        )
+                    }
                 };
-                db.hot_db.put(&CONFIG_KEY, &new_config)?;
-            }
-
-            db.store_schema_version(to)?;
-
-            Ok(())
-        }
-        // Migration for adding `execution_status` field to the fork choice store.
-        (SchemaVersion(5), SchemaVersion(6)) => {
-            // Database operations to be done atomically
-            let mut ops = vec![];
-
-            // The top-level `PersistedForkChoice` struct is still V1 but will have its internal
-            // bytes for the fork choice updated to V6.
-            let fork_choice_opt = db.get_item::<PersistedForkChoiceV1>(&FORK_CHOICE_DB_KEY)?;
-            if let Some(mut persisted_fork_choice) = fork_choice_opt {
-                migration_schema_v6::update_execution_statuses::<T>(&mut persisted_fork_choice)
-                    .map_err(StoreError::SchemaMigrationError)?;
-
-                // Store the converted fork choice store under the same key.
-                ops.push(persisted_fork_choice.as_kv_store_op(FORK_CHOICE_DB_KEY));
-            }
-
-            db.store_schema_version_atomically(to, ops)?;
-
-            Ok(())
-        }
-        // 1. Add `proposer_boost_root`.
-        // 2. Update `justified_epoch` to `justified_checkpoint` and `finalized_epoch` to
-        //  `finalized_checkpoint`.
-        // 3. This migration also includes a potential update to the justified
-        //  checkpoint in case the fork choice store's justified checkpoint and finalized checkpoint
-        //  combination does not actually exist for any blocks in fork choice. This was possible in
-        //  the consensus spec prior to v1.1.6.
-        //
-        // Relevant issues:
-        //
-        // https://github.com/sigp/lighthouse/issues/2741
-        // https://github.com/ethereum/consensus-specs/pull/2727
-        // https://github.com/ethereum/consensus-specs/pull/2730
-        (SchemaVersion(6), SchemaVersion(7)) => {
-            // Database operations to be done atomically
-            let mut ops = vec![];
-
-            let fork_choice_opt = db.get_item::<PersistedForkChoiceV1>(&FORK_CHOICE_DB_KEY)?;
-            if let Some(persisted_fork_choice_v1) = fork_choice_opt {
-                // This migrates the `PersistedForkChoiceStore`, adding the `proposer_boost_root` field.
-                let mut persisted_fork_choice_v7 = persisted_fork_choice_v1.into();
-
-                let result = migration_schema_v7::update_fork_choice::<T>(
-                    &mut persisted_fork_choice_v7,
-                    db.clone(),
-                );
-
-                // Fall back to re-initializing fork choice from an anchor state if necessary.
-                if let Err(e) = result {
-                    warn!(log, "Unable to migrate to database schema 7, re-initializing fork choice"; "error" => ?e);
-                    migration_schema_v7::update_with_reinitialized_fork_choice::<T>(
-                        &mut persisted_fork_choice_v7,
-                        db.clone(),
-                    )
-                    .map_err(StoreError::SchemaMigrationError)?;
-                }
-
-                // Store the converted fork choice store under the same key.
-                ops.push(persisted_fork_choice_v7.as_kv_store_op(FORK_CHOICE_DB_KEY));
-            }
-
-            db.store_schema_version_atomically(to, ops)?;
-
-            Ok(())
-        }
-        // Migration to add an `epoch` key to the fork choice's balances cache.
-        (SchemaVersion(7), SchemaVersion(8)) => {
-            let mut ops = vec![];
-            let fork_choice_opt = db.get_item::<PersistedForkChoiceV7>(&FORK_CHOICE_DB_KEY)?;
-            if let Some(fork_choice) = fork_choice_opt {
-                let updated_fork_choice =
-                    migration_schema_v8::update_fork_choice::<T>(fork_choice, db.clone())?;
-
-                ops.push(updated_fork_choice.as_kv_store_op(FORK_CHOICE_DB_KEY));
+                ops.push(downgraded_eth1_cache.as_kv_store_op(ETH1_CACHE_DB_KEY)?);
             }
 
             db.store_schema_version_atomically(to, ops)?;
@@ -189,24 +131,3 @@ pub fn migrate_schema<T: BeaconChainTypes>(
         .into()),
     }
 }
-
-// Store config used in v4 schema and earlier.
-#[derive(Debug, Clone, PartialEq, Eq, Encode, Decode)]
-pub struct OnDiskStoreConfigV4 {
-    pub slots_per_restore_point: u64,
-    pub _block_cache_size: usize,
-}
-
-impl StoreItem for OnDiskStoreConfigV4 {
-    fn db_column() -> DBColumn {
-        DBColumn::BeaconMeta
-    }
-
-    fn as_store_bytes(&self) -> Vec<u8> {
-        self.as_ssz_bytes()
-    }
-
-    fn from_store_bytes(bytes: &[u8]) -> Result<Self, StoreError> {
-        Ok(Self::from_ssz_bytes(bytes)?)
-    }
-}

beacon_node/beacon_chain/src/schema_change/migration_schema_v12.rs

@@ -0,0 +1,226 @@
+use crate::beacon_chain::{BeaconChainTypes, FORK_CHOICE_DB_KEY, OP_POOL_DB_KEY};
+use crate::persisted_fork_choice::PersistedForkChoiceV11;
+use operation_pool::{PersistedOperationPool, PersistedOperationPoolV12, PersistedOperationPoolV5};
+use slog::{debug, info, Logger};
+use state_processing::{
+    common::get_indexed_attestation, per_block_processing::is_valid_indexed_attestation,
+    VerifyOperation, VerifySignatures,
+};
+use std::sync::Arc;
+use store::{Error, HotColdDB, KeyValueStoreOp, StoreItem};
+
+pub fn upgrade_to_v12<T: BeaconChainTypes>(
+    db: Arc<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>,
+    log: Logger,
+) -> Result<Vec<KeyValueStoreOp>, Error> {
+    let spec = db.get_chain_spec();
+
+    // Load a V5 op pool and transform it to V12.
+    let PersistedOperationPoolV5 {
+        attestations_v5,
+        sync_contributions,
+        attester_slashings_v5,
+        proposer_slashings_v5,
+        voluntary_exits_v5,
+    } = if let Some(op_pool) = db.get_item(&OP_POOL_DB_KEY)? {
+        op_pool
+    } else {
+        debug!(log, "Nothing to do, no operation pool stored");
+        return Ok(vec![]);
+    };
+
+    // Load the persisted fork choice so we can grab the state of the justified block and use
+    // it to verify the stored attestations, slashings and exits.
+    let fork_choice = db
+        .get_item::<PersistedForkChoiceV11>(&FORK_CHOICE_DB_KEY)?
+        .ok_or_else(|| Error::SchemaMigrationError("fork choice missing from database".into()))?;
+    let justified_block_root = fork_choice
+        .fork_choice_store
+        .unrealized_justified_checkpoint
+        .root;
+    let justified_block = db
+        .get_blinded_block(&justified_block_root, None)?
+        .ok_or_else(|| {
+            Error::SchemaMigrationError(format!(
+                "unrealized justified block missing for migration: {justified_block_root:?}",
+            ))
+        })?;
+    let justified_state_root = justified_block.state_root();
+    let mut state = db
+        .get_state(&justified_state_root, Some(justified_block.slot()))?
+        .ok_or_else(|| {
+            Error::SchemaMigrationError(format!(
+                "justified state missing for migration: {justified_state_root:?}"
+            ))
+        })?;
+    state.build_all_committee_caches(spec).map_err(|e| {
+        Error::SchemaMigrationError(format!("unable to build committee caches: {e:?}"))
+    })?;
+
+    // Re-verify attestations while adding attesting indices.
+    let attestations = attestations_v5
+        .into_iter()
+        .flat_map(|(_, attestations)| attestations)
+        .filter_map(|attestation| {
+            let res = state
+                .get_beacon_committee(attestation.data.slot, attestation.data.index)
+                .map_err(Into::into)
+                .and_then(|committee| get_indexed_attestation(committee.committee, &attestation))
+                .and_then(|indexed_attestation| {
+                    is_valid_indexed_attestation(
+                        &state,
+                        &indexed_attestation,
+                        VerifySignatures::True,
+                        spec,
+                    )?;
+                    Ok(indexed_attestation)
+                });
+
+            match res {
+                Ok(indexed) => Some((attestation, indexed.attesting_indices.into())),
+                Err(e) => {
+                    debug!(
+                        log,
+                        "Dropping attestation on migration";
+                        "err" => ?e,
+                        "head_block" => ?attestation.data.beacon_block_root,
+                    );
+                    None
+                }
+            }
+        })
+        .collect::<Vec<_>>();
+
+    let attester_slashings = attester_slashings_v5
+        .iter()
+        .filter_map(|(slashing, _)| {
+            slashing
+                .clone()
+                .validate(&state, spec)
+                .map_err(|e| {
+                    debug!(
+                        log,
+                        "Dropping attester slashing on migration";
+                        "err" => ?e,
+                        "slashing" => ?slashing,
+                    );
+                })
+                .ok()
+        })
+        .collect::<Vec<_>>();
+
+    let proposer_slashings = proposer_slashings_v5
+        .iter()
+        .filter_map(|slashing| {
+            slashing
+                .clone()
+                .validate(&state, spec)
+                .map_err(|e| {
+                    debug!(
+                        log,
+                        "Dropping proposer slashing on migration";
+                        "err" => ?e,
+                        "slashing" => ?slashing,
+                    );
+                })
+                .ok()
+        })
+        .collect::<Vec<_>>();
+
+    let voluntary_exits = voluntary_exits_v5
+        .iter()
+        .filter_map(|exit| {
+            exit.clone()
+                .validate(&state, spec)
+                .map_err(|e| {
+                    debug!(
+                        log,
+                        "Dropping voluntary exit on migration";
+                        "err" => ?e,
+                        "exit" => ?exit,
+                    );
+                })
+                .ok()
+        })
+        .collect::<Vec<_>>();
+
+    debug!(
+        log,
+        "Migrated op pool";
+        "attestations" => attestations.len(),
+        "attester_slashings" => attester_slashings.len(),
+        "proposer_slashings" => proposer_slashings.len(),
+        "voluntary_exits" => voluntary_exits.len()
+    );
+
+    let v12 = PersistedOperationPool::V12(PersistedOperationPoolV12 {
+        attestations,
+        sync_contributions,
+        attester_slashings,
+        proposer_slashings,
+        voluntary_exits,
+    });
+    Ok(vec![v12.as_kv_store_op(OP_POOL_DB_KEY)?])
+}
+
+pub fn downgrade_from_v12<T: BeaconChainTypes>(
+    db: Arc<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>,
+    log: Logger,
+) -> Result<Vec<KeyValueStoreOp>, Error> {
+    // Load a V12 op pool and transform it to V5.
+    let PersistedOperationPoolV12 {
+        attestations,
+        sync_contributions,
+        attester_slashings,
+        proposer_slashings,
+        voluntary_exits,
+    } = if let Some(PersistedOperationPool::<T::EthSpec>::V12(op_pool)) =
+        db.get_item(&OP_POOL_DB_KEY)?
+    {
+        op_pool
+    } else {
+        debug!(log, "Nothing to do, no operation pool stored");
+        return Ok(vec![]);
+    };
+
+    info!(
+        log,
+        "Dropping attestations from pool";
+        "count" => attestations.len(),
+    );
+
+    let attester_slashings_v5 = attester_slashings
+        .into_iter()
+        .filter_map(|slashing| {
+            let fork_version = slashing.first_fork_verified_against()?;
+            Some((slashing.into_inner(), fork_version))
+        })
+        .collect::<Vec<_>>();
+
+    let proposer_slashings_v5 = proposer_slashings
+        .into_iter()
+        .map(|slashing| slashing.into_inner())
+        .collect::<Vec<_>>();
+
+    let voluntary_exits_v5 = voluntary_exits
+        .into_iter()
+        .map(|exit| exit.into_inner())
+        .collect::<Vec<_>>();
+
+    info!(
+        log,
+        "Migrated slashings and exits";
+        "attester_slashings" => attester_slashings_v5.len(),
+        "proposer_slashings" => proposer_slashings_v5.len(),
+        "voluntary_exits" => voluntary_exits_v5.len(),
+    );
+
+    let v5 = PersistedOperationPoolV5 {
+        attestations_v5: vec![],
+        sync_contributions,
+        attester_slashings_v5,
+        proposer_slashings_v5,
+        voluntary_exits_v5,
+    };
+    Ok(vec![v5.as_kv_store_op(OP_POOL_DB_KEY)?])
+}

beacon_node/beacon_chain/src/schema_change/migration_schema_v13.rs

@@ -0,0 +1,150 @@
+use crate::eth1_chain::SszEth1;
+use eth1::{BlockCache, SszDepositCacheV1, SszDepositCacheV13, SszEth1CacheV1, SszEth1CacheV13};
+use ssz::{Decode, Encode};
+use state_processing::common::DepositDataTree;
+use store::Error;
+use types::DEPOSIT_TREE_DEPTH;
+
+pub fn update_eth1_cache(persisted_eth1_v1: SszEth1) -> Result<SszEth1, Error> {
+    if persisted_eth1_v1.use_dummy_backend {
+        // backend_bytes is empty when using dummy backend
+        return Ok(persisted_eth1_v1);
+    }
+
+    let SszEth1 {
+        use_dummy_backend,
+        backend_bytes,
+    } = persisted_eth1_v1;
+
+    let ssz_eth1_cache_v1 = SszEth1CacheV1::from_ssz_bytes(&backend_bytes)?;
+    let SszEth1CacheV1 {
+        block_cache,
+        deposit_cache: deposit_cache_v1,
+        last_processed_block,
+    } = ssz_eth1_cache_v1;
+
+    let SszDepositCacheV1 {
+        logs,
+        leaves,
+        deposit_contract_deploy_block,
+        deposit_roots,
+    } = deposit_cache_v1;
+
+    let deposit_cache_v13 = SszDepositCacheV13 {
+        logs,
+        leaves,
+        deposit_contract_deploy_block,
+        finalized_deposit_count: 0,
+        finalized_block_height: deposit_contract_deploy_block.saturating_sub(1),
+        deposit_tree_snapshot: None,
+        deposit_roots,
+    };
+
+    let ssz_eth1_cache_v13 = SszEth1CacheV13 {
+        block_cache,
+        deposit_cache: deposit_cache_v13,
+        last_processed_block,
+    };
+
+    let persisted_eth1_v13 = SszEth1 {
+        use_dummy_backend,
+        backend_bytes: ssz_eth1_cache_v13.as_ssz_bytes(),
+    };
+
+    Ok(persisted_eth1_v13)
+}
+
+pub fn downgrade_eth1_cache(persisted_eth1_v13: SszEth1) -> Result<Option<SszEth1>, Error> {
+    if persisted_eth1_v13.use_dummy_backend {
+        // backend_bytes is empty when using dummy backend
+        return Ok(Some(persisted_eth1_v13));
+    }
+
+    let SszEth1 {
+        use_dummy_backend,
+        backend_bytes,
+    } = persisted_eth1_v13;
+
+    let ssz_eth1_cache_v13 = SszEth1CacheV13::from_ssz_bytes(&backend_bytes)?;
+    let SszEth1CacheV13 {
+        block_cache,
+        deposit_cache: deposit_cache_v13,
+        last_processed_block,
+    } = ssz_eth1_cache_v13;
+
+    let SszDepositCacheV13 {
+        logs,
+        leaves,
+        deposit_contract_deploy_block,
+        finalized_deposit_count,
+        finalized_block_height: _,
+        deposit_tree_snapshot,
+        deposit_roots,
+    } = deposit_cache_v13;
+
+    if finalized_deposit_count == 0 && deposit_tree_snapshot.is_none() {
+        // This tree was never finalized and can be directly downgraded to v1 without re-initializing
+        let deposit_cache_v1 = SszDepositCacheV1 {
+            logs,
+            leaves,
+            deposit_contract_deploy_block,
+            deposit_roots,
+        };
+        let ssz_eth1_cache_v1 = SszEth1CacheV1 {
+            block_cache,
+            deposit_cache: deposit_cache_v1,
+            last_processed_block,
+        };
+        return Ok(Some(SszEth1 {
+            use_dummy_backend,
+            backend_bytes: ssz_eth1_cache_v1.as_ssz_bytes(),
+        }));
+    }
+    // deposit cache was finalized; can't downgrade
+    Ok(None)
+}
+
+pub fn reinitialized_eth1_cache_v13(deposit_contract_deploy_block: u64) -> SszEth1 {
+    let empty_tree = DepositDataTree::create(&[], 0, DEPOSIT_TREE_DEPTH);
+    let deposit_cache_v13 = SszDepositCacheV13 {
+        logs: vec![],
+        leaves: vec![],
+        deposit_contract_deploy_block,
+        finalized_deposit_count: 0,
+        finalized_block_height: deposit_contract_deploy_block.saturating_sub(1),
+        deposit_tree_snapshot: empty_tree.get_snapshot(),
+        deposit_roots: vec![empty_tree.root()],
+    };
+
+    let ssz_eth1_cache_v13 = SszEth1CacheV13 {
+        block_cache: BlockCache::default(),
+        deposit_cache: deposit_cache_v13,
+        last_processed_block: None,
+    };
+
+    SszEth1 {
+        use_dummy_backend: false,
+        backend_bytes: ssz_eth1_cache_v13.as_ssz_bytes(),
+    }
+}
+
+pub fn reinitialized_eth1_cache_v1(deposit_contract_deploy_block: u64) -> SszEth1 {
+    let empty_tree = DepositDataTree::create(&[], 0, DEPOSIT_TREE_DEPTH);
+    let deposit_cache_v1 = SszDepositCacheV1 {
+        logs: vec![],
+        leaves: vec![],
+        deposit_contract_deploy_block,
+        deposit_roots: vec![empty_tree.root()],
+    };
+
+    let ssz_eth1_cache_v1 = SszEth1CacheV1 {
+        block_cache: BlockCache::default(),
+        deposit_cache: deposit_cache_v1,
+        last_processed_block: None,
+    };
+
+    SszEth1 {
+        use_dummy_backend: false,
+        backend_bytes: ssz_eth1_cache_v1.as_ssz_bytes(),
+    }
+}

beacon_node/beacon_chain/src/schema_change/migration_schema_v20.rs

@@ -0,0 +1,273 @@
+// FIXME(sproul): implement migration
+#![allow(unused)]
+
+use crate::{
+    beacon_chain::{BeaconChainTypes, BEACON_CHAIN_DB_KEY},
+    persisted_beacon_chain::PersistedBeaconChain,
+};
+use slog::{debug, info, Logger};
+use std::collections::HashMap;
+use std::sync::Arc;
+use store::{
+    get_key_for_col,
+    hot_cold_store::{HotColdDBError, HotStateSummaryV1, HotStateSummaryV10},
+    metadata::SchemaVersion,
+    DBColumn, Error, HotColdDB, KeyValueStoreOp, StoreItem,
+};
+use types::{milhouse::Diff, BeaconState, BeaconStateDiff, EthSpec, Hash256, Slot};
+
+fn get_summary_v1<T: BeaconChainTypes>(
+    db: &HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>,
+    state_root: Hash256,
+) -> Result<HotStateSummaryV1, Error> {
+    db.get_item(&state_root)?
+        .ok_or_else(|| HotColdDBError::MissingHotStateSummary(state_root).into())
+}
+
+fn get_state_by_replay<T: BeaconChainTypes>(
+    db: &HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>,
+    state_root: Hash256,
+) -> Result<BeaconState<T::EthSpec>, Error> {
+    /* FIXME(sproul): fix migration
+    // Load state summary.
+    let HotStateSummaryV1 {
+        slot,
+        latest_block_root,
+        epoch_boundary_state_root,
+    } = get_summary_v1::<T>(db, state_root)?;
+
+    // Load full state from the epoch boundary.
+    let (epoch_boundary_state, _) = db.load_hot_state_full(&epoch_boundary_state_root)?;
+
+    // Replay blocks to reach the target state.
+    let blocks = db.load_blocks_to_replay(epoch_boundary_state.slot(), slot, latest_block_root)?;
+
+    db.replay_blocks(epoch_boundary_state, blocks, slot, std::iter::empty(), None)
+    */
+    panic!()
+}
+
+pub fn upgrade_to_v20<T: BeaconChainTypes>(
+    db: Arc<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>,
+    log: Logger,
+) -> Result<(), Error> {
+    /* FIXME(sproul): fix this
+    let mut ops = vec![];
+
+    // Translate hot state summaries to new format:
+    // - Rewrite epoch boundary root to previous epoch boundary root.
+    // - Add previous state root.
+    //
+    // Replace most epoch boundary states by diffs.
+    let split = db.get_split_info();
+    let finalized_slot = split.slot;
+    let finalized_state_root = split.state_root;
+    let slots_per_epoch = T::EthSpec::slots_per_epoch();
+
+    let ssz_head_tracker = db
+        .get_item::<PersistedBeaconChain>(&BEACON_CHAIN_DB_KEY)?
+        .ok_or(Error::MissingPersistedBeaconChain)?
+        .ssz_head_tracker;
+
+    let mut new_summaries = HashMap::new();
+
+    for (head_block_root, head_state_slot) in ssz_head_tracker
+        .roots
+        .into_iter()
+        .zip(ssz_head_tracker.slots)
+    {
+        let block = db
+            .get_blinded_block(&head_block_root, Some(head_state_slot))?
+            .ok_or(Error::BlockNotFound(head_block_root))?;
+        let head_state_root = block.state_root();
+
+        debug!(
+            log,
+            "Re-writing state summaries for head";
+            "block_root" => ?head_block_root,
+            "state_root" => ?head_state_root,
+            "slot" => head_state_slot
+        );
+        let mut current_state = get_state_by_replay::<T>(&db, head_state_root)?;
+        let mut current_state_root = head_state_root;
+
+        new_summaries.insert(
+            head_state_root,
+            HotStateSummaryV10::new(&head_state_root, &current_state)?,
+        );
+
+        for slot in (finalized_slot.as_u64()..current_state.slot().as_u64())
+            .rev()
+            .map(Slot::new)
+        {
+            let epoch_boundary_slot = (slot - 1) / slots_per_epoch * slots_per_epoch;
+
+            let state_root = *current_state.get_state_root(slot)?;
+            let latest_block_root = *current_state.get_block_root(slot)?;
+            let prev_state_root = *current_state.get_state_root(slot - 1)?;
+            let epoch_boundary_state_root = *current_state.get_state_root(epoch_boundary_slot)?;
+
+            // FIXME(sproul): rename V10 variant
+            let summary = HotStateSummaryV10 {
+                slot,
+                latest_block_root,
+                epoch_boundary_state_root,
+                prev_state_root,
+            };
+
+            // Stage the updated state summary for storage.
+            // If we've reached a known segment of chain then we can stop and continue to the next
+            // head.
+            if new_summaries.insert(state_root, summary).is_some() {
+                debug!(
+                    log,
+                    "Finished migrating chain tip";
+                    "head_block_root" => ?head_block_root,
+                    "reason" => format!("reached common state {:?}", state_root),
+                );
+                break;
+            } else {
+                debug!(
+                    log,
+                    "Rewriting hot state summary";
+                    "state_root" => ?state_root,
+                    "slot" => slot,
+                    "epoch_boundary_state_root" => ?epoch_boundary_state_root,
+                    "prev_state_root" => ?prev_state_root,
+                );
+            }
+
+            // If the state reached is an epoch boundary state, then load it so that we can continue
+            // backtracking from it and storing diffs.
+            if slot % slots_per_epoch == 0 {
+                debug!(
+                    log,
+                    "Loading epoch boundary state";
+                    "state_root" => ?state_root,
+                    "slot" => slot,
+                );
+                let backtrack_state = get_state_by_replay::<T>(&db, state_root)?;
+
+                // If the current state is an epoch boundary state too then we might need to convert
+                // it to a diff relative to the backtrack state.
+                if current_state.slot() % slots_per_epoch == 0
+                    && !db.is_stored_as_full_state(current_state_root, current_state.slot())?
+                {
+                    debug!(
+                        log,
+                        "Converting full state to diff";
+                        "prev_state_root" => ?state_root,
+                        "state_root" => ?current_state_root,
+                        "slot" => current_state.slot(),
+                    );
+
+                    let diff = BeaconStateDiff::compute_diff(&backtrack_state, &current_state)?;
+
+                    // Store diff.
+                    ops.push(db.state_diff_as_kv_store_op(&current_state_root, &diff)?);
+
+                    // Delete full state.
+                    let state_key = get_key_for_col(
+                        DBColumn::BeaconState.into(),
+                        current_state_root.as_bytes(),
+                    );
+                    ops.push(KeyValueStoreOp::DeleteKey(state_key));
+                }
+
+                current_state = backtrack_state;
+                current_state_root = state_root;
+            }
+
+            if slot == finalized_slot {
+                // FIXME(sproul): remove assert
+                assert_eq!(finalized_state_root, state_root);
+                debug!(
+                    log,
+                    "Finished migrating chain tip";
+                    "head_block_root" => ?head_block_root,
+                    "reason" => format!("reached finalized state {:?}", finalized_state_root),
+                );
+                break;
+            }
+        }
+    }
+
+    ops.reserve(new_summaries.len());
+    for (state_root, summary) in new_summaries {
+        ops.push(summary.as_kv_store_op(state_root)?);
+    }
+
+    db.store_schema_version_atomically(SchemaVersion(20), ops)
+    */
+    panic!()
+}
+
+pub fn downgrade_from_v20<T: BeaconChainTypes>(
+    db: Arc<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>,
+    log: Logger,
+) -> Result<(), Error> {
+    /* FIXME(sproul): broken
+    let slots_per_epoch = T::EthSpec::slots_per_epoch();
+
+    // Iterate hot state summaries and re-write them so that:
+    //
+    // - The previous state root is removed.
+    // - The epoch boundary root points to the most recent epoch boundary root rather than the
+    //   previous epoch boundary root. We exploit the fact that they are the same except when the slot
+    //   of the summary itself lies on an epoch boundary.
+    let mut summaries = db
+        .iter_hot_state_summaries()
+        .collect::<Result<Vec<_>, _>>()?;
+
+    // Sort by slot ascending so that the state cache has a better chance of hitting.
+    summaries.sort_unstable_by(|(_, summ1), (_, summ2)| summ1.slot.cmp(&summ2.slot));
+
+    info!(log, "Rewriting {} state summaries", summaries.len());
+
+    let mut ops = Vec::with_capacity(summaries.len());
+
+    for (state_root, summary) in summaries {
+        let epoch_boundary_state_root = if summary.slot % slots_per_epoch == 0 {
+            info!(
+                log,
+                "Ensuring state is stored as full state";
+                "state_root" => ?state_root,
+                "slot" => summary.slot
+            );
+            let state = db
+                .get_hot_state(&state_root)?
+                .ok_or(Error::MissingState(state_root))?;
+
+            // Delete state diff.
+            let state_key =
+                get_key_for_col(DBColumn::BeaconStateDiff.into(), state_root.as_bytes());
+            ops.push(KeyValueStoreOp::DeleteKey(state_key));
+
+            // Store full state.
+            db.store_full_state_in_batch(&state_root, &state, &mut ops)?;
+
+            // This state root is its own most recent epoch boundary root.
+            state_root
+        } else {
+            summary.epoch_boundary_state_root
+        };
+        let summary_v1 = HotStateSummaryV1 {
+            slot: summary.slot,
+            latest_block_root: summary.latest_block_root,
+            epoch_boundary_state_root,
+        };
+        debug!(
+            log,
+            "Rewriting state summary";
+            "slot" => summary_v1.slot,
+            "latest_block_root" => ?summary_v1.latest_block_root,
+            "epoch_boundary_state_root" => ?summary_v1.epoch_boundary_state_root,
+        );
+
+        ops.push(summary_v1.as_kv_store_op(state_root)?);
+    }
+
+    db.store_schema_version_atomically(SchemaVersion(8), ops)
+    */
+    panic!()
+}

beacon_node/beacon_chain/src/schema_change/migration_schema_v6.rs

@@ -1,28 +0,0 @@
-///! These functions and structs are only relevant to the database migration from schema 5 to 6.
-use crate::persisted_fork_choice::PersistedForkChoiceV1;
-use crate::schema_change::types::{SszContainerV1, SszContainerV6};
-use crate::BeaconChainTypes;
-use ssz::four_byte_option_impl;
-use ssz::{Decode, Encode};
-
-// Define a "legacy" implementation of `Option<usize>` which uses four bytes for encoding the union
-// selector.
-four_byte_option_impl!(four_byte_option_usize, usize);
-
-pub(crate) fn update_execution_statuses<T: BeaconChainTypes>(
-    persisted_fork_choice: &mut PersistedForkChoiceV1,
-) -> Result<(), String> {
-    let ssz_container_v1 =
-        SszContainerV1::from_ssz_bytes(&persisted_fork_choice.fork_choice.proto_array_bytes)
-            .map_err(|e| {
-                format!(
-                    "Failed to decode ProtoArrayForkChoice during schema migration: {:?}",
-                    e
-                )
-            })?;
-
-    let ssz_container_v6: SszContainerV6 = ssz_container_v1.into();
-
-    persisted_fork_choice.fork_choice.proto_array_bytes = ssz_container_v6.as_ssz_bytes();
-    Ok(())
-}

beacon_node/beacon_chain/src/schema_change/migration_schema_v7.rs

@@ -1,327 +0,0 @@
-///! These functions and structs are only relevant to the database migration from schema 6 to 7.
-use crate::beacon_chain::BeaconChainTypes;
-use crate::beacon_fork_choice_store::{PersistedForkChoiceStoreV1, PersistedForkChoiceStoreV7};
-use crate::persisted_fork_choice::{PersistedForkChoiceV1, PersistedForkChoiceV7};
-use crate::schema_change::types::{ProtoNodeV6, SszContainerV6, SszContainerV7};
-use crate::types::{Checkpoint, Epoch, Hash256};
-use crate::types::{EthSpec, Slot};
-use crate::{BeaconForkChoiceStore, BeaconSnapshot};
-use fork_choice::ForkChoice;
-use proto_array::{core::ProtoNode, core::SszContainer, ProtoArrayForkChoice};
-use ssz::four_byte_option_impl;
-use ssz::{Decode, Encode};
-use std::collections::{HashMap, HashSet};
-use std::sync::Arc;
-use store::hot_cold_store::HotColdDB;
-use store::iter::BlockRootsIterator;
-use store::Error as StoreError;
-
-// Define a "legacy" implementation of `Option<usize>` which uses four bytes for encoding the union
-// selector.
-four_byte_option_impl!(four_byte_option_usize, usize);
-
-/// This method is used to re-initialize fork choice from the finalized state in case we hit an
-/// error during this migration.
-pub(crate) fn update_with_reinitialized_fork_choice<T: BeaconChainTypes>(
-    persisted_fork_choice: &mut PersistedForkChoiceV7,
-    db: Arc<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>,
-) -> Result<(), String> {
-    let anchor_block_root = persisted_fork_choice
-        .fork_choice_store
-        .finalized_checkpoint
-        .root;
-    let anchor_block = db
-        .get_block(&anchor_block_root)
-        .map_err(|e| format!("{:?}", e))?
-        .ok_or_else(|| "Missing anchor beacon block".to_string())?;
-    let anchor_state = db
-        .get_state(&anchor_block.state_root(), Some(anchor_block.slot()))
-        .map_err(|e| format!("{:?}", e))?
-        .ok_or_else(|| "Missing anchor beacon state".to_string())?;
-    let snapshot = BeaconSnapshot {
-        beacon_block: anchor_block,
-        beacon_block_root: anchor_block_root,
-        beacon_state: anchor_state,
-    };
-    let store = BeaconForkChoiceStore::get_forkchoice_store(db, &snapshot);
-    let fork_choice = ForkChoice::from_anchor(
-        store,
-        anchor_block_root,
-        &snapshot.beacon_block,
-        &snapshot.beacon_state,
-    )
-    .map_err(|e| format!("{:?}", e))?;
-    persisted_fork_choice.fork_choice = fork_choice.to_persisted();
-    Ok(())
-}
-
-pub(crate) fn update_fork_choice<T: BeaconChainTypes>(
-    persisted_fork_choice: &mut PersistedForkChoiceV7,
-    db: Arc<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>,
-) -> Result<(), StoreError> {
-    // `PersistedForkChoice` stores the `ProtoArray` as a `Vec<u8>`. Deserialize these
-    // bytes assuming the legacy struct, and transform them to the new struct before
-    // re-serializing.
-    let ssz_container_v6 =
-        SszContainerV6::from_ssz_bytes(&persisted_fork_choice.fork_choice.proto_array_bytes)
-            .map_err(|e| {
-                StoreError::SchemaMigrationError(format!(
-                    "Failed to decode ProtoArrayForkChoice during schema migration: {:?}",
-                    e
-                ))
-            })?;
-
-    // Clone the V6 proto nodes in order to maintain information about `node.justified_epoch`
-    // and `node.finalized_epoch`.
-    let nodes_v6 = ssz_container_v6.nodes.clone();
-
-    let justified_checkpoint = persisted_fork_choice.fork_choice_store.justified_checkpoint;
-    let finalized_checkpoint = persisted_fork_choice.fork_choice_store.finalized_checkpoint;
-
-    // These transformations instantiate `node.justified_checkpoint` and `node.finalized_checkpoint`
-    // to `None`.
-    let ssz_container_v7: SszContainerV7 =
-        ssz_container_v6.into_ssz_container_v7(justified_checkpoint, finalized_checkpoint);
-    let ssz_container: SszContainer = ssz_container_v7.into();
-    let mut fork_choice: ProtoArrayForkChoice = ssz_container.into();
-
-    update_checkpoints::<T>(finalized_checkpoint.root, &nodes_v6, &mut fork_choice, db)
-        .map_err(StoreError::SchemaMigrationError)?;
-
-    // Update the justified checkpoint in the store in case we have a discrepancy
-    // between the store and the proto array nodes.
-    update_store_justified_checkpoint(persisted_fork_choice, &mut fork_choice)
-        .map_err(StoreError::SchemaMigrationError)?;
-
-    Ok(())
-}
-
-struct HeadInfo {
-    index: usize,
-    root: Hash256,
-    slot: Slot,
-}
-
-fn update_checkpoints<T: BeaconChainTypes>(
-    finalized_root: Hash256,
-    nodes_v6: &[ProtoNodeV6],
-    fork_choice: &mut ProtoArrayForkChoice,
-    db: Arc<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>,
-) -> Result<(), String> {
-    let heads = find_finalized_descendant_heads(finalized_root, fork_choice);
-
-    // For each head, first gather all epochs we will need to find justified or finalized roots for.
-    for head in heads {
-        // `relevant_epochs` are epochs for which we will need to find the root at the start slot.
-        // We don't need to worry about whether the are finalized or justified epochs.
-        let mut relevant_epochs = HashSet::new();
-        let relevant_epoch_finder = |index, _: &mut ProtoNode| {
-            let (justified_epoch, finalized_epoch) = nodes_v6
-                .get(index)
-                .map(|node: &ProtoNodeV6| (node.justified_epoch, node.finalized_epoch))
-                .ok_or_else(|| "Index not found in legacy proto nodes".to_string())?;
-            relevant_epochs.insert(justified_epoch);
-            relevant_epochs.insert(finalized_epoch);
-            Ok(())
-        };
-
-        apply_to_chain_of_ancestors(
-            finalized_root,
-            head.index,
-            fork_choice,
-            relevant_epoch_finder,
-        )?;
-
-        // find the block roots associated with each relevant epoch.
-        let roots_by_epoch =
-            map_relevant_epochs_to_roots::<T>(head.root, head.slot, relevant_epochs, db.clone())?;
-
-        // Apply this mutator to the chain of descendants from this head, adding justified
-        // and finalized checkpoints for each.
-        let node_mutator = |index, node: &mut ProtoNode| {
-            let (justified_epoch, finalized_epoch) = nodes_v6
-                .get(index)
-                .map(|node: &ProtoNodeV6| (node.justified_epoch, node.finalized_epoch))
-                .ok_or_else(|| "Index not found in legacy proto nodes".to_string())?;
-
-            // Update the checkpoints only if they haven't already been populated.
-            if node.justified_checkpoint.is_none() {
-                let justified_checkpoint =
-                    roots_by_epoch
-                        .get(&justified_epoch)
-                        .map(|&root| Checkpoint {
-                            epoch: justified_epoch,
-                            root,
-                        });
-                node.justified_checkpoint = justified_checkpoint;
-            }
-            if node.finalized_checkpoint.is_none() {
-                let finalized_checkpoint =
-                    roots_by_epoch
-                        .get(&finalized_epoch)
-                        .map(|&root| Checkpoint {
-                            epoch: finalized_epoch,
-                            root,
-                        });
-                node.finalized_checkpoint = finalized_checkpoint;
-            }
-
-            Ok(())
-        };
-
-        apply_to_chain_of_ancestors(finalized_root, head.index, fork_choice, node_mutator)?;
-    }
-    Ok(())
-}
-
-/// Coverts the given `HashSet<Epoch>` to a `Vec<Epoch>` then reverse sorts by `Epoch`. Next, a
-/// single `BlockRootsIterator` is created which is used to iterate backwards from the given
-/// `head_root` and `head_slot`, finding the block root at the start slot of each epoch.
-fn map_relevant_epochs_to_roots<T: BeaconChainTypes>(
-    head_root: Hash256,
-    head_slot: Slot,
-    epochs: HashSet<Epoch>,
-    db: Arc<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>,
-) -> Result<HashMap<Epoch, Hash256>, String> {
-    // Convert the `HashSet` to a `Vec` and reverse sort the epochs.
-    let mut relevant_epochs = epochs.into_iter().collect::<Vec<_>>();
-    relevant_epochs.sort_unstable_by(|a, b| b.cmp(a));
-
-    // Iterate backwards from the given `head_root` and `head_slot` and find the block root at each epoch.
-    let mut iter = std::iter::once(Ok((head_root, head_slot)))
-        .chain(BlockRootsIterator::from_block(&db, head_root).map_err(|e| format!("{:?}", e))?);
-    let mut roots_by_epoch = HashMap::new();
-    for epoch in relevant_epochs {
-        let start_slot = epoch.start_slot(T::EthSpec::slots_per_epoch());
-
-        let root = iter
-            .find_map(|next| match next {
-                Ok((root, slot)) => (slot == start_slot).then(|| Ok(root)),
-                Err(e) => Some(Err(format!("{:?}", e))),
-            })
-            .transpose()?
-            .ok_or_else(|| "Justified root not found".to_string())?;
-        roots_by_epoch.insert(epoch, root);
-    }
-    Ok(roots_by_epoch)
-}
-
-/// Applies a mutator to every node in a chain, starting from the node at the given
-/// `head_index` and iterating through ancestors until the `finalized_root` is reached.
-fn apply_to_chain_of_ancestors<F>(
-    finalized_root: Hash256,
-    head_index: usize,
-    fork_choice: &mut ProtoArrayForkChoice,
-    mut node_mutator: F,
-) -> Result<(), String>
-where
-    F: FnMut(usize, &mut ProtoNode) -> Result<(), String>,
-{
-    let head = fork_choice
-        .core_proto_array_mut()
-        .nodes
-        .get_mut(head_index)
-        .ok_or_else(|| "Head index not found in proto nodes".to_string())?;
-
-    node_mutator(head_index, head)?;
-
-    let mut parent_index_opt = head.parent;
-    let mut parent_opt =
-        parent_index_opt.and_then(|index| fork_choice.core_proto_array_mut().nodes.get_mut(index));
-
-    // Iterate backwards through all parents until there is no reference to a parent or we reach
-    // the `finalized_root` node.
-    while let (Some(parent), Some(parent_index)) = (parent_opt, parent_index_opt) {
-        node_mutator(parent_index, parent)?;
-
-        // Break out of this while loop *after* the `node_mutator` has been applied to the finalized
-        // node.
-        if parent.root == finalized_root {
-            break;
-        }
-
-        // Update parent values
-        parent_index_opt = parent.parent;
-        parent_opt = parent_index_opt
-            .and_then(|index| fork_choice.core_proto_array_mut().nodes.get_mut(index));
-    }
-    Ok(())
-}
-
-/// Finds all heads by finding all nodes in the proto array that are not referenced as parents. Then
-/// checks that these nodes are descendants of the finalized root in order to determine if they are
-/// relevant.
-fn find_finalized_descendant_heads(
-    finalized_root: Hash256,
-    fork_choice: &ProtoArrayForkChoice,
-) -> Vec<HeadInfo> {
-    let nodes_referenced_as_parents: HashSet<usize> = fork_choice
-        .core_proto_array()
-        .nodes
-        .iter()
-        .filter_map(|node| node.parent)
-        .collect::<HashSet<_>>();
-
-    fork_choice
-        .core_proto_array()
-        .nodes
-        .iter()
-        .enumerate()
-        .filter_map(|(index, node)| {
-            (!nodes_referenced_as_parents.contains(&index)
-                && fork_choice.is_descendant(finalized_root, node.root))
-            .then(|| HeadInfo {
-                index,
-                root: node.root,
-                slot: node.slot,
-            })
-        })
-        .collect::<Vec<_>>()
-}
-
-fn update_store_justified_checkpoint(
-    persisted_fork_choice: &mut PersistedForkChoiceV7,
-    fork_choice: &mut ProtoArrayForkChoice,
-) -> Result<(), String> {
-    let justified_checkpoint = fork_choice
-        .core_proto_array()
-        .nodes
-        .iter()
-        .filter_map(|node| {
-            (node.finalized_checkpoint
-                == Some(persisted_fork_choice.fork_choice_store.finalized_checkpoint))
-            .then(|| node.justified_checkpoint)
-            .flatten()
-        })
-        .max_by_key(|justified_checkpoint| justified_checkpoint.epoch)
-        .ok_or("Proto node with current finalized checkpoint not found")?;
-
-    fork_choice.core_proto_array_mut().justified_checkpoint = justified_checkpoint;
-    persisted_fork_choice.fork_choice.proto_array_bytes = fork_choice.as_bytes();
-    persisted_fork_choice.fork_choice_store.justified_checkpoint = justified_checkpoint;
-    Ok(())
-}
-
-// Add a zero `proposer_boost_root` when migrating from V1-6 to V7.
-impl From<PersistedForkChoiceStoreV1> for PersistedForkChoiceStoreV7 {
-    fn from(other: PersistedForkChoiceStoreV1) -> Self {
-        Self {
-            balances_cache: other.balances_cache,
-            time: other.time,
-            finalized_checkpoint: other.finalized_checkpoint,
-            justified_checkpoint: other.justified_checkpoint,
-            justified_balances: other.justified_balances,
-            best_justified_checkpoint: other.best_justified_checkpoint,
-            proposer_boost_root: Hash256::zero(),
-        }
-    }
-}
-
-impl From<PersistedForkChoiceV1> for PersistedForkChoiceV7 {
-    fn from(other: PersistedForkChoiceV1) -> Self {
-        Self {
-            fork_choice: other.fork_choice,
-            fork_choice_store: other.fork_choice_store.into(),
-        }
-    }
-}

beacon_node/beacon_chain/src/schema_change/migration_schema_v8.rs

@@ -1,50 +0,0 @@
-use crate::beacon_chain::BeaconChainTypes;
-use crate::beacon_fork_choice_store::{
-    BalancesCacheV8, CacheItemV8, PersistedForkChoiceStoreV7, PersistedForkChoiceStoreV8,
-};
-use crate::persisted_fork_choice::{PersistedForkChoiceV7, PersistedForkChoiceV8};
-use std::sync::Arc;
-use store::{Error as StoreError, HotColdDB};
-use types::EthSpec;
-
-pub fn update_fork_choice<T: BeaconChainTypes>(
-    fork_choice: PersistedForkChoiceV7,
-    db: Arc<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>,
-) -> Result<PersistedForkChoiceV8, StoreError> {
-    let PersistedForkChoiceStoreV7 {
-        balances_cache,
-        time,
-        finalized_checkpoint,
-        justified_checkpoint,
-        justified_balances,
-        best_justified_checkpoint,
-        proposer_boost_root,
-    } = fork_choice.fork_choice_store;
-    let mut fork_choice_store = PersistedForkChoiceStoreV8 {
-        balances_cache: BalancesCacheV8::default(),
-        time,
-        finalized_checkpoint,
-        justified_checkpoint,
-        justified_balances,
-        best_justified_checkpoint,
-        proposer_boost_root,
-    };
-
-    // Add epochs to the balances cache. It's safe to just use the block's epoch because
-    // before schema v8 the cache would always miss on skipped slots.
-    for item in balances_cache.items {
-        // Drop any blocks that aren't found, they're presumably too old and this is only a cache.
-        if let Some(block) = db.get_block(&item.block_root)? {
-            fork_choice_store.balances_cache.items.push(CacheItemV8 {
-                block_root: item.block_root,
-                epoch: block.slot().epoch(T::EthSpec::slots_per_epoch()),
-                balances: item.balances,
-            });
-        }
-    }
-
-    Ok(PersistedForkChoiceV8 {
-        fork_choice: fork_choice.fork_choice,
-        fork_choice_store,
-    })
-}

beacon_node/beacon_chain/src/schema_change/types.rs

@@ -1,192 +0,0 @@
-use crate::types::{AttestationShufflingId, Checkpoint, Epoch, Hash256, Slot};
-use proto_array::core::{ProposerBoost, ProtoNode, SszContainer, VoteTracker};
-use proto_array::ExecutionStatus;
-use ssz::four_byte_option_impl;
-use ssz::Encode;
-use ssz_derive::{Decode, Encode};
-use superstruct::superstruct;
-
-// Define a "legacy" implementation of `Option<usize>` which uses four bytes for encoding the union
-// selector.
-four_byte_option_impl!(four_byte_option_usize, usize);
-four_byte_option_impl!(four_byte_option_checkpoint, Checkpoint);
-
-#[superstruct(
-    variants(V1, V6, V7),
-    variant_attributes(derive(Clone, PartialEq, Debug, Encode, Decode)),
-    no_enum
-)]
-pub struct ProtoNode {
-    pub slot: Slot,
-    pub state_root: Hash256,
-    pub target_root: Hash256,
-    pub current_epoch_shuffling_id: AttestationShufflingId,
-    pub next_epoch_shuffling_id: AttestationShufflingId,
-    pub root: Hash256,
-    #[ssz(with = "four_byte_option_usize")]
-    pub parent: Option<usize>,
-    #[superstruct(only(V1, V6))]
-    pub justified_epoch: Epoch,
-    #[superstruct(only(V1, V6))]
-    pub finalized_epoch: Epoch,
-    #[ssz(with = "four_byte_option_checkpoint")]
-    #[superstruct(only(V7))]
-    pub justified_checkpoint: Option<Checkpoint>,
-    #[ssz(with = "four_byte_option_checkpoint")]
-    #[superstruct(only(V7))]
-    pub finalized_checkpoint: Option<Checkpoint>,
-    pub weight: u64,
-    #[ssz(with = "four_byte_option_usize")]
-    pub best_child: Option<usize>,
-    #[ssz(with = "four_byte_option_usize")]
-    pub best_descendant: Option<usize>,
-    #[superstruct(only(V6, V7))]
-    pub execution_status: ExecutionStatus,
-}
-
-impl Into<ProtoNodeV6> for ProtoNodeV1 {
-    fn into(self) -> ProtoNodeV6 {
-        ProtoNodeV6 {
-            slot: self.slot,
-            state_root: self.state_root,
-            target_root: self.target_root,
-            current_epoch_shuffling_id: self.current_epoch_shuffling_id,
-            next_epoch_shuffling_id: self.next_epoch_shuffling_id,
-            root: self.root,
-            parent: self.parent,
-            justified_epoch: self.justified_epoch,
-            finalized_epoch: self.finalized_epoch,
-            weight: self.weight,
-            best_child: self.best_child,
-            best_descendant: self.best_descendant,
-            // We set the following execution value as if the block is a pre-merge-fork block. This
-            // is safe as long as we never import a merge block with the old version of proto-array.
-            // This will be safe since we can't actually process merge blocks until we've made this
-            // change to fork choice.
-            execution_status: ExecutionStatus::irrelevant(),
-        }
-    }
-}
-
-impl Into<ProtoNodeV7> for ProtoNodeV6 {
-    fn into(self) -> ProtoNodeV7 {
-        ProtoNodeV7 {
-            slot: self.slot,
-            state_root: self.state_root,
-            target_root: self.target_root,
-            current_epoch_shuffling_id: self.current_epoch_shuffling_id,
-            next_epoch_shuffling_id: self.next_epoch_shuffling_id,
-            root: self.root,
-            parent: self.parent,
-            justified_checkpoint: None,
-            finalized_checkpoint: None,
-            weight: self.weight,
-            best_child: self.best_child,
-            best_descendant: self.best_descendant,
-            execution_status: self.execution_status,
-        }
-    }
-}
-
-impl Into<ProtoNode> for ProtoNodeV7 {
-    fn into(self) -> ProtoNode {
-        ProtoNode {
-            slot: self.slot,
-            state_root: self.state_root,
-            target_root: self.target_root,
-            current_epoch_shuffling_id: self.current_epoch_shuffling_id,
-            next_epoch_shuffling_id: self.next_epoch_shuffling_id,
-            root: self.root,
-            parent: self.parent,
-            justified_checkpoint: self.justified_checkpoint,
-            finalized_checkpoint: self.finalized_checkpoint,
-            weight: self.weight,
-            best_child: self.best_child,
-            best_descendant: self.best_descendant,
-            execution_status: self.execution_status,
-        }
-    }
-}
-
-#[superstruct(
-    variants(V1, V6, V7),
-    variant_attributes(derive(Encode, Decode)),
-    no_enum
-)]
-#[derive(Encode, Decode)]
-pub struct SszContainer {
-    pub votes: Vec<VoteTracker>,
-    pub balances: Vec<u64>,
-    pub prune_threshold: usize,
-    #[superstruct(only(V1, V6))]
-    pub justified_epoch: Epoch,
-    #[superstruct(only(V1, V6))]
-    pub finalized_epoch: Epoch,
-    #[superstruct(only(V7))]
-    pub justified_checkpoint: Checkpoint,
-    #[superstruct(only(V7))]
-    pub finalized_checkpoint: Checkpoint,
-    #[superstruct(only(V1))]
-    pub nodes: Vec<ProtoNodeV1>,
-    #[superstruct(only(V6))]
-    pub nodes: Vec<ProtoNodeV6>,
-    #[superstruct(only(V7))]
-    pub nodes: Vec<ProtoNodeV7>,
-    pub indices: Vec<(Hash256, usize)>,
-    #[superstruct(only(V7))]
-    pub previous_proposer_boost: ProposerBoost,
-}
-
-impl Into<SszContainerV6> for SszContainerV1 {
-    fn into(self) -> SszContainerV6 {
-        let nodes = self.nodes.into_iter().map(Into::into).collect();
-
-        SszContainerV6 {
-            votes: self.votes,
-            balances: self.balances,
-            prune_threshold: self.prune_threshold,
-            justified_epoch: self.justified_epoch,
-            finalized_epoch: self.finalized_epoch,
-            nodes,
-            indices: self.indices,
-        }
-    }
-}
-
-impl SszContainerV6 {
-    pub(crate) fn into_ssz_container_v7(
-        self,
-        justified_checkpoint: Checkpoint,
-        finalized_checkpoint: Checkpoint,
-    ) -> SszContainerV7 {
-        let nodes = self.nodes.into_iter().map(Into::into).collect();
-
-        SszContainerV7 {
-            votes: self.votes,
-            balances: self.balances,
-            prune_threshold: self.prune_threshold,
-            justified_checkpoint,
-            finalized_checkpoint,
-            nodes,
-            indices: self.indices,
-            previous_proposer_boost: ProposerBoost::default(),
-        }
-    }
-}
-
-impl Into<SszContainer> for SszContainerV7 {
-    fn into(self) -> SszContainer {
-        let nodes = self.nodes.into_iter().map(Into::into).collect();
-
-        SszContainer {
-            votes: self.votes,
-            balances: self.balances,
-            prune_threshold: self.prune_threshold,
-            justified_checkpoint: self.justified_checkpoint,
-            finalized_checkpoint: self.finalized_checkpoint,
-            nodes,
-            indices: self.indices,
-            previous_proposer_boost: self.previous_proposer_boost,
-        }
-    }
-}

beacon_node/beacon_chain/src/shuffling_cache.rs

@@ -1,5 +1,7 @@
-use crate::metrics;
+use crate::{metrics, BeaconChainError};
 use lru::LruCache;
+use oneshot_broadcast::{oneshot, Receiver, Sender};
+use std::sync::Arc;
 use types::{beacon_state::CommitteeCache, AttestationShufflingId, Epoch, Hash256};
 
 /// The size of the LRU cache that stores committee caches for quicker verification.
@@ -9,12 +11,46 @@ use types::{beacon_state::CommitteeCache, AttestationShufflingId, Epoch, Hash256
 /// ignores a few extra bytes in the caches that should be insignificant compared to the indices).
 const CACHE_SIZE: usize = 16;
 
+/// The maximum number of concurrent committee cache "promises" that can be issued. In effect, this
+/// limits the number of concurrent states that can be loaded into memory for the committee cache.
+/// This prevents excessive memory usage at the cost of rejecting some attestations.
+///
+/// We set this value to 2 since states can be quite large and have a significant impact on memory
+/// usage. A healthy network cannot have more than a few committee caches and those caches should
+/// always be inserted during block import. Unstable networks with a high degree of forking might
+/// see some attestations dropped due to this concurrency limit, however I propose that this is
+/// better than low-resource nodes going OOM.
+const MAX_CONCURRENT_PROMISES: usize = 2;
+
+#[derive(Clone)]
+pub enum CacheItem {
+    /// A committee.
+    Committee(Arc<CommitteeCache>),
+    /// A promise for a future committee.
+    Promise(Receiver<Arc<CommitteeCache>>),
+}
+
+impl CacheItem {
+    pub fn is_promise(&self) -> bool {
+        matches!(self, CacheItem::Promise(_))
+    }
+
+    pub fn wait(self) -> Result<Arc<CommitteeCache>, BeaconChainError> {
+        match self {
+            CacheItem::Committee(cache) => Ok(cache),
+            CacheItem::Promise(receiver) => receiver
+                .recv()
+                .map_err(BeaconChainError::CommitteePromiseFailed),
+        }
+    }
+}
+
 /// Provides an LRU cache for `CommitteeCache`.
 ///
 /// It has been named `ShufflingCache` because `CommitteeCacheCache` is a bit weird and looks like
 /// a find/replace error.
 pub struct ShufflingCache {
-    cache: LruCache<AttestationShufflingId, CommitteeCache>,
+    cache: LruCache<AttestationShufflingId, CacheItem>,
 }
 
 impl ShufflingCache {
@@ -24,27 +60,120 @@ impl ShufflingCache {
         }
     }
 
-    pub fn get(&mut self, key: &AttestationShufflingId) -> Option<&CommitteeCache> {
-        let opt = self.cache.get(key);
-
-        if opt.is_some() {
-            metrics::inc_counter(&metrics::SHUFFLING_CACHE_HITS);
-        } else {
-            metrics::inc_counter(&metrics::SHUFFLING_CACHE_MISSES);
+    pub fn get(&mut self, key: &AttestationShufflingId) -> Option<CacheItem> {
+        match self.cache.get(key) {
+            // The cache contained the committee cache, return it.
+            item @ Some(CacheItem::Committee(_)) => {
+                metrics::inc_counter(&metrics::SHUFFLING_CACHE_HITS);
+                item.cloned()
+            }
+            // The cache contains a promise for the committee cache. Check to see if the promise has
+            // already been resolved, without waiting for it.
+            item @ Some(CacheItem::Promise(receiver)) => match receiver.try_recv() {
+                // The promise has already been resolved. Replace the entry in the cache with a
+                // `Committee` entry and then return the committee.
+                Ok(Some(committee)) => {
+                    metrics::inc_counter(&metrics::SHUFFLING_CACHE_PROMISE_HITS);
+                    metrics::inc_counter(&metrics::SHUFFLING_CACHE_HITS);
+                    let ready = CacheItem::Committee(committee);
+                    self.cache.put(key.clone(), ready.clone());
+                    Some(ready)
+                }
+                // The promise has not yet been resolved. Return the promise so the caller can await
+                // it.
+                Ok(None) => {
+                    metrics::inc_counter(&metrics::SHUFFLING_CACHE_PROMISE_HITS);
+                    metrics::inc_counter(&metrics::SHUFFLING_CACHE_HITS);
+                    item.cloned()
+                }
+                // The sender has been dropped without sending a committee. There was most likely an
+                // error computing the committee cache. Drop the key from the cache and return
+                // `None` so the caller can recompute the committee.
+                //
+                // It's worth noting that this is the only place where we removed unresolved
+                // promises from the cache. This means unresolved promises will only be removed if
+                // we try to access them again. This is OK, since the promises don't consume much
+                // memory and the nature of the LRU cache means that future, relevant entries will
+                // still be added to the cache. We expect that *all* promises should be resolved,
+                // unless there is a programming or database error.
+                Err(oneshot_broadcast::Error::SenderDropped) => {
+                    metrics::inc_counter(&metrics::SHUFFLING_CACHE_PROMISE_FAILS);
+                    metrics::inc_counter(&metrics::SHUFFLING_CACHE_MISSES);
+                    self.cache.pop(key);
+                    None
+                }
+            },
+            // The cache does not have this committee and it's not already promised to be computed.
+            None => {
+                metrics::inc_counter(&metrics::SHUFFLING_CACHE_MISSES);
+                None
+            }
         }
-
-        opt
     }
 
     pub fn contains(&self, key: &AttestationShufflingId) -> bool {
         self.cache.contains(key)
     }
 
-    pub fn insert(&mut self, key: AttestationShufflingId, committee_cache: &CommitteeCache) {
-        if !self.cache.contains(&key) {
-            self.cache.put(key, committee_cache.clone());
+    pub fn insert_committee_cache<T: ToArcCommitteeCache>(
+        &mut self,
+        key: AttestationShufflingId,
+        committee_cache: &T,
+    ) {
+        if self
+            .cache
+            .get(&key)
+            // Replace the committee if it's not present or if it's a promise. A bird in the hand is
+            // worth two in the promise-bush!
+            .map_or(true, CacheItem::is_promise)
+        {
+            self.cache.put(
+                key,
+                CacheItem::Committee(committee_cache.to_arc_committee_cache()),
+            );
         }
     }
+
+    pub fn create_promise(
+        &mut self,
+        key: AttestationShufflingId,
+    ) -> Result<Sender<Arc<CommitteeCache>>, BeaconChainError> {
+        let num_active_promises = self
+            .cache
+            .iter()
+            .filter(|(_, item)| item.is_promise())
+            .count();
+        if num_active_promises >= MAX_CONCURRENT_PROMISES {
+            return Err(BeaconChainError::MaxCommitteePromises(num_active_promises));
+        }
+
+        let (sender, receiver) = oneshot();
+        self.cache.put(key, CacheItem::Promise(receiver));
+        Ok(sender)
+    }
+}
+
+/// A helper trait to allow lazy-cloning of the committee cache when inserting into the cache.
+pub trait ToArcCommitteeCache {
+    fn to_arc_committee_cache(&self) -> Arc<CommitteeCache>;
+}
+
+impl ToArcCommitteeCache for CommitteeCache {
+    fn to_arc_committee_cache(&self) -> Arc<CommitteeCache> {
+        Arc::new(self.clone())
+    }
+}
+
+impl ToArcCommitteeCache for Arc<CommitteeCache> {
+    fn to_arc_committee_cache(&self) -> Arc<CommitteeCache> {
+        self.clone()
+    }
+}
+
+impl Default for ShufflingCache {
+    fn default() -> Self {
+        Self::new()
+    }
 }
 
 /// Contains the shuffling IDs for a beacon block.
@@ -73,3 +202,177 @@ impl BlockShufflingIds {
         }
     }
 }
+
+// Disable tests in debug since the beacon chain harness is slow unless in release.
+#[cfg(not(debug_assertions))]
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::test_utils::EphemeralHarnessType;
+    use types::*;
+
+    type BeaconChainHarness =
+        crate::test_utils::BeaconChainHarness<EphemeralHarnessType<MinimalEthSpec>>;
+
+    /// Returns two different committee caches for testing.
+    fn committee_caches() -> (Arc<CommitteeCache>, Arc<CommitteeCache>) {
+        let harness = BeaconChainHarness::builder(MinimalEthSpec)
+            .default_spec()
+            .deterministic_keypairs(8)
+            .fresh_ephemeral_store()
+            .build();
+        let (mut state, _) = harness.get_current_state_and_root();
+        state
+            .build_committee_cache(RelativeEpoch::Current, &harness.chain.spec)
+            .unwrap();
+        state
+            .build_committee_cache(RelativeEpoch::Next, &harness.chain.spec)
+            .unwrap();
+        let committee_a = state
+            .committee_cache(RelativeEpoch::Current)
+            .unwrap()
+            .clone();
+        let committee_b = state.committee_cache(RelativeEpoch::Next).unwrap().clone();
+        assert!(committee_a != committee_b);
+        (committee_a, committee_b)
+    }
+
+    /// Builds a deterministic but incoherent shuffling ID from a `u64`.
+    fn shuffling_id(id: u64) -> AttestationShufflingId {
+        AttestationShufflingId {
+            shuffling_epoch: id.into(),
+            shuffling_decision_block: Hash256::from_low_u64_be(id),
+        }
+    }
+
+    #[test]
+    fn resolved_promise() {
+        let (committee_a, _) = committee_caches();
+        let id_a = shuffling_id(1);
+        let mut cache = ShufflingCache::new();
+
+        // Create a promise.
+        let sender = cache.create_promise(id_a.clone()).unwrap();
+
+        // Retrieve the newly created promise.
+        let item = cache.get(&id_a).unwrap();
+        assert!(
+            matches!(item, CacheItem::Promise(_)),
+            "the item should be a promise"
+        );
+
+        // Resolve the promise.
+        sender.send(committee_a.clone());
+
+        // Ensure the promise has been resolved.
+        let item = cache.get(&id_a).unwrap();
+        assert!(
+            matches!(item, CacheItem::Committee(committee) if committee == committee_a),
+            "the promise should be resolved"
+        );
+        assert_eq!(cache.cache.len(), 1, "the cache should have one entry");
+    }
+
+    #[test]
+    fn unresolved_promise() {
+        let id_a = shuffling_id(1);
+        let mut cache = ShufflingCache::new();
+
+        // Create a promise.
+        let sender = cache.create_promise(id_a.clone()).unwrap();
+
+        // Retrieve the newly created promise.
+        let item = cache.get(&id_a).unwrap();
+        assert!(
+            matches!(item, CacheItem::Promise(_)),
+            "the item should be a promise"
+        );
+
+        // Drop the sender without resolving the promise, simulating an error computing the
+        // committee.
+        drop(sender);
+
+        // Ensure the key now indicates an empty slot.
+        assert!(cache.get(&id_a).is_none(), "the slot should be empty");
+        assert!(cache.cache.is_empty(), "the cache should be empty");
+    }
+
+    #[test]
+    fn two_promises() {
+        let (committee_a, committee_b) = committee_caches();
+        let (id_a, id_b) = (shuffling_id(1), shuffling_id(2));
+        let mut cache = ShufflingCache::new();
+
+        // Create promise A.
+        let sender_a = cache.create_promise(id_a.clone()).unwrap();
+
+        // Retrieve promise A.
+        let item = cache.get(&id_a).unwrap();
+        assert!(
+            matches!(item, CacheItem::Promise(_)),
+            "item a should be a promise"
+        );
+
+        // Create promise B.
+        let sender_b = cache.create_promise(id_b.clone()).unwrap();
+
+        // Retrieve promise B.
+        let item = cache.get(&id_b).unwrap();
+        assert!(
+            matches!(item, CacheItem::Promise(_)),
+            "item b should be a promise"
+        );
+
+        // Resolve promise A.
+        sender_a.send(committee_a.clone());
+        // Ensure promise A has been resolved.
+        let item = cache.get(&id_a).unwrap();
+        assert!(
+            matches!(item, CacheItem::Committee(committee) if committee == committee_a),
+            "promise A should be resolved"
+        );
+
+        // Resolve promise B.
+        sender_b.send(committee_b.clone());
+        // Ensure promise B has been resolved.
+        let item = cache.get(&id_b).unwrap();
+        assert!(
+            matches!(item, CacheItem::Committee(committee) if committee == committee_b),
+            "promise B should be resolved"
+        );
+
+        // Check both entries again.
+        assert!(
+            matches!(cache.get(&id_a).unwrap(), CacheItem::Committee(committee) if committee == committee_a),
+            "promise A should remain resolved"
+        );
+        assert!(
+            matches!(cache.get(&id_b).unwrap(), CacheItem::Committee(committee) if committee == committee_b),
+            "promise B should remain resolved"
+        );
+        assert_eq!(cache.cache.len(), 2, "the cache should have two entries");
+    }
+
+    #[test]
+    fn too_many_promises() {
+        let mut cache = ShufflingCache::new();
+
+        for i in 0..MAX_CONCURRENT_PROMISES {
+            cache.create_promise(shuffling_id(i as u64)).unwrap();
+        }
+
+        // Ensure that the next promise returns an error. It is important for the application to
+        // dump his ass when he can't keep his promises, you're a queen and you deserve better.
+        assert!(matches!(
+            cache.create_promise(shuffling_id(MAX_CONCURRENT_PROMISES as u64)),
+            Err(BeaconChainError::MaxCommitteePromises(
+                MAX_CONCURRENT_PROMISES
+            ))
+        ));
+        assert_eq!(
+            cache.cache.len(),
+            MAX_CONCURRENT_PROMISES,
+            "the cache should have two entries"
+        );
+    }
+}

beacon_node/beacon_chain/src/snapshot_cache.rs

@@ -1,520 +0,0 @@
-use crate::BeaconSnapshot;
-use itertools::process_results;
-use std::cmp;
-use std::time::Duration;
-use types::{
-    beacon_state::CloneConfig, BeaconState, ChainSpec, Epoch, EthSpec, Hash256, SignedBeaconBlock,
-    Slot,
-};
-
-/// The default size of the cache.
-pub const DEFAULT_SNAPSHOT_CACHE_SIZE: usize = 4;
-
-/// The minimum block delay to clone the state in the cache instead of removing it.
-/// This helps keep block processing fast during re-orgs from late blocks.
-const MINIMUM_BLOCK_DELAY_FOR_CLONE: Duration = Duration::from_secs(6);
-
-/// This snapshot is to be used for verifying a child of `self.beacon_block`.
-#[derive(Debug)]
-pub struct PreProcessingSnapshot<T: EthSpec> {
-    /// This state is equivalent to the `self.beacon_block.state_root()` state that has been
-    /// advanced forward one slot using `per_slot_processing`. This state is "primed and ready" for
-    /// the application of another block.
-    pub pre_state: BeaconState<T>,
-    /// This value is only set to `Some` if the `pre_state` was *not* advanced forward.
-    pub beacon_state_root: Option<Hash256>,
-    pub beacon_block: SignedBeaconBlock<T>,
-    pub beacon_block_root: Hash256,
-}
-
-impl<T: EthSpec> From<BeaconSnapshot<T>> for PreProcessingSnapshot<T> {
-    fn from(snapshot: BeaconSnapshot<T>) -> Self {
-        let beacon_state_root = Some(snapshot.beacon_state_root());
-        Self {
-            pre_state: snapshot.beacon_state,
-            beacon_state_root,
-            beacon_block: snapshot.beacon_block,
-            beacon_block_root: snapshot.beacon_block_root,
-        }
-    }
-}
-
-impl<T: EthSpec> CacheItem<T> {
-    pub fn new_without_pre_state(snapshot: BeaconSnapshot<T>) -> Self {
-        Self {
-            beacon_block: snapshot.beacon_block,
-            beacon_block_root: snapshot.beacon_block_root,
-            beacon_state: snapshot.beacon_state,
-            pre_state: None,
-        }
-    }
-
-    fn clone_to_snapshot_with(&self, clone_config: CloneConfig) -> BeaconSnapshot<T> {
-        BeaconSnapshot {
-            beacon_state: self.beacon_state.clone_with(clone_config),
-            beacon_block: self.beacon_block.clone(),
-            beacon_block_root: self.beacon_block_root,
-        }
-    }
-
-    pub fn into_pre_state(self) -> PreProcessingSnapshot<T> {
-        // Do not include the beacon state root if the state has been advanced.
-        let beacon_state_root =
-            Some(self.beacon_block.state_root()).filter(|_| self.pre_state.is_none());
-
-        PreProcessingSnapshot {
-            beacon_block: self.beacon_block,
-            beacon_block_root: self.beacon_block_root,
-            pre_state: self.pre_state.unwrap_or(self.beacon_state),
-            beacon_state_root,
-        }
-    }
-
-    pub fn clone_as_pre_state(&self) -> PreProcessingSnapshot<T> {
-        // Do not include the beacon state root if the state has been advanced.
-        let beacon_state_root =
-            Some(self.beacon_block.state_root()).filter(|_| self.pre_state.is_none());
-
-        PreProcessingSnapshot {
-            beacon_block: self.beacon_block.clone(),
-            beacon_block_root: self.beacon_block_root,
-            pre_state: self
-                .pre_state
-                .as_ref()
-                .map_or_else(|| self.beacon_state.clone(), |pre_state| pre_state.clone()),
-            beacon_state_root,
-        }
-    }
-}
-
-/// The information required for block production.
-pub struct BlockProductionPreState<T: EthSpec> {
-    /// This state may or may not have been advanced forward a single slot.
-    ///
-    /// See the documentation in the `crate::state_advance_timer` module for more information.
-    pub pre_state: BeaconState<T>,
-    /// This value will only be `Some` if `self.pre_state` was **not** advanced forward a single
-    /// slot.
-    ///
-    /// This value can be used to avoid tree-hashing the state during the first call to
-    /// `per_slot_processing`.
-    pub state_root: Option<Hash256>,
-}
-
-pub enum StateAdvance<T: EthSpec> {
-    /// The cache does not contain the supplied block root.
-    BlockNotFound,
-    /// The cache contains the supplied block root but the state has already been advanced.
-    AlreadyAdvanced,
-    /// The cache contains the supplied block root and the state has not yet been advanced.
-    State {
-        state: Box<BeaconState<T>>,
-        state_root: Hash256,
-        block_slot: Slot,
-    },
-}
-
-/// The item stored in the `SnapshotCache`.
-pub struct CacheItem<T: EthSpec> {
-    beacon_block: SignedBeaconBlock<T>,
-    beacon_block_root: Hash256,
-    /// This state is equivalent to `self.beacon_block.state_root()`.
-    beacon_state: BeaconState<T>,
-    /// This state is equivalent to `self.beacon_state` that has had `per_slot_processing` applied
-    /// to it. This state assists in optimizing block processing.
-    pre_state: Option<BeaconState<T>>,
-}
-
-impl<T: EthSpec> Into<BeaconSnapshot<T>> for CacheItem<T> {
-    fn into(self) -> BeaconSnapshot<T> {
-        BeaconSnapshot {
-            beacon_state: self.beacon_state,
-            beacon_block: self.beacon_block,
-            beacon_block_root: self.beacon_block_root,
-        }
-    }
-}
-
-/// Provides a cache of `BeaconSnapshot` that is intended primarily for block processing.
-///
-/// ## Cache Queuing
-///
-/// The cache has a non-standard queue mechanism (specifically, it is not LRU).
-///
-/// The cache has a max number of elements (`max_len`). Until `max_len` is achieved, all snapshots
-/// are simply added to the queue. Once `max_len` is achieved, adding a new snapshot will cause an
-/// existing snapshot to be ejected. The ejected snapshot will:
-///
-/// - Never be the `head_block_root`.
-/// - Be the snapshot with the lowest `state.slot` (ties broken arbitrarily).
-pub struct SnapshotCache<T: EthSpec> {
-    max_len: usize,
-    head_block_root: Hash256,
-    snapshots: Vec<CacheItem<T>>,
-}
-
-impl<T: EthSpec> SnapshotCache<T> {
-    /// Instantiate a new cache which contains the `head` snapshot.
-    ///
-    /// Setting `max_len = 0` is equivalent to setting `max_len = 1`.
-    pub fn new(max_len: usize, head: BeaconSnapshot<T>) -> Self {
-        Self {
-            max_len: cmp::max(max_len, 1),
-            head_block_root: head.beacon_block_root,
-            snapshots: vec![CacheItem::new_without_pre_state(head)],
-        }
-    }
-
-    /// The block roots of all snapshots contained in `self`.
-    pub fn beacon_block_roots(&self) -> Vec<Hash256> {
-        self.snapshots.iter().map(|s| s.beacon_block_root).collect()
-    }
-
-    /// The number of snapshots contained in `self`.
-    pub fn len(&self) -> usize {
-        self.snapshots.len()
-    }
-
-    /// Insert a snapshot, potentially removing an existing snapshot if `self` is at capacity (see
-    /// struct-level documentation for more info).
-    pub fn insert(
-        &mut self,
-        snapshot: BeaconSnapshot<T>,
-        pre_state: Option<BeaconState<T>>,
-        spec: &ChainSpec,
-    ) {
-        let parent_root = snapshot.beacon_block.message().parent_root();
-        let item = CacheItem {
-            beacon_block: snapshot.beacon_block,
-            beacon_block_root: snapshot.beacon_block_root,
-            beacon_state: snapshot.beacon_state,
-            pre_state,
-        };
-
-        // Remove the grandparent of the block that was just inserted.
-        //
-        // Assuming it's unlikely to see re-orgs deeper than one block, this method helps keep the
-        // cache small by removing any states that already have more than one descendant.
-        //
-        // Remove the grandparent first to free up room in the cache.
-        let grandparent_result =
-            process_results(item.beacon_state.rev_iter_block_roots(spec), |iter| {
-                iter.map(|(_slot, root)| root)
-                    .find(|root| *root != item.beacon_block_root && *root != parent_root)
-            });
-        if let Ok(Some(grandparent_root)) = grandparent_result {
-            let head_block_root = self.head_block_root;
-            self.snapshots.retain(|snapshot| {
-                let root = snapshot.beacon_block_root;
-                root == head_block_root || root != grandparent_root
-            });
-        }
-
-        if self.snapshots.len() < self.max_len {
-            self.snapshots.push(item);
-        } else {
-            let insert_at = self
-                .snapshots
-                .iter()
-                .enumerate()
-                .filter_map(|(i, snapshot)| {
-                    if snapshot.beacon_block_root != self.head_block_root {
-                        Some((i, snapshot.beacon_state.slot()))
-                    } else {
-                        None
-                    }
-                })
-                .min_by_key(|(_i, slot)| *slot)
-                .map(|(i, _slot)| i);
-
-            if let Some(i) = insert_at {
-                self.snapshots[i] = item;
-            }
-        }
-    }
-
-    /// If available, returns a `CacheItem` that should be used for importing/processing a block.
-    /// The method will remove the block from `self`, carrying across any caches that may or may not
-    /// be built.
-    ///
-    /// In the event the block being processed was observed late, clone the cache instead of
-    /// moving it. This allows us to process the next block quickly in the case of a re-org.
-    /// Additionally, if the slot was skipped, clone the cache. This ensures blocks that are
-    /// later than 1 slot still have access to the cache and can be processed quickly.
-    pub fn get_state_for_block_processing(
-        &mut self,
-        block_root: Hash256,
-        block_slot: Slot,
-        block_delay: Option<Duration>,
-        spec: &ChainSpec,
-    ) -> Option<(PreProcessingSnapshot<T>, bool)> {
-        self.snapshots
-            .iter()
-            .position(|snapshot| snapshot.beacon_block_root == block_root)
-            .map(|i| {
-                if let Some(cache) = self.snapshots.get(i) {
-                    if block_slot > cache.beacon_block.slot() + 1 {
-                        return (cache.clone_as_pre_state(), true);
-                    }
-                    if let Some(delay) = block_delay {
-                        if delay >= MINIMUM_BLOCK_DELAY_FOR_CLONE
-                            && delay <= Duration::from_secs(spec.seconds_per_slot) * 4
-                        {
-                            return (cache.clone_as_pre_state(), true);
-                        }
-                    }
-                }
-                (self.snapshots.remove(i).into_pre_state(), false)
-            })
-    }
-
-    /// If available, obtains a clone of a `BeaconState` that should be used for block production.
-    /// The clone will use `CloneConfig:all()`, ensuring any tree-hash cache is cloned too.
-    ///
-    /// ## Note
-    ///
-    /// This method clones the `BeaconState` (instead of removing it) since we assume that any block
-    /// we produce will soon be pushed to the `BeaconChain` for importing/processing. Keeping a copy
-    /// of that `BeaconState` in `self` will greatly help with import times.
-    pub fn get_state_for_block_production(
-        &self,
-        block_root: Hash256,
-    ) -> Option<BlockProductionPreState<T>> {
-        self.snapshots
-            .iter()
-            .find(|snapshot| snapshot.beacon_block_root == block_root)
-            .map(|snapshot| {
-                if let Some(pre_state) = &snapshot.pre_state {
-                    BlockProductionPreState {
-                        pre_state: pre_state.clone_with(CloneConfig::all()),
-                        state_root: None,
-                    }
-                } else {
-                    BlockProductionPreState {
-                        pre_state: snapshot.beacon_state.clone_with(CloneConfig::all()),
-                        state_root: Some(snapshot.beacon_block.state_root()),
-                    }
-                }
-            })
-    }
-
-    /// If there is a snapshot with `block_root`, clone it and return the clone.
-    pub fn get_cloned(
-        &self,
-        block_root: Hash256,
-        clone_config: CloneConfig,
-    ) -> Option<BeaconSnapshot<T>> {
-        self.snapshots
-            .iter()
-            .find(|snapshot| snapshot.beacon_block_root == block_root)
-            .map(|snapshot| snapshot.clone_to_snapshot_with(clone_config))
-    }
-
-    pub fn get_for_state_advance(&mut self, block_root: Hash256) -> StateAdvance<T> {
-        if let Some(snapshot) = self
-            .snapshots
-            .iter_mut()
-            .find(|snapshot| snapshot.beacon_block_root == block_root)
-        {
-            if snapshot.pre_state.is_some() {
-                StateAdvance::AlreadyAdvanced
-            } else {
-                let cloned = snapshot
-                    .beacon_state
-                    .clone_with(CloneConfig::committee_caches_only());
-
-                StateAdvance::State {
-                    state: Box::new(std::mem::replace(&mut snapshot.beacon_state, cloned)),
-                    state_root: snapshot.beacon_block.state_root(),
-                    block_slot: snapshot.beacon_block.slot(),
-                }
-            }
-        } else {
-            StateAdvance::BlockNotFound
-        }
-    }
-
-    pub fn update_pre_state(&mut self, block_root: Hash256, state: BeaconState<T>) -> Option<()> {
-        self.snapshots
-            .iter_mut()
-            .find(|snapshot| snapshot.beacon_block_root == block_root)
-            .map(|snapshot| {
-                snapshot.pre_state = Some(state);
-            })
-    }
-
-    /// Removes all snapshots from the queue that are less than or equal to the finalized epoch.
-    pub fn prune(&mut self, finalized_epoch: Epoch) {
-        self.snapshots.retain(|snapshot| {
-            snapshot.beacon_state.slot() > finalized_epoch.start_slot(T::slots_per_epoch())
-        })
-    }
-
-    /// Inform the cache that the head of the beacon chain has changed.
-    ///
-    /// The snapshot that matches this `head_block_root` will never be ejected from the cache
-    /// during `Self::insert`.
-    pub fn update_head(&mut self, head_block_root: Hash256) {
-        self.head_block_root = head_block_root
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-    use crate::test_utils::{BeaconChainHarness, EphemeralHarnessType};
-    use types::{
-        test_utils::generate_deterministic_keypair, BeaconBlock, Epoch, MainnetEthSpec,
-        SignedBeaconBlock, Slot,
-    };
-
-    fn get_harness() -> BeaconChainHarness<EphemeralHarnessType<MainnetEthSpec>> {
-        let harness = BeaconChainHarness::builder(MainnetEthSpec)
-            .default_spec()
-            .deterministic_keypairs(1)
-            .fresh_ephemeral_store()
-            .build();
-
-        harness.advance_slot();
-
-        harness
-    }
-
-    const CACHE_SIZE: usize = 4;
-
-    fn get_snapshot(i: u64) -> BeaconSnapshot<MainnetEthSpec> {
-        let spec = MainnetEthSpec::default_spec();
-
-        let beacon_state = get_harness().chain.head_beacon_state().unwrap();
-
-        let signed_beacon_block = SignedBeaconBlock::from_block(
-            BeaconBlock::empty(&spec),
-            generate_deterministic_keypair(0)
-                .sk
-                .sign(Hash256::from_low_u64_be(42)),
-        );
-
-        BeaconSnapshot {
-            beacon_state,
-            beacon_block: signed_beacon_block,
-            beacon_block_root: Hash256::from_low_u64_be(i),
-        }
-    }
-
-    #[test]
-    fn insert_get_prune_update() {
-        let spec = MainnetEthSpec::default_spec();
-        let mut cache = SnapshotCache::new(CACHE_SIZE, get_snapshot(0));
-
-        // Insert a bunch of entries in the cache. It should look like this:
-        //
-        // Index    Root
-        // 0        0     <--head
-        // 1        1
-        // 2        2
-        // 3        3
-        for i in 1..CACHE_SIZE as u64 {
-            let mut snapshot = get_snapshot(i);
-
-            // Each snapshot should be one slot into an epoch, with each snapshot one epoch apart.
-            *snapshot.beacon_state.slot_mut() =
-                Slot::from(i * MainnetEthSpec::slots_per_epoch() + 1);
-
-            cache.insert(snapshot, None, &spec);
-
-            assert_eq!(
-                cache.snapshots.len(),
-                i as usize + 1,
-                "cache length should be as expected"
-            );
-            assert_eq!(cache.head_block_root, Hash256::from_low_u64_be(0));
-        }
-
-        // Insert a new value in the cache. Afterwards it should look like:
-        //
-        // Index    Root
-        // 0        0     <--head
-        // 1        42
-        // 2        2
-        // 3        3
-        assert_eq!(cache.snapshots.len(), CACHE_SIZE);
-        cache.insert(get_snapshot(42), None, &spec);
-        assert_eq!(cache.snapshots.len(), CACHE_SIZE);
-
-        assert!(
-            cache
-                .get_state_for_block_processing(
-                    Hash256::from_low_u64_be(1),
-                    Slot::new(0),
-                    None,
-                    &spec
-                )
-                .is_none(),
-            "the snapshot with the lowest slot should have been removed during the insert function"
-        );
-        assert!(cache
-            .get_cloned(Hash256::from_low_u64_be(1), CloneConfig::none())
-            .is_none());
-
-        assert_eq!(
-            cache
-                .get_cloned(Hash256::from_low_u64_be(0), CloneConfig::none())
-                .expect("the head should still be in the cache")
-                .beacon_block_root,
-            Hash256::from_low_u64_be(0),
-            "get_cloned should get the correct snapshot"
-        );
-        assert_eq!(
-            cache
-                .get_state_for_block_processing(
-                    Hash256::from_low_u64_be(0),
-                    Slot::new(0),
-                    None,
-                    &spec
-                )
-                .expect("the head should still be in the cache")
-                .0
-                .beacon_block_root,
-            Hash256::from_low_u64_be(0),
-            "get_state_for_block_processing should get the correct snapshot"
-        );
-
-        assert_eq!(
-            cache.snapshots.len(),
-            CACHE_SIZE - 1,
-            "get_state_for_block_processing should shorten the cache"
-        );
-
-        // Prune the cache. Afterwards it should look like:
-        //
-        // Index    Root
-        // 0        2
-        // 1        3
-        cache.prune(Epoch::new(2));
-
-        assert_eq!(cache.snapshots.len(), 2);
-
-        cache.update_head(Hash256::from_low_u64_be(2));
-
-        // Over-fill the cache so it needs to eject some old values on insert.
-        for i in 0..CACHE_SIZE as u64 {
-            cache.insert(get_snapshot(u64::max_value() - i), None, &spec);
-        }
-
-        // Ensure that the new head value was not removed from the cache.
-        assert_eq!(
-            cache
-                .get_state_for_block_processing(
-                    Hash256::from_low_u64_be(2),
-                    Slot::new(0),
-                    None,
-                    &spec
-                )
-                .expect("the new head should still be in the cache")
-                .0
-                .beacon_block_root,
-            Hash256::from_low_u64_be(2),
-            "get_state_for_block_processing should get the correct snapshot"
-        );
-    }
-}

beacon_node/beacon_chain/src/state_advance_timer.rs

@@ -15,8 +15,7 @@
 //! 2. There's a possibility that the head block is never built upon, causing wasted CPU cycles.
 use crate::validator_monitor::HISTORIC_EPOCHS as VALIDATOR_MONITOR_HISTORIC_EPOCHS;
 use crate::{
-    beacon_chain::{ATTESTATION_CACHE_LOCK_TIMEOUT, BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT},
-    snapshot_cache::StateAdvance,
+    beacon_chain::ATTESTATION_CACHE_LOCK_TIMEOUT, chain_config::FORK_CHOICE_LOOKAHEAD_FACTOR,
     BeaconChain, BeaconChainError, BeaconChainTypes,
 };
 use slog::{debug, error, warn, Logger};
@@ -27,8 +26,8 @@ use std::sync::{
     Arc,
 };
 use task_executor::TaskExecutor;
-use tokio::time::sleep;
-use types::{AttestationShufflingId, EthSpec, Hash256, RelativeEpoch, Slot};
+use tokio::time::{sleep, sleep_until, Instant};
+use types::{AttestationShufflingId, BeaconStateError, EthSpec, Hash256, RelativeEpoch, Slot};
 
 /// If the head slot is more than `MAX_ADVANCE_DISTANCE` from the current slot, then don't perform
 /// the state advancement.
@@ -37,9 +36,18 @@ use types::{AttestationShufflingId, EthSpec, Hash256, RelativeEpoch, Slot};
 /// for some period of time.
 const MAX_ADVANCE_DISTANCE: u64 = 4;
 
+/// Similarly for fork choice: avoid the fork choice lookahead during sync.
+///
+/// The value is set to 256 since this would be just over one slot (12.8s) when syncing at
+/// 20 slots/second. Having a single fork-choice run interrupt syncing would have very little
+/// impact whilst having 8 epochs without a block is a comfortable grace period.
+const MAX_FORK_CHOICE_DISTANCE: u64 = 256;
+
 #[derive(Debug)]
 enum Error {
     BeaconChain(BeaconChainError),
+    BeaconState(BeaconStateError),
+    Store(store::Error),
     HeadMissingFromSnapshotCache(Hash256),
     MaxDistanceExceeded {
         current_slot: Slot,
@@ -50,7 +58,7 @@ enum Error {
     },
     BadStateSlot {
         _state_slot: Slot,
-        _block_slot: Slot,
+        _current_slot: Slot,
     },
 }
 
@@ -60,6 +68,18 @@ impl From<BeaconChainError> for Error {
     }
 }
 
+impl From<BeaconStateError> for Error {
+    fn from(e: BeaconStateError) -> Self {
+        Self::BeaconState(e)
+    }
+}
+
+impl From<store::Error> for Error {
+    fn from(e: store::Error) -> Self {
+        Self::Store(e)
+    }
+}
+
 /// Provides a simple thread-safe lock to be used for task co-ordination. Practically equivalent to
 /// `Mutex<()>`.
 #[derive(Clone)]
@@ -105,8 +125,8 @@ async fn state_advance_timer<T: BeaconChainTypes>(
     let slot_duration = slot_clock.slot_duration();
 
     loop {
-        match beacon_chain.slot_clock.duration_to_next_slot() {
-            Some(duration) => sleep(duration + (slot_duration / 4) * 3).await,
+        let duration_to_next_slot = match beacon_chain.slot_clock.duration_to_next_slot() {
+            Some(duration) => duration,
             None => {
                 error!(log, "Failed to read slot clock");
                 // If we can't read the slot clock, just wait another slot.
@@ -115,7 +135,45 @@ async fn state_advance_timer<T: BeaconChainTypes>(
             }
         };
 
-        // Only start spawn the state advance task if the lock was previously free.
+        // Run the state advance 3/4 of the way through the slot (9s on mainnet).
+        let state_advance_offset = slot_duration / 4;
+        let state_advance_instant = if duration_to_next_slot > state_advance_offset {
+            Instant::now() + duration_to_next_slot - state_advance_offset
+        } else {
+            // Skip the state advance for the current slot and wait until the next one.
+            Instant::now() + duration_to_next_slot + slot_duration - state_advance_offset
+        };
+
+        // Run fork choice 23/24s of the way through the slot (11.5s on mainnet).
+        // We need to run after the state advance, so use the same condition as above.
+        let fork_choice_offset = slot_duration / FORK_CHOICE_LOOKAHEAD_FACTOR;
+        let fork_choice_instant = if duration_to_next_slot > state_advance_offset {
+            Instant::now() + duration_to_next_slot - fork_choice_offset
+        } else {
+            Instant::now() + duration_to_next_slot + slot_duration - fork_choice_offset
+        };
+
+        // Wait for the state advance.
+        sleep_until(state_advance_instant).await;
+
+        // Compute the current slot here at approx 3/4 through the slot. Even though this slot is
+        // only used by fork choice we need to calculate it here rather than after the state
+        // advance, in case the state advance flows over into the next slot.
+        let current_slot = match beacon_chain.slot() {
+            Ok(slot) => slot,
+            Err(e) => {
+                warn!(
+                    log,
+                    "Unable to determine slot in state advance timer";
+                    "error" => ?e
+                );
+                // If we can't read the slot clock, just wait another slot.
+                sleep(slot_duration).await;
+                continue;
+            }
+        };
+
+        // Only spawn the state advance task if the lock was previously free.
         if !is_running.lock() {
             let log = log.clone();
             let beacon_chain = beacon_chain.clone();
@@ -163,16 +221,62 @@ async fn state_advance_timer<T: BeaconChainTypes>(
                 "msg" => "system resources may be overloaded"
             )
         }
+
+        // Run fork choice pre-emptively for the next slot. This processes most of the attestations
+        // from this slot off the hot path of block verification and production.
+        // Wait for the fork choice instant (which may already be past).
+        sleep_until(fork_choice_instant).await;
+
+        let log = log.clone();
+        let beacon_chain = beacon_chain.clone();
+        let next_slot = current_slot + 1;
+        executor.spawn(
+            async move {
+                // Don't run fork choice during sync.
+                if beacon_chain.best_slot() + MAX_FORK_CHOICE_DISTANCE < current_slot {
+                    return;
+                }
+
+                // Re-compute the head, dequeuing attestations for the current slot early.
+                beacon_chain.recompute_head_at_slot(next_slot).await;
+
+                // Prepare proposers so that the node can send payload attributes in the case where
+                // it decides to abandon a proposer boost re-org.
+                if let Err(e) = beacon_chain.prepare_beacon_proposer(current_slot).await {
+                    warn!(
+                        log,
+                        "Unable to prepare proposer with lookahead";
+                        "error" => ?e,
+                        "slot" => next_slot,
+                    );
+                }
+
+                // Use a blocking task to avoid blocking the core executor whilst waiting for locks
+                // in `ForkChoiceSignalTx`.
+                beacon_chain.task_executor.clone().spawn_blocking(
+                    move || {
+                        // Signal block proposal for the next slot (if it happens to be waiting).
+                        if let Some(tx) = &beacon_chain.fork_choice_signal_tx {
+                            if let Err(e) = tx.notify_fork_choice_complete(next_slot) {
+                                warn!(
+                                    log,
+                                    "Error signalling fork choice waiter";
+                                    "error" => ?e,
+                                    "slot" => next_slot,
+                                );
+                            }
+                        }
+                    },
+                    "fork_choice_advance_signal_tx",
+                );
+            },
+            "fork_choice_advance",
+        );
     }
 }
 
-/// Reads the `snapshot_cache` from the `beacon_chain` and attempts to take a clone of the
-/// `BeaconState` of the head block. If it obtains this clone, the state will be advanced a single
-/// slot then placed back in the `snapshot_cache` to be used for block verification.
-///
-/// See the module-level documentation for rationale.
 fn advance_head<T: BeaconChainTypes>(
-    beacon_chain: &BeaconChain<T>,
+    beacon_chain: &Arc<BeaconChain<T>>,
     log: &Logger,
 ) -> Result<(), Error> {
     let current_slot = beacon_chain.slot()?;
@@ -182,7 +286,7 @@ fn advance_head<T: BeaconChainTypes>(
     //
     // Fork-choice is not run *before* this function to avoid unnecessary calls whilst syncing.
     {
-        let head_slot = beacon_chain.head_info()?.slot;
+        let head_slot = beacon_chain.best_slot();
 
         // Don't run this when syncing or if lagging too far behind.
         if head_slot + MAX_ADVANCE_DISTANCE < current_slot {
@@ -193,53 +297,38 @@ fn advance_head<T: BeaconChainTypes>(
         }
     }
 
-    // Run fork choice so we get the latest view of the head.
-    //
-    // This is useful since it's quite likely that the last time we ran fork choice was shortly
-    // after receiving the latest gossip block, but not necessarily after we've received the
-    // majority of attestations.
-    beacon_chain.fork_choice()?;
-
-    let head_root = beacon_chain.head_info()?.block_root;
-
-    let (head_slot, head_state_root, mut state) = match beacon_chain
-        .snapshot_cache
-        .try_write_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT)
-        .ok_or(BeaconChainError::SnapshotCacheLockTimeout)?
-        .get_for_state_advance(head_root)
-    {
-        StateAdvance::AlreadyAdvanced => {
-            return Err(Error::StateAlreadyAdvanced {
-                block_root: head_root,
-            })
-        }
-        StateAdvance::BlockNotFound => return Err(Error::HeadMissingFromSnapshotCache(head_root)),
-        StateAdvance::State {
-            state,
-            state_root,
-            block_slot,
-        } => (block_slot, state_root, *state),
+    let (head_block_root, head_block_state_root) = {
+        let snapshot = beacon_chain.head_snapshot();
+        (snapshot.beacon_block_root, snapshot.beacon_state_root())
     };
 
-    let initial_slot = state.slot();
-    let initial_epoch = state.current_epoch();
+    let (head_state_root, mut state) = beacon_chain
+        .store
+        .get_advanced_state(head_block_root, current_slot, head_block_state_root)?
+        .ok_or(Error::HeadMissingFromSnapshotCache(head_block_root))?;
 
-    let state_root = if state.slot() == head_slot {
-        Some(head_state_root)
-    } else {
+    if state.slot() == current_slot + 1 {
+        return Err(Error::StateAlreadyAdvanced {
+            block_root: head_block_root,
+        });
+    } else if state.slot() != current_slot {
         // Protect against advancing a state more than a single slot.
         //
         // Advancing more than one slot without storing the intermediate state would corrupt the
         // database. Future works might store temporary, intermediate states inside this function.
         return Err(Error::BadStateSlot {
-            _block_slot: head_slot,
             _state_slot: state.slot(),
+            _current_slot: current_slot,
         });
-    };
+    }
+
+    let initial_slot = state.slot();
+    let initial_epoch = state.current_epoch();
 
     // Advance the state a single slot.
-    if let Some(summary) = per_slot_processing(&mut state, state_root, &beacon_chain.spec)
-        .map_err(BeaconChainError::from)?
+    if let Some(summary) =
+        per_slot_processing(&mut state, Some(head_state_root), &beacon_chain.spec)
+            .map_err(BeaconChainError::from)?
     {
         // Expose Prometheus metrics.
         if let Err(e) = summary.observe_metrics() {
@@ -273,7 +362,7 @@ fn advance_head<T: BeaconChainTypes>(
     debug!(
         log,
         "Advanced head state one slot";
-        "head_root" => ?head_root,
+        "head_block_root" => ?head_block_root,
         "state_slot" => state.slot(),
         "current_slot" => current_slot,
     );
@@ -292,14 +381,14 @@ fn advance_head<T: BeaconChainTypes>(
     if initial_epoch < state.current_epoch() {
         // Update the proposer cache.
         //
-        // We supply the `head_root` as the decision block since the prior `if` statement guarantees
+        // We supply the `head_block_root` as the decision block since the prior `if` statement guarantees
         // the head root is the latest block from the prior epoch.
         beacon_chain
             .beacon_proposer_cache
             .lock()
             .insert(
                 state.current_epoch(),
-                head_root,
+                head_block_root,
                 state
                     .get_beacon_proposer_indices(&beacon_chain.spec)
                     .map_err(BeaconChainError::from)?,
@@ -308,8 +397,9 @@ fn advance_head<T: BeaconChainTypes>(
             .map_err(BeaconChainError::from)?;
 
         // Update the attester cache.
-        let shuffling_id = AttestationShufflingId::new(head_root, &state, RelativeEpoch::Next)
-            .map_err(BeaconChainError::from)?;
+        let shuffling_id =
+            AttestationShufflingId::new(head_block_root, &state, RelativeEpoch::Next)
+                .map_err(BeaconChainError::from)?;
         let committee_cache = state
             .committee_cache(RelativeEpoch::Next)
             .map_err(BeaconChainError::from)?;
@@ -317,12 +407,12 @@ fn advance_head<T: BeaconChainTypes>(
             .shuffling_cache
             .try_write_for(ATTESTATION_CACHE_LOCK_TIMEOUT)
             .ok_or(BeaconChainError::AttestationCacheLockTimeout)?
-            .insert(shuffling_id.clone(), committee_cache);
+            .insert_committee_cache(shuffling_id.clone(), committee_cache);
 
         debug!(
             log,
             "Primed proposer and attester caches";
-            "head_root" => ?head_root,
+            "head_block_root" => ?head_block_root,
             "next_epoch_shuffling_root" => ?shuffling_id.shuffling_decision_block,
             "state_epoch" => state.current_epoch(),
             "current_epoch" => current_slot.epoch(T::EthSpec::slots_per_epoch()),
@@ -332,44 +422,19 @@ fn advance_head<T: BeaconChainTypes>(
     // Apply the state to the attester cache, if the cache deems it interesting.
     beacon_chain
         .attester_cache
-        .maybe_cache_state(&state, head_root, &beacon_chain.spec)
+        .maybe_cache_state(&state, head_block_root, &beacon_chain.spec)
         .map_err(BeaconChainError::from)?;
 
     let final_slot = state.slot();
 
-    // Insert the advanced state back into the snapshot cache.
-    beacon_chain
-        .snapshot_cache
-        .try_write_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT)
-        .ok_or(BeaconChainError::SnapshotCacheLockTimeout)?
-        .update_pre_state(head_root, state)
-        .ok_or(Error::HeadMissingFromSnapshotCache(head_root))?;
-
-    // If we have moved into the next slot whilst processing the state then this function is going
-    // to become ineffective and likely become a hindrance as we're stealing the tree hash cache
-    // from the snapshot cache (which may force the next block to rebuild a new one).
-    //
-    // If this warning occurs very frequently on well-resourced machines then we should consider
-    // starting it earlier in the slot. Otherwise, it's a good indication that the machine is too
-    // slow/overloaded and will be useful information for the user.
-    let starting_slot = current_slot;
-    let current_slot = beacon_chain.slot()?;
-    if starting_slot < current_slot {
-        warn!(
-            log,
-            "State advance too slow";
-            "head_root" => %head_root,
-            "advanced_slot" => final_slot,
-            "current_slot" => current_slot,
-            "starting_slot" => starting_slot,
-            "msg" => "system resources may be overloaded",
-        );
-    }
+    // Write the advanced state to the database.
+    let advanced_state_root = state.update_tree_hash_cache()?;
+    beacon_chain.store.put_state(&advanced_state_root, &state)?;
 
     debug!(
         log,
         "Completed state advance";
-        "head_root" => ?head_root,
+        "head_block_root" => ?head_block_root,
         "advanced_slot" => final_slot,
         "initial_slot" => initial_slot,
     );

beacon_node/beacon_chain/src/sync_committee_verification.rs

@@ -343,7 +343,7 @@ impl<T: BeaconChainTypes> VerifiedSyncContribution<T> {
         let participant_pubkeys = sync_subcommittee_pubkeys
             .into_iter()
             .zip(contribution.aggregation_bits.iter())
-            .filter_map(|(pubkey, bit)| bit.then(|| pubkey))
+            .filter_map(|(pubkey, bit)| bit.then_some(pubkey))
             .collect::<Vec<_>>();
 
         // Ensure that all signatures are valid.
@@ -635,7 +635,7 @@ pub fn verify_sync_committee_message<T: BeaconChainTypes>(
     let pubkey = pubkey_cache
         .get_pubkey_from_pubkey_bytes(pubkey_bytes)
         .map(Cow::Borrowed)
-        .ok_or_else(|| Error::UnknownValidatorPubkey(*pubkey_bytes))?;
+        .ok_or(Error::UnknownValidatorPubkey(*pubkey_bytes))?;
 
     let next_slot_epoch = (sync_message.get_slot() + 1).epoch(T::EthSpec::slots_per_epoch());
     let fork = chain.spec.fork_at_epoch(next_slot_epoch);

beacon_node/beacon_chain/src/validator_pubkey_cache.rs

@@ -1,472 +0,0 @@
-use crate::errors::BeaconChainError;
-use crate::{BeaconChainTypes, BeaconStore};
-use ssz::{Decode, DecodeError, Encode};
-use std::collections::HashMap;
-use std::convert::TryInto;
-use std::fs::{File, OpenOptions};
-use std::io::{self, Read, Write};
-use std::path::Path;
-use store::{DBColumn, Error as StoreError, StoreItem};
-use types::{BeaconState, Hash256, PublicKey, PublicKeyBytes};
-
-/// Provides a mapping of `validator_index -> validator_publickey`.
-///
-/// This cache exists for two reasons:
-///
-/// 1. To avoid reading a `BeaconState` from disk each time we need a public key.
-/// 2. To reduce the amount of public key _decompression_ required. A `BeaconState` stores public
-///    keys in compressed form and they are needed in decompressed form for signature verification.
-///    Decompression is expensive when many keys are involved.
-///
-/// The cache has a `backing` that it uses to maintain a persistent, on-disk
-/// copy of itself. This allows it to be restored between process invocations.
-pub struct ValidatorPubkeyCache<T: BeaconChainTypes> {
-    pubkeys: Vec<PublicKey>,
-    indices: HashMap<PublicKeyBytes, usize>,
-    pubkey_bytes: Vec<PublicKeyBytes>,
-    backing: PubkeyCacheBacking<T>,
-}
-
-/// Abstraction over on-disk backing.
-///
-/// `File` backing is legacy, `Database` is current.
-enum PubkeyCacheBacking<T: BeaconChainTypes> {
-    File(ValidatorPubkeyCacheFile),
-    Database(BeaconStore<T>),
-}
-
-impl<T: BeaconChainTypes> ValidatorPubkeyCache<T> {
-    /// Create a new public key cache using the keys in `state.validators`.
-    ///
-    /// Also creates a new persistence file, returning an error if there is already a file at
-    /// `persistence_path`.
-    pub fn new(
-        state: &BeaconState<T::EthSpec>,
-        store: BeaconStore<T>,
-    ) -> Result<Self, BeaconChainError> {
-        let mut cache = Self {
-            pubkeys: vec![],
-            indices: HashMap::new(),
-            pubkey_bytes: vec![],
-            backing: PubkeyCacheBacking::Database(store),
-        };
-
-        cache.import_new_pubkeys(state)?;
-
-        Ok(cache)
-    }
-
-    /// Load the pubkey cache from the given on-disk database.
-    pub fn load_from_store(store: BeaconStore<T>) -> Result<Self, BeaconChainError> {
-        let mut pubkeys = vec![];
-        let mut indices = HashMap::new();
-        let mut pubkey_bytes = vec![];
-
-        for validator_index in 0.. {
-            if let Some(DatabasePubkey(pubkey)) =
-                store.get_item(&DatabasePubkey::key_for_index(validator_index))?
-            {
-                pubkeys.push((&pubkey).try_into().map_err(Error::PubkeyDecode)?);
-                pubkey_bytes.push(pubkey);
-                indices.insert(pubkey, validator_index);
-            } else {
-                break;
-            }
-        }
-
-        Ok(ValidatorPubkeyCache {
-            pubkeys,
-            indices,
-            pubkey_bytes,
-            backing: PubkeyCacheBacking::Database(store),
-        })
-    }
-
-    /// DEPRECATED: used only for migration
-    pub fn load_from_file<P: AsRef<Path>>(path: P) -> Result<Self, BeaconChainError> {
-        ValidatorPubkeyCacheFile::open(&path)
-            .and_then(ValidatorPubkeyCacheFile::into_cache)
-            .map_err(Into::into)
-    }
-
-    /// Convert a cache using `File` backing to one using `Database` backing.
-    ///
-    /// This will write all of the keys from `existing_cache` to `store`.
-    pub fn convert(existing_cache: Self, store: BeaconStore<T>) -> Result<Self, BeaconChainError> {
-        let mut result = ValidatorPubkeyCache {
-            pubkeys: Vec::with_capacity(existing_cache.pubkeys.len()),
-            indices: HashMap::with_capacity(existing_cache.indices.len()),
-            pubkey_bytes: Vec::with_capacity(existing_cache.indices.len()),
-            backing: PubkeyCacheBacking::Database(store),
-        };
-        result.import(existing_cache.pubkeys.iter().map(PublicKeyBytes::from))?;
-        Ok(result)
-    }
-
-    /// Scan the given `state` and add any new validator public keys.
-    ///
-    /// Does not delete any keys from `self` if they don't appear in `state`.
-    pub fn import_new_pubkeys(
-        &mut self,
-        state: &BeaconState<T::EthSpec>,
-    ) -> Result<(), BeaconChainError> {
-        if state.validators().len() > self.pubkeys.len() {
-            self.import(
-                state.validators()[self.pubkeys.len()..]
-                    .iter()
-                    .map(|v| v.pubkey),
-            )
-        } else {
-            Ok(())
-        }
-    }
-
-    /// Adds zero or more validators to `self`.
-    fn import<I>(&mut self, validator_keys: I) -> Result<(), BeaconChainError>
-    where
-        I: Iterator<Item = PublicKeyBytes> + ExactSizeIterator,
-    {
-        self.pubkey_bytes.reserve(validator_keys.len());
-        self.pubkeys.reserve(validator_keys.len());
-        self.indices.reserve(validator_keys.len());
-
-        for pubkey in validator_keys {
-            let i = self.pubkeys.len();
-
-            if self.indices.contains_key(&pubkey) {
-                return Err(BeaconChainError::DuplicateValidatorPublicKey);
-            }
-
-            // The item is written to disk _before_ it is written into
-            // the local struct.
-            //
-            // This means that a pubkey cache read from disk will always be equivalent to or
-            // _later than_ the cache that was running in the previous instance of Lighthouse.
-            //
-            // The motivation behind this ordering is that we do not want to have states that
-            // reference a pubkey that is not in our cache. However, it's fine to have pubkeys
-            // that are never referenced in a state.
-            match &mut self.backing {
-                PubkeyCacheBacking::File(persistence_file) => {
-                    persistence_file.append(i, &pubkey)?;
-                }
-                PubkeyCacheBacking::Database(store) => {
-                    store.put_item(&DatabasePubkey::key_for_index(i), &DatabasePubkey(pubkey))?;
-                }
-            }
-
-            self.pubkeys.push(
-                (&pubkey)
-                    .try_into()
-                    .map_err(BeaconChainError::InvalidValidatorPubkeyBytes)?,
-            );
-            self.pubkey_bytes.push(pubkey);
-
-            self.indices.insert(pubkey, i);
-        }
-
-        Ok(())
-    }
-
-    /// Get the public key for a validator with index `i`.
-    pub fn get(&self, i: usize) -> Option<&PublicKey> {
-        self.pubkeys.get(i)
-    }
-
-    /// Get the `PublicKey` for a validator with `PublicKeyBytes`.
-    pub fn get_pubkey_from_pubkey_bytes(&self, pubkey: &PublicKeyBytes) -> Option<&PublicKey> {
-        self.get_index(pubkey)
-            .map(|index| self.get(index))
-            .flatten()
-    }
-
-    /// Get the public key (in bytes form) for a validator with index `i`.
-    pub fn get_pubkey_bytes(&self, i: usize) -> Option<&PublicKeyBytes> {
-        self.pubkey_bytes.get(i)
-    }
-
-    /// Get the index of a validator with `pubkey`.
-    pub fn get_index(&self, pubkey: &PublicKeyBytes) -> Option<usize> {
-        self.indices.get(pubkey).copied()
-    }
-
-    /// Returns the number of validators in the cache.
-    pub fn len(&self) -> usize {
-        self.indices.len()
-    }
-}
-
-/// Wrapper for a public key stored in the database.
-///
-/// Keyed by the validator index as `Hash256::from_low_u64_be(index)`.
-struct DatabasePubkey(PublicKeyBytes);
-
-impl StoreItem for DatabasePubkey {
-    fn db_column() -> DBColumn {
-        DBColumn::PubkeyCache
-    }
-
-    fn as_store_bytes(&self) -> Vec<u8> {
-        self.0.as_ssz_bytes()
-    }
-
-    fn from_store_bytes(bytes: &[u8]) -> Result<Self, StoreError> {
-        Ok(Self(PublicKeyBytes::from_ssz_bytes(bytes)?))
-    }
-}
-
-impl DatabasePubkey {
-    fn key_for_index(index: usize) -> Hash256 {
-        Hash256::from_low_u64_be(index as u64)
-    }
-}
-
-/// Allows for maintaining an on-disk copy of the `ValidatorPubkeyCache`. The file is raw SSZ bytes
-/// (not ASCII encoded).
-///
-/// ## Writes
-///
-/// Each entry is simply appended to the file.
-///
-/// ## Reads
-///
-/// The whole file is parsed as an SSZ "variable list" of objects.
-///
-/// This parsing method is possible because the items in the list are fixed-length SSZ objects.
-struct ValidatorPubkeyCacheFile(File);
-
-#[derive(Debug)]
-enum Error {
-    Io(io::Error),
-    Ssz(DecodeError),
-    PubkeyDecode(bls::Error),
-    /// The file read from disk does not have a contiguous list of validator public keys. The file
-    /// has become corrupted.
-    InconsistentIndex {
-        _expected: Option<usize>,
-        _found: usize,
-    },
-}
-
-impl From<Error> for BeaconChainError {
-    fn from(e: Error) -> BeaconChainError {
-        BeaconChainError::ValidatorPubkeyCacheFileError(format!("{:?}", e))
-    }
-}
-
-impl ValidatorPubkeyCacheFile {
-    /// Opens an existing file for reading and writing.
-    pub fn open<P: AsRef<Path>>(path: P) -> Result<Self, Error> {
-        OpenOptions::new()
-            .read(true)
-            .write(true)
-            .create(false)
-            .append(true)
-            .open(path)
-            .map(Self)
-            .map_err(Error::Io)
-    }
-
-    /// Append a public key to file.
-    ///
-    /// The provided `index` should each be one greater than the previous and start at 0.
-    /// Otherwise, the file will become corrupted and unable to be converted into a cache .
-    pub fn append(&mut self, index: usize, pubkey: &PublicKeyBytes) -> Result<(), Error> {
-        append_to_file(&mut self.0, index, pubkey)
-    }
-
-    /// Creates a `ValidatorPubkeyCache` by reading and parsing the underlying file.
-    pub fn into_cache<T: BeaconChainTypes>(mut self) -> Result<ValidatorPubkeyCache<T>, Error> {
-        let mut bytes = vec![];
-        self.0.read_to_end(&mut bytes).map_err(Error::Io)?;
-
-        let list: Vec<(usize, PublicKeyBytes)> = Vec::from_ssz_bytes(&bytes).map_err(Error::Ssz)?;
-
-        let mut last = None;
-        let mut pubkeys = Vec::with_capacity(list.len());
-        let mut indices = HashMap::with_capacity(list.len());
-        let mut pubkey_bytes = Vec::with_capacity(list.len());
-
-        for (index, pubkey) in list {
-            let expected = last.map(|n| n + 1);
-            if expected.map_or(true, |expected| index == expected) {
-                last = Some(index);
-                pubkeys.push((&pubkey).try_into().map_err(Error::PubkeyDecode)?);
-                pubkey_bytes.push(pubkey);
-                indices.insert(pubkey, index);
-            } else {
-                return Err(Error::InconsistentIndex {
-                    _expected: expected,
-                    _found: index,
-                });
-            }
-        }
-
-        Ok(ValidatorPubkeyCache {
-            pubkeys,
-            indices,
-            pubkey_bytes,
-            backing: PubkeyCacheBacking::File(self),
-        })
-    }
-}
-
-fn append_to_file(file: &mut File, index: usize, pubkey: &PublicKeyBytes) -> Result<(), Error> {
-    let mut line = Vec::with_capacity(index.ssz_bytes_len() + pubkey.ssz_bytes_len());
-
-    index.ssz_append(&mut line);
-    pubkey.ssz_append(&mut line);
-
-    file.write_all(&line).map_err(Error::Io)
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-    use crate::test_utils::{BeaconChainHarness, EphemeralHarnessType};
-    use logging::test_logger;
-    use std::sync::Arc;
-    use store::HotColdDB;
-    use tempfile::tempdir;
-    use types::{
-        test_utils::generate_deterministic_keypair, BeaconState, EthSpec, Keypair, MainnetEthSpec,
-    };
-
-    type E = MainnetEthSpec;
-    type T = EphemeralHarnessType<E>;
-
-    fn get_state(validator_count: usize) -> (BeaconState<E>, Vec<Keypair>) {
-        let harness = BeaconChainHarness::builder(MainnetEthSpec)
-            .default_spec()
-            .deterministic_keypairs(validator_count)
-            .fresh_ephemeral_store()
-            .build();
-
-        harness.advance_slot();
-
-        (harness.get_current_state(), harness.validator_keypairs)
-    }
-
-    fn get_store() -> BeaconStore<T> {
-        Arc::new(
-            HotColdDB::open_ephemeral(<_>::default(), E::default_spec(), test_logger()).unwrap(),
-        )
-    }
-
-    #[allow(clippy::needless_range_loop)]
-    fn check_cache_get(cache: &ValidatorPubkeyCache<T>, keypairs: &[Keypair]) {
-        let validator_count = keypairs.len();
-
-        for i in 0..validator_count + 1 {
-            if i < validator_count {
-                let pubkey = cache.get(i).expect("pubkey should be present");
-                assert_eq!(pubkey, &keypairs[i].pk, "pubkey should match cache");
-
-                let pubkey_bytes: PublicKeyBytes = pubkey.clone().into();
-
-                assert_eq!(
-                    i,
-                    cache
-                        .get_index(&pubkey_bytes)
-                        .expect("should resolve index"),
-                    "index should match cache"
-                );
-            } else {
-                assert_eq!(
-                    cache.get(i),
-                    None,
-                    "should not get pubkey for out of bounds index",
-                );
-            }
-        }
-    }
-
-    #[test]
-    fn basic_operation() {
-        let (state, keypairs) = get_state(8);
-
-        let store = get_store();
-
-        let mut cache = ValidatorPubkeyCache::new(&state, store).expect("should create cache");
-
-        check_cache_get(&cache, &keypairs[..]);
-
-        // Try adding a state with the same number of keypairs.
-        let (state, keypairs) = get_state(8);
-        cache
-            .import_new_pubkeys(&state)
-            .expect("should import pubkeys");
-        check_cache_get(&cache, &keypairs[..]);
-
-        // Try adding a state with less keypairs.
-        let (state, _) = get_state(1);
-        cache
-            .import_new_pubkeys(&state)
-            .expect("should import pubkeys");
-        check_cache_get(&cache, &keypairs[..]);
-
-        // Try adding a state with more keypairs.
-        let (state, keypairs) = get_state(12);
-        cache
-            .import_new_pubkeys(&state)
-            .expect("should import pubkeys");
-        check_cache_get(&cache, &keypairs[..]);
-    }
-
-    #[test]
-    fn persistence() {
-        let (state, keypairs) = get_state(8);
-
-        let store = get_store();
-
-        // Create a new cache.
-        let cache = ValidatorPubkeyCache::new(&state, store.clone()).expect("should create cache");
-        check_cache_get(&cache, &keypairs[..]);
-        drop(cache);
-
-        // Re-init the cache from the file.
-        let mut cache =
-            ValidatorPubkeyCache::load_from_store(store.clone()).expect("should open cache");
-        check_cache_get(&cache, &keypairs[..]);
-
-        // Add some more keypairs.
-        let (state, keypairs) = get_state(12);
-        cache
-            .import_new_pubkeys(&state)
-            .expect("should import pubkeys");
-        check_cache_get(&cache, &keypairs[..]);
-        drop(cache);
-
-        // Re-init the cache from the file.
-        let cache = ValidatorPubkeyCache::load_from_store(store).expect("should open cache");
-        check_cache_get(&cache, &keypairs[..]);
-    }
-
-    #[test]
-    fn invalid_persisted_file() {
-        let dir = tempdir().expect("should create tempdir");
-        let path = dir.path().join("cache.ssz");
-        let pubkey = generate_deterministic_keypair(0).pk.into();
-
-        let mut file = File::create(&path).expect("should create file");
-        append_to_file(&mut file, 0, &pubkey).expect("should write to file");
-        drop(file);
-
-        let cache = ValidatorPubkeyCache::<T>::load_from_file(&path).expect("should open cache");
-        drop(cache);
-
-        let mut file = OpenOptions::new()
-            .write(true)
-            .append(true)
-            .open(&path)
-            .expect("should open file");
-
-        append_to_file(&mut file, 42, &pubkey).expect("should write bad data to file");
-        drop(file);
-
-        assert!(
-            ValidatorPubkeyCache::<T>::load_from_file(&path).is_err(),
-            "should not parse invalid file"
-        );
-    }
-}

beacon_node/beacon_chain/tests/attestation_production.rs

@@ -3,6 +3,7 @@
 use beacon_chain::test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy};
 use beacon_chain::{StateSkipConfig, WhenSlotSkipped};
 use lazy_static::lazy_static;
+use std::sync::Arc;
 use tree_hash::TreeHash;
 use types::{AggregateSignature, EthSpec, Keypair, MainnetEthSpec, RelativeEpoch, Slot};
 
@@ -17,8 +18,8 @@ lazy_static! {
 /// attestation at each slot from genesis through to three epochs past the head.
 ///
 /// It checks the produced attestation against some locally computed values.
-#[test]
-fn produces_attestations() {
+#[tokio::test]
+async fn produces_attestations() {
     let num_blocks_produced = MainnetEthSpec::slots_per_epoch() * 4;
     let additional_slots_tested = MainnetEthSpec::slots_per_epoch() * 3;
 
@@ -26,6 +27,7 @@ fn produces_attestations() {
         .default_spec()
         .keypairs(KEYPAIRS[..].to_vec())
         .fresh_ephemeral_store()
+        .mock_execution_layer()
         .build();
 
     let chain = &harness.chain;
@@ -36,11 +38,13 @@ fn produces_attestations() {
         if slot > 0 && slot <= num_blocks_produced {
             harness.advance_slot();
 
-            harness.extend_chain(
-                1,
-                BlockStrategy::OnCanonicalHead,
-                AttestationStrategy::AllValidators,
-            );
+            harness
+                .extend_chain(
+                    1,
+                    BlockStrategy::OnCanonicalHead,
+                    AttestationStrategy::AllValidators,
+                )
+                .await;
         }
 
         let slot = Slot::from(slot);
@@ -54,11 +58,15 @@ fn produces_attestations() {
             Slot::from(num_blocks_produced)
         };
 
-        let block = chain
+        let blinded_block = chain
             .block_at_slot(block_slot, WhenSlotSkipped::Prev)
             .expect("should get block")
             .expect("block should not be skipped");
-        let block_root = block.message().tree_hash_root();
+        let block_root = blinded_block.message().tree_hash_root();
+        let block = chain
+            .store
+            .make_full_block(&block_root, blinded_block)
+            .unwrap();
 
         let epoch_boundary_slot = state
             .current_epoch()
@@ -124,10 +132,20 @@ fn produces_attestations() {
             assert_eq!(data.target.root, target_root, "bad target root");
 
             let early_attestation = {
-                let proto_block = chain.fork_choice.read().get_block(&block_root).unwrap();
+                let proto_block = chain
+                    .canonical_head
+                    .fork_choice_read_lock()
+                    .get_block(&block_root)
+                    .unwrap();
                 chain
                     .early_attester_cache
-                    .add_head_block(block_root, block.clone(), proto_block, &state, &chain.spec)
+                    .add_head_block(
+                        block_root,
+                        Arc::new(block.clone()),
+                        proto_block,
+                        &state,
+                        &chain.spec,
+                    )
                     .unwrap();
                 chain
                     .early_attester_cache
@@ -143,3 +161,60 @@ fn produces_attestations() {
         }
     }
 }
+
+/// Ensures that the early attester cache wont create an attestation to a block in a later slot than
+/// the one requested.
+#[tokio::test]
+async fn early_attester_cache_old_request() {
+    let harness = BeaconChainHarness::builder(MainnetEthSpec)
+        .default_spec()
+        .keypairs(KEYPAIRS[..].to_vec())
+        .fresh_ephemeral_store()
+        .mock_execution_layer()
+        .build();
+
+    harness.advance_slot();
+
+    harness
+        .extend_chain(
+            2,
+            BlockStrategy::OnCanonicalHead,
+            AttestationStrategy::AllValidators,
+        )
+        .await;
+
+    let head = harness.chain.head_snapshot();
+    assert_eq!(head.beacon_block.slot(), 2);
+    let head_proto_block = harness
+        .chain
+        .canonical_head
+        .fork_choice_read_lock()
+        .get_block(&head.beacon_block_root)
+        .unwrap();
+
+    harness
+        .chain
+        .early_attester_cache
+        .add_head_block(
+            head.beacon_block_root,
+            head.beacon_block.clone(),
+            head_proto_block,
+            &head.beacon_state,
+            &harness.chain.spec,
+        )
+        .unwrap();
+
+    let attest_slot = head.beacon_block.slot() - 1;
+    let attestation = harness
+        .chain
+        .produce_unaggregated_attestation(attest_slot, 0)
+        .unwrap();
+
+    assert_eq!(attestation.data.slot, attest_slot);
+    let attested_block = harness
+        .chain
+        .get_blinded_block(&attestation.data.beacon_block_root)
+        .unwrap()
+        .unwrap();
+    assert_eq!(attested_block.slot(), attest_slot);
+}

beacon_node/beacon_chain/tests/attestation_verification.rs

@@ -42,6 +42,7 @@ fn get_harness(validator_count: usize) -> BeaconChainHarness<EphemeralHarnessTyp
         .spec(spec)
         .keypairs(KEYPAIRS[0..validator_count].to_vec())
         .fresh_ephemeral_store()
+        .mock_execution_layer()
         .build();
 
     harness.advance_slot();
@@ -55,7 +56,7 @@ fn get_harness(validator_count: usize) -> BeaconChainHarness<EphemeralHarnessTyp
 fn get_valid_unaggregated_attestation<T: BeaconChainTypes>(
     chain: &BeaconChain<T>,
 ) -> (Attestation<T::EthSpec>, usize, usize, SecretKey, SubnetId) {
-    let head = chain.head().expect("should get head");
+    let head = chain.head_snapshot();
     let current_slot = chain.slot().expect("should get slot");
 
     let mut valid_attestation = chain
@@ -105,7 +106,8 @@ fn get_valid_aggregated_attestation<T: BeaconChainTypes>(
     chain: &BeaconChain<T>,
     aggregate: Attestation<T::EthSpec>,
 ) -> (SignedAggregateAndProof<T::EthSpec>, usize, SecretKey) {
-    let state = &chain.head().expect("should get head").beacon_state;
+    let head = chain.head_snapshot();
+    let state = &head.beacon_state;
     let current_slot = chain.slot().expect("should get slot");
 
     let committee = state
@@ -154,7 +156,8 @@ fn get_non_aggregator<T: BeaconChainTypes>(
     chain: &BeaconChain<T>,
     aggregate: &Attestation<T::EthSpec>,
 ) -> (usize, SecretKey) {
-    let state = &chain.head().expect("should get head").beacon_state;
+    let head = chain.head_snapshot();
+    let state = &head.beacon_state;
     let current_slot = chain.slot().expect("should get slot");
 
     let committee = state
@@ -212,15 +215,17 @@ struct GossipTester {
 }
 
 impl GossipTester {
-    pub fn new() -> Self {
+    pub async fn new() -> Self {
         let harness = get_harness(VALIDATOR_COUNT);
 
         // Extend the chain out a few epochs so we have some chain depth to play with.
-        harness.extend_chain(
-            MainnetEthSpec::slots_per_epoch() as usize * 3 - 1,
-            BlockStrategy::OnCanonicalHead,
-            AttestationStrategy::AllValidators,
-        );
+        harness
+            .extend_chain(
+                MainnetEthSpec::slots_per_epoch() as usize * 3 - 1,
+                BlockStrategy::OnCanonicalHead,
+                AttestationStrategy::AllValidators,
+            )
+            .await;
 
         // Advance into a slot where there have not been blocks or attestations produced.
         harness.advance_slot();
@@ -394,9 +399,10 @@ impl GossipTester {
     }
 }
 /// Tests verification of `SignedAggregateAndProof` from the gossip network.
-#[test]
-fn aggregated_gossip_verification() {
+#[tokio::test]
+async fn aggregated_gossip_verification() {
     GossipTester::new()
+        .await
         /*
          * The following two tests ensure:
          *
@@ -510,8 +516,7 @@ fn aggregated_gossip_verification() {
                 let committee_len = tester
                     .harness
                     .chain
-                    .head()
-                    .unwrap()
+                    .head_snapshot()
                     .beacon_state
                     .get_beacon_committee(tester.slot(), a.message.aggregate.data.index)
                     .expect("should get committees")
@@ -611,7 +616,7 @@ fn aggregated_gossip_verification() {
                     tester.valid_aggregate.message.aggregate.clone(),
                     None,
                     &sk,
-                    &chain.head_info().unwrap().fork,
+                    &chain.canonical_head.cached_head().head_fork(),
                     chain.genesis_validators_root,
                     &chain.spec,
                 )
@@ -668,9 +673,10 @@ fn aggregated_gossip_verification() {
 }
 
 /// Tests the verification conditions for an unaggregated attestation on the gossip network.
-#[test]
-fn unaggregated_gossip_verification() {
+#[tokio::test]
+async fn unaggregated_gossip_verification() {
     GossipTester::new()
+        .await
         /*
          * The following test ensures:
          *
@@ -683,8 +689,7 @@ fn unaggregated_gossip_verification() {
                 a.data.index = tester
                     .harness
                     .chain
-                    .head()
-                    .unwrap()
+                    .head_snapshot()
                     .beacon_state
                     .get_committee_count_at_slot(a.data.slot)
                     .unwrap()
@@ -923,16 +928,18 @@ fn unaggregated_gossip_verification() {
 /// Ensures that an attestation that skips epochs can still be processed.
 ///
 /// This also checks that we can do a state lookup if we don't get a hit from the shuffling cache.
-#[test]
-fn attestation_that_skips_epochs() {
+#[tokio::test]
+async fn attestation_that_skips_epochs() {
     let harness = get_harness(VALIDATOR_COUNT);
 
     // Extend the chain out a few epochs so we have some chain depth to play with.
-    harness.extend_chain(
-        MainnetEthSpec::slots_per_epoch() as usize * 3 + 1,
-        BlockStrategy::OnCanonicalHead,
-        AttestationStrategy::SomeValidators(vec![]),
-    );
+    harness
+        .extend_chain(
+            MainnetEthSpec::slots_per_epoch() as usize * 3 + 1,
+            BlockStrategy::OnCanonicalHead,
+            AttestationStrategy::SomeValidators(vec![]),
+        )
+        .await;
 
     let current_slot = harness.chain.slot().expect("should get slot");
     let current_epoch = harness.chain.epoch().expect("should get epoch");
@@ -974,7 +981,7 @@ fn attestation_that_skips_epochs() {
     let block_slot = harness
         .chain
         .store
-        .get_block(&block_root)
+        .get_blinded_block(&block_root, None)
         .expect("should not error getting block")
         .expect("should find attestation block")
         .message()
@@ -991,16 +998,18 @@ fn attestation_that_skips_epochs() {
         .expect("should gossip verify attestation that skips slots");
 }
 
-#[test]
-fn attestation_to_finalized_block() {
+#[tokio::test]
+async fn attestation_to_finalized_block() {
     let harness = get_harness(VALIDATOR_COUNT);
 
     // Extend the chain out a few epochs so we have some chain depth to play with.
-    harness.extend_chain(
-        MainnetEthSpec::slots_per_epoch() as usize * 4 + 1,
-        BlockStrategy::OnCanonicalHead,
-        AttestationStrategy::AllValidators,
-    );
+    harness
+        .extend_chain(
+            MainnetEthSpec::slots_per_epoch() as usize * 4 + 1,
+            BlockStrategy::OnCanonicalHead,
+            AttestationStrategy::AllValidators,
+        )
+        .await;
 
     let finalized_checkpoint = harness
         .chain
@@ -1066,16 +1075,18 @@ fn attestation_to_finalized_block() {
         .contains(earlier_block_root));
 }
 
-#[test]
-fn verify_aggregate_for_gossip_doppelganger_detection() {
+#[tokio::test]
+async fn verify_aggregate_for_gossip_doppelganger_detection() {
     let harness = get_harness(VALIDATOR_COUNT);
 
     // Extend the chain out a few epochs so we have some chain depth to play with.
-    harness.extend_chain(
-        MainnetEthSpec::slots_per_epoch() as usize * 3 - 1,
-        BlockStrategy::OnCanonicalHead,
-        AttestationStrategy::AllValidators,
-    );
+    harness
+        .extend_chain(
+            MainnetEthSpec::slots_per_epoch() as usize * 3 - 1,
+            BlockStrategy::OnCanonicalHead,
+            AttestationStrategy::AllValidators,
+        )
+        .await;
 
     // Advance into a slot where there have not been blocks or attestations produced.
     harness.advance_slot();
@@ -1123,16 +1134,18 @@ fn verify_aggregate_for_gossip_doppelganger_detection() {
         .expect("should check if gossip aggregator was observed"));
 }
 
-#[test]
-fn verify_attestation_for_gossip_doppelganger_detection() {
+#[tokio::test]
+async fn verify_attestation_for_gossip_doppelganger_detection() {
     let harness = get_harness(VALIDATOR_COUNT);
 
     // Extend the chain out a few epochs so we have some chain depth to play with.
-    harness.extend_chain(
-        MainnetEthSpec::slots_per_epoch() as usize * 3 - 1,
-        BlockStrategy::OnCanonicalHead,
-        AttestationStrategy::AllValidators,
-    );
+    harness
+        .extend_chain(
+            MainnetEthSpec::slots_per_epoch() as usize * 3 - 1,
+            BlockStrategy::OnCanonicalHead,
+            AttestationStrategy::AllValidators,
+        )
+        .await;
 
     // Advance into a slot where there have not been blocks or attestations produced.
     harness.advance_slot();

beacon_node/beacon_chain/tests/main.rs

@@ -3,6 +3,7 @@ mod attestation_verification;
 mod block_verification;
 mod merge;
 mod op_verification;
+mod payload_invalidation;
 mod store_tests;
 mod sync_committee_verification;
 mod tests;

beacon_node/beacon_chain/tests/merge.rs

@@ -1,34 +1,38 @@
 #![cfg(not(debug_assertions))] // Tests run too slow in debug.
 
 use beacon_chain::test_utils::BeaconChainHarness;
-use execution_layer::test_utils::{generate_pow_block, DEFAULT_TERMINAL_BLOCK};
+use execution_layer::test_utils::{generate_pow_block, Block, DEFAULT_TERMINAL_BLOCK};
 use types::*;
 
 const VALIDATOR_COUNT: usize = 32;
 
 type E = MainnetEthSpec;
 
-fn verify_execution_payload_chain<T: EthSpec>(chain: &[ExecutionPayload<T>]) {
-    let mut prev_ep: Option<ExecutionPayload<T>> = None;
+fn verify_execution_payload_chain<T: EthSpec>(chain: &[FullPayload<T>]) {
+    let mut prev_ep: Option<FullPayload<T>> = None;
 
     for ep in chain {
-        assert!(*ep != ExecutionPayload::default());
-        assert!(ep.block_hash != Hash256::zero());
+        assert!(*ep != FullPayload::default());
+        assert!(ep.block_hash() != ExecutionBlockHash::zero());
 
         // Check against previous `ExecutionPayload`.
         if let Some(prev_ep) = prev_ep {
-            assert_eq!(prev_ep.block_hash, ep.parent_hash);
-            assert_eq!(prev_ep.block_number + 1, ep.block_number);
+            assert_eq!(prev_ep.block_hash(), ep.execution_payload.parent_hash);
+            assert_eq!(
+                prev_ep.execution_payload.block_number + 1,
+                ep.execution_payload.block_number
+            );
+            assert!(ep.execution_payload.timestamp > prev_ep.execution_payload.timestamp);
         }
         prev_ep = Some(ep.clone());
     }
 }
 
-#[test]
+#[tokio::test]
 // TODO(merge): This isn't working cause the non-zero values in `initialize_beacon_state_from_eth1`
 // are causing failed lookups to the execution node. I need to come back to this.
 #[should_panic]
-fn merge_with_terminal_block_hash_override() {
+async fn merge_with_terminal_block_hash_override() {
     let altair_fork_epoch = Epoch::new(0);
     let bellatrix_fork_epoch = Epoch::new(0);
 
@@ -40,7 +44,7 @@ fn merge_with_terminal_block_hash_override() {
         spec.terminal_total_difficulty,
         DEFAULT_TERMINAL_BLOCK,
         0,
-        Hash256::zero(),
+        ExecutionBlockHash::zero(),
     )
     .unwrap()
     .block_hash;
@@ -49,6 +53,7 @@ fn merge_with_terminal_block_hash_override() {
 
     let harness = BeaconChainHarness::builder(E::default())
         .spec(spec)
+        .logger(logging::test_logger())
         .deterministic_keypairs(VALIDATOR_COUNT)
         .fresh_ephemeral_store()
         .mock_execution_layer()
@@ -67,8 +72,7 @@ fn merge_with_terminal_block_hash_override() {
     assert!(
         harness
             .chain
-            .head()
-            .unwrap()
+            .head_snapshot()
             .beacon_block
             .as_merge()
             .is_ok(),
@@ -77,22 +81,22 @@ fn merge_with_terminal_block_hash_override() {
 
     let mut execution_payloads = vec![];
     for i in 0..E::slots_per_epoch() * 3 {
-        harness.extend_slots(1);
+        harness.extend_slots(1).await;
 
-        let block = harness.chain.head().unwrap().beacon_block;
+        let block = &harness.chain.head_snapshot().beacon_block;
 
         let execution_payload = block.message().body().execution_payload().unwrap().clone();
         if i == 0 {
-            assert_eq!(execution_payload.block_hash, genesis_pow_block_hash);
+            assert_eq!(execution_payload.block_hash(), genesis_pow_block_hash);
         }
         execution_payloads.push(execution_payload);
     }
 
-    verify_execution_payload_chain(&execution_payloads);
+    verify_execution_payload_chain(execution_payloads.as_slice());
 }
 
-#[test]
-fn base_altair_merge_with_terminal_block_after_fork() {
+#[tokio::test]
+async fn base_altair_merge_with_terminal_block_after_fork() {
     let altair_fork_epoch = Epoch::new(4);
     let altair_fork_slot = altair_fork_epoch.start_slot(E::slots_per_epoch());
     let bellatrix_fork_epoch = Epoch::new(8);
@@ -106,6 +110,7 @@ fn base_altair_merge_with_terminal_block_after_fork() {
 
     let harness = BeaconChainHarness::builder(E::default())
         .spec(spec)
+        .logger(logging::test_logger())
         .deterministic_keypairs(VALIDATOR_COUNT)
         .fresh_ephemeral_store()
         .mock_execution_layer()
@@ -115,15 +120,15 @@ fn base_altair_merge_with_terminal_block_after_fork() {
      * Start with the base fork.
      */
 
-    assert!(harness.chain.head().unwrap().beacon_block.as_base().is_ok());
+    assert!(harness.chain.head_snapshot().beacon_block.as_base().is_ok());
 
     /*
      * Do the Altair fork.
      */
 
-    harness.extend_to_slot(altair_fork_slot);
+    harness.extend_to_slot(altair_fork_slot).await;
 
-    let altair_head = harness.chain.head().unwrap().beacon_block;
+    let altair_head = &harness.chain.head_snapshot().beacon_block;
     assert!(altair_head.as_altair().is_ok());
     assert_eq!(altair_head.slot(), altair_fork_slot);
 
@@ -131,30 +136,30 @@ fn base_altair_merge_with_terminal_block_after_fork() {
      * Do the merge fork, without a terminal PoW block.
      */
 
-    harness.extend_to_slot(merge_fork_slot);
+    harness.extend_to_slot(merge_fork_slot).await;
 
-    let merge_head = harness.chain.head().unwrap().beacon_block;
+    let merge_head = &harness.chain.head_snapshot().beacon_block;
     assert!(merge_head.as_merge().is_ok());
     assert_eq!(merge_head.slot(), merge_fork_slot);
     assert_eq!(
         *merge_head.message().body().execution_payload().unwrap(),
-        ExecutionPayload::default()
+        FullPayload::default()
     );
 
     /*
      * Next merge block shouldn't include an exec payload.
      */
 
-    harness.extend_slots(1);
+    harness.extend_slots(1).await;
 
-    let one_after_merge_head = harness.chain.head().unwrap().beacon_block;
+    let one_after_merge_head = &harness.chain.head_snapshot().beacon_block;
     assert_eq!(
         *one_after_merge_head
             .message()
             .body()
             .execution_payload()
             .unwrap(),
-        ExecutionPayload::default()
+        FullPayload::default()
     );
     assert_eq!(one_after_merge_head.slot(), merge_fork_slot + 1);
 
@@ -167,16 +172,40 @@ fn base_altair_merge_with_terminal_block_after_fork() {
         .move_to_terminal_block()
         .unwrap();
 
+    // Add a slot duration to get to the next slot
+    let timestamp = harness.get_timestamp_at_slot() + harness.spec.seconds_per_slot;
+
+    harness
+        .execution_block_generator()
+        .modify_last_block(|block| {
+            if let Block::PoW(terminal_block) = block {
+                terminal_block.timestamp = timestamp;
+            }
+        });
+
+    harness.extend_slots(1).await;
+
+    let one_after_merge_head = &harness.chain.head_snapshot().beacon_block;
+    assert_eq!(
+        *one_after_merge_head
+            .message()
+            .body()
+            .execution_payload()
+            .unwrap(),
+        FullPayload::default()
+    );
+    assert_eq!(one_after_merge_head.slot(), merge_fork_slot + 2);
+
     /*
      * Next merge block should include an exec payload.
      */
 
     for _ in 0..4 {
-        harness.extend_slots(1);
+        harness.extend_slots(1).await;
 
-        let block = harness.chain.head().unwrap().beacon_block;
+        let block = &harness.chain.head_snapshot().beacon_block;
         execution_payloads.push(block.message().body().execution_payload().unwrap().clone());
     }
 
-    verify_execution_payload_chain(&execution_payloads);
+    verify_execution_payload_chain(execution_payloads.as_slice());
 }

beacon_node/beacon_chain/tests/op_verification.rs

@@ -40,23 +40,26 @@ fn get_harness(store: Arc<HotColdDB>, validator_count: usize) -> TestHarness {
         .default_spec()
         .keypairs(KEYPAIRS[0..validator_count].to_vec())
         .fresh_disk_store(store)
+        .mock_execution_layer()
         .build();
     harness.advance_slot();
     harness
 }
 
-#[test]
-fn voluntary_exit() {
+#[tokio::test]
+async fn voluntary_exit() {
     let db_path = tempdir().unwrap();
     let store = get_store(&db_path);
     let harness = get_harness(store.clone(), VALIDATOR_COUNT);
     let spec = &harness.chain.spec.clone();
 
-    harness.extend_chain(
-        (E::slots_per_epoch() * (spec.shard_committee_period + 1)) as usize,
-        BlockStrategy::OnCanonicalHead,
-        AttestationStrategy::AllValidators,
-    );
+    harness
+        .extend_chain(
+            (E::slots_per_epoch() * (spec.shard_committee_period + 1)) as usize,
+            BlockStrategy::OnCanonicalHead,
+            AttestationStrategy::AllValidators,
+        )
+        .await;
 
     let validator_index1 = VALIDATOR_COUNT - 1;
     let validator_index2 = VALIDATOR_COUNT - 2;

beacon_node/beacon_chain/tests/sync_committee_verification.rs

@@ -30,6 +30,7 @@ fn get_harness(validator_count: usize) -> BeaconChainHarness<EphemeralHarnessTyp
         .spec(spec)
         .keypairs(KEYPAIRS[0..validator_count].to_vec())
         .fresh_ephemeral_store()
+        .mock_execution_layer()
         .build();
 
     harness.advance_slot();
@@ -45,15 +46,8 @@ fn get_valid_sync_committee_message(
     slot: Slot,
     relative_sync_committee: RelativeSyncCommittee,
 ) -> (SyncCommitteeMessage, usize, SecretKey, SyncSubnetId) {
-    let head_state = harness
-        .chain
-        .head_beacon_state()
-        .expect("should get head state");
-    let head_block_root = harness
-        .chain
-        .head()
-        .expect("should get head state")
-        .beacon_block_root;
+    let head_state = harness.chain.head_beacon_state_cloned();
+    let head_block_root = harness.chain.head_snapshot().beacon_block_root;
     let (signature, _) = harness
         .make_sync_committee_messages(&head_state, head_block_root, slot, relative_sync_committee)
         .get(0)
@@ -76,16 +70,9 @@ fn get_valid_sync_contribution(
     harness: &BeaconChainHarness<EphemeralHarnessType<E>>,
     relative_sync_committee: RelativeSyncCommittee,
 ) -> (SignedContributionAndProof<E>, usize, SecretKey) {
-    let head_state = harness
-        .chain
-        .head_beacon_state()
-        .expect("should get head state");
+    let head_state = harness.chain.head_beacon_state_cloned();
 
-    let head_block_root = harness
-        .chain
-        .head()
-        .expect("should get head state")
-        .beacon_block_root;
+    let head_block_root = harness.chain.head_snapshot().beacon_block_root;
     let sync_contributions = harness.make_sync_contributions(
         &head_state,
         head_block_root,
@@ -115,7 +102,7 @@ fn get_non_aggregator(
     harness: &BeaconChainHarness<EphemeralHarnessType<E>>,
     slot: Slot,
 ) -> (usize, SecretKey) {
-    let state = &harness.chain.head().expect("should get head").beacon_state;
+    let state = &harness.chain.head_snapshot().beacon_state;
     let sync_subcommittee_size = E::sync_committee_size()
         .safe_div(SYNC_COMMITTEE_SUBNET_COUNT as usize)
         .expect("should determine sync subcommittee size");
@@ -161,17 +148,19 @@ fn get_non_aggregator(
 }
 
 /// Tests verification of `SignedContributionAndProof` from the gossip network.
-#[test]
-fn aggregated_gossip_verification() {
+#[tokio::test]
+async fn aggregated_gossip_verification() {
     let harness = get_harness(VALIDATOR_COUNT);
     let state = harness.get_current_state();
 
-    harness.add_attested_blocks_at_slots(
-        state,
-        Hash256::zero(),
-        &[Slot::new(1), Slot::new(2)],
-        (0..VALIDATOR_COUNT).collect::<Vec<_>>().as_slice(),
-    );
+    harness
+        .add_attested_blocks_at_slots(
+            state,
+            Hash256::zero(),
+            &[Slot::new(1), Slot::new(2)],
+            (0..VALIDATOR_COUNT).collect::<Vec<_>>().as_slice(),
+        )
+        .await;
 
     let current_slot = harness.chain.slot().expect("should get slot");
 
@@ -179,7 +168,7 @@ fn aggregated_gossip_verification() {
         get_valid_sync_contribution(&harness, RelativeSyncCommittee::Current);
 
     macro_rules! assert_invalid {
-            ($desc: tt, $attn_getter: expr, $($error: pat) |+ $( if $guard: expr )?) => {
+            ($desc: tt, $attn_getter: expr, $($error: pat_param) |+ $( if $guard: expr )?) => {
                 assert!(
                     matches!(
                         harness
@@ -405,7 +394,7 @@ fn aggregated_gossip_verification() {
                 valid_aggregate.message.contribution.clone(),
                 None,
                 &non_aggregator_sk,
-                &harness.chain.head_info().expect("should get head info").fork,
+                &harness.chain.canonical_head.cached_head().head_fork(),
                 harness.chain.genesis_validators_root,
                 &harness.chain.spec,
             )
@@ -473,6 +462,7 @@ fn aggregated_gossip_verification() {
 
     harness
         .add_attested_block_at_slot(target_slot, state, Hash256::zero(), &[])
+        .await
         .expect("should add block");
 
     // **Incorrectly** create a sync contribution using the current sync committee
@@ -487,17 +477,19 @@ fn aggregated_gossip_verification() {
 }
 
 /// Tests the verification conditions for sync committee messages on the gossip network.
-#[test]
-fn unaggregated_gossip_verification() {
+#[tokio::test]
+async fn unaggregated_gossip_verification() {
     let harness = get_harness(VALIDATOR_COUNT);
     let state = harness.get_current_state();
 
-    harness.add_attested_blocks_at_slots(
-        state,
-        Hash256::zero(),
-        &[Slot::new(1), Slot::new(2)],
-        (0..VALIDATOR_COUNT).collect::<Vec<_>>().as_slice(),
-    );
+    harness
+        .add_attested_blocks_at_slots(
+            state,
+            Hash256::zero(),
+            &[Slot::new(1), Slot::new(2)],
+            (0..VALIDATOR_COUNT).collect::<Vec<_>>().as_slice(),
+        )
+        .await;
 
     let current_slot = harness.chain.slot().expect("should get slot");
 
@@ -505,7 +497,7 @@ fn unaggregated_gossip_verification() {
         get_valid_sync_committee_message(&harness, current_slot, RelativeSyncCommittee::Current);
 
     macro_rules! assert_invalid {
-            ($desc: tt, $attn_getter: expr, $subnet_getter: expr, $($error: pat) |+ $( if $guard: expr )?) => {
+            ($desc: tt, $attn_getter: expr, $subnet_getter: expr, $($error: pat_param) |+ $( if $guard: expr )?) => {
                 assert!(
                     matches!(
                         harness
@@ -647,6 +639,7 @@ fn unaggregated_gossip_verification() {
 
     harness
         .add_attested_block_at_slot(target_slot, state, Hash256::zero(), &[])
+        .await
         .expect("should add block");
 
     // **Incorrectly** create a sync message using the current sync committee

beacon_node/beacon_chain/tests/tests.rs

@@ -6,14 +6,17 @@ use beacon_chain::{
         AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType,
         OP_POOL_DB_KEY,
     },
-    StateSkipConfig, WhenSlotSkipped,
+    BeaconChain, NotifyExecutionLayer, StateSkipConfig, WhenSlotSkipped,
 };
+use fork_choice::CountUnrealized;
 use lazy_static::lazy_static;
 use operation_pool::PersistedOperationPool;
 use state_processing::{
     per_slot_processing, per_slot_processing::Error as SlotProcessingError, EpochProcessingError,
 };
-use types::{BeaconStateError, EthSpec, Hash256, Keypair, MinimalEthSpec, RelativeEpoch, Slot};
+use types::{
+    BeaconState, BeaconStateError, EthSpec, Hash256, Keypair, MinimalEthSpec, RelativeEpoch, Slot,
+};
 
 // Should ideally be divisible by 3.
 pub const VALIDATOR_COUNT: usize = 24;
@@ -28,6 +31,7 @@ fn get_harness(validator_count: usize) -> BeaconChainHarness<EphemeralHarnessTyp
         .default_spec()
         .keypairs(KEYPAIRS[0..validator_count].to_vec())
         .fresh_ephemeral_store()
+        .mock_execution_layer()
         .build();
 
     harness.advance_slot();
@@ -39,7 +43,7 @@ fn get_harness(validator_count: usize) -> BeaconChainHarness<EphemeralHarnessTyp
 fn massive_skips() {
     let harness = get_harness(8);
     let spec = &harness.chain.spec;
-    let mut state = harness.chain.head().expect("should get head").beacon_state;
+    let mut state = harness.chain.head_beacon_state_cloned();
 
     // Run per_slot_processing until it returns an error.
     let error = loop {
@@ -59,18 +63,20 @@ fn massive_skips() {
     )
 }
 
-#[test]
-fn iterators() {
+#[tokio::test]
+async fn iterators() {
     let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 2 - 1;
 
     let harness = get_harness(VALIDATOR_COUNT);
 
-    harness.extend_chain(
-        num_blocks_produced as usize,
-        BlockStrategy::OnCanonicalHead,
-        // No need to produce attestations for this test.
-        AttestationStrategy::SomeValidators(vec![]),
-    );
+    harness
+        .extend_chain(
+            num_blocks_produced as usize,
+            BlockStrategy::OnCanonicalHead,
+            // No need to produce attestations for this test.
+            AttestationStrategy::SomeValidators(vec![]),
+        )
+        .await;
 
     let block_roots: Vec<(Hash256, Slot)> = harness
         .chain
@@ -121,7 +127,7 @@ fn iterators() {
         )
     });
 
-    let head = &harness.chain.head().expect("should get head");
+    let head = harness.chain.head_snapshot();
 
     assert_eq!(
         *block_roots.last().expect("should have some block roots"),
@@ -136,20 +142,44 @@ fn iterators() {
     );
 }
 
-#[test]
-fn find_reorgs() {
+fn find_reorg_slot(
+    chain: &BeaconChain<EphemeralHarnessType<MinimalEthSpec>>,
+    new_state: &BeaconState<MinimalEthSpec>,
+    new_block_root: Hash256,
+) -> Slot {
+    let (old_state, old_block_root) = {
+        let head = chain.canonical_head.cached_head();
+        let old_state = head.snapshot.beacon_state.clone();
+        let old_block_root = head.head_block_root();
+        (old_state, old_block_root)
+    };
+    beacon_chain::canonical_head::find_reorg_slot(
+        &old_state,
+        old_block_root,
+        new_state,
+        new_block_root,
+        &chain.spec,
+    )
+    .unwrap()
+}
+
+#[tokio::test]
+async fn find_reorgs() {
     let num_blocks_produced = MinimalEthSpec::slots_per_historical_root() + 1;
 
     let harness = get_harness(VALIDATOR_COUNT);
 
-    harness.extend_chain(
-        num_blocks_produced as usize,
-        BlockStrategy::OnCanonicalHead,
-        // No need to produce attestations for this test.
-        AttestationStrategy::SomeValidators(vec![]),
-    );
+    harness
+        .extend_chain(
+            num_blocks_produced as usize,
+            BlockStrategy::OnCanonicalHead,
+            // No need to produce attestations for this test.
+            AttestationStrategy::SomeValidators(vec![]),
+        )
+        .await;
 
-    let head_state = harness.chain.head_beacon_state().unwrap();
+    let head = harness.chain.head_snapshot();
+    let head_state = &head.beacon_state;
     let head_slot = head_state.slot();
     let genesis_state = harness
         .chain
@@ -159,10 +189,11 @@ fn find_reorgs() {
     // because genesis is more than `SLOTS_PER_HISTORICAL_ROOT` away, this should return with the
     // finalized slot.
     assert_eq!(
-        harness
-            .chain
-            .find_reorg_slot(&genesis_state, harness.chain.genesis_block_root)
-            .unwrap(),
+        find_reorg_slot(
+            &harness.chain,
+            &genesis_state,
+            harness.chain.genesis_block_root
+        ),
         head_state
             .finalized_checkpoint()
             .epoch
@@ -171,13 +202,11 @@ fn find_reorgs() {
 
     // test head
     assert_eq!(
-        harness
-            .chain
-            .find_reorg_slot(
-                &head_state,
-                harness.chain.head_beacon_block().unwrap().canonical_root()
-            )
-            .unwrap(),
+        find_reorg_slot(
+            &harness.chain,
+            &head_state,
+            harness.chain.head_beacon_block().canonical_root()
+        ),
         head_slot
     );
 
@@ -193,16 +222,13 @@ fn find_reorgs() {
         .unwrap()
         .unwrap();
     assert_eq!(
-        harness
-            .chain
-            .find_reorg_slot(&prev_state, prev_block_root)
-            .unwrap(),
+        find_reorg_slot(&harness.chain, &prev_state, prev_block_root),
         prev_slot
     );
 }
 
-#[test]
-fn chooses_fork() {
+#[tokio::test]
+async fn chooses_fork() {
     let harness = get_harness(VALIDATOR_COUNT);
 
     let two_thirds = (VALIDATOR_COUNT / 3) * 2;
@@ -216,22 +242,27 @@ fn chooses_fork() {
     let faulty_fork_blocks = delay + 2;
 
     // Build an initial chain where all validators agree.
-    harness.extend_chain(
-        initial_blocks,
-        BlockStrategy::OnCanonicalHead,
-        AttestationStrategy::AllValidators,
-    );
+    harness
+        .extend_chain(
+            initial_blocks,
+            BlockStrategy::OnCanonicalHead,
+            AttestationStrategy::AllValidators,
+        )
+        .await;
 
-    let (honest_head, faulty_head) = harness.generate_two_forks_by_skipping_a_block(
-        &honest_validators,
-        &faulty_validators,
-        honest_fork_blocks,
-        faulty_fork_blocks,
-    );
+    let (honest_head, faulty_head) = harness
+        .generate_two_forks_by_skipping_a_block(
+            &honest_validators,
+            &faulty_validators,
+            honest_fork_blocks,
+            faulty_fork_blocks,
+        )
+        .await;
 
     assert_ne!(honest_head, faulty_head, "forks should be distinct");
 
-    let state = &harness.chain.head().expect("should get head").beacon_state;
+    let head = harness.chain.head_snapshot();
+    let state = &head.beacon_state;
 
     assert_eq!(
         state.slot(),
@@ -240,29 +271,28 @@ fn chooses_fork() {
     );
 
     assert_eq!(
-        harness
-            .chain
-            .head()
-            .expect("should get head")
-            .beacon_block_root,
+        harness.chain.head_snapshot().beacon_block_root,
         honest_head,
         "the honest chain should be the canonical chain"
     );
 }
 
-#[test]
-fn finalizes_with_full_participation() {
+#[tokio::test]
+async fn finalizes_with_full_participation() {
     let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 5;
 
     let harness = get_harness(VALIDATOR_COUNT);
 
-    harness.extend_chain(
-        num_blocks_produced as usize,
-        BlockStrategy::OnCanonicalHead,
-        AttestationStrategy::AllValidators,
-    );
+    harness
+        .extend_chain(
+            num_blocks_produced as usize,
+            BlockStrategy::OnCanonicalHead,
+            AttestationStrategy::AllValidators,
+        )
+        .await;
 
-    let state = &harness.chain.head().expect("should get head").beacon_state;
+    let head = harness.chain.head_snapshot();
+    let state = &head.beacon_state;
 
     assert_eq!(
         state.slot(),
@@ -286,8 +316,8 @@ fn finalizes_with_full_participation() {
     );
 }
 
-#[test]
-fn finalizes_with_two_thirds_participation() {
+#[tokio::test]
+async fn finalizes_with_two_thirds_participation() {
     let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 5;
 
     let harness = get_harness(VALIDATOR_COUNT);
@@ -295,13 +325,16 @@ fn finalizes_with_two_thirds_participation() {
     let two_thirds = (VALIDATOR_COUNT / 3) * 2;
     let attesters = (0..two_thirds).collect();
 
-    harness.extend_chain(
-        num_blocks_produced as usize,
-        BlockStrategy::OnCanonicalHead,
-        AttestationStrategy::SomeValidators(attesters),
-    );
+    harness
+        .extend_chain(
+            num_blocks_produced as usize,
+            BlockStrategy::OnCanonicalHead,
+            AttestationStrategy::SomeValidators(attesters),
+        )
+        .await;
 
-    let state = &harness.chain.head().expect("should get head").beacon_state;
+    let head = harness.chain.head_snapshot();
+    let state = &head.beacon_state;
 
     assert_eq!(
         state.slot(),
@@ -330,8 +363,8 @@ fn finalizes_with_two_thirds_participation() {
     );
 }
 
-#[test]
-fn does_not_finalize_with_less_than_two_thirds_participation() {
+#[tokio::test]
+async fn does_not_finalize_with_less_than_two_thirds_participation() {
     let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 5;
 
     let harness = get_harness(VALIDATOR_COUNT);
@@ -340,13 +373,16 @@ fn does_not_finalize_with_less_than_two_thirds_participation() {
     let less_than_two_thirds = two_thirds - 1;
     let attesters = (0..less_than_two_thirds).collect();
 
-    harness.extend_chain(
-        num_blocks_produced as usize,
-        BlockStrategy::OnCanonicalHead,
-        AttestationStrategy::SomeValidators(attesters),
-    );
+    harness
+        .extend_chain(
+            num_blocks_produced as usize,
+            BlockStrategy::OnCanonicalHead,
+            AttestationStrategy::SomeValidators(attesters),
+        )
+        .await;
 
-    let state = &harness.chain.head().expect("should get head").beacon_state;
+    let head = harness.chain.head_snapshot();
+    let state = &head.beacon_state;
 
     assert_eq!(
         state.slot(),
@@ -370,19 +406,22 @@ fn does_not_finalize_with_less_than_two_thirds_participation() {
     );
 }
 
-#[test]
-fn does_not_finalize_without_attestation() {
+#[tokio::test]
+async fn does_not_finalize_without_attestation() {
     let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 5;
 
     let harness = get_harness(VALIDATOR_COUNT);
 
-    harness.extend_chain(
-        num_blocks_produced as usize,
-        BlockStrategy::OnCanonicalHead,
-        AttestationStrategy::SomeValidators(vec![]),
-    );
+    harness
+        .extend_chain(
+            num_blocks_produced as usize,
+            BlockStrategy::OnCanonicalHead,
+            AttestationStrategy::SomeValidators(vec![]),
+        )
+        .await;
 
-    let state = &harness.chain.head().expect("should get head").beacon_state;
+    let head = harness.chain.head_snapshot();
+    let state = &head.beacon_state;
 
     assert_eq!(
         state.slot(),
@@ -406,18 +445,20 @@ fn does_not_finalize_without_attestation() {
     );
 }
 
-#[test]
-fn roundtrip_operation_pool() {
+#[tokio::test]
+async fn roundtrip_operation_pool() {
     let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 5;
 
     let harness = get_harness(VALIDATOR_COUNT);
 
     // Add some attestations
-    harness.extend_chain(
-        num_blocks_produced as usize,
-        BlockStrategy::OnCanonicalHead,
-        AttestationStrategy::AllValidators,
-    );
+    harness
+        .extend_chain(
+            num_blocks_produced as usize,
+            BlockStrategy::OnCanonicalHead,
+            AttestationStrategy::AllValidators,
+        )
+        .await;
     assert!(harness.chain.op_pool.num_attestations() > 0);
 
     // TODO: could add some other operations
@@ -438,25 +479,28 @@ fn roundtrip_operation_pool() {
     assert_eq!(harness.chain.op_pool, restored_op_pool);
 }
 
-#[test]
-fn unaggregated_attestations_added_to_fork_choice_some_none() {
+#[tokio::test]
+async fn unaggregated_attestations_added_to_fork_choice_some_none() {
     let num_blocks_produced = MinimalEthSpec::slots_per_epoch() / 2;
 
     let harness = get_harness(VALIDATOR_COUNT);
 
-    harness.extend_chain(
-        num_blocks_produced as usize,
-        BlockStrategy::OnCanonicalHead,
-        AttestationStrategy::AllValidators,
-    );
+    harness
+        .extend_chain(
+            num_blocks_produced as usize,
+            BlockStrategy::OnCanonicalHead,
+            AttestationStrategy::AllValidators,
+        )
+        .await;
 
-    let state = &harness.chain.head().expect("should get head").beacon_state;
-    let mut fork_choice = harness.chain.fork_choice.write();
+    let head = harness.chain.head_snapshot();
+    let state = &head.beacon_state;
+    let mut fork_choice = harness.chain.canonical_head.fork_choice_write_lock();
 
     // Move forward a slot so all queued attestations can be processed.
     harness.advance_slot();
     fork_choice
-        .update_time(harness.chain.slot().unwrap())
+        .update_time(harness.chain.slot().unwrap(), &harness.chain.spec)
         .unwrap();
 
     let validator_slots: Vec<(usize, Slot)> = (0..VALIDATOR_COUNT)
@@ -492,8 +536,8 @@ fn unaggregated_attestations_added_to_fork_choice_some_none() {
     }
 }
 
-#[test]
-fn attestations_with_increasing_slots() {
+#[tokio::test]
+async fn attestations_with_increasing_slots() {
     let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 5;
 
     let harness = get_harness(VALIDATOR_COUNT);
@@ -501,14 +545,16 @@ fn attestations_with_increasing_slots() {
     let mut attestations = vec![];
 
     for _ in 0..num_blocks_produced {
-        harness.extend_chain(
-            2,
-            BlockStrategy::OnCanonicalHead,
-            // Don't produce & include any attestations (we'll collect them later).
-            AttestationStrategy::SomeValidators(vec![]),
-        );
+        harness
+            .extend_chain(
+                2,
+                BlockStrategy::OnCanonicalHead,
+                // Don't produce & include any attestations (we'll collect them later).
+                AttestationStrategy::SomeValidators(vec![]),
+            )
+            .await;
 
-        let head = harness.chain.head().unwrap();
+        let head = harness.chain.head_snapshot();
         let head_state_root = head.beacon_state_root();
 
         attestations.extend(harness.get_unaggregated_attestations(
@@ -547,25 +593,28 @@ fn attestations_with_increasing_slots() {
     }
 }
 
-#[test]
-fn unaggregated_attestations_added_to_fork_choice_all_updated() {
+#[tokio::test]
+async fn unaggregated_attestations_added_to_fork_choice_all_updated() {
     let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 2 - 1;
 
     let harness = get_harness(VALIDATOR_COUNT);
 
-    harness.extend_chain(
-        num_blocks_produced as usize,
-        BlockStrategy::OnCanonicalHead,
-        AttestationStrategy::AllValidators,
-    );
+    harness
+        .extend_chain(
+            num_blocks_produced as usize,
+            BlockStrategy::OnCanonicalHead,
+            AttestationStrategy::AllValidators,
+        )
+        .await;
 
-    let state = &harness.chain.head().expect("should get head").beacon_state;
-    let mut fork_choice = harness.chain.fork_choice.write();
+    let head = harness.chain.head_snapshot();
+    let state = &head.beacon_state;
+    let mut fork_choice = harness.chain.canonical_head.fork_choice_write_lock();
 
     // Move forward a slot so all queued attestations can be processed.
     harness.advance_slot();
     fork_choice
-        .update_time(harness.chain.slot().unwrap())
+        .update_time(harness.chain.slot().unwrap(), &harness.chain.spec)
         .unwrap();
 
     let validators: Vec<usize> = (0..VALIDATOR_COUNT).collect();
@@ -604,7 +653,7 @@ fn unaggregated_attestations_added_to_fork_choice_all_updated() {
     }
 }
 
-fn run_skip_slot_test(skip_slots: u64) {
+async fn run_skip_slot_test(skip_slots: u64) {
     let num_validators = 8;
     let harness_a = get_harness(num_validators);
     let harness_b = get_harness(num_validators);
@@ -614,29 +663,21 @@ fn run_skip_slot_test(skip_slots: u64) {
         harness_b.advance_slot();
     }
 
-    harness_a.extend_chain(
-        1,
-        BlockStrategy::OnCanonicalHead,
-        // No attestation required for test.
-        AttestationStrategy::SomeValidators(vec![]),
-    );
+    harness_a
+        .extend_chain(
+            1,
+            BlockStrategy::OnCanonicalHead,
+            // No attestation required for test.
+            AttestationStrategy::SomeValidators(vec![]),
+        )
+        .await;
 
     assert_eq!(
-        harness_a
-            .chain
-            .head()
-            .expect("should get head")
-            .beacon_block
-            .slot(),
+        harness_a.chain.head_snapshot().beacon_block.slot(),
         Slot::new(skip_slots + 1)
     );
     assert_eq!(
-        harness_b
-            .chain
-            .head()
-            .expect("should get head")
-            .beacon_block
-            .slot(),
+        harness_b.chain.head_snapshot().beacon_block.slot(),
         Slot::new(0)
     );
 
@@ -644,53 +685,39 @@ fn run_skip_slot_test(skip_slots: u64) {
         harness_b
             .chain
             .process_block(
-                harness_a
-                    .chain
-                    .head()
-                    .expect("should get head")
-                    .beacon_block
-                    .clone(),
+                harness_a.chain.head_snapshot().beacon_block_root,
+                harness_a.chain.head_snapshot().beacon_block.clone(),
+                CountUnrealized::True,
+                NotifyExecutionLayer::Yes,
             )
+            .await
             .unwrap(),
-        harness_a
-            .chain
-            .head()
-            .expect("should get head")
-            .beacon_block_root
+        harness_a.chain.head_snapshot().beacon_block_root
     );
 
-    harness_b
-        .chain
-        .fork_choice()
-        .expect("should run fork choice");
+    harness_b.chain.recompute_head_at_current_slot().await;
 
     assert_eq!(
-        harness_b
-            .chain
-            .head()
-            .expect("should get head")
-            .beacon_block
-            .slot(),
+        harness_b.chain.head_snapshot().beacon_block.slot(),
         Slot::new(skip_slots + 1)
     );
 }
 
-#[test]
-fn produces_and_processes_with_genesis_skip_slots() {
+#[tokio::test]
+async fn produces_and_processes_with_genesis_skip_slots() {
     for i in 0..MinimalEthSpec::slots_per_epoch() * 4 {
-        run_skip_slot_test(i)
+        run_skip_slot_test(i).await
     }
 }
 
-#[test]
-fn block_roots_skip_slot_behaviour() {
+#[tokio::test]
+async fn block_roots_skip_slot_behaviour() {
     let harness = get_harness(VALIDATOR_COUNT);
 
     // Test should be longer than the block roots to ensure a DB lookup is triggered.
     let chain_length = harness
         .chain
-        .head()
-        .unwrap()
+        .head_snapshot()
         .beacon_state
         .block_roots()
         .len() as u64
@@ -707,11 +734,13 @@ fn block_roots_skip_slot_behaviour() {
         let slot = harness.chain.slot().unwrap().as_u64();
 
         if !skipped_slots.contains(&slot) {
-            harness.extend_chain(
-                1,
-                BlockStrategy::OnCanonicalHead,
-                AttestationStrategy::AllValidators,
-            );
+            harness
+                .extend_chain(
+                    1,
+                    BlockStrategy::OnCanonicalHead,
+                    AttestationStrategy::AllValidators,
+                )
+                .await;
         }
     }
 
@@ -743,7 +772,11 @@ fn block_roots_skip_slot_behaviour() {
                 "WhenSlotSkipped::Prev should accurately return the prior skipped block"
             );
 
-            let expected_block = harness.chain.get_block(&skipped_root).unwrap().unwrap();
+            let expected_block = harness
+                .chain
+                .get_blinded_block(&skipped_root)
+                .unwrap()
+                .unwrap();
 
             assert_eq!(
                 harness
@@ -781,7 +814,11 @@ fn block_roots_skip_slot_behaviour() {
                 "WhenSlotSkipped::None and WhenSlotSkipped::Prev should be equal on non-skipped slot"
             );
 
-            let expected_block = harness.chain.get_block(&skips_prev).unwrap().unwrap();
+            let expected_block = harness
+                .chain
+                .get_blinded_block(&skips_prev)
+                .unwrap()
+                .unwrap();
 
             assert_eq!(
                 harness
@@ -811,7 +848,7 @@ fn block_roots_skip_slot_behaviour() {
 
     let future_slot = harness.chain.slot().unwrap() + 1;
     assert_eq!(
-        harness.chain.head().unwrap().beacon_block.slot(),
+        harness.chain.head_snapshot().beacon_block.slot(),
         future_slot - 2,
         "test precondition"
     );

beacon_node/builder_client/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "builder_client"
+version = "0.1.0"
+edition = "2021"
+authors = ["Sean Anderson <sean@sigmaprime.io>"]
+
+[dependencies]
+reqwest = { version = "0.11.0", features = ["json","stream"] }
+sensitive_url = { path = "../../common/sensitive_url" }
+eth2 = { path = "../../common/eth2" }
+serde = { version = "1.0.116", features = ["derive"] }
+serde_json = "1.0.58"

beacon_node/builder_client/src/lib.rs

@@ -0,0 +1,204 @@
+use eth2::types::builder_bid::SignedBuilderBid;
+use eth2::types::{
+    BlindedPayload, EthSpec, ExecPayload, ExecutionBlockHash, ExecutionPayload,
+    ForkVersionedResponse, PublicKeyBytes, SignedBeaconBlock, SignedValidatorRegistrationData,
+    Slot,
+};
+pub use eth2::Error;
+use eth2::{ok_or_error, StatusCode};
+use reqwest::{IntoUrl, Response};
+use sensitive_url::SensitiveUrl;
+use serde::de::DeserializeOwned;
+use serde::Serialize;
+use std::time::Duration;
+
+pub const DEFAULT_TIMEOUT_MILLIS: u64 = 15000;
+
+/// This timeout is in accordance with v0.2.0 of the [builder specs](https://github.com/flashbots/mev-boost/pull/20).
+pub const DEFAULT_GET_HEADER_TIMEOUT_MILLIS: u64 = 1000;
+
+#[derive(Clone)]
+pub struct Timeouts {
+    get_header: Duration,
+    post_validators: Duration,
+    post_blinded_blocks: Duration,
+    get_builder_status: Duration,
+}
+
+impl Default for Timeouts {
+    fn default() -> Self {
+        Self {
+            get_header: Duration::from_millis(DEFAULT_GET_HEADER_TIMEOUT_MILLIS),
+            post_validators: Duration::from_millis(DEFAULT_TIMEOUT_MILLIS),
+            post_blinded_blocks: Duration::from_millis(DEFAULT_TIMEOUT_MILLIS),
+            get_builder_status: Duration::from_millis(DEFAULT_TIMEOUT_MILLIS),
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct BuilderHttpClient {
+    client: reqwest::Client,
+    server: SensitiveUrl,
+    timeouts: Timeouts,
+}
+
+impl BuilderHttpClient {
+    pub fn new(server: SensitiveUrl) -> Result<Self, Error> {
+        Ok(Self {
+            client: reqwest::Client::new(),
+            server,
+            timeouts: Timeouts::default(),
+        })
+    }
+
+    pub fn new_with_timeouts(server: SensitiveUrl, timeouts: Timeouts) -> Result<Self, Error> {
+        Ok(Self {
+            client: reqwest::Client::new(),
+            server,
+            timeouts,
+        })
+    }
+
+    async fn get_with_timeout<T: DeserializeOwned, U: IntoUrl>(
+        &self,
+        url: U,
+        timeout: Duration,
+    ) -> Result<T, Error> {
+        self.get_response_with_timeout(url, Some(timeout))
+            .await?
+            .json()
+            .await
+            .map_err(Error::Reqwest)
+    }
+
+    /// Perform a HTTP GET request, returning the `Response` for further processing.
+    async fn get_response_with_timeout<U: IntoUrl>(
+        &self,
+        url: U,
+        timeout: Option<Duration>,
+    ) -> Result<Response, Error> {
+        let mut builder = self.client.get(url);
+        if let Some(timeout) = timeout {
+            builder = builder.timeout(timeout);
+        }
+        let response = builder.send().await.map_err(Error::Reqwest)?;
+        ok_or_error(response).await
+    }
+
+    /// Generic POST function supporting arbitrary responses and timeouts.
+    async fn post_generic<T: Serialize, U: IntoUrl>(
+        &self,
+        url: U,
+        body: &T,
+        timeout: Option<Duration>,
+    ) -> Result<Response, Error> {
+        let mut builder = self.client.post(url);
+        if let Some(timeout) = timeout {
+            builder = builder.timeout(timeout);
+        }
+        let response = builder.json(body).send().await?;
+        ok_or_error(response).await
+    }
+
+    async fn post_with_raw_response<T: Serialize, U: IntoUrl>(
+        &self,
+        url: U,
+        body: &T,
+        timeout: Option<Duration>,
+    ) -> Result<Response, Error> {
+        let mut builder = self.client.post(url);
+        if let Some(timeout) = timeout {
+            builder = builder.timeout(timeout);
+        }
+        let response = builder.json(body).send().await.map_err(Error::Reqwest)?;
+        ok_or_error(response).await
+    }
+
+    /// `POST /eth/v1/builder/validators`
+    pub async fn post_builder_validators(
+        &self,
+        validator: &[SignedValidatorRegistrationData],
+    ) -> Result<(), Error> {
+        let mut path = self.server.full.clone();
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("eth")
+            .push("v1")
+            .push("builder")
+            .push("validators");
+
+        self.post_generic(path, &validator, Some(self.timeouts.post_validators))
+            .await?;
+        Ok(())
+    }
+
+    /// `POST /eth/v1/builder/blinded_blocks`
+    pub async fn post_builder_blinded_blocks<E: EthSpec>(
+        &self,
+        blinded_block: &SignedBeaconBlock<E, BlindedPayload<E>>,
+    ) -> Result<ForkVersionedResponse<ExecutionPayload<E>>, Error> {
+        let mut path = self.server.full.clone();
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("eth")
+            .push("v1")
+            .push("builder")
+            .push("blinded_blocks");
+
+        Ok(self
+            .post_with_raw_response(
+                path,
+                &blinded_block,
+                Some(self.timeouts.post_blinded_blocks),
+            )
+            .await?
+            .json()
+            .await?)
+    }
+
+    /// `GET /eth/v1/builder/header`
+    pub async fn get_builder_header<E: EthSpec, Payload: ExecPayload<E>>(
+        &self,
+        slot: Slot,
+        parent_hash: ExecutionBlockHash,
+        pubkey: &PublicKeyBytes,
+    ) -> Result<Option<ForkVersionedResponse<SignedBuilderBid<E, Payload>>>, Error> {
+        let mut path = self.server.full.clone();
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("eth")
+            .push("v1")
+            .push("builder")
+            .push("header")
+            .push(slot.to_string().as_str())
+            .push(format!("{parent_hash:?}").as_str())
+            .push(pubkey.as_hex_string().as_str());
+
+        let resp = self.get_with_timeout(path, self.timeouts.get_header).await;
+
+        if matches!(resp, Err(Error::StatusCode(StatusCode::NO_CONTENT))) {
+            Ok(None)
+        } else {
+            resp.map(Some)
+        }
+    }
+
+    /// `GET /eth/v1/builder/status`
+    pub async fn get_builder_status<E: EthSpec>(&self) -> Result<(), Error> {
+        let mut path = self.server.full.clone();
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("eth")
+            .push("v1")
+            .push("builder")
+            .push("status");
+
+        self.get_with_timeout(path, self.timeouts.get_builder_status)
+            .await
+    }
+}

beacon_node/client/Cargo.toml

@@ -2,10 +2,10 @@
 name = "client"
 version = "0.2.0"
 authors = ["Sigma Prime <contact@sigmaprime.io>"]
-edition = "2018"
+edition = "2021"
 
 [dev-dependencies]
-toml = "0.5.6"
+serde_yaml = "0.8.13"
 
 [dependencies]
 beacon_chain = { path = "../beacon_chain" }
@@ -13,7 +13,7 @@ store = { path = "../store" }
 network = { path = "../network" }
 timer = { path = "../timer" }
 lighthouse_network = { path = "../lighthouse_network" }
-parking_lot = "0.11.0"
+parking_lot = "0.12.0"
 types = { path = "../../consensus/types" }
 eth2_config = { path = "../../common/eth2_config" }
 slot_clock = { path = "../../common/slot_clock" }

beacon_node/client/src/builder.rs

@@ -1,6 +1,8 @@
 use crate::config::{ClientGenesis, Config as ClientConfig};
 use crate::notifier::spawn_notifier;
 use crate::Client;
+use beacon_chain::otb_verification_service::start_otb_verification_service;
+use beacon_chain::proposer_prep_service::start_proposer_prep_service;
 use beacon_chain::schema_change::migrate_schema;
 use beacon_chain::{
     builder::{BeaconChainBuilder, Witness},
@@ -20,7 +22,7 @@ use execution_layer::ExecutionLayer;
 use genesis::{interop_genesis_state, Eth1GenesisService, DEFAULT_ETH1_BLOCK_HASH};
 use lighthouse_network::{prometheus_client::registry::Registry, NetworkGlobals};
 use monitoring_api::{MonitoringHttpClient, ProcessType};
-use network::{NetworkConfig, NetworkMessage, NetworkService};
+use network::{NetworkConfig, NetworkSenders, NetworkService};
 use slasher::Slasher;
 use slasher_service::SlasherService;
 use slog::{debug, info, warn, Logger};
@@ -29,18 +31,15 @@ use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::Duration;
 use timer::spawn_timer;
-use tokio::sync::{mpsc::UnboundedSender, oneshot};
+use tokio::sync::oneshot;
 use types::{
-    test_utils::generate_deterministic_keypairs, BeaconState, ChainSpec, EthSpec, Hash256,
-    SignedBeaconBlock,
+    test_utils::generate_deterministic_keypairs, BeaconState, ChainSpec, EthSpec,
+    ExecutionBlockHash, Hash256, SignedBeaconBlock,
 };
 
 /// Interval between polling the eth1 node for genesis information.
 pub const ETH1_GENESIS_UPDATE_INTERVAL_MILLIS: u64 = 7_000;
 
-/// Timeout for checkpoint sync HTTP requests.
-pub const CHECKPOINT_SYNC_HTTP_TIMEOUT: Duration = Duration::from_secs(60);
-
 /// Builds a `Client` instance.
 ///
 /// ## Notes
@@ -64,7 +63,7 @@ pub struct ClientBuilder<T: BeaconChainTypes> {
     beacon_chain: Option<Arc<BeaconChain<T>>>,
     eth1_service: Option<Eth1Service>,
     network_globals: Option<Arc<NetworkGlobals<T::EthSpec>>>,
-    network_send: Option<UnboundedSender<NetworkMessage<T::EthSpec>>>,
+    network_senders: Option<NetworkSenders<T::EthSpec>>,
     gossipsub_registry: Option<Registry>,
     db_path: Option<PathBuf>,
     freezer_db_path: Option<PathBuf>,
@@ -96,7 +95,7 @@ where
             beacon_chain: None,
             eth1_service: None,
             network_globals: None,
-            network_send: None,
+            network_senders: None,
             gossipsub_registry: None,
             db_path: None,
             freezer_db_path: None,
@@ -149,11 +148,10 @@ where
             None
         };
 
-        let execution_layer = if let Some(execution_endpoints) = config.execution_endpoints {
+        let execution_layer = if let Some(config) = config.execution_layer {
             let context = runtime_context.service_context("exec".into());
-            let execution_layer = ExecutionLayer::from_urls(
-                execution_endpoints,
-                config.suggested_fee_recipient,
+            let execution_layer = ExecutionLayer::from_config(
+                config,
                 context.executor.clone(),
                 context.log().clone(),
             )
@@ -166,14 +164,17 @@ where
         let builder = BeaconChainBuilder::new(eth_spec_instance)
             .logger(context.log().clone())
             .store(store)
+            .task_executor(context.executor.clone())
             .custom_spec(spec.clone())
             .chain_config(chain_config)
+            .store_migrator_config(config.store_migrator.clone())
             .graffiti(graffiti)
             .event_handler(event_handler)
             .execution_layer(execution_layer)
             .monitor_validators(
                 config.validator_monitor_auto,
                 config.validator_monitor_pubkeys.clone(),
+                config.validator_monitor_individual_tracking_threshold,
                 runtime_context
                     .service_context("val_mon".to_string())
                     .log()
@@ -271,10 +272,60 @@ where
                     "remote_url" => %url,
                 );
 
-                let remote =
-                    BeaconNodeHttpClient::new(url, Timeouts::set_all(CHECKPOINT_SYNC_HTTP_TIMEOUT));
+                let remote = BeaconNodeHttpClient::new(
+                    url,
+                    Timeouts::set_all(Duration::from_secs(
+                        config.chain.checkpoint_sync_url_timeout,
+                    )),
+                );
                 let slots_per_epoch = TEthSpec::slots_per_epoch();
 
+                let deposit_snapshot = if config.sync_eth1_chain {
+                    // We want to fetch deposit snapshot before fetching the finalized beacon state to
+                    // ensure that the snapshot is not newer than the beacon state that satisfies the
+                    // deposit finalization conditions
+                    debug!(context.log(), "Downloading deposit snapshot");
+                    let deposit_snapshot_result = remote
+                        .get_deposit_snapshot()
+                        .await
+                        .map_err(|e| match e {
+                            ApiError::InvalidSsz(e) => format!(
+                                "Unable to parse SSZ: {:?}. Ensure the checkpoint-sync-url refers to a \
+                                node for the correct network",
+                                e
+                            ),
+                            e => format!("Error fetching deposit snapshot from remote: {:?}", e),
+                        });
+                    match deposit_snapshot_result {
+                        Ok(Some(deposit_snapshot)) => {
+                            if deposit_snapshot.is_valid() {
+                                Some(deposit_snapshot)
+                            } else {
+                                warn!(context.log(), "Remote BN sent invalid deposit snapshot!");
+                                None
+                            }
+                        }
+                        Ok(None) => {
+                            warn!(
+                                context.log(),
+                                "Remote BN does not support EIP-4881 fast deposit sync"
+                            );
+                            None
+                        }
+                        Err(e) => {
+                            warn!(
+                                context.log(),
+                                "Remote BN does not support EIP-4881 fast deposit sync";
+                                "error" => e
+                            );
+                            None
+                        }
+                    }
+                } else {
+                    None
+                };
+
+                debug!(context.log(), "Downloading finalized block");
                 // Find a suitable finalized block on an epoch boundary.
                 let mut block = remote
                     .get_beacon_blocks_ssz::<TEthSpec>(BlockId::Finalized, &spec)
@@ -289,6 +340,8 @@ where
                     })?
                     .ok_or("Finalized block missing from remote, it returned 404")?;
 
+                debug!(context.log(), "Downloaded finalized block");
+
                 let mut block_slot = block.slot();
 
                 while block.slot() % slots_per_epoch != 0 {
@@ -300,6 +353,12 @@ where
                         "block_slot" => block_slot,
                     );
 
+                    debug!(
+                        context.log(),
+                        "Searching for aligned checkpoint block";
+                        "block_slot" => block_slot
+                    );
+
                     if let Some(found_block) = remote
                         .get_beacon_blocks_ssz::<TEthSpec>(BlockId::Slot(block_slot), &spec)
                         .await
@@ -311,7 +370,19 @@ where
                     }
                 }
 
+                debug!(
+                    context.log(),
+                    "Downloaded aligned finalized block";
+                    "block_root" => ?block.canonical_root(),
+                    "block_slot" => block.slot(),
+                );
+
                 let state_root = block.state_root();
+                debug!(
+                    context.log(),
+                    "Downloading finalized state";
+                    "state_root" => ?state_root
+                );
                 let state = remote
                     .get_debug_beacon_states_ssz::<TEthSpec>(StateId::Root(state_root), &spec)
                     .await
@@ -325,6 +396,8 @@ where
                         format!("Checkpoint state missing from remote: {:?}", state_root)
                     })?;
 
+                debug!(context.log(), "Downloaded finalized state");
+
                 let genesis_state = BeaconState::from_ssz_bytes(&genesis_state_bytes, &spec)
                     .map_err(|e| format!("Unable to parse genesis state SSZ: {:?}", e))?;
 
@@ -336,15 +409,39 @@ where
                     "state_root" => ?state_root,
                 );
 
+                let service =
+                    deposit_snapshot.and_then(|snapshot| match Eth1Service::from_deposit_snapshot(
+                        config.eth1,
+                        context.log().clone(),
+                        spec,
+                        &snapshot,
+                    ) {
+                        Ok(service) => {
+                            info!(
+                                context.log(),
+                                "Loaded deposit tree snapshot";
+                                "deposits loaded" => snapshot.deposit_count,
+                            );
+                            Some(service)
+                        }
+                        Err(e) => {
+                            warn!(context.log(),
+                                "Unable to load deposit snapshot";
+                                "error" => ?e
+                            );
+                            None
+                        }
+                    });
+
                 builder
                     .weak_subjectivity_state(state, block, genesis_state)
-                    .map(|v| (v, None))?
+                    .map(|v| (v, service))?
             }
             ClientGenesis::DepositContract => {
                 info!(
                     context.log(),
                     "Waiting for eth2 genesis from eth1";
-                    "eth1_endpoints" => format!("{:?}", &config.eth1.endpoints),
+                    "eth1_endpoints" => format!("{:?}", &config.eth1.endpoint),
                     "contract_deploy_block" => config.eth1.deposit_contract_deploy_block,
                     "deposit_contract" => &config.eth1.deposit_contract_address
                 );
@@ -353,7 +450,7 @@ where
                     config.eth1,
                     context.log().clone(),
                     context.eth2_config().spec.clone(),
-                );
+                )?;
 
                 // If the HTTP API server is enabled, start an instance of it where it only
                 // contains a reference to the eth1 service (all non-eth1 endpoints will fail
@@ -371,7 +468,7 @@ where
                     > = Arc::new(http_api::Context {
                         config: self.http_api_config.clone(),
                         chain: None,
-                        network_tx: None,
+                        network_senders: None,
                         network_globals: None,
                         eth1_service: Some(genesis_service.eth1_service.clone()),
                         log: context.log().clone(),
@@ -431,7 +528,9 @@ where
             ClientGenesis::FromStore => builder.resume_from_db().map(|v| (v, None))?,
         };
 
-        self.eth1_service = eth1_service_option;
+        if config.sync_eth1_chain {
+            self.eth1_service = eth1_service_option;
+        }
         self.beacon_chain_builder = Some(beacon_chain_builder);
         Ok(self)
     }
@@ -455,7 +554,7 @@ where
             None
         };
 
-        let (network_globals, network_send) = NetworkService::start(
+        let (network_globals, network_senders) = NetworkService::start(
             beacon_chain,
             config,
             context.executor,
@@ -467,7 +566,7 @@ where
         .map_err(|e| format!("Failed to start network: {:?}", e))?;
 
         self.network_globals = Some(network_globals);
-        self.network_send = Some(network_send);
+        self.network_senders = Some(network_senders);
         self.gossipsub_registry = gossipsub_registry;
 
         Ok(self)
@@ -484,13 +583,8 @@ where
             .beacon_chain
             .clone()
             .ok_or("node timer requires a beacon chain")?;
-        let seconds_per_slot = self
-            .chain_spec
-            .as_ref()
-            .ok_or("node timer requires a chain spec")?
-            .seconds_per_slot;
 
-        spawn_timer(context.executor, beacon_chain, seconds_per_slot)
+        spawn_timer(context.executor, beacon_chain)
             .map_err(|e| format!("Unable to start node timer: {}", e))?;
 
         Ok(self)
@@ -516,16 +610,16 @@ where
             .beacon_chain
             .clone()
             .ok_or("slasher service requires a beacon chain")?;
-        let network_send = self
-            .network_send
+        let network_senders = self
+            .network_senders
             .clone()
-            .ok_or("slasher service requires a network sender")?;
+            .ok_or("slasher service requires network senders")?;
         let context = self
             .runtime_context
             .as_ref()
             .ok_or("slasher requires a runtime_context")?
             .service_context("slasher_service_ctxt".into());
-        SlasherService::new(beacon_chain, network_send).run(&context.executor)
+        SlasherService::new(beacon_chain, network_senders.network_send()).run(&context.executor)
     }
 
     /// Start the explorer client which periodically sends beacon
@@ -595,7 +689,7 @@ where
             let ctx = Arc::new(http_api::Context {
                 config: self.http_api_config.clone(),
                 chain: self.beacon_chain.clone(),
-                network_tx: self.network_send.clone(),
+                network_senders: self.network_senders.clone(),
                 network_globals: self.network_globals.clone(),
                 eth1_service: self.eth1_service.clone(),
                 log: log.clone(),
@@ -662,50 +756,57 @@ where
             );
 
             if let Some(execution_layer) = beacon_chain.execution_layer.as_ref() {
-                let store = beacon_chain.store.clone();
-                let inner_execution_layer = execution_layer.clone();
+                // Only send a head update *after* genesis.
+                if let Ok(current_slot) = beacon_chain.slot() {
+                    let params = beacon_chain
+                        .canonical_head
+                        .cached_head()
+                        .forkchoice_update_parameters();
+                    if params
+                        .head_hash
+                        .map_or(false, |hash| hash != ExecutionBlockHash::zero())
+                    {
+                        // Spawn a new task to update the EE without waiting for it to complete.
+                        let inner_chain = beacon_chain.clone();
+                        runtime_context.executor.spawn(
+                            async move {
+                                let result = inner_chain
+                                    .update_execution_engine_forkchoice(
+                                        current_slot,
+                                        params,
+                                        Default::default(),
+                                    )
+                                    .await;
 
-                let head = beacon_chain
-                    .head_info()
-                    .map_err(|e| format!("Unable to read beacon chain head: {:?}", e))?;
+                                // No need to exit early if setting the head fails. It will be set again if/when the
+                                // node comes online.
+                                if let Err(e) = result {
+                                    warn!(
+                                        log,
+                                        "Failed to update head on execution engines";
+                                        "error" => ?e
+                                    );
+                                }
+                            },
+                            "el_fork_choice_update",
+                        );
+                    }
 
-                // Issue the head to the execution engine on startup. This ensures it can start
-                // syncing.
-                if let Some(block_hash) = head.execution_payload_block_hash {
-                    runtime_context.executor.spawn(
-                        async move {
-                            let result = BeaconChain::<
-                                Witness<TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>,
-                            >::update_execution_engine_forkchoice(
-                                inner_execution_layer,
-                                store,
-                                head.finalized_checkpoint.root,
-                                block_hash,
-                            )
-                            .await;
+                    // Spawn a routine that tracks the status of the execution engines.
+                    execution_layer.spawn_watchdog_routine(beacon_chain.slot_clock.clone());
 
-                            // No need to exit early if setting the head fails. It will be set again if/when the
-                            // node comes online.
-                            if let Err(e) = result {
-                                warn!(
-                                    log,
-                                    "Failed to update head on execution engines";
-                                    "error" => ?e
-                                );
-                            }
-                        },
-                        "el_fork_choice_update",
+                    // Spawn a routine that removes expired proposer preparations.
+                    execution_layer.spawn_clean_proposer_caches_routine::<TSlotClock>(
+                        beacon_chain.slot_clock.clone(),
                     );
+
+                    // Spawns a routine that polls the `exchange_transition_configuration` endpoint.
+                    execution_layer.spawn_transition_configuration_poll(beacon_chain.spec.clone());
                 }
-
-                // Spawn a routine that tracks the status of the execution engines.
-                execution_layer.spawn_watchdog_routine(beacon_chain.slot_clock.clone());
-
-                // Spawn a routine that removes expired proposer preparations.
-                execution_layer.spawn_clean_proposer_preparation_routine::<TSlotClock, TEthSpec>(
-                    beacon_chain.slot_clock.clone(),
-                );
             }
+
+            start_proposer_prep_service(runtime_context.executor.clone(), beacon_chain.clone());
+            start_otb_verification_service(runtime_context.executor.clone(), beacon_chain.clone());
         }
 
         Ok(Client {
@@ -764,7 +865,6 @@ where
     /// Specifies that the `Client` should use a `HotColdDB` database.
     pub fn disk_store(
         mut self,
-        datadir: &Path,
         hot_path: &Path,
         cold_path: &Path,
         config: StoreConfig,
@@ -783,8 +883,22 @@ where
         self.db_path = Some(hot_path.into());
         self.freezer_db_path = Some(cold_path.into());
 
+        let inner_spec = spec.clone();
+        let deposit_contract_deploy_block = context
+            .eth2_network_config
+            .as_ref()
+            .map(|config| config.deposit_contract_deploy_block)
+            .unwrap_or(0);
+
         let schema_upgrade = |db, from, to| {
-            migrate_schema::<Witness<TSlotClock, TEth1Backend, _, _, _>>(db, datadir, from, to, log)
+            migrate_schema::<Witness<TSlotClock, TEth1Backend, _, _, _>>(
+                db,
+                deposit_contract_deploy_block,
+                from,
+                to,
+                log,
+                &inner_spec,
+            )
         };
 
         let store = HotColdDB::open(
@@ -819,7 +933,7 @@ where
             .runtime_context
             .as_ref()
             .ok_or("caching_eth1_backend requires a runtime_context")?
-            .service_context("eth1_rpc".into());
+            .service_context("deposit_contract_rpc".into());
         let beacon_chain_builder = self
             .beacon_chain_builder
             .ok_or("caching_eth1_backend requires a beacon_chain_builder")?;
@@ -843,7 +957,7 @@ where
 
             CachingEth1Backend::from_service(eth1_service_from_genesis)
         } else if config.purge_cache {
-            CachingEth1Backend::new(config, context.log().clone(), spec)
+            CachingEth1Backend::new(config, context.log().clone(), spec)?
         } else {
             beacon_chain_builder
                 .get_persisted_eth1_backend()?
@@ -857,11 +971,7 @@ where
                     .map(|chain| chain.into_backend())
                 })
                 .unwrap_or_else(|| {
-                    Ok(CachingEth1Backend::new(
-                        config,
-                        context.log().clone(),
-                        spec.clone(),
-                    ))
+                    CachingEth1Backend::new(config, context.log().clone(), spec.clone())
                 })?
         };
 

beacon_node/client/src/config.rs

@@ -1,16 +1,18 @@
+use beacon_chain::migrate::MigratorConfig;
+use beacon_chain::validator_monitor::DEFAULT_INDIVIDUAL_TRACKING_THRESHOLD;
 use directory::DEFAULT_ROOT_DIR;
+use environment::LoggerConfig;
 use network::NetworkConfig;
 use sensitive_url::SensitiveUrl;
 use serde_derive::{Deserialize, Serialize};
 use std::fs;
 use std::path::PathBuf;
-use types::{Address, Graffiti, PublicKeyBytes};
-
+use types::{Graffiti, PublicKeyBytes};
 /// Default directory name for the freezer database under the top-level data dir.
 const DEFAULT_FREEZER_DB_DIR: &str = "freezer_db";
 
 /// Defines how the client should initialize the `BeaconChain` and other components.
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
 pub enum ClientGenesis {
     /// Creates a genesis state as per the 2019 Canada interop specifications.
     Interop {
@@ -21,6 +23,7 @@ pub enum ClientGenesis {
     FromStore,
     /// Connects to an eth1 node and waits until it can create the genesis state from the deposit
     /// contract.
+    #[default]
     DepositContract,
     /// Loads the genesis state from SSZ-encoded `BeaconState` bytes.
     ///
@@ -38,16 +41,10 @@ pub enum ClientGenesis {
     },
 }
 
-impl Default for ClientGenesis {
-    fn default() -> Self {
-        Self::DepositContract
-    }
-}
-
 /// The core configuration of a Lighthouse beacon node.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Config {
-    pub data_dir: PathBuf,
+    data_dir: PathBuf,
     /// Name of the directory inside the data directory where the main "hot" DB is located.
     pub db_name: String,
     /// Path where the freezer database will be located.
@@ -64,20 +61,26 @@ pub struct Config {
     pub validator_monitor_auto: bool,
     /// A list of validator pubkeys to monitor.
     pub validator_monitor_pubkeys: Vec<PublicKeyBytes>,
+    /// Once the number of monitored validators goes above this threshold, we
+    /// will stop tracking metrics on a per-validator basis. This prevents large
+    /// validator counts causing infeasibly high cardinailty for Prometheus and
+    /// high log volumes.
+    pub validator_monitor_individual_tracking_threshold: usize,
     #[serde(skip)]
     /// The `genesis` field is not serialized or deserialized by `serde` to ensure it is defined
     /// via the CLI at runtime, instead of from a configuration file saved to disk.
     pub genesis: ClientGenesis,
     pub store: store::StoreConfig,
+    pub store_migrator: MigratorConfig,
     pub network: network::NetworkConfig,
     pub chain: beacon_chain::ChainConfig,
     pub eth1: eth1::Config,
-    pub execution_endpoints: Option<Vec<SensitiveUrl>>,
-    pub suggested_fee_recipient: Option<Address>,
+    pub execution_layer: Option<execution_layer::Config>,
     pub http_api: http_api::Config,
     pub http_metrics: http_metrics::Config,
     pub monitoring_api: Option<monitoring_api::Config>,
     pub slasher: Option<slasher::Config>,
+    pub logger_config: LoggerConfig,
 }
 
 impl Default for Config {
@@ -89,13 +92,13 @@ impl Default for Config {
             log_file: PathBuf::from(""),
             genesis: <_>::default(),
             store: <_>::default(),
+            store_migrator: <_>::default(),
             network: NetworkConfig::default(),
             chain: <_>::default(),
             dummy_eth1_backend: false,
             sync_eth1_chain: false,
             eth1: <_>::default(),
-            execution_endpoints: None,
-            suggested_fee_recipient: None,
+            execution_layer: None,
             graffiti: Graffiti::default(),
             http_api: <_>::default(),
             http_metrics: <_>::default(),
@@ -103,11 +106,24 @@ impl Default for Config {
             slasher: None,
             validator_monitor_auto: false,
             validator_monitor_pubkeys: vec![],
+            validator_monitor_individual_tracking_threshold: DEFAULT_INDIVIDUAL_TRACKING_THRESHOLD,
+            logger_config: LoggerConfig::default(),
         }
     }
 }
 
 impl Config {
+    /// Updates the data directory for the Client.
+    pub fn set_data_dir(&mut self, data_dir: PathBuf) {
+        self.data_dir = data_dir.clone();
+        self.http_api.data_dir = data_dir;
+    }
+
+    /// Gets the config's data_dir.
+    pub fn data_dir(&self) -> &PathBuf {
+        &self.data_dir
+    }
+
     /// Get the database path without initialising it.
     pub fn get_db_path(&self) -> PathBuf {
         self.get_data_dir().join(&self.db_name)
@@ -151,10 +167,8 @@ impl Config {
     pub fn get_existing_legacy_data_dir(&self) -> Option<PathBuf> {
         dirs::home_dir()
             .map(|home_dir| home_dir.join(&self.data_dir))
-            // Return `None` if the directory does not exists.
-            .filter(|dir| dir.exists())
-            // Return `None` if the legacy directory is identical to the modern.
-            .filter(|dir| *dir != self.get_modern_data_dir())
+            // Return `None` if the legacy directory does not exist or if it is identical to the modern.
+            .filter(|dir| dir.exists() && *dir != self.get_modern_data_dir())
     }
 
     /// Returns the core path for the client.
@@ -171,7 +185,7 @@ impl Config {
     /// For more information, see:
     ///
     /// https://github.com/sigp/lighthouse/pull/2843
-    fn get_data_dir(&self) -> PathBuf {
+    pub fn get_data_dir(&self) -> PathBuf {
         let existing_legacy_dir = self.get_existing_legacy_data_dir();
 
         if let Some(legacy_dir) = existing_legacy_dir {
@@ -202,7 +216,8 @@ mod tests {
     #[test]
     fn serde() {
         let config = Config::default();
-        let serialized = toml::to_string(&config).expect("should serde encode default config");
-        toml::from_str::<Config>(&serialized).expect("should serde decode default config");
+        let serialized =
+            serde_yaml::to_string(&config).expect("should serde encode default config");
+        serde_yaml::from_str::<Config>(&serialized).expect("should serde decode default config");
     }
 }

beacon_node/client/src/notifier.rs

@@ -1,13 +1,16 @@
 use crate::metrics;
-use beacon_chain::{BeaconChain, BeaconChainTypes, HeadSafetyStatus};
+use beacon_chain::{
+    merge_readiness::{MergeConfig, MergeReadiness},
+    BeaconChain, BeaconChainTypes, ExecutionStatus,
+};
 use lighthouse_network::{types::SyncState, NetworkGlobals};
-use parking_lot::Mutex;
 use slog::{crit, debug, error, info, warn, Logger};
 use slot_clock::SlotClock;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
+use tokio::sync::Mutex;
 use tokio::time::sleep;
-use types::{EthSpec, Slot};
+use types::*;
 
 /// Create a warning log whenever the peer count is at or below this value.
 pub const WARN_PEER_COUNT: usize = 1;
@@ -30,20 +33,9 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
     seconds_per_slot: u64,
 ) -> Result<(), String> {
     let slot_duration = Duration::from_secs(seconds_per_slot);
-    let duration_to_next_slot = beacon_chain
-        .slot_clock
-        .duration_to_next_slot()
-        .ok_or("slot_notifier unable to determine time to next slot")?;
-
-    // Run this half way through each slot.
-    let start_instant = tokio::time::Instant::now() + duration_to_next_slot + (slot_duration / 2);
-
-    // Run this each slot.
-    let interval_duration = slot_duration;
 
     let speedo = Mutex::new(Speedo::default());
     let log = executor.log().clone();
-    let mut interval = tokio::time::interval_at(start_instant, interval_duration);
 
     // Keep track of sync state and reset the speedo on specific sync state changes.
     // Specifically, if we switch between a sync and a backfill sync, reset the speedo.
@@ -77,8 +69,22 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
 
         // Perform post-genesis logging.
         let mut last_backfill_log_slot = None;
+
         loop {
-            interval.tick().await;
+            // Run the notifier half way through each slot.
+            //
+            // Keep remeasuring the offset rather than using an interval, so that we can correct
+            // for system time clock adjustments.
+            let wait = match beacon_chain.slot_clock.duration_to_next_slot() {
+                Some(duration) => duration + slot_duration / 2,
+                None => {
+                    warn!(log, "Unable to read current slot");
+                    sleep(slot_duration).await;
+                    continue;
+                }
+            };
+            sleep(wait).await;
+
             let connected_peer_count = network.connected_peers();
             let sync_state = network.sync_state();
 
@@ -87,12 +93,12 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
                 match (current_sync_state, &sync_state) {
                     (_, SyncState::BackFillSyncing { .. }) => {
                         // We have transitioned to a backfill sync. Reset the speedo.
-                        let mut speedo = speedo.lock();
+                        let mut speedo = speedo.lock().await;
                         speedo.clear();
                     }
                     (SyncState::BackFillSyncing { .. }, _) => {
                         // We have transitioned from a backfill sync, reset the speedo
-                        let mut speedo = speedo.lock();
+                        let mut speedo = speedo.lock().await;
                         speedo.clear();
                     }
                     (_, _) => {}
@@ -100,15 +106,10 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
                 current_sync_state = sync_state;
             }
 
-            let head_info = match beacon_chain.head_info() {
-                Ok(head_info) => head_info,
-                Err(e) => {
-                    error!(log, "Failed to get beacon chain head info"; "error" => format!("{:?}", e));
-                    break;
-                }
-            };
-
-            let head_slot = head_info.slot;
+            let cached_head = beacon_chain.canonical_head.cached_head();
+            let head_slot = cached_head.head_slot();
+            let head_root = cached_head.head_block_root();
+            let finalized_checkpoint = cached_head.finalized_checkpoint();
 
             metrics::set_gauge(&metrics::NOTIFIER_HEAD_SLOT, head_slot.as_u64() as i64);
 
@@ -125,15 +126,12 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
             };
 
             let current_epoch = current_slot.epoch(T::EthSpec::slots_per_epoch());
-            let finalized_epoch = head_info.finalized_checkpoint.epoch;
-            let finalized_root = head_info.finalized_checkpoint.root;
-            let head_root = head_info.block_root;
 
             // The default is for regular sync but this gets modified if backfill sync is in
             // progress.
             let mut sync_distance = current_slot - head_slot;
 
-            let mut speedo = speedo.lock();
+            let mut speedo = speedo.lock().await;
             match current_sync_state {
                 SyncState::BackFillSyncing { .. } => {
                     // Observe backfilling sync info.
@@ -177,8 +175,8 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
                 log,
                 "Slot timer";
                 "peers" => peer_count_pretty(connected_peer_count),
-                "finalized_root" => format!("{}", finalized_root),
-                "finalized_epoch" => finalized_epoch,
+                "finalized_root" => format!("{}", finalized_checkpoint.root),
+                "finalized_epoch" => finalized_checkpoint.epoch,
                 "head_block" => format!("{}", head_root),
                 "head_slot" => head_slot,
                 "current_slot" => current_slot,
@@ -264,35 +262,29 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
                     head_root.to_string()
                 };
 
-                let block_hash = match beacon_chain.head_safety_status() {
-                    Ok(HeadSafetyStatus::Safe(hash_opt)) => hash_opt
-                        .map(|hash| format!("{} (verified)", hash))
-                        .unwrap_or_else(|| "n/a".to_string()),
-                    Ok(HeadSafetyStatus::Unsafe(block_hash)) => {
+                let block_hash = match beacon_chain.canonical_head.head_execution_status() {
+                    Ok(ExecutionStatus::Irrelevant(_)) => "n/a".to_string(),
+                    Ok(ExecutionStatus::Valid(hash)) => format!("{} (verified)", hash),
+                    Ok(ExecutionStatus::Optimistic(hash)) => {
                         warn!(
                             log,
-                            "Head execution payload is unverified";
-                            "execution_block_hash" => ?block_hash,
+                            "Head is optimistic";
+                            "info" => "chain not fully verified, \
+                                block and attestation production disabled until execution engine syncs",
+                            "execution_block_hash" => ?hash,
                         );
-                        format!("{} (unverified)", block_hash)
+                        format!("{} (unverified)", hash)
                     }
-                    Ok(HeadSafetyStatus::Invalid(block_hash)) => {
+                    Ok(ExecutionStatus::Invalid(hash)) => {
                         crit!(
                             log,
                             "Head execution payload is invalid";
                             "msg" => "this scenario may be unrecoverable",
-                            "execution_block_hash" => ?block_hash,
+                            "execution_block_hash" => ?hash,
                         );
-                        format!("{} (invalid)", block_hash)
-                    }
-                    Err(e) => {
-                        error!(
-                            log,
-                            "Failed to read head safety status";
-                            "error" => ?e
-                        );
-                        "n/a".to_string()
+                        format!("{} (invalid)", hash)
                     }
+                    Err(_) => "unknown".to_string(),
                 };
 
                 info!(
@@ -300,8 +292,8 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
                     "Synced";
                     "peers" => peer_count_pretty(connected_peer_count),
                     "exec_hash" => block_hash,
-                    "finalized_root" => format!("{}", finalized_root),
-                    "finalized_epoch" => finalized_epoch,
+                    "finalized_root" => format!("{}", finalized_checkpoint.root),
+                    "finalized_epoch" => finalized_checkpoint.epoch,
                     "epoch" => current_epoch,
                     "block" => block_info,
                     "slot" => current_slot,
@@ -312,14 +304,15 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
                     log,
                     "Searching for peers";
                     "peers" => peer_count_pretty(connected_peer_count),
-                    "finalized_root" => format!("{}", finalized_root),
-                    "finalized_epoch" => finalized_epoch,
+                    "finalized_root" => format!("{}", finalized_checkpoint.root),
+                    "finalized_epoch" => finalized_checkpoint.epoch,
                     "head_slot" => head_slot,
                     "current_slot" => current_slot,
                 );
             }
 
             eth1_logging(&beacon_chain, &log);
+            merge_readiness_logging(current_slot, &beacon_chain, &log).await;
         }
     };
 
@@ -329,60 +322,152 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
     Ok(())
 }
 
+/// Provides some helpful logging to users to indicate if their node is ready for the Bellatrix
+/// fork and subsequent merge transition.
+async fn merge_readiness_logging<T: BeaconChainTypes>(
+    current_slot: Slot,
+    beacon_chain: &BeaconChain<T>,
+    log: &Logger,
+) {
+    let merge_completed = beacon_chain
+        .canonical_head
+        .cached_head()
+        .snapshot
+        .beacon_block
+        .message()
+        .body()
+        .execution_payload()
+        .map_or(false, |payload| {
+            payload.parent_hash() != ExecutionBlockHash::zero()
+        });
+
+    let has_execution_layer = beacon_chain.execution_layer.is_some();
+
+    if merge_completed && has_execution_layer
+        || !beacon_chain.is_time_to_prepare_for_bellatrix(current_slot)
+    {
+        return;
+    }
+
+    if merge_completed && !has_execution_layer {
+        error!(
+            log,
+            "Execution endpoint required";
+            "info" => "you need an execution engine to validate blocks, see: \
+                       https://lighthouse-book.sigmaprime.io/merge-migration.html"
+        );
+        return;
+    }
+
+    match beacon_chain.check_merge_readiness().await {
+        MergeReadiness::Ready {
+            config,
+            current_difficulty,
+        } => match config {
+            MergeConfig {
+                terminal_total_difficulty: Some(ttd),
+                terminal_block_hash: None,
+                terminal_block_hash_epoch: None,
+            } => {
+                info!(
+                    log,
+                    "Ready for the merge";
+                    "terminal_total_difficulty" => %ttd,
+                    "current_difficulty" => current_difficulty
+                        .map(|d| d.to_string())
+                        .unwrap_or_else(|| "??".into()),
+                )
+            }
+            MergeConfig {
+                terminal_total_difficulty: _,
+                terminal_block_hash: Some(terminal_block_hash),
+                terminal_block_hash_epoch: Some(terminal_block_hash_epoch),
+            } => {
+                info!(
+                    log,
+                    "Ready for the merge";
+                    "info" => "you are using override parameters, please ensure that you \
+                        understand these parameters and their implications.",
+                    "terminal_block_hash" => ?terminal_block_hash,
+                    "terminal_block_hash_epoch" => ?terminal_block_hash_epoch,
+                )
+            }
+            other => error!(
+                log,
+                "Inconsistent merge configuration";
+                "config" => ?other
+            ),
+        },
+        readiness @ MergeReadiness::ExchangeTransitionConfigurationFailed { error: _ } => {
+            error!(
+                log,
+                "Not ready for merge";
+                "info" => %readiness,
+                "hint" => "try updating Lighthouse and/or the execution layer",
+            )
+        }
+        readiness @ MergeReadiness::NotSynced => warn!(
+            log,
+            "Not ready for merge";
+            "info" => %readiness,
+        ),
+        readiness @ MergeReadiness::NoExecutionEndpoint => warn!(
+            log,
+            "Not ready for merge";
+            "info" => %readiness,
+        ),
+    }
+}
+
 fn eth1_logging<T: BeaconChainTypes>(beacon_chain: &BeaconChain<T>, log: &Logger) {
     let current_slot_opt = beacon_chain.slot().ok();
 
-    if let Ok(head_info) = beacon_chain.head_info() {
-        // Perform some logging about the eth1 chain
-        if let Some(eth1_chain) = beacon_chain.eth1_chain.as_ref() {
-            // No need to do logging if using the dummy backend.
-            if eth1_chain.is_dummy_backend() {
-                return;
-            }
-
-            if let Some(status) =
-                eth1_chain.sync_status(head_info.genesis_time, current_slot_opt, &beacon_chain.spec)
-            {
-                debug!(
-                    log,
-                    "Eth1 cache sync status";
-                    "eth1_head_block" => status.head_block_number,
-                    "latest_cached_block_number" => status.latest_cached_block_number,
-                    "latest_cached_timestamp" => status.latest_cached_block_timestamp,
-                    "voting_target_timestamp" => status.voting_target_timestamp,
-                    "ready" => status.lighthouse_is_cached_and_ready
-                );
-
-                if !status.lighthouse_is_cached_and_ready {
-                    let voting_target_timestamp = status.voting_target_timestamp;
-
-                    let distance = status
-                        .latest_cached_block_timestamp
-                        .map(|latest| {
-                            voting_target_timestamp.saturating_sub(latest)
-                                / beacon_chain.spec.seconds_per_eth1_block
-                        })
-                        .map(|distance| distance.to_string())
-                        .unwrap_or_else(|| "initializing deposits".to_string());
-
-                    warn!(
-                        log,
-                        "Syncing eth1 block cache";
-                        "est_blocks_remaining" => distance,
-                    );
-                }
-            } else {
-                error!(
-                    log,
-                    "Unable to determine eth1 sync status";
-                );
-            }
+    // Perform some logging about the eth1 chain
+    if let Some(eth1_chain) = beacon_chain.eth1_chain.as_ref() {
+        // No need to do logging if using the dummy backend.
+        if eth1_chain.is_dummy_backend() {
+            return;
+        }
+
+        if let Some(status) = eth1_chain.sync_status(
+            beacon_chain.genesis_time,
+            current_slot_opt,
+            &beacon_chain.spec,
+        ) {
+            debug!(
+                log,
+                "Eth1 cache sync status";
+                "eth1_head_block" => status.head_block_number,
+                "latest_cached_block_number" => status.latest_cached_block_number,
+                "latest_cached_timestamp" => status.latest_cached_block_timestamp,
+                "voting_target_timestamp" => status.voting_target_timestamp,
+                "ready" => status.lighthouse_is_cached_and_ready
+            );
+
+            if !status.lighthouse_is_cached_and_ready {
+                let voting_target_timestamp = status.voting_target_timestamp;
+
+                let distance = status
+                    .latest_cached_block_timestamp
+                    .map(|latest| {
+                        voting_target_timestamp.saturating_sub(latest)
+                            / beacon_chain.spec.seconds_per_eth1_block
+                    })
+                    .map(|distance| distance.to_string())
+                    .unwrap_or_else(|| "initializing deposits".to_string());
+
+                warn!(
+                    log,
+                    "Syncing deposit contract block cache";
+                    "est_blocks_remaining" => distance,
+                );
+            }
+        } else {
+            error!(
+                log,
+                "Unable to determine deposit contract sync status";
+            );
         }
-    } else {
-        error!(
-            log,
-            "Unable to get head info";
-        );
     }
 }
 

beacon_node/eth1/Cargo.toml

@@ -2,33 +2,34 @@
 name = "eth1"
 version = "0.2.0"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
-edition = "2018"
+edition = "2021"
 
 [dev-dependencies]
 eth1_test_rig = { path = "../../testing/eth1_test_rig" }
-toml = "0.5.6"
-web3 = { version = "0.17.0", default-features = false, features = ["http-tls", "signing", "ws-tls-tokio"] }
+serde_yaml = "0.8.13"
+web3 = { version = "0.18.0", default-features = false, features = ["http-tls", "signing", "ws-tls-tokio"] }
 sloggers = { version = "2.1.1", features = ["json"] }
 environment = { path = "../../lighthouse/environment" }
 
 [dependencies]
 reqwest = { version = "0.11.0", features = ["native-tls-vendored"] }
+execution_layer = { path = "../execution_layer" }
 futures = "0.3.7"
 serde_json = "1.0.58"
 serde = { version = "1.0.116", features = ["derive"] }
 hex = "0.4.2"
 types = { path = "../../consensus/types"}
 merkle_proof = { path = "../../consensus/merkle_proof"}
-eth2_ssz = "0.4.1"
-eth2_ssz_derive = "0.3.0"
-tree_hash = "0.4.1"
-parking_lot = "0.11.0"
+ethereum_ssz = "0.5.0"
+ethereum_ssz_derive = "0.5.0"
+tree_hash = "0.5.0"
+parking_lot = "0.12.0"
 slog = "2.5.2"
+superstruct = "0.7.0"
 tokio = { version = "1.14.0", features = ["full"] }
 state_processing = { path = "../../consensus/state_processing" }
 lighthouse_metrics = { path = "../../common/lighthouse_metrics"}
 lazy_static = "1.4.0"
 task_executor = { path = "../../common/task_executor" }
 eth2 = { path = "../../common/eth2" }
-fallback = { path = "../../common/fallback" }
 sensitive_url = { path = "../../common/sensitive_url" }

beacon_node/eth1/src/block_cache.rs

@@ -1,7 +1,10 @@
 use ssz_derive::{Decode, Encode};
+use std::collections::HashMap;
 use std::ops::RangeInclusive;
 
 pub use eth2::lighthouse::Eth1Block;
+use eth2::types::Hash256;
+use std::sync::Arc;
 
 #[derive(Debug, PartialEq, Clone)]
 pub enum Error {
@@ -20,7 +23,9 @@ pub enum Error {
 /// timestamp.
 #[derive(Debug, PartialEq, Clone, Default, Encode, Decode)]
 pub struct BlockCache {
-    blocks: Vec<Eth1Block>,
+    blocks: Vec<Arc<Eth1Block>>,
+    #[ssz(skip_serializing, skip_deserializing)]
+    by_hash: HashMap<Hash256, Arc<Eth1Block>>,
 }
 
 impl BlockCache {
@@ -36,12 +41,12 @@ impl BlockCache {
 
     /// Returns the earliest (lowest timestamp) block, if any.
     pub fn earliest_block(&self) -> Option<&Eth1Block> {
-        self.blocks.first()
+        self.blocks.first().map(|ptr| ptr.as_ref())
     }
 
     /// Returns the latest (highest timestamp) block, if any.
     pub fn latest_block(&self) -> Option<&Eth1Block> {
-        self.blocks.last()
+        self.blocks.last().map(|ptr| ptr.as_ref())
     }
 
     /// Returns the timestamp of the earliest block in the cache (if any).
@@ -71,7 +76,7 @@ impl BlockCache {
     /// - Monotonically increasing block numbers.
     /// - Non-uniformly increasing block timestamps.
     pub fn iter(&self) -> impl DoubleEndedIterator<Item = &Eth1Block> + Clone {
-        self.blocks.iter()
+        self.blocks.iter().map(|ptr| ptr.as_ref())
     }
 
     /// Shortens the cache, keeping the latest (by block number) `len` blocks while dropping the
@@ -80,7 +85,11 @@ impl BlockCache {
     /// If `len` is greater than the vector's current length, this has no effect.
     pub fn truncate(&mut self, len: usize) {
         if len < self.blocks.len() {
-            self.blocks = self.blocks.split_off(self.blocks.len() - len);
+            let remaining = self.blocks.split_off(self.blocks.len() - len);
+            for block in &self.blocks {
+                self.by_hash.remove(&block.hash);
+            }
+            self.blocks = remaining;
         }
     }
 
@@ -92,12 +101,27 @@ impl BlockCache {
 
     /// Returns a block with the corresponding number, if any.
     pub fn block_by_number(&self, block_number: u64) -> Option<&Eth1Block> {
-        self.blocks.get(
-            self.blocks
-                .as_slice()
-                .binary_search_by(|block| block.number.cmp(&block_number))
-                .ok()?,
-        )
+        self.blocks
+            .get(
+                self.blocks
+                    .as_slice()
+                    .binary_search_by(|block| block.number.cmp(&block_number))
+                    .ok()?,
+            )
+            .map(|ptr| ptr.as_ref())
+    }
+
+    /// Returns a block with the corresponding hash, if any.
+    pub fn block_by_hash(&self, block_hash: &Hash256) -> Option<&Eth1Block> {
+        self.by_hash.get(block_hash).map(|ptr| ptr.as_ref())
+    }
+
+    /// Rebuilds the by_hash map
+    pub fn rebuild_by_hash_map(&mut self) {
+        self.by_hash.clear();
+        for block in self.blocks.iter() {
+            self.by_hash.insert(block.hash, block.clone());
+        }
     }
 
     /// Insert an `Eth1Snapshot` into `self`, allowing future queries.
@@ -161,7 +185,9 @@ impl BlockCache {
             }
         }
 
-        self.blocks.push(block);
+        let ptr = Arc::new(block);
+        self.by_hash.insert(ptr.hash, ptr.clone());
+        self.blocks.push(ptr);
 
         Ok(())
     }
@@ -269,6 +295,8 @@ mod tests {
                 .expect("should add consecutive blocks with duplicate timestamps");
         }
 
+        let blocks = blocks.into_iter().map(Arc::new).collect::<Vec<_>>();
+
         assert_eq!(cache.blocks, blocks, "should have added all blocks");
     }
 }