v1.5.0-rc.1 (#2516)

## Issue Addressed

NA

## Proposed Changes

- Bump version

## Additional Info

NA

.dockerignore

@@ -1,2 +1,4 @@
-tests/ef_tests/eth2.0-spec-tests
+testing/ef_tests/eth2.0-spec-tests
 target/
+*.data
+*.tar.gz

.github/ISSUE_TEMPLATE.md

@@ -2,6 +2,11 @@
 
 Please provide a brief description of the issue.
 
+## Version
+
+Please provide your Lighthouse and Rust version. Are you building from
+`stable` or `unstable`, which commit?
+
 ## Present Behaviour
 
 Describe the present behaviour of the application, with regards to this

.github/custom/config.toml

@@ -0,0 +1,3 @@
+# Custom Cargo config to be used for the udeps CI job
+[http]
+multiplexing = false

.github/workflows/book.yml

@@ -0,0 +1,30 @@
+name: mdbook
+
+on:
+  push:
+    branches:
+      - unstable
+
+jobs:
+  build-and-upload-to-s3:
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@master
+
+      - name: Setup mdBook
+        uses: peaceiris/actions-mdbook@v1
+        with:
+          mdbook-version: 'latest'
+
+      - run: mdbook build
+        working-directory: book
+
+      - uses: jakejarvis/s3-sync-action@be0c4ab89158cac4278689ebedd8407dd5f35a83
+        with:
+          args: --follow-symlinks --delete
+        env:
+          AWS_S3_BUCKET: ${{ secrets.AWS_S3_BOOK_BUCKET }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_REGION: 'ap-southeast-2'
+          SOURCE_DIR: 'book/book'

.github/workflows/cancel-previous-runs.yml

@@ -0,0 +1,14 @@
+name: cancel previous runs
+on: [push]
+jobs:
+  cancel:
+    name: 'Cancel Previous Runs'
+    runs-on: ubuntu-latest
+    timeout-minutes: 3
+    steps:
+      # https://github.com/styfle/cancel-workflow-action/releases
+      - uses: styfle/cancel-workflow-action@514c783324374c6940d1b92bfb962d0763d22de3 # 0.7.0
+        with:
+          # https://api.github.com/repos/sigp/lighthouse/actions/workflows
+          workflow_id: 697364,2434944,4462424,308241,2883401,316
+          access_token: ${{ github.token }}

.github/workflows/docker.yml

@@ -0,0 +1,118 @@
+name: docker
+
+on:
+    push:
+        branches:
+            - unstable
+            - stable
+
+env:
+    DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+    DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+    IMAGE_NAME: ${{ github.repository_owner}}/lighthouse
+    LCLI_IMAGE_NAME: ${{ github.repository_owner }}/lcli
+
+jobs:
+    extract-branch-name:
+        runs-on: ubuntu-18.04
+        steps:
+            - name: Extract branch name
+              run: echo "::set-output name=BRANCH_NAME::$(echo ${GITHUB_REF#refs/heads/})"
+              id: extract_branch
+        outputs:
+            BRANCH_NAME: ${{ steps.extract_branch.outputs.BRANCH_NAME }}
+    build-docker-arm64:
+        runs-on: ubuntu-18.04
+        needs: [extract-branch-name]
+        # We need to enable experimental docker features in order to use `docker buildx`
+        env:
+            DOCKER_CLI_EXPERIMENTAL: enabled
+        steps:
+            - uses: actions/checkout@v2
+            - name: Update Rust
+              run: rustup update stable
+            - name: Dockerhub login
+              run: |
+                  echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin
+            - name: Cross build lighthouse binary
+              run:  |
+                  cargo install cross
+                  make build-aarch64-portable
+            - name: Move cross-built ARM binary into Docker scope
+              run: |
+                  mkdir ./bin;
+                  mv ./target/aarch64-unknown-linux-gnu/release/lighthouse ./bin;
+            - name: Set Env
+              if: needs.extract-branch-name.outputs.BRANCH_NAME == 'unstable'
+              run: |
+                  echo "TAG_SUFFIX=-unstable" >> $GITHUB_ENV;
+            # Install dependencies for emulation. Have to create a new builder to pick up emulation support.
+            - name: Build ARM64 dockerfile (with push)
+              run: |
+                  docker run --privileged --rm tonistiigi/binfmt --install arm64
+                  docker buildx create --use --name cross-builder
+                  docker buildx build \
+                      --platform=linux/arm64 \
+                      --file ./Dockerfile.cross . \
+                      --tag ${IMAGE_NAME}:latest-arm64${TAG_SUFFIX} \
+                      --push
+    build-docker-amd64:
+        runs-on: ubuntu-18.04
+        needs: [extract-branch-name]
+        steps:
+            - uses: actions/checkout@v2
+            - name: Update Rust
+              run: rustup update stable
+            - name: Dockerhub login
+              run: |
+                  echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin
+            - name: Set Env
+              if: needs.extract-branch-name.outputs.BRANCH_NAME == 'unstable'
+              run: |
+                  echo "TAG_SUFFIX=-unstable" >> $GITHUB_ENV;
+            - name: Build AMD64 dockerfile (with push)
+              run: |
+                  docker build \
+                      --build-arg PORTABLE=true \
+                      --tag ${IMAGE_NAME}:latest-amd64${TAG_SUFFIX} \
+                      --file ./Dockerfile .
+                  docker push ${IMAGE_NAME}:latest-amd64${TAG_SUFFIX}
+    build-docker-multiarch:
+        runs-on: ubuntu-18.04
+        needs: [build-docker-arm64, build-docker-amd64, extract-branch-name]
+        # We need to enable experimental docker features in order to use `docker manifest`
+        env:
+            DOCKER_CLI_EXPERIMENTAL: enabled
+        steps:
+            - name: Dockerhub login
+              run: |
+                  echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin
+            - name: Set Env
+              if: needs.extract-branch-name.outputs.BRANCH_NAME == 'unstable'
+              run: |
+                  echo "TAG_SUFFIX=-unstable" >> $GITHUB_ENV;
+            - name: Create and push multiarch manifest
+              run: |
+                  docker manifest create ${IMAGE_NAME}:latest${TAG_SUFFIX} \
+                      --amend ${IMAGE_NAME}:latest-arm64${TAG_SUFFIX} \
+                      --amend ${IMAGE_NAME}:latest-amd64${TAG_SUFFIX};
+                  docker manifest push ${IMAGE_NAME}:latest${TAG_SUFFIX}
+    build-docker-lcli:
+        runs-on: ubuntu-18.04
+        needs: [extract-branch-name]
+        steps:
+            - uses: actions/checkout@v2
+            - name: Dockerhub login
+              run: |
+                  echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin
+            - name: Set Env
+              if: needs.extract-branch-name.outputs.BRANCH_NAME == 'unstable'
+              run: |
+                  echo "TAG_SUFFIX=-unstable" >> $GITHUB_ENV;
+            - name: Build lcli dockerfile (with push)
+              run: |
+                  docker build \
+                      --build-arg PORTABLE=true \
+                      --tag ${LCLI_IMAGE_NAME}:latest${TAG_SUFFIX} \
+                      --file ./lcli/Dockerfile .
+                  docker push ${LCLI_IMAGE_NAME}:latest${TAG_SUFFIX}

.github/workflows/release.yml

@@ -0,0 +1,289 @@
+name:   Release Suite
+
+on:
+    push:
+        tags:
+            - v*
+
+env:
+    DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+    DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+    REPO_NAME: sigp/lighthouse
+    IMAGE_NAME: sigp/lighthouse
+
+jobs:
+    extract-version:
+        runs-on: ubuntu-latest
+        steps:
+            - name: Extract version
+              run: echo "::set-output name=VERSION::$(echo ${GITHUB_REF#refs/tags/})"
+              id: extract_version
+        outputs:
+            VERSION: ${{ steps.extract_version.outputs.VERSION }}
+
+    build-docker-arm64:
+        runs-on: ubuntu-18.04
+        needs: [extract-version]
+        # We need to enable experimental docker features in order to use `docker buildx`
+        env:
+            DOCKER_CLI_EXPERIMENTAL: enabled
+            VERSION: ${{ needs.extract-version.outputs.VERSION }}
+        steps:
+            - uses: actions/checkout@v2
+            - name: Update Rust
+              run: rustup update stable
+            - name: Dockerhub login
+              run: |
+                  echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin
+            - name: Cross build lighthouse binary
+              run:  |
+                  cargo install cross
+                  make build-aarch64-portable
+            - name: Move cross-built ARM binary into Docker scope
+              run: |
+                  mkdir ./bin;
+                  mv ./target/aarch64-unknown-linux-gnu/release/lighthouse ./bin;
+            # Install dependencies for emulation. Have to create a new builder to pick up emulation support.
+            - name: Build ARM64 dockerfile (with push)
+              run: |
+                  docker run --privileged --rm tonistiigi/binfmt --install arm64
+                  docker buildx create --use --name cross-builder
+                  docker buildx build \
+                      --platform=linux/arm64 \
+                      --file ./Dockerfile.cross . \
+                      --tag ${IMAGE_NAME}:${{ env.VERSION }}-arm64 \
+                      --push
+    build-docker-amd64:
+        runs-on: ubuntu-18.04
+        needs: [extract-version]
+        env:
+            DOCKER_CLI_EXPERIMENTAL: enabled
+            VERSION: ${{ needs.extract-version.outputs.VERSION }}
+        steps:
+            - uses: actions/checkout@v2
+            - name: Dockerhub login
+              run: |
+                  echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin
+            - name: Build AMD64 dockerfile (with push)
+              run: |
+                  docker build \
+                      --build-arg PORTABLE=true \
+                      --tag ${IMAGE_NAME}:${{ env.VERSION }}-amd64 \
+                      --file ./Dockerfile .
+                  docker push ${IMAGE_NAME}:${{ env.VERSION }}-amd64
+    build-docker-multiarch:
+        runs-on: ubuntu-18.04
+        needs: [build-docker-arm64, build-docker-amd64, extract-version]
+        # We need to enable experimental docker features in order to use `docker manifest`
+        env:
+            DOCKER_CLI_EXPERIMENTAL: enabled
+            VERSION: ${{ needs.extract-version.outputs.VERSION }}
+        steps:
+            - name: Dockerhub login
+              run: |
+                  echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin
+            - name: Create and push multiarch manifest
+              run: |
+                  docker manifest create ${IMAGE_NAME}:${{ env.VERSION }} \
+                      --amend ${IMAGE_NAME}:${{ env.VERSION }}-arm64 \
+                      --amend ${IMAGE_NAME}:${{ env.VERSION }}-amd64;
+                  docker manifest push ${IMAGE_NAME}:${{ env.VERSION }}
+    build:
+        name:   Build Release
+        strategy:
+            matrix:
+                arch: [aarch64-unknown-linux-gnu,
+                       aarch64-unknown-linux-gnu-portable,
+                       x86_64-unknown-linux-gnu,
+                       x86_64-unknown-linux-gnu-portable,
+                       x86_64-apple-darwin,
+                       x86_64-apple-darwin-portable]
+                include:
+                    -   arch: aarch64-unknown-linux-gnu
+                        platform: ubuntu-latest
+                    -   arch: aarch64-unknown-linux-gnu-portable
+                        platform: ubuntu-latest
+                    -   arch: x86_64-unknown-linux-gnu
+                        platform: ubuntu-latest
+                    -   arch: x86_64-unknown-linux-gnu-portable
+                        platform: ubuntu-latest
+                    -   arch: x86_64-apple-darwin
+                        platform: macos-latest
+                    -   arch: x86_64-apple-darwin-portable
+                        platform: macos-latest
+
+        runs-on:    ${{ matrix.platform }}
+        needs: extract-version
+        steps:
+            - name: Checkout sources
+              uses: actions/checkout@v2
+            - name: Build toolchain
+              uses: actions-rs/toolchain@v1
+              with:
+                  toolchain: stable
+                  profile:  minimal
+                  override: true
+
+            # ==============================
+            #       Builds
+            # ==============================
+
+            - name: Build Lighthouse for aarch64-unknown-linux-gnu-portable
+              if:   matrix.arch == 'aarch64-unknown-linux-gnu-portable'
+              run:  |
+                cargo install cross
+                make build-aarch64-portable
+
+            - name: Build Lighthouse for aarch64-unknown-linux-gnu
+              if:   matrix.arch == 'aarch64-unknown-linux-gnu'
+              run:  |
+                cargo install cross
+                make build-aarch64
+
+            - name: Build Lighthouse for x86_64-unknown-linux-gnu-portable
+              if:   matrix.arch == 'x86_64-unknown-linux-gnu-portable'
+              run:  |
+                cargo install cross
+                make build-x86_64-portable
+
+            - name: Build Lighthouse for x86_64-unknown-linux-gnu
+              if:   matrix.arch == 'x86_64-unknown-linux-gnu'
+              run:  |
+                cargo install cross
+                make build-x86_64
+
+            - name: Move cross-compiled binary
+              if:  startsWith(matrix.arch, 'aarch64')
+              run: mv target/aarch64-unknown-linux-gnu/release/lighthouse ~/.cargo/bin/lighthouse
+
+            - name: Move cross-compiled binary
+              if:  startsWith(matrix.arch, 'x86_64-unknown-linux-gnu')
+              run: mv target/x86_64-unknown-linux-gnu/release/lighthouse ~/.cargo/bin/lighthouse
+
+            - name: Build Lighthouse for x86_64-apple-darwin portable
+              if:   matrix.arch == 'x86_64-apple-darwin-portable'
+              run:  cargo install --path lighthouse --force --locked --features portable
+
+            - name: Build Lighthouse for x86_64-apple-darwin modern
+              if:   matrix.arch == 'x86_64-apple-darwin'
+              run:  cargo install --path lighthouse --force --locked --features modern
+
+            - name: Configure GPG and create artifacts
+              env:
+                  GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
+                  GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
+              run: |
+                  export GPG_TTY=$(tty)
+                  echo "$GPG_SIGNING_KEY" | gpg --batch --import
+                  mkdir artifacts
+                  mv ~/.cargo/bin/lighthouse ./artifacts
+                  cd artifacts
+                  tar -czf lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz lighthouse
+                  echo "$GPG_PASSPHRASE" | gpg --passphrase-fd 0 --pinentry-mode loopback --batch -ab lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz
+                  mv *tar.gz* ..
+
+            # =======================================================================
+            # Upload artifacts
+            # This is required to share artifacts between different jobs
+            # =======================================================================
+
+            - name:  Upload artifact
+              uses:  actions/upload-artifact@v2
+              with:
+                  name: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz
+                  path: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz
+
+            - name: Upload signature
+              uses: actions/upload-artifact@v2
+              with:
+                  name: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc
+                  path: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc
+
+    draft-release:
+        name:   Draft Release
+        needs:  [build, extract-version]
+        runs-on: ubuntu-latest
+        env:
+            VERSION:  ${{ needs.extract-version.outputs.VERSION }}
+        steps:
+            # This is necessary for generating the changelog. It has to come before "Download Artifacts" or else it deletes the artifacts.
+            - name: Checkout sources
+              uses: actions/checkout@v2
+              with:
+                  fetch-depth: 0
+
+            # ==============================
+            #       Download artifacts
+            # ==============================
+
+            - name: Download artifacts
+              uses: actions/download-artifact@v2
+
+            # ==============================
+            #       Create release draft
+            # ==============================
+
+            - name: Generate Full Changelog
+              id: changelog
+              run: echo "::set-output name=CHANGELOG::$(git log --pretty=format:"- %s" $(git describe --tags --abbrev=0 ${{ env.VERSION }}^)..${{ env.VERSION }})"
+
+            - name: Create Release Draft
+              env:
+                  GITHUB_USER: sigp
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+              # The formatting here is borrowed from OpenEthereum: https://github.com/openethereum/openethereum/blob/main/.github/workflows/build.yml
+              run: |
+                  body=$(cat <<- "ENDBODY"
+                  <Rick and Morty character>
+
+                  ## Testing Checklist (DELETE ME)
+
+                  - [ ] Run on synced Pyrmont Sigma Prime nodes.
+                  - [ ] Run on synced Prater Sigma Prime nodes.
+                  - [ ] Run on synced Canary (mainnet) Sigma Prime nodes.
+                  - [ ] Resync a Pyrmont node.
+                  - [ ] Resync a Prater node.
+                  - [ ] Resync a mainnet node.
+
+                  ## Release Checklist (DELETE ME)
+
+                  - [ ] Merge `unstable` -> `stable`.
+                  - [ ] Ensure docker images are published (check `latest` and the version tag).
+                  - [ ] Prepare Discord post.
+                  - [ ] Prepare Twitter post.
+                  - [ ] Prepare mailing list email.
+
+                  ## Summary
+
+                  Add a summary.
+
+                  ## All Changes
+
+                  ${{ steps.changelog.outputs.CHANGELOG }}
+
+                  ## Binaries
+
+                  [See pre-built binaries documentation.](https://lighthouse-book.sigmaprime.io/installation-binaries.html)
+
+                  The binaries are signed with Sigma Prime's PGP key: `15E66D941F697E28F49381F426416DC3F30674B0`
+
+                  | System | Architecture | Binary | PGP Signature |
+                  |:---:|:---:|:---:|:---|
+                  | <img src="https://simpleicons.org/icons/apple.svg" style="width: 32px;"/> | x86_64 | [lighthouse-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz.asc) |
+                  | <img src="https://simpleicons.org/icons/apple.svg" style="width: 32px;"/> | x86_64 | [lighthouse-${{ env.VERSION }}-x86_64-apple-darwin-portable.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-apple-darwin-portable.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-apple-darwin-portable.tar.gz.asc) |
+                  | <img src="https://simpleicons.org/icons/linux.svg" style="width: 32px;"/> | x86_64 | [lighthouse-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz.asc) |
+                  | <img src="https://simpleicons.org/icons/linux.svg" style="width: 32px;"/> | x86_64 | [lighthouse-${{ env.VERSION }}-x86_64-unknown-linux-gnu-portable.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-unknown-linux-gnu-portable.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-unknown-linux-gnu-portable.tar.gz.asc) |
+                  | <img src="https://simpleicons.org/icons/raspberrypi.svg" style="width: 32px;"/> | aarch64 | [lighthouse-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz.asc) |
+                  | <img src="https://simpleicons.org/icons/raspberrypi.svg" style="width: 32px;"/> | aarch64 | [lighthouse-${{ env.VERSION }}-aarch64-unknown-linux-gnu-portable.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64-unknown-linux-gnu-portable.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64-unknown-linux-gnu-portable.tar.gz.asc) |
+                  | | | | |
+                  | **System** | **Option** | - | **Resource** |
+                  | <img src="https://simpleicons.org/icons/docker.svg" style="width: 32px;"/> | Docker | [${{ env.VERSION }}](https://hub.docker.com/r/${{ env.IMAGE_NAME }}/tags?page=1&ordering=last_updated&name=${{ env.VERSION }}) | [${{ env.IMAGE_NAME }}](https://hub.docker.com/r/${{ env.IMAGE_NAME }}) |
+                  ENDBODY
+                  )
+                  assets=()
+                  for asset in ./lighthouse-*.tar.gz*; do
+                      assets+=("-a" "$asset/$asset")
+                  done
+                  tag_name="${{ env.VERSION }}"
+                  echo "$body" | hub release create --draft "${assets[@]}" -F "-" "$tag_name"

.github/workflows/test-suite.yml

@@ -1,9 +1,28 @@
 name: test-suite
 
-on: [push]
-
+on:
+  push:
+    branches:
+      - stable
+      - staging
+      - trying
+      - 'pr/*'
+  pull_request:
+env:
+  # Deny warnings in CI
+  RUSTFLAGS: "-D warnings"
+  # The Nightly version used for cargo-udeps, might need updating from time to time.
+  PINNED_NIGHTLY: nightly-2021-06-09
 jobs:
+  target-branch-check:
+    name: target-branch-check
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+    steps:
+        - name: Check that pull request is targeting unstable branch
+          run: test ${{ github.base_ref }} = "unstable"
   cargo-fmt:
+    name: cargo-fmt
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v1
@@ -12,6 +31,7 @@ jobs:
     - name: Check formatting with cargo fmt
       run: make cargo-fmt
   release-tests-ubuntu:
+    name: release-tests-ubuntu
     runs-on: ubuntu-latest
     needs: cargo-fmt
     steps:
@@ -22,7 +42,34 @@ jobs:
       run: sudo npm install -g ganache-cli
     - name: Run tests in release
       run: make test-release
+  release-tests-windows:
+    name: release-tests-windows
+    runs-on: windows-latest
+    needs: cargo-fmt
+    steps:
+    - uses: actions/checkout@v1
+    - name: Get latest version of stable Rust
+      run: rustup update stable
+    - name: Install ganache-cli
+      run: npm install -g ganache-cli
+    - name: Install make
+      run: choco install -y make
+    - name: Run tests in release
+      run: make test-release
+  beacon-chain-tests:
+    name: beacon-chain-tests
+    runs-on: ubuntu-latest
+    needs: cargo-fmt
+    steps:
+    - uses: actions/checkout@v1
+    - name: Get latest version of stable Rust
+      run: rustup update stable
+    - name: Run beacon_chain tests for base hard fork
+      run: make test-beacon-chain-base
+    - name: Run beacon_chain tests for Altair hard fork
+      run: make test-beacon-chain-altair
   debug-tests-ubuntu:
+    name: debug-tests-ubuntu
     runs-on: ubuntu-latest
     needs: cargo-fmt
     steps:
@@ -33,19 +80,162 @@ jobs:
       run: sudo npm install -g ganache-cli
     - name: Run tests in debug
       run: make test-debug
-  ef-tests-ubuntu:
+  state-transition-vectors-ubuntu:
+    name: state-transition-vectors-ubuntu
     runs-on: ubuntu-latest
     needs: cargo-fmt
     steps:
     - uses: actions/checkout@v1
     - name: Get latest version of stable Rust
       run: rustup update stable
-    - name: Run eth2.0-spec-tests with and without fake_crypto
-      run: make test-ef
-  dockerfile-ubuntu:
+    - name: Run state_transition_vectors in release.
+      run: make run-state-transition-tests
+  ef-tests-ubuntu:
+    name: ef-tests-ubuntu
     runs-on: ubuntu-latest
     needs: cargo-fmt
     steps:
     - uses: actions/checkout@v1
+    - name: Get latest version of stable Rust
+      run: rustup update stable
+    - name: Run eth2.0-spec-tests with blst, milagro and fake_crypto
+      run: make test-ef
+  dockerfile-ubuntu:
+    name: dockerfile-ubuntu
+    runs-on: ubuntu-latest
+    needs: cargo-fmt
+    steps:
+    - uses: actions/checkout@v1
+    - name: Get latest version of stable Rust
+      run: rustup update stable
     - name: Build the root Dockerfile
       run: docker build .
+  eth1-simulator-ubuntu:
+    name: eth1-simulator-ubuntu
+    runs-on: ubuntu-latest
+    needs: cargo-fmt
+    steps:
+    - uses: actions/checkout@v1
+    - name: Get latest version of stable Rust
+      run: rustup update stable
+    - name: Install ganache-cli
+      run: sudo npm install -g ganache-cli
+    - name: Run the beacon chain sim that starts from an eth1 contract
+      run: cargo run --release --bin simulator eth1-sim
+  no-eth1-simulator-ubuntu:
+    name: no-eth1-simulator-ubuntu
+    runs-on: ubuntu-latest
+    needs: cargo-fmt
+    steps:
+    - uses: actions/checkout@v1
+    - name: Get latest version of stable Rust
+      run: rustup update stable
+    - name: Install ganache-cli
+      run: sudo npm install -g ganache-cli
+    - name: Run the beacon chain sim without an eth1 connection
+      run: cargo run --release --bin simulator no-eth1-sim
+  syncing-simulator-ubuntu:
+    name: syncing-simulator-ubuntu
+    runs-on: ubuntu-latest
+    needs: cargo-fmt
+    steps:
+      - uses: actions/checkout@v1
+      - name: Get latest version of stable Rust
+        run: rustup update stable
+      - name: Install ganache-cli
+        run: sudo npm install -g ganache-cli
+      - name: Run the syncing simulator
+        run: cargo run --release --bin simulator syncing-sim
+  doppelganger-protection-test:
+      name: doppelganger-protection-test
+      runs-on: ubuntu-latest
+      needs: cargo-fmt
+      steps:
+          - uses: actions/checkout@v1
+          - name: Get latest version of stable Rust
+            run: rustup update stable
+          - name: Install ganache-cli
+            run: sudo npm install -g ganache-cli
+          - name: Install lighthouse and lcli
+            run: |
+                make
+                make install-lcli
+          - name: Run the doppelganger protection success test script
+            run: |
+                cd scripts/tests
+                ./doppelganger_protection.sh success
+          - name: Run the doppelganger protection failure test script
+            run: |
+                cd scripts/tests
+                ./doppelganger_protection.sh failure
+  check-benchmarks:
+    name: check-benchmarks
+    runs-on: ubuntu-latest
+    needs: cargo-fmt
+    steps:
+    - uses: actions/checkout@v1
+    - name: Get latest version of stable Rust
+      run: rustup update stable
+    - name: Typecheck benchmark code without running it
+      run: make check-benches
+  check-consensus:
+    name: check-consensus
+    runs-on: ubuntu-latest
+    needs: cargo-fmt
+    steps:
+    - uses: actions/checkout@v1
+    - name: Get latest version of stable Rust
+      run: rustup update stable
+    - name: Typecheck consensus code in strict mode
+      run: make check-consensus
+  clippy:
+    name: clippy
+    runs-on: ubuntu-latest
+    needs: cargo-fmt
+    steps:
+    - uses: actions/checkout@v1
+    - name: Get latest version of stable Rust
+      run: rustup update stable
+    - name: Lint code for quality and style with Clippy
+      run: make lint
+    - name: Certify Cargo.lock freshness
+      run: git diff --exit-code Cargo.lock
+  arbitrary-check:
+    name: arbitrary-check
+    runs-on: ubuntu-latest
+    needs: cargo-fmt
+    steps:
+    - uses: actions/checkout@v1
+    - name: Get latest version of stable Rust
+      run: rustup update stable
+    - name: Validate state_processing feature arbitrary-fuzz
+      run:  make arbitrary-fuzz
+  cargo-audit:
+    name: cargo-audit
+    runs-on: ubuntu-latest
+    needs: cargo-fmt
+    steps:
+    - uses: actions/checkout@v1
+    - name: Get latest version of stable Rust
+      run: rustup update stable
+    - name: Run cargo audit to identify known security vulnerabilities reported to the RustSec Advisory Database
+      run:  make audit
+  cargo-udeps:
+    name: cargo-udeps
+    runs-on: ubuntu-latest
+    needs: cargo-fmt
+    steps:
+    - uses: actions/checkout@v1
+    - name: Install Rust (${{ env.PINNED_NIGHTLY }})
+      run: rustup toolchain install $PINNED_NIGHTLY
+    - name: Install cargo-udeps
+      run: cargo install cargo-udeps --locked
+    - name: Create Cargo config dir
+      run: mkdir -p .cargo
+    - name: Install custom Cargo config
+      run: cp -f .github/custom/config.toml .cargo/config.toml
+    - name: Run cargo udeps to identify unused crates in the dependency graph
+      run: make udeps
+    env:
+      # Allow warnings on Nightly
+      RUSTFLAGS: ""

.gitignore

@@ -6,3 +6,5 @@ target/
 flamegraph.svg
 perf.data*
 *.tar.gz
+/bin
+genesis.ssz

CONTRIBUTING.md

@@ -1,6 +1,6 @@
 # Contributors Guide
 
-Lighthouse is an open-source Ethereum 2.0 client. We we're community driven and
+Lighthouse is an open-source Ethereum 2.0 client. We're community driven and
 welcome all contribution. We aim to provide a constructive, respectful and fun
 environment for collaboration.
 
@@ -31,7 +31,7 @@ same. We operate like a typical open-source project operating on GitHub: the
 repository [Issues](https://github.com/sigp/lighthouse/issues) is where we
 track what needs to be done and [Pull
 Requests](https://github.com/sigp/lighthouse/pulls) is where code gets
-reviewed. We use [gitter](https://gitter.im/sigp/lighthouse) to chat
+reviewed. We use [discord](https://discord.gg/cyAszAh) to chat
 informally.
 
 ### General Work-Flow
@@ -48,7 +48,7 @@ questions.
    your changes in to the main repository.
 4. Wait for the repository maintainers to **review your changes** to ensure the
    issue is addressed satisfactorily. Optionally, mention your PR on
-[gitter](https://gitter.im/sigp/lighthouse).
+[discord](https://discord.gg/cyAszAh).
 5. If the issue is addressed the repository maintainers will **merge your
    pull-request** and you'll be an official contributor!
 
@@ -76,18 +76,19 @@ https://github.com/sigp/lighthouse.git` (pro-tip: [use SSH](https://help.github.
    name of your branch isn't critical but it should be short and instructive.
 E.g., if you're fixing a bug with serialization, you could name your branch
 `fix_serialization_bug`.
-4. Commit your changes and push them to your fork with `$ git push origin
+4. Make sure you sign your commits. See [relevant doc](https://help.github.com/en/github/authenticating-to-github/about-commit-signature-verification).
+5. Commit your changes and push them to your fork with `$ git push origin
    your_feature_name`.
-5. Go to your fork on github.com and use the web interface to create a pull
+6. Go to your fork on github.com and use the web interface to create a pull
    request into the sigp/lighthouse repo.
 
 From there, the repository maintainers will review the PR and either accept it
 or provide some constructive feedback.
 
-There's great
+There's a great
 [guide](https://akrabat.com/the-beginners-guide-to-contributing-to-a-github-project/)
 by Rob Allen that provides much more detail on each of these steps, if you're
-having trouble. As always, jump on [gitter](https://gitter.im/sigp/lighthouse)
+having trouble. As always, jump on [discord](https://discord.gg/cyAszAh)
 if you get stuck.
 
 

Cargo.toml

@@ -1,58 +1,97 @@
 [workspace]
 members = [
-    "eth2/lmd_ghost",
-    "eth2/operation_pool",
-    "eth2/state_processing",
-    "eth2/types",
-    "eth2/utils/bls",
-    "eth2/utils/compare_fields",
-    "eth2/utils/compare_fields_derive",
-    "eth2/utils/deposit_contract",
-    "eth2/utils/eth2_config",
-    "eth2/utils/eth2_interop_keypairs",
-    "eth2/utils/eth2_testnet_config",
-    "eth2/utils/logging",
-    "eth2/utils/eth2_hashing",
-    "eth2/utils/lighthouse_metrics",
-    "eth2/utils/lighthouse_bootstrap",
-    "eth2/utils/merkle_proof",
-    "eth2/utils/int_to_bytes",
-    "eth2/utils/serde_hex",
-    "eth2/utils/slot_clock",
-    "eth2/utils/ssz",
-    "eth2/utils/ssz_derive",
-    "eth2/utils/ssz_types",
-    "eth2/utils/swap_or_not_shuffle",
-    "eth2/utils/cached_tree_hash",
-    "eth2/utils/tree_hash",
-    "eth2/utils/tree_hash_derive",
-    "eth2/utils/test_random_derive",
-    "beacon_node",
-    "beacon_node/store",
-    "beacon_node/client",
-    "beacon_node/rest_api",
-    "beacon_node/network",
-    "beacon_node/eth2-libp2p",
-    "beacon_node/version",
-    "beacon_node/eth1",
-    "beacon_node/beacon_chain",
-    "beacon_node/websocket_server",
-    "tests/beacon_chain_sim",
-    "tests/ef_tests",
-    "tests/eth1_test_rig",
-    "tests/node_test_rig",
-    "lcli",
-    "validator_client",
     "account_manager",
+
+    "beacon_node",
+    "beacon_node/beacon_chain",
+    "beacon_node/client",
+    "beacon_node/eth1",
+    "beacon_node/eth2_libp2p",
+    "beacon_node/http_api",
+    "beacon_node/http_metrics",
+    "beacon_node/network",
+    "beacon_node/store",
+    "beacon_node/timer",
+
+	"boot_node",
+
+    "common/account_utils",
+    "common/clap_utils",
+    "common/compare_fields",
+    "common/compare_fields_derive",
+    "common/deposit_contract",
+    "common/directory",
+    "common/eth2",
+    "common/eth2_config",
+    "common/eth2_interop_keypairs",
+    "common/eth2_network_config",
+    "common/eth2_wallet_manager",
+    "common/hashset_delay",
+    "common/lighthouse_metrics",
+    "common/lighthouse_version",
+    "common/lockfile",
+    "common/logging",
+    "common/lru_cache",
+    "common/malloc_utils",
+    "common/remote_signer_consumer",
+    "common/sensitive_url",
+    "common/slot_clock",
+    "common/task_executor",
+    "common/target_check",
+    "common/test_random_derive",
+    "common/validator_dir",
+    "common/warp_utils",
+    "common/fallback",
+    "common/monitoring_api",
+
+    "consensus/cached_tree_hash",
+    "consensus/int_to_bytes",
+    "consensus/fork_choice",
+    "consensus/proto_array",
+    "consensus/safe_arith",
+    "consensus/ssz",
+    "consensus/ssz_derive",
+    "consensus/ssz_types",
+    "consensus/serde_utils",
+    "consensus/state_processing",
+    "consensus/swap_or_not_shuffle",
+    "consensus/tree_hash",
+    "consensus/tree_hash_derive",
+
+    "crypto/bls",
+    "crypto/eth2_hashing",
+    "crypto/eth2_key_derivation",
+    "crypto/eth2_keystore",
+    "crypto/eth2_wallet",
+
+    "lcli",
+
     "lighthouse",
-    "lighthouse/environment"
+    "lighthouse/environment",
+
+    "remote_signer",
+    "remote_signer/backend",
+    "remote_signer/client",
+
+    "slasher",
+    "slasher/service",
+
+    "testing/ef_tests",
+    "testing/eth1_test_rig",
+    "testing/node_test_rig",
+    "testing/remote_signer_test",
+    "testing/simulator",
+    "testing/state_transition_vectors",
+
+    "validator_client",
+    "validator_client/slashing_protection",
 ]
 
 [patch]
 [patch.crates-io]
-tree_hash = { path = "eth2/utils/tree_hash" }
-tree_hash_derive = { path = "eth2/utils/tree_hash_derive" }
-eth2_ssz = { path = "eth2/utils/ssz" }
-eth2_ssz_derive = { path = "eth2/utils/ssz_derive" }
-eth2_ssz_types = { path = "eth2/utils/ssz_types" }
-eth2_hashing = { path = "eth2/utils/eth2_hashing" }
+tree_hash = { path = "consensus/tree_hash" }
+tree_hash_derive = { path = "consensus/tree_hash_derive" }
+eth2_ssz = { path = "consensus/ssz" }
+eth2_ssz_derive = { path = "consensus/ssz_derive" }
+eth2_ssz_types = { path = "consensus/ssz_types" }
+eth2_hashing = { path = "crypto/eth2_hashing" }

Cross.toml

@@ -0,0 +1,4 @@
+[build.env]
+passthrough = [
+    "RUSTFLAGS",
+]

Dockerfile

@@ -1,4 +1,14 @@
-FROM rust:latest
-
+FROM rust:1.53.0 AS builder
+RUN apt-get update && apt-get -y upgrade && apt-get install -y cmake
 COPY . lighthouse
+ARG PORTABLE
+ENV PORTABLE $PORTABLE
 RUN cd lighthouse && make
+
+FROM debian:buster-slim
+RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends \
+  libssl-dev \
+  ca-certificates \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+COPY --from=builder /usr/local/cargo/bin/lighthouse /usr/local/bin/lighthouse

Dockerfile.cross

@@ -0,0 +1,10 @@
+# This image is meant to enable cross-architecture builds.
+# It assumes the lighthouse binary has already been
+# compiled for `$TARGETPLATFORM` and moved to `./bin`.
+FROM --platform=$TARGETPLATFORM debian:buster-slim
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  libssl-dev \
+  ca-certificates \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+COPY ./bin/lighthouse /usr/local/bin/lighthouse

Makefile

@@ -1,31 +1,121 @@
 .PHONY: tests
 
-EF_TESTS = "tests/ef_tests"
+EF_TESTS = "testing/ef_tests"
+BEACON_CHAIN_CRATE = "beacon_node/beacon_chain"
+STATE_TRANSITION_VECTORS = "testing/state_transition_vectors"
+GIT_TAG := $(shell git describe --tags --candidates 1)
+BIN_DIR = "bin"
 
-# Builds the entire workspace in release (optimized).
+X86_64_TAG = "x86_64-unknown-linux-gnu"
+BUILD_PATH_X86_64 = "target/$(X86_64_TAG)/release"
+AARCH64_TAG = "aarch64-unknown-linux-gnu"
+BUILD_PATH_AARCH64 = "target/$(AARCH64_TAG)/release"
+
+PINNED_NIGHTLY ?= nightly
+
+# Builds the Lighthouse binary in release (optimized).
 #
 # Binaries will most likely be found in `./target/release`
 install:
-	cargo install --path lighthouse --force
+ifeq ($(PORTABLE), true)
+	cargo install --path lighthouse --force --locked --features portable
+else
+	cargo install --path lighthouse --force --locked
+endif
+
+# Builds the lcli binary in release (optimized).
+install-lcli:
+ifeq ($(PORTABLE), true)
+	cargo install --path lcli --force --locked --features portable
+else
+	cargo install --path lcli --force --locked
+endif
+
+# The following commands use `cross` to build a cross-compile.
+#
+# These commands require that:
+#
+# - `cross` is installed (`cargo install cross`).
+# - Docker is running.
+# - The current user is in the `docker` group.
+#
+# The resulting binaries will be created in the `target/` directory.
+#
+# The *-portable options compile the blst library *without* the use of some
+# optimized CPU functions that may not be available on some systems. This
+# results in a more portable binary with ~20% slower BLS verification.
+build-x86_64:
+	cross build --release --manifest-path lighthouse/Cargo.toml --target x86_64-unknown-linux-gnu --features modern
+build-x86_64-portable:
+	cross build --release --manifest-path lighthouse/Cargo.toml --target x86_64-unknown-linux-gnu --features portable
+build-aarch64:
+	cross build --release --manifest-path lighthouse/Cargo.toml --target aarch64-unknown-linux-gnu
+build-aarch64-portable:
+	cross build --release --manifest-path lighthouse/Cargo.toml --target aarch64-unknown-linux-gnu --features portable
+
+# Create a `.tar.gz` containing a binary for a specific target.
+define tarball_release_binary
+	cp $(1)/lighthouse $(BIN_DIR)/lighthouse
+	cd $(BIN_DIR) && \
+		tar -czf lighthouse-$(GIT_TAG)-$(2)$(3).tar.gz lighthouse && \
+		rm lighthouse
+endef
+
+# Create a series of `.tar.gz` files in the BIN_DIR directory, each containing
+# a `lighthouse` binary for a different target.
+#
+# The current git tag will be used as the version in the output file names. You
+# will likely need to use `git tag` and create a semver tag (e.g., `v0.2.3`).
+build-release-tarballs:
+	[ -d $(BIN_DIR) ] || mkdir -p $(BIN_DIR)
+	$(MAKE) build-x86_64
+	$(call tarball_release_binary,$(BUILD_PATH_X86_64),$(X86_64_TAG),"")
+	$(MAKE) build-x86_64-portable
+	$(call tarball_release_binary,$(BUILD_PATH_X86_64),$(X86_64_TAG),"-portable")
+	$(MAKE) build-aarch64
+	$(call tarball_release_binary,$(BUILD_PATH_AARCH64),$(AARCH64_TAG),"")
+	$(MAKE) build-aarch64-portable
+	$(call tarball_release_binary,$(BUILD_PATH_AARCH64),$(AARCH64_TAG),"-portable")
 
 # Runs the full workspace tests in **release**, without downloading any additional
 # test vectors.
 test-release:
-	cargo test --all --release --exclude ef_tests
+	cargo test --workspace --release --exclude ef_tests --exclude beacon_chain
 
 # Runs the full workspace tests in **debug**, without downloading any additional test
 # vectors.
 test-debug:
-	cargo test --all --exclude ef_tests
+	cargo test --workspace --exclude ef_tests --exclude beacon_chain
 
 # Runs cargo-fmt (linter).
 cargo-fmt:
 	cargo fmt --all -- --check
 
+# Typechecks benchmark code
+check-benches:
+	cargo check --workspace --benches
+
+# Typechecks consensus code *without* allowing deprecated legacy arithmetic or metrics.
+check-consensus:
+	cargo check --manifest-path=consensus/state_processing/Cargo.toml --no-default-features
+
 # Runs only the ef-test vectors.
 run-ef-tests:
+	rm -rf $(EF_TESTS)/.accessed_file_log.txt
 	cargo test --release --manifest-path=$(EF_TESTS)/Cargo.toml --features "ef_tests"
 	cargo test --release --manifest-path=$(EF_TESTS)/Cargo.toml --features "ef_tests,fake_crypto"
+	cargo test --release --manifest-path=$(EF_TESTS)/Cargo.toml --features "ef_tests,milagro"
+	./$(EF_TESTS)/check_all_files_accessed.py $(EF_TESTS)/.accessed_file_log.txt $(EF_TESTS)/eth2.0-spec-tests
+
+# Run the tests in the `beacon_chain` crate.
+test-beacon-chain: test-beacon-chain-base test-beacon-chain-altair
+
+test-beacon-chain-%:
+	env FORK_NAME=$* cargo test --release --features fork_from_env --manifest-path=$(BEACON_CHAIN_CRATE)/Cargo.toml
+
+# Runs only the tests/state_transition_vectors tests.
+run-state-transition-tests:
+	make -C $(STATE_TRANSITION_VECTORS) test
 
 # Downloads and runs the EF test vectors.
 test-ef: make-ef-tests run-ef-tests
@@ -37,6 +127,15 @@ test: test-release
 # Runs the entire test suite, downloading test vectors if required.
 test-full: cargo-fmt test-release test-debug test-ef
 
+# Lints the code for bad style and potentially unsafe arithmetic using Clippy.
+# Clippy lints are opt-in per-crate for now. By default, everything is allowed except for performance and correctness lints.
+lint:
+	cargo clippy --workspace --tests -- \
+        -D warnings \
+        -A clippy::from-over-into \
+        -A clippy::upper-case-acronyms \
+        -A clippy::vec-init-then-push
+
 # Runs the makefile in the `ef_tests` repo.
 #
 # May download and extract an archive of test vectors from the ethereum
@@ -45,7 +144,21 @@ test-full: cargo-fmt test-release test-debug test-ef
 make-ef-tests:
 	make -C $(EF_TESTS)
 
+# Verifies that state_processing feature arbitrary-fuzz will compile
+arbitrary-fuzz:
+	cargo check --manifest-path=consensus/state_processing/Cargo.toml --features arbitrary-fuzz
+
+# Runs cargo audit (Audit Cargo.lock files for crates with security vulnerabilities reported to the RustSec Advisory Database)
+audit:
+	cargo install --force cargo-audit
+	cargo audit
+
+# Runs `cargo udeps` to check for unused dependencies
+udeps:
+	cargo +$(PINNED_NIGHTLY) udeps --tests --all-targets --release
+
 # Performs a `cargo` clean and cleans the `ef_tests` directory.
 clean:
 	cargo clean
 	make -C $(EF_TESTS) clean
+	make -C $(STATE_TRANSITION_VECTORS) clean

README.md

@@ -2,78 +2,65 @@
 
 An open-source Ethereum 2.0 client, written in Rust and maintained by Sigma Prime.
 
-[![Build Status]][Build Link] [![Book Status]][Book Link] [![RustDoc Status]][RustDoc Link] [![Chat Badge]][Chat Link] [![Swagger Badge]][Swagger Link]
+[![Build Status]][Build Link] [![Book Status]][Book Link] [![Chat Badge]][Chat Link]
 
-[Build Status]: https://github.com/sigp/lighthouse/workflows/test-suite/badge.svg?branch=master
+[Build Status]: https://github.com/sigp/lighthouse/workflows/test-suite/badge.svg?branch=stable
 [Build Link]: https://github.com/sigp/lighthouse/actions
 [Chat Badge]: https://img.shields.io/badge/chat-discord-%237289da
 [Chat Link]: https://discord.gg/cyAszAh
-[Book Status]:https://img.shields.io/badge/user--docs-master-informational
-[Book Link]: http://lighthouse-book.sigmaprime.io/
-[RustDoc Status]:https://img.shields.io/badge/code--docs-master-orange
-[RustDoc Link]: http://lighthouse-docs.sigmaprime.io/
-[Swagger Badge]: https://img.shields.io/badge/Open%20API-0.2.0-success
-[Swagger Link]: https://app.swaggerhub.com/apis-docs/spble/lighthouse_rest_api/0.2.0
+[Book Status]:https://img.shields.io/badge/user--docs-unstable-informational
+[Book Link]: https://lighthouse-book.sigmaprime.io
+[stable]: https://github.com/sigp/lighthouse/tree/stable
+[unstable]: https://github.com/sigp/lighthouse/tree/unstable
+[blog]: https://lighthouse.sigmaprime.io
 
-[Documentation](http://lighthouse-book.sigmaprime.io/)
+[Documentation](https://lighthouse-book.sigmaprime.io)
 
-![terminalize](https://i.postimg.cc/kG11dpCW/lighthouse-cli-png.gif)
+![Banner](https://i.postimg.cc/hjdTGKPd/photo-2020-10-23-09-52-16.jpg)
 
 ## Overview
 
 Lighthouse is:
 
+- Ready for use on Eth2 mainnet.
 - Fully open-source, licensed under Apache 2.0.
-- Security-focused. Fuzzing has begun and security reviews are planned
-	for late-2019.
-- Built in [Rust](https://www.rust-lang.org/), a modern language providing unique safety guarantees and
+- Security-focused. Fuzzing techniques have been continuously applied and several external security reviews have been performed.
+- Built in [Rust](https://www.rust-lang.org), a modern language providing unique safety guarantees and
 	excellent performance (comparable to C++).
 - Funded by various organisations, including Sigma Prime, the
-	Ethereum Foundation, ConsenSys and private individuals.
-- Actively involved in the specification and security analysis of the emerging
-    Ethereum 2.0 specification.
+	Ethereum Foundation, ConsenSys, the Decentralization Foundation and private individuals.
+- Actively involved in the specification and security analysis of the
+	Ethereum 2.0 specification.
 
-Like all Ethereum 2.0 clients, Lighthouse is a work-in-progress.
-
-## Development Status
-
-Current development overview:
-
-- Specification `v0.8.3` implemented, optimized and passing test vectors.
-- Rust-native libp2p with Gossipsub and Discv5.
-- RESTful JSON API via HTTP server.
-- Events via WebSocket.
-- Metrics via Prometheus.
-
-### Roadmap
-
-- ~~**April 2019**: Inital single-client testnets.~~
-- ~~**September 2019**: Inter-operability with other Ethereum 2.0 clients.~~
-- **Q4 2019**: `lighthouse-0.0.1` release: All major phase 0
-     features implemented.
-- **Q4 2019**: Public, multi-client testnet with user-facing functionality.
-- **Q4 2019**: Third-party security review.
-- **Q1 2020**: Production Beacon Chain testnet (tentative).
+## Eth2 Deposit Contract
 
+The Lighthouse team acknowledges
+[`0x00000000219ab540356cBB839Cbe05303d7705Fa`](https://etherscan.io/address/0x00000000219ab540356cbb839cbe05303d7705fa)
+as the canonical Eth2 deposit contract address.
 
 ## Documentation
 
-The [Lighthouse Book](http://lighthouse-book.sigmaprime.io/) contains information
-for testnet users and developers.
+The [Lighthouse Book](https://lighthouse-book.sigmaprime.io) contains information for users and
+developers.
 
-Code documentation is generated via `cargo doc` and hosted at
-[lighthouse-docs.sigmaprime.io](http://lighthouse-docs.sigmaprime.io/).
+The Lighthouse team maintains a blog at [lighthouse.sigmaprime.io][blog] which contains periodical
+progress updates, roadmap insights and interesting findings.
 
-If you'd like some background on Sigma Prime, please see the [Lighthouse Update
-\#00](https://lighthouse.sigmaprime.io/update-00.html) blog post or
-[sigmaprime.io](https://sigmaprime.io).
+## Branches
+
+Lighthouse maintains two permanent branches:
+
+- [`stable`][stable]: Always points to the latest stable release.
+  - This is ideal for most users.
+- [`unstable`][unstable]: Used for development, contains the latest PRs.
+  - Developers should base their PRs on this branch.
 
 ## Contributing
 
 Lighthouse welcomes contributors.
 
 If you are looking to contribute, please head to the
-[Contributing](http://lighthouse-book.sigmaprime.io/contributing.html) section
+[Contributing](https://lighthouse-book.sigmaprime.io/contributing.html) section
 of the Lighthouse book.
 
 ## Contact
@@ -82,8 +69,11 @@ The best place for discussion is the [Lighthouse Discord
 server](https://discord.gg/cyAszAh). Alternatively, you may use the
 [sigp/lighthouse gitter](https://gitter.im/sigp/lighthouse).
 
+Sign up to the [Lighthouse Development Updates](http://eepurl.com/dh9Lvb) mailing list for email
+notifications about releases, network status and other important information.
+
 Encrypt sensitive messages using our [PGP
-key](https://keybase.io/sigp/pgp_keys.asc?fingerprint=dcf37e025d6c9d42ea795b119e7c6cf9988604be).
+key](https://keybase.io/sigp/pgp_keys.asc?fingerprint=15e66d941f697e28f49381f426416dc3f30674b0).
 
 ## Donations
 

account_manager/Cargo.toml

@@ -1,28 +1,41 @@
 [package]
 name = "account_manager"
-version = "0.0.1"
+version = "0.3.5"
 authors = ["Paul Hauner <paul@paulhauner.com>", "Luke Anderson <luke@sigmaprime.io>"]
 edition = "2018"
 
-[dev-dependencies]
-tempdir = "0.3"
-
 [dependencies]
-bls = { path = "../eth2/utils/bls" }
-clap = "2.33.0"
+bls = { path = "../crypto/bls" }
+clap = "2.33.3"
 slog = "2.5.2"
-slog-term = "2.4.2"
-slog-async = "2.3.0"
-types = { path = "../eth2/types" }
-dirs = "2.0.2"
+slog-term = "2.6.0"
+slog-async = "2.5.0"
+types = { path = "../consensus/types" }
+dirs = "3.0.1"
 environment = { path = "../lighthouse/environment" }
-deposit_contract = { path = "../eth2/utils/deposit_contract" }
-libc = "0.2.65"
-eth2_ssz = { path = "../eth2/utils/ssz" }
-eth2_ssz_derive = { path = "../eth2/utils/ssz_derive" }
-hex = "0.3"
-validator_client = { path = "../validator_client" }
-rayon = "1.2.0"
-eth2_testnet_config = { path = "../eth2/utils/eth2_testnet_config" }
-web3 = "0.8.0"
-futures = "0.1.25"
+deposit_contract = { path = "../common/deposit_contract" }
+libc = "0.2.79"
+eth2_ssz = "0.1.2"
+eth2_ssz_derive = "0.1.0"
+hex = "0.4.2"
+rayon = "1.4.1"
+eth2_network_config = { path = "../common/eth2_network_config" }
+futures = "0.3.7"
+clap_utils = { path = "../common/clap_utils" }
+directory = { path = "../common/directory" }
+eth2_wallet = { path = "../crypto/eth2_wallet" }
+eth2_wallet_manager = { path = "../common/eth2_wallet_manager" }
+rand = "0.7.3"
+validator_dir = { path = "../common/validator_dir" }
+tokio = { version = "1.10.0", features = ["full"] }
+eth2_keystore = { path = "../crypto/eth2_keystore" }
+account_utils = { path = "../common/account_utils" }
+slashing_protection = { path = "../validator_client/slashing_protection" }
+eth2 = {path = "../common/eth2"}
+safe_arith = {path = "../consensus/safe_arith"}
+slot_clock = { path = "../common/slot_clock" }
+filesystem = { path = "../common/filesystem" }
+sensitive_url = { path = "../common/sensitive_url" }
+
+[dev-dependencies]
+tempfile = "3.1.0"

account_manager/src/cli.rs

@@ -1,97 +0,0 @@
-use clap::{App, Arg, SubCommand};
-
-pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
-    App::new("account_manager")
-        .visible_aliases(&["a", "am", "account", "account_manager"])
-        .about("Utilities for generating and managing Ethereum 2.0 accounts.")
-        .subcommand(
-            SubCommand::with_name("validator")
-                .about("Generate or manage Etheruem 2.0 validators.")
-                .subcommand(
-                    SubCommand::with_name("new")
-                        .about("Create a new Ethereum 2.0 validator.")
-                        .arg(
-                            Arg::with_name("deposit-value")
-                                .short("v")
-                                .long("deposit-value")
-                                .value_name("GWEI")
-                                .takes_value(true)
-                                .default_value("3200000000")
-                                .help("The deposit amount in Gwei (not Wei). Default is 3.2 ETH."),
-                        )
-                        .arg(
-                            Arg::with_name("send-deposits")
-                                .long("send-deposits")
-                                .help("If present, submit validator deposits to an eth1 endpoint /
-                                       defined by the --eth1-endpoint. Requires either the /
-                                       --deposit-contract or --testnet-dir flag.")
-                        )
-                        .arg(
-                            Arg::with_name("eth1-endpoint")
-                                .short("e")
-                                .long("eth1-endpoint")
-                                .value_name("HTTP_SERVER")
-                                .takes_value(true)
-                                .default_value("http://localhost:8545")
-                                .help("The URL to the Eth1 JSON-RPC HTTP API (e.g., Geth/Parity-Ethereum)."),
-                        )
-                        .arg(
-                            Arg::with_name("account-index")
-                                .short("i")
-                                .long("account-index")
-                                .value_name("INDEX")
-                                .takes_value(true)
-                                .default_value("0")
-                                .help("The eth1 accounts[] index which will send the transaction"),
-                        )
-                        .arg(
-                            Arg::with_name("password")
-                                .short("p")
-                                .long("password")
-                                .value_name("FILE")
-                                .takes_value(true)
-                                .help("The password file to unlock the eth1 account (see --index)"),
-                        )
-                        .arg(
-                            Arg::with_name("testnet-dir")
-                                .long("testnet-dir")
-                                .value_name("DIRECTORY")
-                                .takes_value(true)
-                                .help("The directory from which to read the deposit contract /
-                                       address. Defaults to the current Lighthouse testnet."),
-                        )
-                        .subcommand(
-                            SubCommand::with_name("insecure")
-                                .about("Produce insecure, ephemeral validators. DO NOT USE TO STORE VALUE.")
-                                .arg(
-                                    Arg::with_name("first")
-                                        .index(1)
-                                        .value_name("INDEX")
-                                        .help("Index of the first validator")
-                                        .takes_value(true)
-                                        .required(true),
-                                )
-                                .arg(
-                                    Arg::with_name("last")
-                                        .index(2)
-                                        .value_name("INDEX")
-                                        .help("Index of the last validator")
-                                        .takes_value(true)
-                                        .required(true),
-                                ),
-                        )
-                        .subcommand(
-                            SubCommand::with_name("random")
-                                .about("Produces public keys using entropy from the Rust 'rand' library.")
-                                .arg(
-                                    Arg::with_name("validator_count")
-                                        .index(1)
-                                        .value_name("INTEGER")
-                                        .help("The number of new validators to generate.")
-                                        .takes_value(true)
-                                        .default_value("1"),
-                                ),
-                        )
-                )
-        )
-}

account_manager/src/common.rs

@@ -0,0 +1,67 @@
+use account_utils::PlainText;
+use account_utils::{read_input_from_user, strip_off_newlines};
+use eth2_wallet::bip39::{Language, Mnemonic};
+use std::fs;
+use std::path::PathBuf;
+use std::str::from_utf8;
+use std::thread::sleep;
+use std::time::Duration;
+
+pub const MNEMONIC_PROMPT: &str = "Enter the mnemonic phrase:";
+pub const WALLET_NAME_PROMPT: &str = "Enter wallet name:";
+
+pub fn read_mnemonic_from_cli(
+    mnemonic_path: Option<PathBuf>,
+    stdin_inputs: bool,
+) -> Result<Mnemonic, String> {
+    let mnemonic = match mnemonic_path {
+        Some(path) => fs::read(&path)
+            .map_err(|e| format!("Unable to read {:?}: {:?}", path, e))
+            .and_then(|bytes| {
+                let bytes_no_newlines: PlainText = strip_off_newlines(bytes).into();
+                let phrase = from_utf8(bytes_no_newlines.as_ref())
+                    .map_err(|e| format!("Unable to derive mnemonic: {:?}", e))?;
+                Mnemonic::from_phrase(phrase, Language::English).map_err(|e| {
+                    format!(
+                        "Unable to derive mnemonic from string {:?}: {:?}",
+                        phrase, e
+                    )
+                })
+            })?,
+        None => loop {
+            eprintln!();
+            eprintln!("{}", MNEMONIC_PROMPT);
+
+            let mnemonic = read_input_from_user(stdin_inputs)?;
+
+            match Mnemonic::from_phrase(mnemonic.as_str(), Language::English) {
+                Ok(mnemonic_m) => {
+                    eprintln!("Valid mnemonic provided.");
+                    eprintln!();
+                    sleep(Duration::from_secs(1));
+                    break mnemonic_m;
+                }
+                Err(_) => {
+                    eprintln!("Invalid mnemonic");
+                }
+            }
+        },
+    };
+    Ok(mnemonic)
+}
+
+/// Reads in a wallet name from the user. If the `--wallet-name` flag is provided, use it. Otherwise
+/// read from an interactive prompt using tty unless the `--stdin-inputs` flag is provided.
+pub fn read_wallet_name_from_cli(
+    wallet_name: Option<String>,
+    stdin_inputs: bool,
+) -> Result<String, String> {
+    match wallet_name {
+        Some(name) => Ok(name),
+        None => {
+            eprintln!("{}", WALLET_NAME_PROMPT);
+
+            read_input_from_user(stdin_inputs)
+        }
+    }
+}

account_manager/src/lib.rs

@@ -1,441 +1,37 @@
-mod cli;
+mod common;
+pub mod validator;
+pub mod wallet;
 
+use clap::App;
 use clap::ArgMatches;
-use deposit_contract::DEPOSIT_GAS;
-use environment::{Environment, RuntimeContext};
-use eth2_testnet_config::Eth2TestnetConfig;
-use futures::{future, Future, IntoFuture, Stream};
-use rayon::prelude::*;
-use slog::{crit, error, info, Logger};
-use std::fs;
-use std::fs::File;
-use std::io::Read;
-use std::path::PathBuf;
-use types::{ChainSpec, EthSpec};
-use validator_client::validator_directory::{ValidatorDirectory, ValidatorDirectoryBuilder};
-use web3::{
-    transports::Http,
-    types::{Address, TransactionRequest, U256},
-    Web3,
-};
+use environment::Environment;
+use types::EthSpec;
 
-pub use cli::cli_app;
+pub const CMD: &str = "account_manager";
+pub const SECRETS_DIR_FLAG: &str = "secrets-dir";
+pub const VALIDATOR_DIR_FLAG: &str = "validator-dir";
+pub const WALLETS_DIR_FLAG: &str = "wallets-dir";
 
-/// Run the account manager, logging an error if the operation did not succeed.
-pub fn run<T: EthSpec>(matches: &ArgMatches, mut env: Environment<T>) {
-    let log = env.core_context().log.clone();
-    match run_account_manager(matches, env) {
-        Ok(()) => (),
-        Err(e) => crit!(log, "Account manager failed"; "error" => e),
-    }
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD)
+        .visible_aliases(&["a", "am", "account", CMD])
+        .about("Utilities for generating and managing Ethereum 2.0 accounts.")
+        .subcommand(wallet::cli_app())
+        .subcommand(validator::cli_app())
 }
 
 /// Run the account manager, returning an error if the operation did not succeed.
-fn run_account_manager<T: EthSpec>(
-    matches: &ArgMatches,
-    mut env: Environment<T>,
-) -> Result<(), String> {
-    let context = env.core_context();
-    let log = context.log.clone();
-
-    // If the `datadir` was not provided, default to the home directory. If the home directory is
-    // not known, use the current directory.
-    let datadir = matches
-        .value_of("datadir")
-        .map(PathBuf::from)
-        .unwrap_or_else(|| {
-            dirs::home_dir()
-                .unwrap_or_else(|| PathBuf::from("."))
-                .join(".lighthouse")
-                .join("validators")
-        });
-
-    fs::create_dir_all(&datadir).map_err(|e| format!("Failed to create datadir: {}", e))?;
-
-    info!(
-        log,
-        "Located data directory";
-        "path" => format!("{:?}", datadir)
-    );
-
+pub fn run<T: EthSpec>(matches: &ArgMatches<'_>, env: Environment<T>) -> Result<(), String> {
     match matches.subcommand() {
-        ("validator", Some(matches)) => match matches.subcommand() {
-            ("new", Some(matches)) => run_new_validator_subcommand(matches, datadir, env)?,
-            _ => {
-                return Err("Invalid 'validator new' command. See --help.".to_string());
-            }
-        },
-        _ => {
-            return Err("Invalid 'validator' command. See --help.".to_string());
+        (wallet::CMD, Some(matches)) => wallet::cli_run(matches)?,
+        (validator::CMD, Some(matches)) => validator::cli_run(matches, env)?,
+        (unknown, _) => {
+            return Err(format!(
+                "{} is not a valid {} command. See --help.",
+                unknown, CMD
+            ));
         }
     }
 
     Ok(())
 }
-
-/// Describes the crypto key generation methods for a validator.
-enum KeygenMethod {
-    /// Produce an insecure "deterministic" keypair. Used only for interop and testing.
-    Insecure(usize),
-    /// Generate a new key from the `rand` thread random RNG.
-    ThreadRandom,
-}
-
-/// Process the subcommand for creating new validators.
-fn run_new_validator_subcommand<T: EthSpec>(
-    matches: &ArgMatches,
-    datadir: PathBuf,
-    mut env: Environment<T>,
-) -> Result<(), String> {
-    let context = env.core_context();
-    let log = context.log.clone();
-
-    let methods: Vec<KeygenMethod> = match matches.subcommand() {
-        ("insecure", Some(matches)) => {
-            let first = matches
-                .value_of("first")
-                .ok_or_else(|| "No first index".to_string())?
-                .parse::<usize>()
-                .map_err(|e| format!("Unable to parse first index: {}", e))?;
-            let last = matches
-                .value_of("last")
-                .ok_or_else(|| "No last index".to_string())?
-                .parse::<usize>()
-                .map_err(|e| format!("Unable to parse first index: {}", e))?;
-
-            (first..last).map(KeygenMethod::Insecure).collect()
-        }
-        ("random", Some(matches)) => {
-            let count = matches
-                .value_of("validator_count")
-                .ok_or_else(|| "No validator count".to_string())?
-                .parse::<usize>()
-                .map_err(|e| format!("Unable to parse validator count: {}", e))?;
-
-            (0..count).map(|_| KeygenMethod::ThreadRandom).collect()
-        }
-        _ => {
-            return Err("Invalid 'validator' command. See --help.".to_string());
-        }
-    };
-
-    let deposit_value = matches
-        .value_of("deposit-value")
-        .ok_or_else(|| "No deposit-value".to_string())?
-        .parse::<u64>()
-        .map_err(|e| format!("Unable to parse deposit-value: {}", e))?;
-
-    let validators = make_validators(
-        datadir.clone(),
-        &methods,
-        deposit_value,
-        &context.eth2_config.spec,
-    )?;
-
-    if matches.is_present("send-deposits") {
-        let eth1_endpoint = matches
-            .value_of("eth1-endpoint")
-            .ok_or_else(|| "No eth1-endpoint".to_string())?;
-        let account_index = matches
-            .value_of("account-index")
-            .ok_or_else(|| "No account-index".to_string())?
-            .parse::<usize>()
-            .map_err(|e| format!("Unable to parse account-index: {}", e))?;
-
-        // If supplied, load the eth1 account password from file.
-        let password = if let Some(password_path) = matches.value_of("password") {
-            Some(
-                File::open(password_path)
-                    .map_err(|e| format!("Unable to open password file: {:?}", e))
-                    .and_then(|mut file| {
-                        let mut password = String::new();
-                        file.read_to_string(&mut password)
-                            .map_err(|e| format!("Unable to read password file to string: {:?}", e))
-                            .map(|_| password)
-                    })
-                    .map(|password| {
-                        // Trim the line feed from the end of the password file, if present.
-                        if password.ends_with("\n") {
-                            password[0..password.len() - 1].to_string()
-                        } else {
-                            password
-                        }
-                    })?,
-            )
-        } else {
-            None
-        };
-
-        info!(
-            log,
-            "Submitting validator deposits";
-            "eth1_node_http_endpoint" => eth1_endpoint
-        );
-
-        // Load the testnet configuration from disk, or use the default testnet.
-        let eth2_testnet_config: Eth2TestnetConfig<T> = if let Some(testnet_dir_str) =
-            matches.value_of("testnet-dir")
-        {
-            let testnet_dir = testnet_dir_str
-                .parse::<PathBuf>()
-                .map_err(|e| format!("Unable to parse testnet-dir: {}", e))?;
-
-            if !testnet_dir.exists() {
-                return Err(format!(
-                    "Testnet directory at {:?} does not exist",
-                    testnet_dir
-                ));
-            }
-
-            info!(
-                log,
-                "Loading deposit contract address";
-                "testnet_dir" => format!("{:?}", &testnet_dir)
-            );
-
-            Eth2TestnetConfig::load(testnet_dir.clone())
-                .map_err(|e| format!("Failed to load testnet dir at {:?}: {}", testnet_dir, e))?
-        } else {
-            info!(
-                log,
-                "Using Lighthouse testnet deposit contract";
-            );
-
-            Eth2TestnetConfig::hard_coded()
-                .map_err(|e| format!("Failed to load hard_coded testnet dir: {}", e))?
-        };
-
-        // Convert from `types::Address` to `web3::types::Address`.
-        let deposit_contract = Address::from_slice(
-            eth2_testnet_config
-                .deposit_contract_address()?
-                .as_fixed_bytes(),
-        );
-
-        if let Err(()) = env.runtime().block_on(deposit_validators(
-            context.clone(),
-            eth1_endpoint.to_string(),
-            deposit_contract,
-            validators.clone(),
-            account_index,
-            deposit_value,
-            password,
-        )) {
-            error!(
-                log,
-                "Created validators but could not submit deposits";
-            )
-        } else {
-            info!(
-                log,
-                "Validator deposits complete";
-            );
-        }
-    }
-
-    info!(
-        log,
-        "Generated validator directories";
-        "base_path" => format!("{:?}", datadir),
-        "count" => validators.len(),
-    );
-
-    Ok(())
-}
-
-/// Produces a validator directory for each of the key generation methods provided in `methods`.
-fn make_validators(
-    datadir: PathBuf,
-    methods: &[KeygenMethod],
-    deposit_value: u64,
-    spec: &ChainSpec,
-) -> Result<Vec<ValidatorDirectory>, String> {
-    methods
-        .par_iter()
-        .map(|method| {
-            let mut builder = ValidatorDirectoryBuilder::default()
-                .spec(spec.clone())
-                .custom_deposit_amount(deposit_value);
-
-            builder = match method {
-                KeygenMethod::Insecure(index) => builder.insecure_keypairs(*index),
-                KeygenMethod::ThreadRandom => builder.thread_random_keypairs(),
-            };
-
-            builder
-                .create_directory(datadir.clone())?
-                .write_keypair_files()?
-                .write_eth1_data_file()?
-                .build()
-        })
-        .collect()
-}
-
-/// For each `ValidatorDirectory`, submit a deposit transaction to the `eth1_endpoint`.
-///
-/// Returns success as soon as the eth1 endpoint accepts the transaction (i.e., does not wait for
-/// transaction success/revert).
-fn deposit_validators<E: EthSpec>(
-    context: RuntimeContext<E>,
-    eth1_endpoint: String,
-    deposit_contract: Address,
-    validators: Vec<ValidatorDirectory>,
-    account_index: usize,
-    deposit_value: u64,
-    password: Option<String>,
-) -> impl Future<Item = (), Error = ()> {
-    let log_1 = context.log.clone();
-    let log_2 = context.log.clone();
-
-    Http::new(&eth1_endpoint)
-        .map_err(move |e| {
-            error!(
-                log_1,
-                "Failed to start web3 HTTP transport";
-                "error" => format!("{:?}", e)
-            )
-        })
-        .into_future()
-        /*
-         * Loop through the validator directories and submit the deposits.
-         */
-        .and_then(move |(event_loop, transport)| {
-            let web3 = Web3::new(transport);
-
-            futures::stream::iter_ok(validators)
-                .for_each(move |validator| {
-                    let web3 = web3.clone();
-                    let log = log_2.clone();
-                    let password = password.clone();
-
-                    deposit_validator(
-                        web3,
-                        deposit_contract,
-                        &validator,
-                        deposit_value,
-                        account_index,
-                        password,
-                        log,
-                    )
-                })
-                .map(|_| event_loop)
-        })
-        // Web3 gives errors if the event loop is dropped whilst performing requests.
-        .map(|event_loop| drop(event_loop))
-}
-
-/// For the given `ValidatorDirectory`, submit a deposit transaction to the `web3` node.
-///
-/// Returns success as soon as the eth1 endpoint accepts the transaction (i.e., does not wait for
-/// transaction success/revert).
-fn deposit_validator(
-    web3: Web3<Http>,
-    deposit_contract: Address,
-    validator: &ValidatorDirectory,
-    deposit_amount: u64,
-    account_index: usize,
-    password_opt: Option<String>,
-    log: Logger,
-) -> impl Future<Item = (), Error = ()> {
-    validator
-        .voting_keypair
-        .clone()
-        .ok_or_else(|| error!(log, "Validator does not have voting keypair"))
-        .and_then(|voting_keypair| {
-            validator
-                .deposit_data
-                .clone()
-                .ok_or_else(|| error!(log, "Validator does not have deposit data"))
-                .map(|deposit_data| (voting_keypair, deposit_data))
-        })
-        .into_future()
-        .and_then(move |(voting_keypair, deposit_data)| {
-            let pubkey_1 = voting_keypair.pk.clone();
-            let pubkey_2 = voting_keypair.pk.clone();
-
-            let web3_1 = web3.clone();
-            let web3_2 = web3.clone();
-
-            let log_1 = log.clone();
-            let log_2 = log.clone();
-
-            web3.eth()
-                .accounts()
-                .map_err(|e| format!("Failed to get accounts: {:?}", e))
-                .and_then(move |accounts| {
-                    accounts
-                        .get(account_index)
-                        .cloned()
-                        .ok_or_else(|| "Insufficient accounts for deposit".to_string())
-                })
-                /*
-                 * If a password was supplied, unlock the account.
-                 */
-                .and_then(move |from_address| {
-                    let future: Box<dyn Future<Item = Address, Error = String> + Send> =
-                        if let Some(password) = password_opt {
-                            // Unlock for only a single transaction.
-                            let duration = None;
-
-                            let future = web3_1
-                                .personal()
-                                .unlock_account(from_address, &password, duration)
-                                .then(move |result| match result {
-                                    Ok(true) => Ok(from_address),
-                                    Ok(false) => {
-                                        Err("Eth1 node refused to unlock account. Check password."
-                                            .to_string())
-                                    }
-                                    Err(e) => Err(format!("Eth1 unlock request failed: {:?}", e)),
-                                });
-
-                            Box::new(future)
-                        } else {
-                            Box::new(future::ok(from_address))
-                        };
-
-                    future
-                })
-                /*
-                 * Submit the deposit transaction.
-                 */
-                .and_then(move |from| {
-                    let tx_request = TransactionRequest {
-                        from,
-                        to: Some(deposit_contract),
-                        gas: Some(U256::from(DEPOSIT_GAS)),
-                        gas_price: None,
-                        value: Some(U256::from(from_gwei(deposit_amount))),
-                        data: Some(deposit_data.into()),
-                        nonce: None,
-                        condition: None,
-                    };
-
-                    web3_2
-                        .eth()
-                        .send_transaction(tx_request)
-                        .map_err(|e| format!("Failed to call deposit fn: {:?}", e))
-                })
-                .map(move |tx| {
-                    info!(
-                        log_1,
-                        "Validator deposit successful";
-                        "eth1_tx_hash" => format!("{:?}", tx),
-                        "validator_voting_pubkey" => format!("{:?}", pubkey_1)
-                    )
-                })
-                .map_err(move |e| {
-                    error!(
-                        log_2,
-                        "Validator deposit_failed";
-                        "error" => e,
-                        "validator_voting_pubkey" => format!("{:?}", pubkey_2)
-                    )
-                })
-        })
-}
-
-/// Converts gwei to wei.
-fn from_gwei(gwei: u64) -> U256 {
-    U256::from(gwei) * U256::exp10(9)
-}

account_manager/src/validator/create.rs

@@ -0,0 +1,297 @@
+use crate::common::read_wallet_name_from_cli;
+use crate::wallet::create::STDIN_INPUTS_FLAG;
+use crate::{SECRETS_DIR_FLAG, WALLETS_DIR_FLAG};
+use account_utils::{
+    random_password, read_password_from_user, strip_off_newlines, validator_definitions, PlainText,
+};
+use clap::{App, Arg, ArgMatches};
+use directory::{
+    ensure_dir_exists, parse_path_or_default_with_flag, DEFAULT_SECRET_DIR, DEFAULT_WALLET_DIR,
+};
+use environment::Environment;
+use eth2_wallet_manager::WalletManager;
+use slashing_protection::{SlashingDatabase, SLASHING_PROTECTION_FILENAME};
+use std::ffi::OsStr;
+use std::fs;
+use std::path::{Path, PathBuf};
+use types::EthSpec;
+use validator_dir::Builder as ValidatorDirBuilder;
+
+pub const CMD: &str = "create";
+pub const WALLET_NAME_FLAG: &str = "wallet-name";
+pub const WALLET_PASSWORD_FLAG: &str = "wallet-password";
+pub const DEPOSIT_GWEI_FLAG: &str = "deposit-gwei";
+pub const STORE_WITHDRAW_FLAG: &str = "store-withdrawal-keystore";
+pub const COUNT_FLAG: &str = "count";
+pub const AT_MOST_FLAG: &str = "at-most";
+pub const WALLET_PASSWORD_PROMPT: &str = "Enter your wallet's password:";
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD)
+        .about(
+            "Creates new validators from an existing EIP-2386 wallet using the EIP-2333 HD key \
+            derivation scheme.",
+        )
+        .arg(
+            Arg::with_name(WALLET_NAME_FLAG)
+                .long(WALLET_NAME_FLAG)
+                .value_name("WALLET_NAME")
+                .help("Use the wallet identified by this name")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(WALLET_PASSWORD_FLAG)
+                .long(WALLET_PASSWORD_FLAG)
+                .value_name("WALLET_PASSWORD_PATH")
+                .help("A path to a file containing the password which will unlock the wallet.")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(WALLETS_DIR_FLAG)
+                .long(WALLETS_DIR_FLAG)
+                .value_name(WALLETS_DIR_FLAG)
+                .help("A path containing Eth2 EIP-2386 wallets. Defaults to ~/.lighthouse/{network}/wallets")
+                .takes_value(true)
+                .conflicts_with("datadir"),
+        )
+        .arg(
+            Arg::with_name(SECRETS_DIR_FLAG)
+                .long(SECRETS_DIR_FLAG)
+                .value_name("SECRETS_DIR")
+                .help(
+                    "The path where the validator keystore passwords will be stored. \
+                    Defaults to ~/.lighthouse/{network}/secrets",
+                )
+                .conflicts_with("datadir")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(DEPOSIT_GWEI_FLAG)
+                .long(DEPOSIT_GWEI_FLAG)
+                .value_name("DEPOSIT_GWEI")
+                .help(
+                    "The GWEI value of the deposit amount. Defaults to the minimum amount \
+                    required for an active validator (MAX_EFFECTIVE_BALANCE)",
+                )
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(STORE_WITHDRAW_FLAG)
+                .long(STORE_WITHDRAW_FLAG)
+                .help(
+                    "If present, the withdrawal keystore will be stored alongside the voting \
+                    keypair. It is generally recommended to *not* store the withdrawal key and \
+                    instead generate them from the wallet seed when required.",
+                ),
+        )
+        .arg(
+            Arg::with_name(COUNT_FLAG)
+                .long(COUNT_FLAG)
+                .value_name("VALIDATOR_COUNT")
+                .help("The number of validators to create, regardless of how many already exist")
+                .conflicts_with("at-most")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(AT_MOST_FLAG)
+                .long(AT_MOST_FLAG)
+                .value_name("AT_MOST_VALIDATORS")
+                .help(
+                    "Observe the number of validators in --validator-dir, only creating enough to \
+                    reach the given count. Never deletes an existing validator.",
+                )
+                .conflicts_with("count")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(STDIN_INPUTS_FLAG)
+                .takes_value(false)
+                .hidden(cfg!(windows))
+                .long(STDIN_INPUTS_FLAG)
+                .help("If present, read all user inputs from stdin instead of tty."),
+        )
+}
+
+pub fn cli_run<T: EthSpec>(
+    matches: &ArgMatches,
+    mut env: Environment<T>,
+    validator_dir: PathBuf,
+) -> Result<(), String> {
+    let spec = env.core_context().eth2_config.spec;
+
+    let name: Option<String> = clap_utils::parse_optional(matches, WALLET_NAME_FLAG)?;
+    let stdin_inputs = cfg!(windows) || matches.is_present(STDIN_INPUTS_FLAG);
+
+    let wallet_base_dir = if matches.value_of("datadir").is_some() {
+        let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
+        path.join(DEFAULT_WALLET_DIR)
+    } else {
+        parse_path_or_default_with_flag(matches, WALLETS_DIR_FLAG, DEFAULT_WALLET_DIR)?
+    };
+    let secrets_dir = if matches.value_of("datadir").is_some() {
+        let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
+        path.join(DEFAULT_SECRET_DIR)
+    } else {
+        parse_path_or_default_with_flag(matches, SECRETS_DIR_FLAG, DEFAULT_SECRET_DIR)?
+    };
+
+    let deposit_gwei = clap_utils::parse_optional(matches, DEPOSIT_GWEI_FLAG)?
+        .unwrap_or(spec.max_effective_balance);
+    let count: Option<usize> = clap_utils::parse_optional(matches, COUNT_FLAG)?;
+    let at_most: Option<usize> = clap_utils::parse_optional(matches, AT_MOST_FLAG)?;
+
+    // The command will always fail if the wallet dir does not exist.
+    if !wallet_base_dir.exists() {
+        return Err(format!(
+            "No wallet directory at {:?}. Use the `lighthouse --network {} {} {} {}` command to create a wallet",
+            wallet_base_dir,
+            matches.value_of("network").unwrap_or("<NETWORK>"),
+            crate::CMD,
+            crate::wallet::CMD,
+            crate::wallet::create::CMD
+        ));
+    }
+
+    ensure_dir_exists(&validator_dir)?;
+    ensure_dir_exists(&secrets_dir)?;
+
+    eprintln!("secrets-dir path {:?}", secrets_dir);
+    eprintln!("wallets-dir path {:?}", wallet_base_dir);
+
+    let starting_validator_count = existing_validator_count(&validator_dir)?;
+
+    let n = match (count, at_most) {
+        (Some(_), Some(_)) => Err(format!(
+            "Cannot supply --{} and --{}",
+            COUNT_FLAG, AT_MOST_FLAG
+        )),
+        (None, None) => Err(format!(
+            "Must supply either --{} or --{}",
+            COUNT_FLAG, AT_MOST_FLAG
+        )),
+        (Some(count), None) => Ok(count),
+        (None, Some(at_most)) => Ok(at_most.saturating_sub(starting_validator_count)),
+    }?;
+
+    if n == 0 {
+        eprintln!(
+            "No validators to create. {}={:?}, {}={:?}",
+            COUNT_FLAG, count, AT_MOST_FLAG, at_most
+        );
+        return Ok(());
+    }
+
+    let wallet_password_path: Option<PathBuf> =
+        clap_utils::parse_optional(matches, WALLET_PASSWORD_FLAG)?;
+
+    let wallet_name = read_wallet_name_from_cli(name, stdin_inputs)?;
+    let wallet_password = read_wallet_password_from_cli(wallet_password_path, stdin_inputs)?;
+
+    let mgr = WalletManager::open(&wallet_base_dir)
+        .map_err(|e| format!("Unable to open --{}: {:?}", WALLETS_DIR_FLAG, e))?;
+
+    let mut wallet = mgr
+        .wallet_by_name(&wallet_name)
+        .map_err(|e| format!("Unable to open wallet: {:?}", e))?;
+
+    let slashing_protection_path = validator_dir.join(SLASHING_PROTECTION_FILENAME);
+    let slashing_protection =
+        SlashingDatabase::open_or_create(&slashing_protection_path).map_err(|e| {
+            format!(
+                "Unable to open or create slashing protection database at {}: {:?}",
+                slashing_protection_path.display(),
+                e
+            )
+        })?;
+
+    // Create an empty transaction and drops it. Used to test if the database is locked.
+    slashing_protection.test_transaction().map_err(|e| {
+        format!(
+            "Cannot create keys while the validator client is running: {:?}",
+            e
+        )
+    })?;
+
+    for i in 0..n {
+        let voting_password = random_password();
+        let withdrawal_password = random_password();
+
+        let keystores = wallet
+            .next_validator(
+                wallet_password.as_bytes(),
+                voting_password.as_bytes(),
+                withdrawal_password.as_bytes(),
+            )
+            .map_err(|e| format!("Unable to create validator keys: {:?}", e))?;
+
+        let voting_pubkey = keystores.voting.public_key().ok_or_else(|| {
+            format!(
+                "Keystore public key is invalid: {}",
+                keystores.voting.pubkey()
+            )
+        })?;
+
+        slashing_protection
+            .register_validator(voting_pubkey.compress())
+            .map_err(|e| {
+                format!(
+                    "Error registering validator {}: {:?}",
+                    voting_pubkey.as_hex_string(),
+                    e
+                )
+            })?;
+
+        ValidatorDirBuilder::new(validator_dir.clone())
+            .password_dir(secrets_dir.clone())
+            .voting_keystore(keystores.voting, voting_password.as_bytes())
+            .withdrawal_keystore(keystores.withdrawal, withdrawal_password.as_bytes())
+            .create_eth1_tx_data(deposit_gwei, &spec)
+            .store_withdrawal_keystore(matches.is_present(STORE_WITHDRAW_FLAG))
+            .build()
+            .map_err(|e| format!("Unable to build validator directory: {:?}", e))?;
+
+        println!("{}/{}\t{}", i + 1, n, voting_pubkey.as_hex_string());
+    }
+
+    Ok(())
+}
+
+/// Returns the number of validators that exist in the given `validator_dir`.
+///
+/// This function just assumes all files and directories, excluding the validator definitions YAML
+/// and slashing protection database are validator directories, making it likely to return a higher
+/// number than accurate but never a lower one.
+fn existing_validator_count<P: AsRef<Path>>(validator_dir: P) -> Result<usize, String> {
+    fs::read_dir(validator_dir.as_ref())
+        .map(|iter| {
+            iter.filter_map(|e| e.ok())
+                .filter(|e| {
+                    e.file_name() != OsStr::new(validator_definitions::CONFIG_FILENAME)
+                        && e.file_name()
+                            != OsStr::new(slashing_protection::SLASHING_PROTECTION_FILENAME)
+                })
+                .count()
+        })
+        .map_err(|e| format!("Unable to read {:?}: {}", validator_dir.as_ref(), e))
+}
+
+/// Used when a user is accessing an existing wallet. Read in a wallet password from a file if the password file
+/// path is provided. Otherwise, read from an interactive prompt using tty unless the `--stdin-inputs`
+/// flag is provided.
+pub fn read_wallet_password_from_cli(
+    password_file_path: Option<PathBuf>,
+    stdin_inputs: bool,
+) -> Result<PlainText, String> {
+    match password_file_path {
+        Some(path) => fs::read(&path)
+            .map_err(|e| format!("Unable to read {:?}: {:?}", path, e))
+            .map(|bytes| strip_off_newlines(bytes).into()),
+        None => {
+            eprintln!();
+            eprintln!("{}", WALLET_PASSWORD_PROMPT);
+            let password =
+                PlainText::from(read_password_from_user(stdin_inputs)?.as_ref().to_vec());
+            Ok(password)
+        }
+    }
+}

account_manager/src/validator/exit.rs

@@ -0,0 +1,413 @@
+use crate::wallet::create::STDIN_INPUTS_FLAG;
+use bls::{Keypair, PublicKey};
+use clap::{App, Arg, ArgMatches};
+use environment::Environment;
+use eth2::{
+    types::{GenesisData, StateId, ValidatorData, ValidatorId, ValidatorStatus},
+    BeaconNodeHttpClient, Timeouts,
+};
+use eth2_keystore::Keystore;
+use eth2_network_config::Eth2NetworkConfig;
+use safe_arith::SafeArith;
+use sensitive_url::SensitiveUrl;
+use slot_clock::{SlotClock, SystemTimeSlotClock};
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+use tokio::time::sleep;
+use types::{ChainSpec, Epoch, EthSpec, Fork, VoluntaryExit};
+
+pub const CMD: &str = "exit";
+pub const KEYSTORE_FLAG: &str = "keystore";
+pub const PASSWORD_FILE_FLAG: &str = "password-file";
+pub const BEACON_SERVER_FLAG: &str = "beacon-node";
+pub const NO_WAIT: &str = "no-wait";
+pub const PASSWORD_PROMPT: &str = "Enter the keystore password";
+
+pub const DEFAULT_BEACON_NODE: &str = "http://localhost:5052/";
+pub const CONFIRMATION_PHRASE: &str = "Exit my validator";
+pub const WEBSITE_URL: &str = "https://lighthouse-book.sigmaprime.io/voluntary-exit.html";
+pub const PROMPT: &str = "WARNING: WITHDRAWING STAKED ETH IS NOT CURRENTLY POSSIBLE";
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new("exit")
+        .about("Submits a VoluntaryExit to the beacon chain for a given validator keystore.")
+        .arg(
+            Arg::with_name(KEYSTORE_FLAG)
+                .long(KEYSTORE_FLAG)
+                .value_name("KEYSTORE_PATH")
+                .help("The path to the EIP-2335 voting keystore for the validator")
+                .takes_value(true)
+                .required(true),
+        )
+        .arg(
+            Arg::with_name(PASSWORD_FILE_FLAG)
+                .long(PASSWORD_FILE_FLAG)
+                .value_name("PASSWORD_FILE_PATH")
+                .help("The path to the password file which unlocks the validator voting keystore")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(BEACON_SERVER_FLAG)
+                .long(BEACON_SERVER_FLAG)
+                .value_name("NETWORK_ADDRESS")
+                .help("Address to a beacon node HTTP API")
+                .default_value(DEFAULT_BEACON_NODE)
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(NO_WAIT)
+                .long(NO_WAIT)
+                .help("Exits after publishing the voluntary exit without waiting for confirmation that the exit was included in the beacon chain")
+        )
+        .arg(
+            Arg::with_name(STDIN_INPUTS_FLAG)
+                .takes_value(false)
+                .hidden(cfg!(windows))
+                .long(STDIN_INPUTS_FLAG)
+                .help("If present, read all user inputs from stdin instead of tty."),
+        )
+}
+
+pub fn cli_run<E: EthSpec>(matches: &ArgMatches, env: Environment<E>) -> Result<(), String> {
+    let keystore_path: PathBuf = clap_utils::parse_required(matches, KEYSTORE_FLAG)?;
+    let password_file_path: Option<PathBuf> =
+        clap_utils::parse_optional(matches, PASSWORD_FILE_FLAG)?;
+
+    let stdin_inputs = cfg!(windows) || matches.is_present(STDIN_INPUTS_FLAG);
+    let no_wait = matches.is_present(NO_WAIT);
+
+    let spec = env.eth2_config().spec.clone();
+    let server_url: String = clap_utils::parse_required(matches, BEACON_SERVER_FLAG)?;
+    let client = BeaconNodeHttpClient::new(
+        SensitiveUrl::parse(&server_url)
+            .map_err(|e| format!("Failed to parse beacon http server: {:?}", e))?,
+        Timeouts::set_all(Duration::from_secs(env.eth2_config.spec.seconds_per_slot)),
+    );
+
+    let testnet_config = env
+        .testnet
+        .clone()
+        .expect("network should have a valid config");
+
+    env.runtime().block_on(publish_voluntary_exit::<E>(
+        &keystore_path,
+        password_file_path.as_ref(),
+        &client,
+        &spec,
+        stdin_inputs,
+        &testnet_config,
+        no_wait,
+    ))?;
+
+    Ok(())
+}
+
+/// Gets the keypair and validator_index for every validator and calls `publish_voluntary_exit` on it.
+async fn publish_voluntary_exit<E: EthSpec>(
+    keystore_path: &Path,
+    password_file_path: Option<&PathBuf>,
+    client: &BeaconNodeHttpClient,
+    spec: &ChainSpec,
+    stdin_inputs: bool,
+    testnet_config: &Eth2NetworkConfig,
+    no_wait: bool,
+) -> Result<(), String> {
+    let genesis_data = get_geneisis_data(client).await?;
+    let testnet_genesis_root = testnet_config
+        .beacon_state::<E>()
+        .as_ref()
+        .expect("network should have valid genesis state")
+        .genesis_validators_root();
+
+    // Verify that the beacon node and validator being exited are on the same network.
+    if genesis_data.genesis_validators_root != testnet_genesis_root {
+        return Err(
+            "Invalid genesis state. Please ensure that your beacon node is on the same network \
+                 as the validator you are publishing an exit for"
+                .to_string(),
+        );
+    }
+
+    // Return immediately if beacon node is not synced
+    if is_syncing(client).await? {
+        return Err("Beacon node is still syncing".to_string());
+    }
+
+    let keypair = load_voting_keypair(keystore_path, password_file_path, stdin_inputs)?;
+
+    let epoch = get_current_epoch::<E>(genesis_data.genesis_time, spec)
+        .ok_or("Failed to get current epoch. Please check your system time")?;
+    let validator_index = get_validator_index_for_exit(client, &keypair.pk, epoch, spec).await?;
+
+    let fork = get_beacon_state_fork(client).await?;
+    let voluntary_exit = VoluntaryExit {
+        epoch,
+        validator_index,
+    };
+
+    eprintln!(
+        "Publishing a voluntary exit for validator: {} \n",
+        keypair.pk
+    );
+    eprintln!("WARNING: THIS IS AN IRREVERSIBLE OPERATION\n");
+    eprintln!("{}\n", PROMPT);
+    eprintln!(
+        "PLEASE VISIT {} TO MAKE SURE YOU UNDERSTAND THE IMPLICATIONS OF A VOLUNTARY EXIT.",
+        WEBSITE_URL
+    );
+    eprintln!("Enter the exit phrase from the above URL to confirm the voluntary exit: ");
+
+    let confirmation = account_utils::read_input_from_user(stdin_inputs)?;
+    if confirmation == CONFIRMATION_PHRASE {
+        // Sign and publish the voluntary exit to network
+        let signed_voluntary_exit = voluntary_exit.sign(
+            &keypair.sk,
+            &fork,
+            genesis_data.genesis_validators_root,
+            spec,
+        );
+        client
+            .post_beacon_pool_voluntary_exits(&signed_voluntary_exit)
+            .await
+            .map_err(|e| format!("Failed to publish voluntary exit: {}", e))?;
+        tokio::time::sleep(std::time::Duration::from_secs(1)).await; // Provides nicer UX.
+        eprintln!(
+            "Successfully validated and published voluntary exit for validator {}",
+            keypair.pk
+        );
+    } else {
+        eprintln!(
+            "Did not publish voluntary exit for validator {}. Please check that you entered the correct exit phrase.",
+            keypair.pk
+        );
+        return Ok(());
+    }
+
+    if no_wait {
+        return Ok(());
+    }
+
+    loop {
+        // Sleep for a slot duration and then check if voluntary exit was processed
+        // by checking the validator status.
+        sleep(Duration::from_secs(spec.seconds_per_slot)).await;
+
+        let validator_data = get_validator_data(client, &keypair.pk).await?;
+        match validator_data.status {
+            ValidatorStatus::ActiveExiting => {
+                let exit_epoch = validator_data.validator.exit_epoch;
+                let withdrawal_epoch = validator_data.validator.withdrawable_epoch;
+                let current_epoch = get_current_epoch::<E>(genesis_data.genesis_time, spec)
+                    .ok_or("Failed to get current epoch. Please check your system time")?;
+                eprintln!("Voluntary exit has been accepted into the beacon chain, but not yet finalized. \
+                        Finalization may take several minutes or longer. Before finalization there is a low \
+                        probability that the exit may be reverted.");
+                eprintln!(
+                    "Current epoch: {}, Exit epoch: {}, Withdrawable epoch: {}",
+                    current_epoch, exit_epoch, withdrawal_epoch
+                );
+                eprintln!("Please keep your validator running till exit epoch");
+                eprintln!(
+                    "Exit epoch in approximately {} secs",
+                    (exit_epoch - current_epoch) * spec.seconds_per_slot * E::slots_per_epoch()
+                );
+                break;
+            }
+            ValidatorStatus::ExitedSlashed | ValidatorStatus::ExitedUnslashed => {
+                eprintln!(
+                    "Validator has exited on epoch: {}",
+                    validator_data.validator.exit_epoch
+                );
+                break;
+            }
+            _ => eprintln!("Waiting for voluntary exit to be accepted into the beacon chain..."),
+        }
+    }
+
+    Ok(())
+}
+
+/// Get the validator index of a given the validator public key by querying the beacon node endpoint.
+///
+/// Returns an error if the beacon endpoint returns an error or given validator is not eligible for an exit.
+async fn get_validator_index_for_exit(
+    client: &BeaconNodeHttpClient,
+    validator_pubkey: &PublicKey,
+    epoch: Epoch,
+    spec: &ChainSpec,
+) -> Result<u64, String> {
+    let validator_data = get_validator_data(client, validator_pubkey).await?;
+
+    match validator_data.status {
+        ValidatorStatus::ActiveOngoing => {
+            let eligible_epoch = validator_data
+                .validator
+                .activation_epoch
+                .safe_add(spec.shard_committee_period)
+                .map_err(|e| format!("Failed to calculate eligible epoch, validator activation epoch too high: {:?}", e))?;
+
+            if epoch >= eligible_epoch {
+                Ok(validator_data.index)
+            } else {
+                Err(format!(
+                    "Validator {:?} is not eligible for exit. It will become eligible on epoch {}",
+                    validator_pubkey, eligible_epoch
+                ))
+            }
+        }
+        status => Err(format!(
+            "Validator {:?} is not eligible for voluntary exit. Validator status: {:?}",
+            validator_pubkey, status
+        )),
+    }
+}
+
+/// Returns the validator data by querying the beacon node client.
+async fn get_validator_data(
+    client: &BeaconNodeHttpClient,
+    validator_pubkey: &PublicKey,
+) -> Result<ValidatorData, String> {
+    Ok(client
+        .get_beacon_states_validator_id(
+            StateId::Head,
+            &ValidatorId::PublicKey(validator_pubkey.into()),
+        )
+        .await
+        .map_err(|e| format!("Failed to get validator details: {:?}", e))?
+        .ok_or_else(|| {
+            format!(
+                "Validator {} is not present in the beacon state. \
+                Please ensure that your beacon node is synced and the validator has been deposited.",
+                validator_pubkey
+            )
+        })?
+        .data)
+}
+
+/// Get genesis data by querying the beacon node client.
+async fn get_geneisis_data(client: &BeaconNodeHttpClient) -> Result<GenesisData, String> {
+    Ok(client
+        .get_beacon_genesis()
+        .await
+        .map_err(|e| format!("Failed to get beacon genesis: {}", e))?
+        .data)
+}
+
+/// Gets syncing status from beacon node client and returns true if syncing and false otherwise.
+async fn is_syncing(client: &BeaconNodeHttpClient) -> Result<bool, String> {
+    Ok(client
+        .get_node_syncing()
+        .await
+        .map_err(|e| format!("Failed to get sync status: {:?}", e))?
+        .data
+        .is_syncing)
+}
+
+/// Get fork object for the current state by querying the beacon node client.
+async fn get_beacon_state_fork(client: &BeaconNodeHttpClient) -> Result<Fork, String> {
+    Ok(client
+        .get_beacon_states_fork(StateId::Head)
+        .await
+        .map_err(|e| format!("Failed to get get fork: {:?}", e))?
+        .ok_or("Failed to get fork, state not found")?
+        .data)
+}
+
+/// Calculates the current epoch from the genesis time and current time.
+fn get_current_epoch<E: EthSpec>(genesis_time: u64, spec: &ChainSpec) -> Option<Epoch> {
+    let slot_clock = SystemTimeSlotClock::new(
+        spec.genesis_slot,
+        Duration::from_secs(genesis_time),
+        Duration::from_secs(spec.seconds_per_slot),
+    );
+    slot_clock.now().map(|s| s.epoch(E::slots_per_epoch()))
+}
+
+/// Load the voting keypair by loading and decrypting the keystore.
+///
+/// If the `password_file_path` is Some, unlock keystore using password in given file
+/// otherwise, prompts user for a password to unlock the keystore.
+fn load_voting_keypair(
+    voting_keystore_path: &Path,
+    password_file_path: Option<&PathBuf>,
+    stdin_inputs: bool,
+) -> Result<Keypair, String> {
+    let keystore = Keystore::from_json_file(&voting_keystore_path).map_err(|e| {
+        format!(
+            "Unable to read keystore JSON {:?}: {:?}",
+            voting_keystore_path, e
+        )
+    })?;
+
+    // Get password from password file.
+    if let Some(password_file) = password_file_path {
+        validator_dir::unlock_keypair_from_password_path(voting_keystore_path, password_file)
+            .map_err(|e| format!("Error while decrypting keypair: {:?}", e))
+    } else {
+        // Prompt password from user.
+        eprintln!();
+        eprintln!(
+            "{} for validator in {:?}: ",
+            PASSWORD_PROMPT, voting_keystore_path
+        );
+        let password = account_utils::read_password_from_user(stdin_inputs)?;
+        match keystore.decrypt_keypair(password.as_ref()) {
+            Ok(keypair) => {
+                eprintln!("Password is correct.");
+                eprintln!();
+                std::thread::sleep(std::time::Duration::from_secs(1)); // Provides nicer UX.
+                Ok(keypair)
+            }
+            Err(eth2_keystore::Error::InvalidPassword) => Err("Invalid password".to_string()),
+            Err(e) => Err(format!("Error while decrypting keypair: {:?}", e)),
+        }
+    }
+}
+
+#[cfg(test)]
+#[cfg(not(debug_assertions))]
+mod tests {
+    use super::*;
+    use eth2_keystore::KeystoreBuilder;
+    use std::fs::File;
+    use std::io::Write;
+    use tempfile::{tempdir, TempDir};
+
+    const PASSWORD: &str = "cats";
+    const KEYSTORE_NAME: &str = "keystore-m_12381_3600_0_0_0-1595406747.json";
+    const PASSWORD_FILE: &str = "password.pass";
+
+    fn create_and_save_keystore(dir: &TempDir, save_password: bool) -> PublicKey {
+        let keypair = Keypair::random();
+        let keystore = KeystoreBuilder::new(&keypair, PASSWORD.as_bytes(), "".into())
+            .unwrap()
+            .build()
+            .unwrap();
+
+        // Create a keystore.
+        File::create(dir.path().join(KEYSTORE_NAME))
+            .map(|mut file| keystore.to_json_writer(&mut file).unwrap())
+            .unwrap();
+        if save_password {
+            File::create(dir.path().join(PASSWORD_FILE))
+                .map(|mut file| file.write_all(PASSWORD.as_bytes()).unwrap())
+                .unwrap();
+        }
+        keystore.public_key().unwrap()
+    }
+
+    #[test]
+    fn test_load_keypair_password_file() {
+        let dir = tempdir().unwrap();
+        let expected_pk = create_and_save_keystore(&dir, true);
+
+        let kp = load_voting_keypair(
+            &dir.path().join(KEYSTORE_NAME),
+            Some(&dir.path().join(PASSWORD_FILE)),
+            false,
+        )
+        .unwrap();
+
+        assert_eq!(expected_pk, kp.pk.into());
+    }
+}

account_manager/src/validator/import.rs

@@ -0,0 +1,275 @@
+use crate::wallet::create::{PASSWORD_FLAG, STDIN_INPUTS_FLAG};
+use account_utils::{
+    eth2_keystore::Keystore,
+    read_password_from_user,
+    validator_definitions::{
+        recursively_find_voting_keystores, ValidatorDefinition, ValidatorDefinitions,
+        CONFIG_FILENAME,
+    },
+    ZeroizeString,
+};
+use clap::{App, Arg, ArgMatches};
+use slashing_protection::{SlashingDatabase, SLASHING_PROTECTION_FILENAME};
+use std::fs;
+use std::path::PathBuf;
+use std::thread::sleep;
+use std::time::Duration;
+
+pub const CMD: &str = "import";
+pub const KEYSTORE_FLAG: &str = "keystore";
+pub const DIR_FLAG: &str = "directory";
+pub const REUSE_PASSWORD_FLAG: &str = "reuse-password";
+
+pub const PASSWORD_PROMPT: &str = "Enter the keystore password, or press enter to omit it:";
+pub const KEYSTORE_REUSE_WARNING: &str = "DO NOT USE THE ORIGINAL KEYSTORES TO VALIDATE WITH \
+                                          ANOTHER CLIENT, OR YOU WILL GET SLASHED.";
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD)
+        .about(
+            "Imports one or more EIP-2335 passwords into a Lighthouse VC directory, \
+            requesting passwords interactively. The directory flag provides a convenient \
+            method for importing a directory of keys generated by the eth2-deposit-cli \
+            Python utility.",
+        )
+        .arg(
+            Arg::with_name(KEYSTORE_FLAG)
+                .long(KEYSTORE_FLAG)
+                .value_name("KEYSTORE_PATH")
+                .help("Path to a single keystore to be imported.")
+                .conflicts_with(DIR_FLAG)
+                .required_unless(DIR_FLAG)
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(DIR_FLAG)
+                .long(DIR_FLAG)
+                .value_name("KEYSTORES_DIRECTORY")
+                .help(
+                    "Path to a directory which contains zero or more keystores \
+                    for import. This directory and all sub-directories will be \
+                    searched and any file name which contains 'keystore' and \
+                    has the '.json' extension will be attempted to be imported.",
+                )
+                .conflicts_with(KEYSTORE_FLAG)
+                .required_unless(KEYSTORE_FLAG)
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(STDIN_INPUTS_FLAG)
+                .takes_value(false)
+                .hidden(cfg!(windows))
+                .long(STDIN_INPUTS_FLAG)
+                .help("If present, read all user inputs from stdin instead of tty."),
+        )
+        .arg(
+            Arg::with_name(REUSE_PASSWORD_FLAG)
+                .long(REUSE_PASSWORD_FLAG)
+                .help("If present, the same password will be used for all imported keystores."),
+        )
+        .arg(
+            Arg::with_name(PASSWORD_FLAG)
+                .long(PASSWORD_FLAG)
+                .value_name("KEYSTORE_PASSWORD_PATH")
+                .requires(REUSE_PASSWORD_FLAG)
+                .help(
+                    "The path to the file containing the password which will unlock all \
+                    keystores being imported. This flag must be used with `--reuse-password`. \
+                    The password will be copied to the `validator_definitions.yml` file, so after \
+                    import we strongly recommend you delete the file at KEYSTORE_PASSWORD_PATH.",
+                )
+                .takes_value(true),
+        )
+}
+
+pub fn cli_run(matches: &ArgMatches, validator_dir: PathBuf) -> Result<(), String> {
+    let keystore: Option<PathBuf> = clap_utils::parse_optional(matches, KEYSTORE_FLAG)?;
+    let keystores_dir: Option<PathBuf> = clap_utils::parse_optional(matches, DIR_FLAG)?;
+    let stdin_inputs = cfg!(windows) || matches.is_present(STDIN_INPUTS_FLAG);
+    let reuse_password = matches.is_present(REUSE_PASSWORD_FLAG);
+    let keystore_password_path: Option<PathBuf> =
+        clap_utils::parse_optional(matches, PASSWORD_FLAG)?;
+
+    let mut defs = ValidatorDefinitions::open_or_create(&validator_dir)
+        .map_err(|e| format!("Unable to open {}: {:?}", CONFIG_FILENAME, e))?;
+
+    let slashing_protection_path = validator_dir.join(SLASHING_PROTECTION_FILENAME);
+    let slashing_protection =
+        SlashingDatabase::open_or_create(&slashing_protection_path).map_err(|e| {
+            format!(
+                "Unable to open or create slashing protection database at {}: {:?}",
+                slashing_protection_path.display(),
+                e
+            )
+        })?;
+
+    // Create an empty transaction and drop it. Used to test if the database is locked.
+    slashing_protection.test_transaction().map_err(|e| {
+        format!(
+            "Cannot import keys while the validator client is running: {:?}",
+            e
+        )
+    })?;
+
+    // Collect the paths for the keystores that should be imported.
+    let keystore_paths = match (keystore, keystores_dir) {
+        (Some(keystore), None) => vec![keystore],
+        (None, Some(keystores_dir)) => {
+            let mut keystores = vec![];
+
+            recursively_find_voting_keystores(&keystores_dir, &mut keystores)
+                .map_err(|e| format!("Unable to search {:?}: {:?}", keystores_dir, e))?;
+
+            if keystores.is_empty() {
+                eprintln!("No keystores found in {:?}", keystores_dir);
+                return Ok(());
+            }
+
+            keystores
+        }
+        _ => {
+            return Err(format!(
+                "Must supply either --{} or --{}",
+                KEYSTORE_FLAG, DIR_FLAG
+            ))
+        }
+    };
+
+    eprintln!("WARNING: {}", KEYSTORE_REUSE_WARNING);
+
+    // For each keystore:
+    //
+    // - Obtain the keystore password, if the user desires.
+    // - Copy the keystore into the `validator_dir`.
+    // - Register the voting key with the slashing protection database.
+    // - Add the keystore to the validator definitions file.
+    //
+    // Skip keystores that already exist, but exit early if any operation fails.
+    // Reuses the same password for all keystores if the `REUSE_PASSWORD_FLAG` flag is set.
+    let mut num_imported_keystores = 0;
+    let mut previous_password: Option<ZeroizeString> = None;
+
+    for src_keystore in &keystore_paths {
+        let keystore = Keystore::from_json_file(src_keystore)
+            .map_err(|e| format!("Unable to read keystore JSON {:?}: {:?}", src_keystore, e))?;
+
+        eprintln!();
+        eprintln!("Keystore found at {:?}:", src_keystore);
+        eprintln!();
+        eprintln!(" - Public key: 0x{}", keystore.pubkey());
+        eprintln!(" - UUID: {}", keystore.uuid());
+        eprintln!();
+        eprintln!(
+            "If you enter the password it will be stored as plain-text in {} so that it is not \
+             required each time the validator client starts.",
+            CONFIG_FILENAME
+        );
+
+        let password_opt = loop {
+            if let Some(password) = previous_password.clone() {
+                eprintln!("Reuse previous password.");
+                break Some(password);
+            }
+            eprintln!();
+            eprintln!("{}", PASSWORD_PROMPT);
+
+            let password = match keystore_password_path.as_ref() {
+                Some(path) => {
+                    let password_from_file: ZeroizeString = fs::read_to_string(&path)
+                        .map_err(|e| format!("Unable to read {:?}: {:?}", path, e))?
+                        .into();
+                    password_from_file.without_newlines()
+                }
+                None => {
+                    let password_from_user = read_password_from_user(stdin_inputs)?;
+                    if password_from_user.as_ref().is_empty() {
+                        eprintln!("Continuing without password.");
+                        sleep(Duration::from_secs(1)); // Provides nicer UX.
+                        break None;
+                    }
+                    password_from_user
+                }
+            };
+
+            match keystore.decrypt_keypair(password.as_ref()) {
+                Ok(_) => {
+                    eprintln!("Password is correct.");
+                    eprintln!();
+                    sleep(Duration::from_secs(1)); // Provides nicer UX.
+                    if reuse_password {
+                        previous_password = Some(password.clone());
+                    }
+                    break Some(password);
+                }
+                Err(eth2_keystore::Error::InvalidPassword) => {
+                    eprintln!("Invalid password");
+                }
+                Err(e) => return Err(format!("Error whilst decrypting keypair: {:?}", e)),
+            }
+        };
+
+        // The keystore is placed in a directory that matches the name of the public key. This
+        // provides some loose protection against adding the same keystore twice.
+        let dest_dir = validator_dir.join(format!("0x{}", keystore.pubkey()));
+        if dest_dir.exists() {
+            eprintln!(
+                "Skipping import of keystore for existing public key: {:?}",
+                src_keystore
+            );
+            continue;
+        }
+
+        fs::create_dir_all(&dest_dir)
+            .map_err(|e| format!("Unable to create import directory: {:?}", e))?;
+
+        // Retain the keystore file name, but place it in the new directory.
+        let dest_keystore = src_keystore
+            .file_name()
+            .and_then(|file_name| file_name.to_str())
+            .map(|file_name_str| dest_dir.join(file_name_str))
+            .ok_or_else(|| format!("Badly formatted file name: {:?}", src_keystore))?;
+
+        // Copy the keystore to the new location.
+        fs::copy(&src_keystore, &dest_keystore)
+            .map_err(|e| format!("Unable to copy keystore: {:?}", e))?;
+
+        // Register with slashing protection.
+        let voting_pubkey = keystore
+            .public_key()
+            .ok_or_else(|| format!("Keystore public key is invalid: {}", keystore.pubkey()))?;
+        slashing_protection
+            .register_validator(voting_pubkey.compress())
+            .map_err(|e| {
+                format!(
+                    "Error registering validator {}: {:?}",
+                    voting_pubkey.as_hex_string(),
+                    e
+                )
+            })?;
+
+        eprintln!("Successfully imported keystore.");
+        num_imported_keystores += 1;
+
+        let validator_def =
+            ValidatorDefinition::new_keystore_with_password(&dest_keystore, password_opt, None)
+                .map_err(|e| format!("Unable to create new validator definition: {:?}", e))?;
+
+        defs.push(validator_def);
+
+        defs.save(&validator_dir)
+            .map_err(|e| format!("Unable to save {}: {:?}", CONFIG_FILENAME, e))?;
+
+        eprintln!("Successfully updated {}.", CONFIG_FILENAME);
+    }
+
+    eprintln!();
+    eprintln!(
+        "Successfully imported {} validators ({} skipped).",
+        num_imported_keystores,
+        keystore_paths.len() - num_imported_keystores
+    );
+    eprintln!();
+    eprintln!("WARNING: {}", KEYSTORE_REUSE_WARNING);
+
+    Ok(())
+}

account_manager/src/validator/list.rs

@@ -0,0 +1,28 @@
+use account_utils::validator_definitions::ValidatorDefinitions;
+use clap::App;
+use std::path::PathBuf;
+
+pub const CMD: &str = "list";
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD).about("Lists the public keys of all validators.")
+}
+
+pub fn cli_run(validator_dir: PathBuf) -> Result<(), String> {
+    let validator_definitions = ValidatorDefinitions::open(&validator_dir).map_err(|e| {
+        format!(
+            "No validator definitions found in {:?}: {:?}",
+            validator_dir, e
+        )
+    })?;
+
+    for def in validator_definitions.as_slice() {
+        println!(
+            "{} ({})",
+            def.voting_public_key,
+            if def.enabled { "enabled" } else { "disabled" }
+        );
+    }
+
+    Ok(())
+}

account_manager/src/validator/mod.rs

@@ -0,0 +1,65 @@
+pub mod create;
+pub mod exit;
+pub mod import;
+pub mod list;
+pub mod modify;
+pub mod recover;
+pub mod slashing_protection;
+
+use crate::VALIDATOR_DIR_FLAG;
+use clap::{App, Arg, ArgMatches};
+use directory::{parse_path_or_default_with_flag, DEFAULT_VALIDATOR_DIR};
+use environment::Environment;
+use std::path::PathBuf;
+use types::EthSpec;
+
+pub const CMD: &str = "validator";
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD)
+        .about("Provides commands for managing Eth2 validators.")
+        .arg(
+            Arg::with_name(VALIDATOR_DIR_FLAG)
+                .long(VALIDATOR_DIR_FLAG)
+                .value_name("VALIDATOR_DIRECTORY")
+                .help(
+                    "The path to search for validator directories. \
+                    Defaults to ~/.lighthouse/{network}/validators",
+                )
+                .takes_value(true)
+                .conflicts_with("datadir"),
+        )
+        .subcommand(create::cli_app())
+        .subcommand(modify::cli_app())
+        .subcommand(import::cli_app())
+        .subcommand(list::cli_app())
+        .subcommand(recover::cli_app())
+        .subcommand(slashing_protection::cli_app())
+        .subcommand(exit::cli_app())
+}
+
+pub fn cli_run<T: EthSpec>(matches: &ArgMatches, env: Environment<T>) -> Result<(), String> {
+    let validator_base_dir = if matches.value_of("datadir").is_some() {
+        let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
+        path.join(DEFAULT_VALIDATOR_DIR)
+    } else {
+        parse_path_or_default_with_flag(matches, VALIDATOR_DIR_FLAG, DEFAULT_VALIDATOR_DIR)?
+    };
+    eprintln!("validator-dir path: {:?}", validator_base_dir);
+
+    match matches.subcommand() {
+        (create::CMD, Some(matches)) => create::cli_run::<T>(matches, env, validator_base_dir),
+        (modify::CMD, Some(matches)) => modify::cli_run(matches, validator_base_dir),
+        (import::CMD, Some(matches)) => import::cli_run(matches, validator_base_dir),
+        (list::CMD, Some(_)) => list::cli_run(validator_base_dir),
+        (recover::CMD, Some(matches)) => recover::cli_run(matches, validator_base_dir),
+        (slashing_protection::CMD, Some(matches)) => {
+            slashing_protection::cli_run(matches, env, validator_base_dir)
+        }
+        (exit::CMD, Some(matches)) => exit::cli_run(matches, env),
+        (unknown, _) => Err(format!(
+            "{} does not have a {} command. See --help",
+            CMD, unknown
+        )),
+    }
+}

account_manager/src/validator/modify.rs

@@ -0,0 +1,100 @@
+use account_utils::validator_definitions::ValidatorDefinitions;
+use bls::PublicKey;
+use clap::{App, Arg, ArgMatches};
+use std::{collections::HashSet, path::PathBuf};
+
+pub const CMD: &str = "modify";
+pub const ENABLE: &str = "enable";
+pub const DISABLE: &str = "disable";
+
+pub const PUBKEY_FLAG: &str = "pubkey";
+pub const ALL: &str = "all";
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD)
+        .about("Modify validator status in validator_definitions.yml.")
+        .subcommand(
+            App::new(ENABLE)
+                .about("Enable validator(s) in validator_definitions.yml.")
+                .arg(
+                    Arg::with_name(PUBKEY_FLAG)
+                        .long(PUBKEY_FLAG)
+                        .value_name("PUBKEY")
+                        .help("Validator pubkey to enable")
+                        .takes_value(true),
+                )
+                .arg(
+                    Arg::with_name(ALL)
+                        .long(ALL)
+                        .help("Enable all validators in the validator directory")
+                        .takes_value(false)
+                        .conflicts_with(PUBKEY_FLAG),
+                ),
+        )
+        .subcommand(
+            App::new(DISABLE)
+                .about("Disable validator(s) in validator_definitions.yml.")
+                .arg(
+                    Arg::with_name(PUBKEY_FLAG)
+                        .long(PUBKEY_FLAG)
+                        .value_name("PUBKEY")
+                        .help("Validator pubkey to disable")
+                        .takes_value(true),
+                )
+                .arg(
+                    Arg::with_name(ALL)
+                        .long(ALL)
+                        .help("Disable all validators in the validator directory")
+                        .takes_value(false)
+                        .conflicts_with(PUBKEY_FLAG),
+                ),
+        )
+}
+
+pub fn cli_run(matches: &ArgMatches, validator_dir: PathBuf) -> Result<(), String> {
+    // `true` implies we are setting `validator_definition.enabled = true` and
+    // vice versa.
+    let (enabled, sub_matches) = match matches.subcommand() {
+        (ENABLE, Some(sub_matches)) => (true, sub_matches),
+        (DISABLE, Some(sub_matches)) => (false, sub_matches),
+        (unknown, _) => {
+            return Err(format!(
+                "{} does not have a {} command. See --help",
+                CMD, unknown
+            ))
+        }
+    };
+    let mut defs = ValidatorDefinitions::open(&validator_dir).map_err(|e| {
+        format!(
+            "No validator definitions found in {:?}: {:?}",
+            validator_dir, e
+        )
+    })?;
+    let pubkeys_to_modify = if sub_matches.is_present(ALL) {
+        defs.as_slice()
+            .iter()
+            .map(|def| def.voting_public_key.clone())
+            .collect::<HashSet<_>>()
+    } else {
+        let public_key: PublicKey = clap_utils::parse_required(sub_matches, PUBKEY_FLAG)?;
+        std::iter::once(public_key).collect::<HashSet<PublicKey>>()
+    };
+
+    // Modify required entries from  validator_definitions.
+    for def in defs.as_mut_slice() {
+        if pubkeys_to_modify.contains(&def.voting_public_key) {
+            def.enabled = enabled;
+            eprintln!(
+                "Validator {} {}",
+                def.voting_public_key,
+                if enabled { "enabled" } else { "disabled" }
+            );
+        }
+    }
+
+    defs.save(&validator_dir)
+        .map_err(|e| format!("Unable to modify validator definitions: {:?}", e))?;
+
+    eprintln!("\nSuccessfully modified validator_definitions.yml");
+    Ok(())
+}

account_manager/src/validator/recover.rs

@@ -0,0 +1,149 @@
+use super::create::STORE_WITHDRAW_FLAG;
+use crate::common::read_mnemonic_from_cli;
+use crate::validator::create::COUNT_FLAG;
+use crate::wallet::create::STDIN_INPUTS_FLAG;
+use crate::SECRETS_DIR_FLAG;
+use account_utils::eth2_keystore::{keypair_from_secret, Keystore, KeystoreBuilder};
+use account_utils::random_password;
+use clap::{App, Arg, ArgMatches};
+use directory::ensure_dir_exists;
+use directory::{parse_path_or_default_with_flag, DEFAULT_SECRET_DIR};
+use eth2_wallet::bip39::Seed;
+use eth2_wallet::{recover_validator_secret_from_mnemonic, KeyType, ValidatorKeystores};
+use std::path::PathBuf;
+use validator_dir::Builder as ValidatorDirBuilder;
+pub const CMD: &str = "recover";
+pub const FIRST_INDEX_FLAG: &str = "first-index";
+pub const MNEMONIC_FLAG: &str = "mnemonic-path";
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD)
+        .about(
+            "Recovers validator private keys given a BIP-39 mnemonic phrase. \
+            If you did not specify a `--first-index` or count `--count`, by default this will \
+            only recover the keys associated with the validator at index 0 for an HD wallet \
+            in accordance with the EIP-2333 spec.")
+        .arg(
+            Arg::with_name(FIRST_INDEX_FLAG)
+                .long(FIRST_INDEX_FLAG)
+                .value_name("FIRST_INDEX")
+                .help("The first of consecutive key indexes you wish to recover.")
+                .takes_value(true)
+                .required(false)
+                .default_value("0"),
+        )
+        .arg(
+            Arg::with_name(COUNT_FLAG)
+                .long(COUNT_FLAG)
+                .value_name("COUNT")
+                .help("The number of validator keys you wish to recover. Counted consecutively from the provided `--first_index`.")
+                .takes_value(true)
+                .required(false)
+                .default_value("1"),
+        )
+        .arg(
+            Arg::with_name(MNEMONIC_FLAG)
+                .long(MNEMONIC_FLAG)
+                .value_name("MNEMONIC_PATH")
+                .help(
+                    "If present, the mnemonic will be read in from this file.",
+                )
+                .takes_value(true)
+        )
+        .arg(
+            Arg::with_name(SECRETS_DIR_FLAG)
+                .long(SECRETS_DIR_FLAG)
+                .value_name("SECRETS_DIR")
+                .help(
+                    "The path where the validator keystore passwords will be stored. \
+                    Defaults to ~/.lighthouse/{network}/secrets",
+                )
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(STORE_WITHDRAW_FLAG)
+                .long(STORE_WITHDRAW_FLAG)
+                .help(
+                    "If present, the withdrawal keystore will be stored alongside the voting \
+                    keypair. It is generally recommended to *not* store the withdrawal key and \
+                    instead generate them from the wallet seed when required.",
+                ),
+        )
+        .arg(
+            Arg::with_name(STDIN_INPUTS_FLAG)
+                .takes_value(false)
+                .hidden(cfg!(windows))
+                .long(STDIN_INPUTS_FLAG)
+                .help("If present, read all user inputs from stdin instead of tty."),
+        )
+}
+
+pub fn cli_run(matches: &ArgMatches, validator_dir: PathBuf) -> Result<(), String> {
+    let secrets_dir = if matches.value_of("datadir").is_some() {
+        let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
+        path.join(DEFAULT_SECRET_DIR)
+    } else {
+        parse_path_or_default_with_flag(matches, SECRETS_DIR_FLAG, DEFAULT_SECRET_DIR)?
+    };
+    let first_index: u32 = clap_utils::parse_required(matches, FIRST_INDEX_FLAG)?;
+    let count: u32 = clap_utils::parse_required(matches, COUNT_FLAG)?;
+    let mnemonic_path: Option<PathBuf> = clap_utils::parse_optional(matches, MNEMONIC_FLAG)?;
+    let stdin_inputs = cfg!(windows) || matches.is_present(STDIN_INPUTS_FLAG);
+
+    eprintln!("secrets-dir path: {:?}", secrets_dir);
+
+    ensure_dir_exists(&validator_dir)?;
+    ensure_dir_exists(&secrets_dir)?;
+
+    eprintln!();
+    eprintln!("WARNING: KEY RECOVERY CAN LEAD TO DUPLICATING VALIDATORS KEYS, WHICH CAN LEAD TO SLASHING.");
+    eprintln!();
+
+    let mnemonic = read_mnemonic_from_cli(mnemonic_path, stdin_inputs)?;
+
+    let seed = Seed::new(&mnemonic, "");
+
+    for index in first_index..first_index + count {
+        let voting_password = random_password();
+        let withdrawal_password = random_password();
+
+        let derive = |key_type: KeyType, password: &[u8]| -> Result<Keystore, String> {
+            let (secret, path) =
+                recover_validator_secret_from_mnemonic(seed.as_bytes(), index, key_type)
+                    .map_err(|e| format!("Unable to recover validator keys: {:?}", e))?;
+
+            let keypair = keypair_from_secret(secret.as_bytes())
+                .map_err(|e| format!("Unable build keystore: {:?}", e))?;
+
+            KeystoreBuilder::new(&keypair, password, format!("{}", path))
+                .map_err(|e| format!("Unable build keystore: {:?}", e))?
+                .build()
+                .map_err(|e| format!("Unable build keystore: {:?}", e))
+        };
+
+        let keystores = ValidatorKeystores {
+            voting: derive(KeyType::Voting, voting_password.as_bytes())?,
+            withdrawal: derive(KeyType::Withdrawal, withdrawal_password.as_bytes())?,
+        };
+
+        let voting_pubkey = keystores.voting.pubkey().to_string();
+
+        ValidatorDirBuilder::new(validator_dir.clone())
+            .password_dir(secrets_dir.clone())
+            .voting_keystore(keystores.voting, voting_password.as_bytes())
+            .withdrawal_keystore(keystores.withdrawal, withdrawal_password.as_bytes())
+            .store_withdrawal_keystore(matches.is_present(STORE_WITHDRAW_FLAG))
+            .build()
+            .map_err(|e| format!("Unable to build validator directory: {:?}", e))?;
+
+        println!(
+            "{}/{}\tIndex: {}\t0x{}",
+            index - first_index,
+            count - first_index,
+            index,
+            voting_pubkey
+        );
+    }
+
+    Ok(())
+}

account_manager/src/validator/slashing_protection.rs

@@ -0,0 +1,246 @@
+use clap::{App, Arg, ArgMatches};
+use environment::Environment;
+use slashing_protection::{
+    interchange::Interchange, InterchangeError, InterchangeImportOutcome, SlashingDatabase,
+    SLASHING_PROTECTION_FILENAME,
+};
+use std::fs::File;
+use std::path::PathBuf;
+use types::{BeaconState, Epoch, EthSpec, Slot};
+
+pub const CMD: &str = "slashing-protection";
+pub const IMPORT_CMD: &str = "import";
+pub const EXPORT_CMD: &str = "export";
+
+pub const IMPORT_FILE_ARG: &str = "IMPORT-FILE";
+pub const EXPORT_FILE_ARG: &str = "EXPORT-FILE";
+
+pub const MINIFY_FLAG: &str = "minify";
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD)
+        .about("Import or export slashing protection data to or from another client")
+        .subcommand(
+            App::new(IMPORT_CMD)
+                .about("Import an interchange file")
+                .arg(
+                    Arg::with_name(IMPORT_FILE_ARG)
+                        .takes_value(true)
+                        .value_name("FILE")
+                        .help("The slashing protection interchange file to import (.json)"),
+                )
+                .arg(
+                    Arg::with_name(MINIFY_FLAG)
+                        .long(MINIFY_FLAG)
+                        .takes_value(true)
+                        .default_value("true")
+                        .possible_values(&["false", "true"])
+                        .help(
+                            "Minify the input file before processing. This is *much* faster, \
+                             but will not detect slashable data in the input.",
+                        ),
+                ),
+        )
+        .subcommand(
+            App::new(EXPORT_CMD)
+                .about("Export an interchange file")
+                .arg(
+                    Arg::with_name(EXPORT_FILE_ARG)
+                        .takes_value(true)
+                        .value_name("FILE")
+                        .help("The filename to export the interchange file to"),
+                )
+                .arg(
+                    Arg::with_name(MINIFY_FLAG)
+                        .long(MINIFY_FLAG)
+                        .takes_value(true)
+                        .default_value("false")
+                        .possible_values(&["false", "true"])
+                        .help(
+                            "Minify the output file. This will make it smaller and faster to \
+                             import, but not faster to generate.",
+                        ),
+                ),
+        )
+}
+
+pub fn cli_run<T: EthSpec>(
+    matches: &ArgMatches<'_>,
+    env: Environment<T>,
+    validator_base_dir: PathBuf,
+) -> Result<(), String> {
+    let slashing_protection_db_path = validator_base_dir.join(SLASHING_PROTECTION_FILENAME);
+
+    let testnet_config = env
+        .testnet
+        .ok_or("Unable to get testnet configuration from the environment")?;
+
+    let genesis_validators_root = testnet_config
+        .beacon_state::<T>()
+        .map(|state: BeaconState<T>| state.genesis_validators_root())
+        .map_err(|e| {
+            format!(
+                "Unable to get genesis state, has genesis occurred? Detail: {:?}",
+                e
+            )
+        })?;
+
+    match matches.subcommand() {
+        (IMPORT_CMD, Some(matches)) => {
+            let import_filename: PathBuf = clap_utils::parse_required(matches, IMPORT_FILE_ARG)?;
+            let minify: bool = clap_utils::parse_required(matches, MINIFY_FLAG)?;
+            let import_file = File::open(&import_filename).map_err(|e| {
+                format!(
+                    "Unable to open import file at {}: {:?}",
+                    import_filename.display(),
+                    e
+                )
+            })?;
+
+            eprint!("Loading JSON file into memory & deserializing");
+            let mut interchange = Interchange::from_json_reader(&import_file)
+                .map_err(|e| format!("Error parsing file for import: {:?}", e))?;
+            eprintln!(" [done].");
+
+            if minify {
+                eprint!("Minifying input file for faster loading");
+                interchange = interchange
+                    .minify()
+                    .map_err(|e| format!("Minification failed: {:?}", e))?;
+                eprintln!(" [done].");
+            }
+
+            let slashing_protection_database =
+                SlashingDatabase::open_or_create(&slashing_protection_db_path).map_err(|e| {
+                    format!(
+                        "Unable to open database at {}: {:?}",
+                        slashing_protection_db_path.display(),
+                        e
+                    )
+                })?;
+
+            let display_slot = |slot: Option<Slot>| {
+                slot.map_or("none".to_string(), |slot| format!("{}", slot.as_u64()))
+            };
+            let display_epoch = |epoch: Option<Epoch>| {
+                epoch.map_or("?".to_string(), |epoch| format!("{}", epoch.as_u64()))
+            };
+            let display_attestation = |source, target| match (source, target) {
+                (None, None) => "none".to_string(),
+                (source, target) => format!("{}=>{}", display_epoch(source), display_epoch(target)),
+            };
+
+            match slashing_protection_database
+                .import_interchange_info(interchange, genesis_validators_root)
+            {
+                Ok(outcomes) => {
+                    eprintln!("All records imported successfully:");
+                    for outcome in &outcomes {
+                        match outcome {
+                            InterchangeImportOutcome::Success { pubkey, summary } => {
+                                eprintln!("- {:?}", pubkey);
+                                eprintln!(
+                                    "    - min block: {}",
+                                    display_slot(summary.min_block_slot)
+                                );
+                                eprintln!(
+                                    "    - min attestation: {}",
+                                    display_attestation(
+                                        summary.min_attestation_source,
+                                        summary.min_attestation_target
+                                    )
+                                );
+                                eprintln!(
+                                    "    - max attestation: {}",
+                                    display_attestation(
+                                        summary.max_attestation_source,
+                                        summary.max_attestation_target
+                                    )
+                                );
+                            }
+                            InterchangeImportOutcome::Failure { pubkey, error } => {
+                                panic!(
+                                    "import should be atomic, but key {:?} was imported despite error: {:?}",
+                                    pubkey, error
+                                );
+                            }
+                        }
+                    }
+                }
+                Err(InterchangeError::AtomicBatchAborted(outcomes)) => {
+                    eprintln!("ERROR, slashable data in input:");
+                    for outcome in &outcomes {
+                        if let InterchangeImportOutcome::Failure { pubkey, error } = outcome {
+                            eprintln!("- {:?}", pubkey);
+                            eprintln!("    - error: {:?}", error);
+                        }
+                    }
+                    return Err(
+                        "ERROR: import aborted due to slashable data, see above.\n\
+                         Please see https://lighthouse-book.sigmaprime.io/slashing-protection.html#slashable-data-in-import\n\
+                         IT IS NOT SAFE TO START VALIDATING".to_string()
+                    );
+                }
+                Err(e) => {
+                    return Err(format!(
+                        "Fatal error during import: {:?}\n\
+                         IT IS NOT SAFE TO START VALIDATING",
+                        e
+                    ));
+                }
+            }
+
+            eprintln!("Import completed successfully.");
+            eprintln!(
+                "Please double-check that the minimum and maximum blocks and attestations above \
+                 match your expectations."
+            );
+
+            Ok(())
+        }
+        (EXPORT_CMD, Some(matches)) => {
+            let export_filename: PathBuf = clap_utils::parse_required(matches, EXPORT_FILE_ARG)?;
+            let minify: bool = clap_utils::parse_required(matches, MINIFY_FLAG)?;
+
+            if !slashing_protection_db_path.exists() {
+                return Err(format!(
+                    "No slashing protection database exists at: {}",
+                    slashing_protection_db_path.display()
+                ));
+            }
+
+            let slashing_protection_database = SlashingDatabase::open(&slashing_protection_db_path)
+                .map_err(|e| {
+                    format!(
+                        "Unable to open database at {}: {:?}",
+                        slashing_protection_db_path.display(),
+                        e
+                    )
+                })?;
+
+            let mut interchange = slashing_protection_database
+                .export_interchange_info(genesis_validators_root)
+                .map_err(|e| format!("Error during export: {:?}", e))?;
+
+            if minify {
+                eprintln!("Minifying output file");
+                interchange = interchange
+                    .minify()
+                    .map_err(|e| format!("Unable to minify output: {:?}", e))?;
+            }
+
+            let output_file = File::create(export_filename)
+                .map_err(|e| format!("Error creating output file: {:?}", e))?;
+
+            interchange
+                .write_to(&output_file)
+                .map_err(|e| format!("Error writing output file: {:?}", e))?;
+
+            eprintln!("Export completed successfully");
+
+            Ok(())
+        }
+        ("", _) => Err("No subcommand provided, see --help for options".to_string()),
+        (command, _) => Err(format!("No such subcommand `{}`", command)),
+    }
+}

account_manager/src/wallet/create.rs

@@ -0,0 +1,238 @@
+use crate::common::read_wallet_name_from_cli;
+use crate::WALLETS_DIR_FLAG;
+use account_utils::{
+    is_password_sufficiently_complex, random_password, read_password_from_user, strip_off_newlines,
+};
+use clap::{App, Arg, ArgMatches};
+use eth2_wallet::{
+    bip39::{Language, Mnemonic, MnemonicType},
+    PlainText,
+};
+use eth2_wallet_manager::{LockedWallet, WalletManager, WalletType};
+use filesystem::create_with_600_perms;
+use std::ffi::OsStr;
+use std::fs;
+use std::path::{Path, PathBuf};
+
+pub const CMD: &str = "create";
+pub const HD_TYPE: &str = "hd";
+pub const NAME_FLAG: &str = "name";
+pub const PASSWORD_FLAG: &str = "password-file";
+pub const TYPE_FLAG: &str = "type";
+pub const MNEMONIC_FLAG: &str = "mnemonic-output-path";
+pub const STDIN_INPUTS_FLAG: &str = "stdin-inputs";
+pub const MNEMONIC_LENGTH_FLAG: &str = "mnemonic-length";
+pub const MNEMONIC_TYPES: &[MnemonicType] = &[
+    MnemonicType::Words12,
+    MnemonicType::Words15,
+    MnemonicType::Words18,
+    MnemonicType::Words21,
+    MnemonicType::Words24,
+];
+pub const NEW_WALLET_PASSWORD_PROMPT: &str =
+    "Enter a password for your new wallet that is at least 12 characters long:";
+pub const RETYPE_PASSWORD_PROMPT: &str = "Please re-enter your wallet's new password:";
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD)
+        .about("Creates a new HD (hierarchical-deterministic) EIP-2386 wallet.")
+        .arg(
+            Arg::with_name(NAME_FLAG)
+                .long(NAME_FLAG)
+                .value_name("WALLET_NAME")
+                .help(
+                    "The wallet will be created with this name. It is not allowed to \
+                            create two wallets with the same name for the same --base-dir.",
+                )
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(PASSWORD_FLAG)
+                .long(PASSWORD_FLAG)
+                .value_name("WALLET_PASSWORD_PATH")
+                .help(
+                    "A path to a file containing the password which will unlock the wallet. \
+                    If the file does not exist, a random password will be generated and \
+                    saved at that path. To avoid confusion, if the file does not already \
+                    exist it must include a '.pass' suffix.",
+                )
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(TYPE_FLAG)
+                .long(TYPE_FLAG)
+                .value_name("WALLET_TYPE")
+                .help(
+                    "The type of wallet to create. Only HD (hierarchical-deterministic) \
+                            wallets are supported presently..",
+                )
+                .takes_value(true)
+                .possible_values(&[HD_TYPE])
+                .default_value(HD_TYPE),
+        )
+        .arg(
+            Arg::with_name(MNEMONIC_FLAG)
+                .long(MNEMONIC_FLAG)
+                .value_name("MNEMONIC_PATH")
+                .help(
+                    "If present, the mnemonic will be saved to this file. DO NOT SHARE THE MNEMONIC.",
+                )
+                .takes_value(true)
+        )
+        .arg(
+            Arg::with_name(STDIN_INPUTS_FLAG)
+                .takes_value(false)
+                .hidden(cfg!(windows))
+                .long(STDIN_INPUTS_FLAG)
+                .help("If present, read all user inputs from stdin instead of tty."),
+        )
+        .arg(
+            Arg::with_name(MNEMONIC_LENGTH_FLAG)
+                .long(MNEMONIC_LENGTH_FLAG)
+                .value_name("MNEMONIC_LENGTH")
+                .help("The number of words to use for the mnemonic phrase.")
+                .takes_value(true)
+                .validator(|len| {
+                    match len.parse::<usize>().ok().and_then(|words| MnemonicType::for_word_count(words).ok()) {
+                        Some(_) => Ok(()),
+                        None => Err(format!("Mnemonic length must be one of {}", MNEMONIC_TYPES.iter().map(|t| t.word_count().to_string()).collect::<Vec<_>>().join(", "))),
+                    }
+                })
+                .default_value("24"),
+        )
+}
+
+pub fn cli_run(matches: &ArgMatches, wallet_base_dir: PathBuf) -> Result<(), String> {
+    let mnemonic_output_path: Option<PathBuf> = clap_utils::parse_optional(matches, MNEMONIC_FLAG)?;
+
+    // Create a new random mnemonic.
+    //
+    // The `tiny-bip39` crate uses `thread_rng()` for this entropy.
+    let mnemonic_length = clap_utils::parse_required(matches, MNEMONIC_LENGTH_FLAG)?;
+    let mnemonic = Mnemonic::new(
+        MnemonicType::for_word_count(mnemonic_length).expect("Mnemonic length already validated"),
+        Language::English,
+    );
+
+    let wallet = create_wallet_from_mnemonic(matches, wallet_base_dir.as_path(), &mnemonic)?;
+
+    if let Some(path) = mnemonic_output_path {
+        create_with_600_perms(&path, mnemonic.phrase().as_bytes())
+            .map_err(|e| format!("Unable to write mnemonic to {:?}: {:?}", path, e))?;
+    }
+
+    println!("Your wallet's {}-word BIP-39 mnemonic is:", mnemonic_length);
+    println!();
+    println!("\t{}", mnemonic.phrase());
+    println!();
+    println!("This mnemonic can be used to fully restore your wallet, should ");
+    println!("you lose the JSON file or your password. ");
+    println!();
+    println!("It is very important that you DO NOT SHARE this mnemonic as it will ");
+    println!("reveal the private keys of all validators and keys generated with  ");
+    println!("this wallet. That would be catastrophic.");
+    println!();
+    println!("It is also important to store a backup of this mnemonic so you can ");
+    println!("recover your private keys in the case of data loss. Writing it on ");
+    println!("a piece of paper and storing it in a safe place would be prudent.");
+    println!();
+    println!("Your wallet's UUID is:");
+    println!();
+    println!("\t{}", wallet.wallet().uuid());
+    println!();
+    println!("You do not need to backup your UUID or keep it secret.");
+
+    Ok(())
+}
+
+pub fn create_wallet_from_mnemonic(
+    matches: &ArgMatches,
+    wallet_base_dir: &Path,
+    mnemonic: &Mnemonic,
+) -> Result<LockedWallet, String> {
+    let name: Option<String> = clap_utils::parse_optional(matches, NAME_FLAG)?;
+    let wallet_password_path: Option<PathBuf> = clap_utils::parse_optional(matches, PASSWORD_FLAG)?;
+    let type_field: String = clap_utils::parse_required(matches, TYPE_FLAG)?;
+    let stdin_inputs = cfg!(windows) || matches.is_present(STDIN_INPUTS_FLAG);
+    let wallet_type = match type_field.as_ref() {
+        HD_TYPE => WalletType::Hd,
+        unknown => return Err(format!("--{} {} is not supported", TYPE_FLAG, unknown)),
+    };
+
+    let mgr = WalletManager::open(&wallet_base_dir)
+        .map_err(|e| format!("Unable to open --{}: {:?}", WALLETS_DIR_FLAG, e))?;
+
+    let wallet_password: PlainText = match wallet_password_path {
+        Some(path) => {
+            // Create a random password if the file does not exist.
+            if !path.exists() {
+                // To prevent users from accidentally supplying their password to the PASSWORD_FLAG and
+                // create a file with that name, we require that the password has a .pass suffix.
+                if path.extension() != Some(OsStr::new("pass")) {
+                    return Err(format!(
+                        "Only creates a password file if that file ends in .pass: {:?}",
+                        path
+                    ));
+                }
+
+                create_with_600_perms(&path, random_password().as_bytes())
+                    .map_err(|e| format!("Unable to write to {:?}: {:?}", path, e))?;
+            }
+            read_new_wallet_password_from_cli(Some(path), stdin_inputs)?
+        }
+        None => read_new_wallet_password_from_cli(None, stdin_inputs)?,
+    };
+
+    let wallet_name = read_wallet_name_from_cli(name, stdin_inputs)?;
+
+    let wallet = mgr
+        .create_wallet(
+            wallet_name,
+            wallet_type,
+            mnemonic,
+            wallet_password.as_bytes(),
+        )
+        .map_err(|e| format!("Unable to create wallet: {:?}", e))?;
+    Ok(wallet)
+}
+
+/// Used when a user is creating a new wallet. Read in a wallet password from a file if the password file
+/// path is provided. Otherwise, read from an interactive prompt using tty unless the `--stdin-inputs`
+/// flag is provided. This verifies the password complexity and verifies the password is correctly re-entered.
+pub fn read_new_wallet_password_from_cli(
+    password_file_path: Option<PathBuf>,
+    stdin_inputs: bool,
+) -> Result<PlainText, String> {
+    match password_file_path {
+        Some(path) => {
+            let password: PlainText = fs::read(&path)
+                .map_err(|e| format!("Unable to read {:?}: {:?}", path, e))
+                .map(|bytes| strip_off_newlines(bytes).into())?;
+
+            // Ensure the password meets the minimum requirements.
+            is_password_sufficiently_complex(password.as_bytes())?;
+            Ok(password)
+        }
+        None => loop {
+            eprintln!();
+            eprintln!("{}", NEW_WALLET_PASSWORD_PROMPT);
+            let password =
+                PlainText::from(read_password_from_user(stdin_inputs)?.as_ref().to_vec());
+
+            // Ensure the password meets the minimum requirements.
+            match is_password_sufficiently_complex(password.as_bytes()) {
+                Ok(_) => {
+                    eprintln!("{}", RETYPE_PASSWORD_PROMPT);
+                    let retyped_password =
+                        PlainText::from(read_password_from_user(stdin_inputs)?.as_ref().to_vec());
+                    if retyped_password == password {
+                        break Ok(password);
+                    } else {
+                        eprintln!("Passwords do not match.");
+                    }
+                }
+                Err(message) => eprintln!("{}", message),
+            }
+        },
+    }
+}

account_manager/src/wallet/list.rs

@@ -0,0 +1,24 @@
+use crate::WALLETS_DIR_FLAG;
+use clap::App;
+use eth2_wallet_manager::WalletManager;
+use std::path::PathBuf;
+
+pub const CMD: &str = "list";
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD).about("Lists the names of all wallets.")
+}
+
+pub fn cli_run(wallet_base_dir: PathBuf) -> Result<(), String> {
+    let mgr = WalletManager::open(&wallet_base_dir)
+        .map_err(|e| format!("Unable to open --{}: {:?}", WALLETS_DIR_FLAG, e))?;
+
+    for (name, _uuid) in mgr
+        .wallets()
+        .map_err(|e| format!("Unable to list wallets: {:?}", e))?
+    {
+        println!("{}", name)
+    }
+
+    Ok(())
+}

account_manager/src/wallet/mod.rs

@@ -0,0 +1,48 @@
+pub mod create;
+pub mod list;
+pub mod recover;
+
+use crate::WALLETS_DIR_FLAG;
+use clap::{App, Arg, ArgMatches};
+use directory::{ensure_dir_exists, parse_path_or_default_with_flag, DEFAULT_WALLET_DIR};
+use std::path::PathBuf;
+
+pub const CMD: &str = "wallet";
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD)
+        .about("Manage wallets, from which validator keys can be derived.")
+        .arg(
+            Arg::with_name(WALLETS_DIR_FLAG)
+                .long(WALLETS_DIR_FLAG)
+                .value_name("WALLETS_DIRECTORY")
+                .help("A path containing Eth2 EIP-2386 wallets. Defaults to ~/.lighthouse/{network}/wallets")
+                .takes_value(true)
+                .conflicts_with("datadir"),
+        )
+        .subcommand(create::cli_app())
+        .subcommand(list::cli_app())
+        .subcommand(recover::cli_app())
+}
+
+pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
+    let wallet_base_dir = if matches.value_of("datadir").is_some() {
+        let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
+        path.join(DEFAULT_WALLET_DIR)
+    } else {
+        parse_path_or_default_with_flag(matches, WALLETS_DIR_FLAG, DEFAULT_WALLET_DIR)?
+    };
+    ensure_dir_exists(&wallet_base_dir)?;
+
+    eprintln!("wallet-dir path: {:?}", wallet_base_dir);
+
+    match matches.subcommand() {
+        (create::CMD, Some(matches)) => create::cli_run(matches, wallet_base_dir),
+        (list::CMD, Some(_)) => list::cli_run(wallet_base_dir),
+        (recover::CMD, Some(matches)) => recover::cli_run(matches, wallet_base_dir),
+        (unknown, _) => Err(format!(
+            "{} does not have a {} command. See --help",
+            CMD, unknown
+        )),
+    }
+}

account_manager/src/wallet/recover.rs

@@ -0,0 +1,86 @@
+use crate::common::read_mnemonic_from_cli;
+use crate::wallet::create::{create_wallet_from_mnemonic, STDIN_INPUTS_FLAG};
+use crate::wallet::create::{HD_TYPE, NAME_FLAG, PASSWORD_FLAG, TYPE_FLAG};
+use clap::{App, Arg, ArgMatches};
+use std::path::PathBuf;
+
+pub const CMD: &str = "recover";
+pub const MNEMONIC_FLAG: &str = "mnemonic-path";
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD)
+        .about("Recovers an EIP-2386 wallet from a given a BIP-39 mnemonic phrase.")
+        .arg(
+            Arg::with_name(NAME_FLAG)
+                .long(NAME_FLAG)
+                .value_name("WALLET_NAME")
+                .help(
+                    "The wallet will be created with this name. It is not allowed to \
+                            create two wallets with the same name for the same --base-dir.",
+                )
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(PASSWORD_FLAG)
+                .long(PASSWORD_FLAG)
+                .value_name("PASSWORD_FILE_PATH")
+                .help(
+                    "This will be the new password for your recovered wallet. \
+                    A path to a file containing the password which will unlock the wallet. \
+                    If the file does not exist, a random password will be generated and \
+                    saved at that path. To avoid confusion, if the file does not already \
+                    exist it must include a '.pass' suffix.",
+                )
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(MNEMONIC_FLAG)
+                .long(MNEMONIC_FLAG)
+                .value_name("MNEMONIC_PATH")
+                .help("If present, the mnemonic will be read in from this file.")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(TYPE_FLAG)
+                .long(TYPE_FLAG)
+                .value_name("WALLET_TYPE")
+                .help(
+                    "The type of wallet to create. Only HD (hierarchical-deterministic) \
+                            wallets are supported presently..",
+                )
+                .takes_value(true)
+                .possible_values(&[HD_TYPE])
+                .default_value(HD_TYPE),
+        )
+        .arg(
+            Arg::with_name(STDIN_INPUTS_FLAG)
+                .takes_value(false)
+                .hidden(cfg!(windows))
+                .long(STDIN_INPUTS_FLAG)
+                .help("If present, read all user inputs from stdin instead of tty."),
+        )
+}
+
+pub fn cli_run(matches: &ArgMatches, wallet_base_dir: PathBuf) -> Result<(), String> {
+    let mnemonic_path: Option<PathBuf> = clap_utils::parse_optional(matches, MNEMONIC_FLAG)?;
+    let stdin_inputs = cfg!(windows) || matches.is_present(STDIN_INPUTS_FLAG);
+
+    eprintln!();
+    eprintln!("WARNING: KEY RECOVERY CAN LEAD TO DUPLICATING VALIDATORS KEYS, WHICH CAN LEAD TO SLASHING.");
+    eprintln!();
+
+    let mnemonic = read_mnemonic_from_cli(mnemonic_path, stdin_inputs)?;
+
+    let wallet = create_wallet_from_mnemonic(matches, wallet_base_dir.as_path(), &mnemonic)
+        .map_err(|e| format!("Unable to create wallet: {:?}", e))?;
+
+    println!("Your wallet has been successfully recovered.");
+    println!();
+    println!("Your wallet's UUID is:");
+    println!();
+    println!("\t{}", wallet.wallet().uuid());
+    println!();
+    println!("You do not need to backup your UUID or keep it secret.");
+
+    Ok(())
+}

beacon_node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "beacon_node"
-version = "0.1.0"
+version = "1.5.0-rc.1"
 authors = ["Paul Hauner <paul@paulhauner.com>", "Age Manning <Age@AgeManning.com"]
 edition = "2018"
 
@@ -9,31 +9,40 @@ name = "beacon_node"
 path = "src/lib.rs"
 
 [dev-dependencies]
-node_test_rig = { path = "../tests/node_test_rig" }
+node_test_rig = { path = "../testing/node_test_rig" }
+
+[features]
+write_ssz_files = ["beacon_chain/write_ssz_files"]  # Writes debugging .ssz files to /tmp during block processing.
 
 [dependencies]
-eth2_config = { path = "../eth2/utils/eth2_config" }
-lighthouse_bootstrap = { path = "../eth2/utils/lighthouse_bootstrap" }
+eth2_config = { path = "../common/eth2_config" }
 beacon_chain = { path = "beacon_chain" }
-types = { path = "../eth2/types" }
+types = { path = "../consensus/types" }
 store = { path = "./store" }
 client = { path = "client" }
-version = { path = "version" }
-clap = "2.33.0"
-rand = "0.7.2"
+clap = "2.33.3"
+rand = "0.7.3"
 slog = { version = "2.5.2", features = ["max_level_trace", "release_max_level_trace"] }
-slog-term = "2.4.2"
-slog-async = "2.3.0"
-ctrlc = { version = "3.1.3", features = ["termination"] }
-tokio = "0.1.22"
-tokio-timer = "0.2.12"
-exit-future = "0.1.4"
-env_logger = "0.7.1"
-dirs = "2.0.2"
-logging = { path = "../eth2/utils/logging" }
-futures = "0.1.29"
+slog-term = "2.6.0"
+slog-async = "2.5.0"
+ctrlc = { version = "3.1.6", features = ["termination"] }
+tokio = { version = "1.10.0", features = ["time"] }
+exit-future = "0.2.0"
+dirs = "3.0.1"
+logging = { path = "../common/logging" }
+directory = {path = "../common/directory"}
+futures = "0.3.7"
 environment = { path = "../lighthouse/environment" }
+task_executor = { path = "../common/task_executor" }
 genesis = { path = "genesis" }
-eth2_testnet_config = { path = "../eth2/utils/eth2_testnet_config" }
-eth2-libp2p = { path = "./eth2-libp2p" }
-eth2_ssz = { path = "../eth2/utils/ssz" }
+eth2_network_config = { path = "../common/eth2_network_config" }
+eth2_libp2p = { path = "./eth2_libp2p" }
+eth2_ssz = "0.1.2"
+serde = "1.0.116"
+clap_utils = { path = "../common/clap_utils" }
+hyper = "0.14.4"
+lighthouse_version = { path = "../common/lighthouse_version" }
+hex = "0.4.2"
+slasher = { path = "../slasher" }
+monitoring_api = { path = "../common/monitoring_api" }
+sensitive_url = { path = "../common/sensitive_url" }

beacon_node/beacon_chain/Cargo.toml

@@ -1,48 +1,65 @@
 [package]
 name = "beacon_chain"
-version = "0.1.0"
+version = "0.2.0"
 authors = ["Paul Hauner <paul@paulhauner.com>", "Age Manning <Age@AgeManning.com>"]
 edition = "2018"
 
 [features]
-
+default = ["participation_metrics"]
 write_ssz_files = []  # Writes debugging .ssz files to /tmp during block processing.
-
-[dependencies]
-eth2_config = { path = "../../eth2/utils/eth2_config" }
-merkle_proof = { path = "../../eth2/utils/merkle_proof" }
-store = { path = "../store" }
-parking_lot = "0.9.0"
-lazy_static = "1.4.0"
-lighthouse_metrics = { path = "../../eth2/utils/lighthouse_metrics" }
-lighthouse_bootstrap = { path = "../../eth2/utils/lighthouse_bootstrap" }
-log = "0.4.8"
-operation_pool = { path = "../../eth2/operation_pool" }
-rayon = "1.2.0"
-serde = "1.0.102"
-serde_derive = "1.0.102"
-serde_yaml = "0.8.11"
-serde_json = "1.0.41"
-slog = { version = "2.5.2", features = ["max_level_trace"] }
-sloggers = "0.3.4"
-slot_clock = { path = "../../eth2/utils/slot_clock" }
-eth2_hashing = "0.1.0"
-eth2_ssz = "0.1.2"
-eth2_ssz_types = { path = "../../eth2/utils/ssz_types" }
-eth2_ssz_derive = "0.1.0"
-state_processing = { path = "../../eth2/state_processing" }
-tree_hash = "0.1.0"
-types = { path = "../../eth2/types" }
-lmd_ghost = { path = "../../eth2/lmd_ghost" }
-eth1 = { path = "../eth1" }
-websocket_server = { path = "../websocket_server" }
-futures = "0.1.25"
-exit-future = "0.1.3"
-genesis = { path = "../genesis" }
-integer-sqrt = "0.1"
-rand = "0.7.2"
+participation_metrics = []  # Exposes validator participation metrics to Prometheus.
+test_logger = [] # Print log output to stderr when running tests instead of dropping it
+fork_from_env = [] # Initialise the harness chain spec from the FORK_NAME env variable
 
 [dev-dependencies]
-tempfile = "3.1.0"
-lazy_static = "1.4.0"
+maplit = "1.0.2"
 environment = { path = "../../lighthouse/environment" }
+
+[dependencies]
+eth2_config = { path = "../../common/eth2_config" }
+merkle_proof = { path = "../../consensus/merkle_proof" }
+store = { path = "../store" }
+parking_lot = "0.11.0"
+lazy_static = "1.4.0"
+smallvec = "1.6.1"
+lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
+log = "0.4.11"
+operation_pool = { path = "../operation_pool" }
+rayon = "1.4.1"
+serde = "1.0.116"
+serde_derive = "1.0.116"
+serde_yaml = "0.8.13"
+serde_json = "1.0.58"
+slog = { version = "2.5.2", features = ["max_level_trace"] }
+sloggers = "1.0.1"
+slot_clock = { path = "../../common/slot_clock" }
+eth2_hashing = "0.1.0"
+eth2_ssz = "0.1.2"
+eth2_ssz_types = { path = "../../consensus/ssz_types" }
+eth2_ssz_derive = "0.1.0"
+state_processing = { path = "../../consensus/state_processing" }
+tree_hash = "0.1.1"
+types = { path = "../../consensus/types" }
+tokio = "1.10.0"
+eth1 = { path = "../eth1" }
+futures = "0.3.7"
+genesis = { path = "../genesis" }
+integer-sqrt = "0.1.5"
+int_to_bytes = { path = "../../consensus/int_to_bytes" }
+rand = "0.7.3"
+rand_core = "0.6.2"
+proto_array = { path = "../../consensus/proto_array" }
+lru = "0.6.0"
+tempfile = "3.1.0"
+bitvec = "0.19.3"
+bls = { path = "../../crypto/bls" }
+safe_arith = { path = "../../consensus/safe_arith" }
+fork_choice = { path = "../../consensus/fork_choice" }
+task_executor = { path = "../../common/task_executor" }
+derivative = "2.1.1"
+itertools = "0.10.0"
+regex = "1.3.9"
+exit-future = "0.2.0"
+slasher = { path = "../../slasher" }
+eth2 = { path = "../../common/eth2" }
+strum = { version = "0.21.0", features = ["derive"] }

beacon_node/beacon_chain/src/attester_cache.rs

@@ -0,0 +1,377 @@
+//! This module provides the `AttesterCache`, a cache designed for reducing state-reads when
+//! validators produce `AttestationData`.
+//!
+//! This cache is required *as well as* the `ShufflingCache` since the `ShufflingCache` does not
+//! provide any information about the `state.current_justified_checkpoint`. It is not trivial to add
+//! the justified checkpoint to the `ShufflingCache` since that cache is keyed by shuffling decision
+//! root, which is not suitable for the justified checkpoint. Whilst we can know the shuffling for
+//! epoch `n` during `n - 1`, we *cannot* know the justified checkpoint. Instead, we *must* perform
+//! `per_epoch_processing` to transform the state from epoch `n - 1` to epoch `n` so that rewards
+//! and penalties can be computed and the `state.current_justified_checkpoint` can be updated.
+
+use crate::{BeaconChain, BeaconChainError, BeaconChainTypes};
+use parking_lot::RwLock;
+use state_processing::state_advance::{partial_state_advance, Error as StateAdvanceError};
+use std::collections::HashMap;
+use std::ops::Range;
+use types::{
+    beacon_state::{
+        compute_committee_index_in_epoch, compute_committee_range_in_epoch, epoch_committee_count,
+    },
+    BeaconState, BeaconStateError, ChainSpec, Checkpoint, Epoch, EthSpec, Hash256, RelativeEpoch,
+    Slot,
+};
+
+type JustifiedCheckpoint = Checkpoint;
+type CommitteeLength = usize;
+type CommitteeIndex = u64;
+type CacheHashMap = HashMap<AttesterCacheKey, AttesterCacheValue>;
+
+/// The maximum number of `AttesterCacheValues` to be kept in memory.
+///
+/// Each `AttesterCacheValues` is very small (~16 bytes) and the cache will generally be kept small
+/// by pruning on finality.
+///
+/// The value provided here is much larger than will be used during ideal network conditions,
+/// however we make it large since the values are so small.
+const MAX_CACHE_LEN: usize = 1_024;
+
+#[derive(Debug)]
+pub enum Error {
+    BeaconState(BeaconStateError),
+    // Boxed to avoid an infinite-size recursion issue.
+    BeaconChain(Box<BeaconChainError>),
+    MissingBeaconState(Hash256),
+    FailedToTransitionState(StateAdvanceError),
+    CannotAttestToFutureState {
+        state_slot: Slot,
+        request_slot: Slot,
+    },
+    /// Indicates a cache inconsistency.
+    WrongEpoch {
+        request_epoch: Epoch,
+        epoch: Epoch,
+    },
+    InvalidCommitteeIndex {
+        committee_index: u64,
+    },
+    /// Indicates an inconsistency with the beacon state committees.
+    InverseRange {
+        range: Range<usize>,
+    },
+}
+
+impl From<BeaconStateError> for Error {
+    fn from(e: BeaconStateError) -> Self {
+        Error::BeaconState(e)
+    }
+}
+
+impl From<BeaconChainError> for Error {
+    fn from(e: BeaconChainError) -> Self {
+        Error::BeaconChain(Box::new(e))
+    }
+}
+
+/// Stores the minimal amount of data required to compute the committee length for any committee at any
+/// slot in a given `epoch`.
+struct CommitteeLengths {
+    /// The `epoch` to which the lengths pertain.
+    epoch: Epoch,
+    /// The length of the shuffling in `self.epoch`.
+    active_validator_indices_len: usize,
+}
+
+impl CommitteeLengths {
+    /// Instantiate `Self` using `state.current_epoch()`.
+    fn new<T: EthSpec>(state: &BeaconState<T>, spec: &ChainSpec) -> Result<Self, Error> {
+        let active_validator_indices_len = if let Ok(committee_cache) =
+            state.committee_cache(RelativeEpoch::Current)
+        {
+            committee_cache.active_validator_indices().len()
+        } else {
+            // Building the cache like this avoids taking a mutable reference to `BeaconState`.
+            let committee_cache = state.initialize_committee_cache(state.current_epoch(), spec)?;
+            committee_cache.active_validator_indices().len()
+        };
+
+        Ok(Self {
+            epoch: state.current_epoch(),
+            active_validator_indices_len,
+        })
+    }
+
+    /// Get the length of the committee at the given `slot` and `committee_index`.
+    fn get<T: EthSpec>(
+        &self,
+        slot: Slot,
+        committee_index: CommitteeIndex,
+        spec: &ChainSpec,
+    ) -> Result<CommitteeLength, Error> {
+        let slots_per_epoch = T::slots_per_epoch();
+        let request_epoch = slot.epoch(slots_per_epoch);
+
+        // Sanity check.
+        if request_epoch != self.epoch {
+            return Err(Error::WrongEpoch {
+                request_epoch,
+                epoch: self.epoch,
+            });
+        }
+
+        let slots_per_epoch = slots_per_epoch as usize;
+        let committees_per_slot =
+            T::get_committee_count_per_slot(self.active_validator_indices_len, spec)?;
+        let index_in_epoch = compute_committee_index_in_epoch(
+            slot,
+            slots_per_epoch,
+            committees_per_slot,
+            committee_index as usize,
+        );
+        let range = compute_committee_range_in_epoch(
+            epoch_committee_count(committees_per_slot, slots_per_epoch),
+            index_in_epoch,
+            self.active_validator_indices_len,
+        )
+        .ok_or(Error::InvalidCommitteeIndex { committee_index })?;
+
+        range
+            .end
+            .checked_sub(range.start)
+            .ok_or(Error::InverseRange { range })
+    }
+}
+
+/// Provides the following information for some epoch:
+///
+/// - The `state.current_justified_checkpoint` value.
+/// - The committee lengths for all indices and slots.
+///
+/// These values are used during attestation production.
+pub struct AttesterCacheValue {
+    current_justified_checkpoint: Checkpoint,
+    committee_lengths: CommitteeLengths,
+}
+
+impl AttesterCacheValue {
+    /// Instantiate `Self` using `state.current_epoch()`.
+    pub fn new<T: EthSpec>(state: &BeaconState<T>, spec: &ChainSpec) -> Result<Self, Error> {
+        let current_justified_checkpoint = state.current_justified_checkpoint();
+        let committee_lengths = CommitteeLengths::new(state, spec)?;
+        Ok(Self {
+            current_justified_checkpoint,
+            committee_lengths,
+        })
+    }
+
+    /// Get the justified checkpoint and committee length for some `slot` and `committee_index`.
+    fn get<T: EthSpec>(
+        &self,
+        slot: Slot,
+        committee_index: CommitteeIndex,
+        spec: &ChainSpec,
+    ) -> Result<(JustifiedCheckpoint, CommitteeLength), Error> {
+        self.committee_lengths
+            .get::<T>(slot, committee_index, spec)
+            .map(|committee_length| (self.current_justified_checkpoint, committee_length))
+    }
+}
+
+/// The `AttesterCacheKey` is fundamentally the same thing as the proposer shuffling decision root,
+/// however here we use it as an identity for both of the following values:
+///
+/// 1. The `state.current_justified_checkpoint`.
+/// 2. The attester shuffling.
+///
+/// This struct relies upon the premise that the `state.current_justified_checkpoint` in epoch `n`
+/// is determined by the root of the latest block in epoch `n - 1`. Notably, this is identical to
+/// how the proposer shuffling is keyed in `BeaconProposerCache`.
+///
+/// It is also safe, but not maximally efficient, to key the attester shuffling with the same
+/// strategy. For better shuffling keying strategies, see the `ShufflingCache`.
+#[derive(Eq, PartialEq, Hash, Clone, Copy)]
+pub struct AttesterCacheKey {
+    /// The epoch from which the justified checkpoint should be observed.
+    ///
+    /// Attestations which use `self.epoch` as `target.epoch` should use this key.
+    epoch: Epoch,
+    /// The root of the block at the last slot of `self.epoch - 1`.
+    decision_root: Hash256,
+}
+
+impl AttesterCacheKey {
+    /// Instantiate `Self` to key `state.current_epoch()`.
+    ///
+    /// The `latest_block_root` should be the latest block that has been applied to `state`. This
+    /// parameter is required since the state does not store the block root for any block with the
+    /// same slot as `state.slot()`.
+    ///
+    /// ## Errors
+    ///
+    /// May error if `epoch` is out of the range of `state.block_roots`.
+    pub fn new<T: EthSpec>(
+        epoch: Epoch,
+        state: &BeaconState<T>,
+        latest_block_root: Hash256,
+    ) -> Result<Self, Error> {
+        let slots_per_epoch = T::slots_per_epoch();
+        let decision_slot = epoch.start_slot(slots_per_epoch).saturating_sub(1_u64);
+
+        let decision_root = if decision_slot.epoch(slots_per_epoch) == epoch {
+            // This scenario is only possible during the genesis epoch. In this scenario, all-zeros
+            // is used as an alias to the genesis block.
+            Hash256::zero()
+        } else if epoch > state.current_epoch() {
+            // If the requested epoch is higher than the current epoch, the latest block will always
+            // be the decision root.
+            latest_block_root
+        } else {
+            *state.get_block_root(decision_slot)?
+        };
+
+        Ok(Self {
+            epoch,
+            decision_root,
+        })
+    }
+}
+
+/// Provides a cache for the justified checkpoint and committee length when producing an
+/// attestation.
+///
+/// See the module-level documentation for more information.
+#[derive(Default)]
+pub struct AttesterCache {
+    cache: RwLock<CacheHashMap>,
+}
+
+impl AttesterCache {
+    /// Get the justified checkpoint and committee length for the `slot` and `committee_index` in
+    /// the state identified by the cache `key`.
+    pub fn get<T: EthSpec>(
+        &self,
+        key: &AttesterCacheKey,
+        slot: Slot,
+        committee_index: CommitteeIndex,
+        spec: &ChainSpec,
+    ) -> Result<Option<(JustifiedCheckpoint, CommitteeLength)>, Error> {
+        self.cache
+            .read()
+            .get(key)
+            .map(|cache_item| cache_item.get::<T>(slot, committee_index, spec))
+            .transpose()
+    }
+
+    /// Cache the `state.current_epoch()` values if they are not already present in the state.
+    pub fn maybe_cache_state<T: EthSpec>(
+        &self,
+        state: &BeaconState<T>,
+        latest_block_root: Hash256,
+        spec: &ChainSpec,
+    ) -> Result<(), Error> {
+        let key = AttesterCacheKey::new(state.current_epoch(), state, latest_block_root)?;
+        let mut cache = self.cache.write();
+        if !cache.contains_key(&key) {
+            let cache_item = AttesterCacheValue::new(state, spec)?;
+            Self::insert_respecting_max_len(&mut cache, key, cache_item);
+        }
+        Ok(())
+    }
+
+    /// Read the state identified by `state_root` from the database, advance it to the required
+    /// slot, use it to prime the cache and return the values for the provided `slot` and
+    /// `committee_index`.
+    ///
+    /// ## Notes
+    ///
+    /// This function takes a write-lock on the internal cache. Prefer attempting a `Self::get` call
+    /// before running this function as `Self::get` only takes a read-lock and is therefore less
+    /// likely to create contention.
+    pub fn load_and_cache_state<T: BeaconChainTypes>(
+        &self,
+        state_root: Hash256,
+        key: AttesterCacheKey,
+        slot: Slot,
+        committee_index: CommitteeIndex,
+        chain: &BeaconChain<T>,
+    ) -> Result<(JustifiedCheckpoint, CommitteeLength), Error> {
+        let spec = &chain.spec;
+        let slots_per_epoch = T::EthSpec::slots_per_epoch();
+        let epoch = slot.epoch(slots_per_epoch);
+
+        // Take a write-lock on the cache before starting the state read.
+        //
+        // Whilst holding the write-lock during the state read will create contention, it prevents
+        // the scenario where multiple requests from separate threads cause duplicate state reads.
+        let mut cache = self.cache.write();
+
+        // Try the cache to see if someone has already primed it between the time the function was
+        // called and when the cache write-lock was obtained. This avoids performing duplicate state
+        // reads.
+        if let Some(value) = cache
+            .get(&key)
+            .map(|cache_item| cache_item.get::<T::EthSpec>(slot, committee_index, spec))
+            .transpose()?
+        {
+            return Ok(value);
+        }
+
+        let mut state: BeaconState<T::EthSpec> = chain
+            .get_state(&state_root, None)?
+            .ok_or(Error::MissingBeaconState(state_root))?;
+
+        if state.slot() > slot {
+            // This indicates an internal inconsistency.
+            return Err(Error::CannotAttestToFutureState {
+                state_slot: state.slot(),
+                request_slot: slot,
+            });
+        } else if state.current_epoch() < epoch {
+            // Only perform a "partial" state advance since we do not require the state roots to be
+            // accurate.
+            partial_state_advance(
+                &mut state,
+                Some(state_root),
+                epoch.start_slot(slots_per_epoch),
+                spec,
+            )
+            .map_err(Error::FailedToTransitionState)?;
+            state.build_committee_cache(RelativeEpoch::Current, spec)?;
+        }
+
+        let cache_item = AttesterCacheValue::new(&state, spec)?;
+        let value = cache_item.get::<T::EthSpec>(slot, committee_index, spec)?;
+        Self::insert_respecting_max_len(&mut cache, key, cache_item);
+        Ok(value)
+    }
+
+    /// Insert a value to `cache`, ensuring it does not exceed the maximum length.
+    ///
+    /// If the cache is already full, the item with the lowest epoch will be removed.
+    fn insert_respecting_max_len(
+        cache: &mut CacheHashMap,
+        key: AttesterCacheKey,
+        value: AttesterCacheValue,
+    ) {
+        while cache.len() >= MAX_CACHE_LEN {
+            if let Some(oldest) = cache
+                .iter()
+                .map(|(key, _)| *key)
+                .min_by_key(|key| key.epoch)
+            {
+                cache.remove(&oldest);
+            } else {
+                break;
+            }
+        }
+
+        cache.insert(key, value);
+    }
+
+    /// Remove all entries where the `key.epoch` is lower than the given `epoch`.
+    ///
+    /// Generally, the provided `epoch` should be the finalized epoch.
+    pub fn prune_below(&self, epoch: Epoch) {
+        self.cache.write().retain(|target, _| target.epoch >= epoch);
+    }
+}

beacon_node/beacon_chain/src/beacon_fork_choice_store.rs

@@ -0,0 +1,351 @@
+//! Defines the `BeaconForkChoiceStore` which provides the persistent storage for the `ForkChoice`
+//! struct.
+//!
+//! Additionally, the private `BalancesCache` struct is defined; a cache designed to avoid database
+//! reads when fork choice requires the validator balances of the justified state.
+
+use crate::{metrics, BeaconSnapshot};
+use fork_choice::ForkChoiceStore;
+use ssz_derive::{Decode, Encode};
+use std::marker::PhantomData;
+use std::sync::Arc;
+use store::{Error as StoreError, HotColdDB, ItemStore};
+use types::{BeaconBlock, BeaconState, BeaconStateError, Checkpoint, EthSpec, Hash256, Slot};
+
+#[derive(Debug)]
+pub enum Error {
+    UnableToReadSlot,
+    UnableToReadTime,
+    InvalidGenesisSnapshot(Slot),
+    AncestorUnknown { ancestor_slot: Slot },
+    UninitializedBestJustifiedBalances,
+    FailedToReadBlock(StoreError),
+    MissingBlock(Hash256),
+    FailedToReadState(StoreError),
+    MissingState(Hash256),
+    InvalidPersistedBytes(ssz::DecodeError),
+    BeaconStateError(BeaconStateError),
+}
+
+impl From<BeaconStateError> for Error {
+    fn from(e: BeaconStateError) -> Self {
+        Error::BeaconStateError(e)
+    }
+}
+
+/// The number of validator balance sets that are cached within `BalancesCache`.
+const MAX_BALANCE_CACHE_SIZE: usize = 4;
+
+/// Returns the effective balances for every validator in the given `state`.
+///
+/// Any validator who is not active in the epoch of the given `state` is assigned a balance of
+/// zero.
+pub fn get_effective_balances<T: EthSpec>(state: &BeaconState<T>) -> Vec<u64> {
+    state
+        .validators()
+        .iter()
+        .map(|validator| {
+            if validator.is_active_at(state.current_epoch()) {
+                validator.effective_balance
+            } else {
+                0
+            }
+        })
+        .collect()
+}
+
+/// An item that is stored in the `BalancesCache`.
+#[derive(PartialEq, Clone, Debug, Encode, Decode)]
+struct CacheItem {
+    /// The block root at which `self.balances` are valid.
+    block_root: Hash256,
+    /// The effective balances from a `BeaconState` validator registry.
+    balances: Vec<u64>,
+}
+
+/// Provides a cache to avoid reading `BeaconState` from disk when updating the current justified
+/// checkpoint.
+///
+/// It is effectively a mapping of `epoch_boundary_block_root -> state.balances`.
+#[derive(PartialEq, Clone, Default, Debug, Encode, Decode)]
+struct BalancesCache {
+    items: Vec<CacheItem>,
+}
+
+impl BalancesCache {
+    /// Inspect the given `state` and determine the root of the block at the first slot of
+    /// `state.current_epoch`. If there is not already some entry for the given block root, then
+    /// add the effective balances from the `state` to the cache.
+    pub fn process_state<E: EthSpec>(
+        &mut self,
+        block_root: Hash256,
+        state: &BeaconState<E>,
+    ) -> Result<(), Error> {
+        // We are only interested in balances from states that are at the start of an epoch,
+        // because this is where the `current_justified_checkpoint.root` will point.
+        if !Self::is_first_block_in_epoch(block_root, state)? {
+            return Ok(());
+        }
+
+        let epoch_boundary_slot = state.current_epoch().start_slot(E::slots_per_epoch());
+        let epoch_boundary_root = if epoch_boundary_slot == state.slot() {
+            block_root
+        } else {
+            // This call remains sensible as long as `state.block_roots` is larger than a single
+            // epoch.
+            *state.get_block_root(epoch_boundary_slot)?
+        };
+
+        if self.position(epoch_boundary_root).is_none() {
+            let item = CacheItem {
+                block_root: epoch_boundary_root,
+                balances: get_effective_balances(state),
+            };
+
+            if self.items.len() == MAX_BALANCE_CACHE_SIZE {
+                self.items.remove(0);
+            }
+
+            self.items.push(item);
+        }
+
+        Ok(())
+    }
+
+    /// Returns `true` if the given `block_root` is the first/only block to have been processed in
+    /// the epoch of the given `state`.
+    ///
+    /// We can determine if it is the first block by looking back through `state.block_roots` to
+    /// see if there is a block in the current epoch with a different root.
+    fn is_first_block_in_epoch<E: EthSpec>(
+        block_root: Hash256,
+        state: &BeaconState<E>,
+    ) -> Result<bool, Error> {
+        let mut prior_block_found = false;
+
+        for slot in state.current_epoch().slot_iter(E::slots_per_epoch()) {
+            if slot < state.slot() {
+                if *state.get_block_root(slot)? != block_root {
+                    prior_block_found = true;
+                    break;
+                }
+            } else {
+                break;
+            }
+        }
+
+        Ok(!prior_block_found)
+    }
+
+    fn position(&self, block_root: Hash256) -> Option<usize> {
+        self.items
+            .iter()
+            .position(|item| item.block_root == block_root)
+    }
+
+    /// Get the balances for the given `block_root`, if any.
+    ///
+    /// If some balances are found, they are removed from the cache.
+    pub fn get(&mut self, block_root: Hash256) -> Option<Vec<u64>> {
+        let i = self.position(block_root)?;
+        Some(self.items.remove(i).balances)
+    }
+}
+
+/// Implements `fork_choice::ForkChoiceStore` in order to provide a persistent backing to the
+/// `fork_choice::ForkChoice` struct.
+#[derive(Debug)]
+pub struct BeaconForkChoiceStore<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> {
+    store: Arc<HotColdDB<E, Hot, Cold>>,
+    balances_cache: BalancesCache,
+    time: Slot,
+    finalized_checkpoint: Checkpoint,
+    justified_checkpoint: Checkpoint,
+    justified_balances: Vec<u64>,
+    best_justified_checkpoint: Checkpoint,
+    _phantom: PhantomData<E>,
+}
+
+impl<E, Hot, Cold> PartialEq for BeaconForkChoiceStore<E, Hot, Cold>
+where
+    E: EthSpec,
+    Hot: ItemStore<E>,
+    Cold: ItemStore<E>,
+{
+    /// This implementation ignores the `store` and `slot_clock`.
+    fn eq(&self, other: &Self) -> bool {
+        self.balances_cache == other.balances_cache
+            && self.time == other.time
+            && self.finalized_checkpoint == other.finalized_checkpoint
+            && self.justified_checkpoint == other.justified_checkpoint
+            && self.justified_balances == other.justified_balances
+            && self.best_justified_checkpoint == other.best_justified_checkpoint
+    }
+}
+
+impl<E, Hot, Cold> BeaconForkChoiceStore<E, Hot, Cold>
+where
+    E: EthSpec,
+    Hot: ItemStore<E>,
+    Cold: ItemStore<E>,
+{
+    /// Initialize `Self` from some `anchor` checkpoint which may or may not be the genesis state.
+    ///
+    /// ## Specification
+    ///
+    /// Equivalent to:
+    ///
+    /// https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/fork-choice.md#get_forkchoice_store
+    ///
+    /// ## Notes:
+    ///
+    /// It is assumed that `anchor` is already persisted in `store`.
+    pub fn get_forkchoice_store(
+        store: Arc<HotColdDB<E, Hot, Cold>>,
+        anchor: &BeaconSnapshot<E>,
+    ) -> Self {
+        let anchor_state = &anchor.beacon_state;
+        let mut anchor_block_header = anchor_state.latest_block_header().clone();
+        if anchor_block_header.state_root == Hash256::zero() {
+            anchor_block_header.state_root = anchor.beacon_state_root();
+        }
+        let anchor_root = anchor_block_header.canonical_root();
+        let anchor_epoch = anchor_state.current_epoch();
+        let justified_checkpoint = Checkpoint {
+            epoch: anchor_epoch,
+            root: anchor_root,
+        };
+        let finalized_checkpoint = justified_checkpoint;
+
+        Self {
+            store,
+            balances_cache: <_>::default(),
+            time: anchor_state.slot(),
+            justified_checkpoint,
+            justified_balances: anchor_state.balances().clone().into(),
+            finalized_checkpoint,
+            best_justified_checkpoint: justified_checkpoint,
+            _phantom: PhantomData,
+        }
+    }
+
+    /// Save the current state of `Self` to a `PersistedForkChoiceStore` which can be stored to the
+    /// on-disk database.
+    pub fn to_persisted(&self) -> PersistedForkChoiceStore {
+        PersistedForkChoiceStore {
+            balances_cache: self.balances_cache.clone(),
+            time: self.time,
+            finalized_checkpoint: self.finalized_checkpoint,
+            justified_checkpoint: self.justified_checkpoint,
+            justified_balances: self.justified_balances.clone(),
+            best_justified_checkpoint: self.best_justified_checkpoint,
+        }
+    }
+
+    /// Restore `Self` from a previously-generated `PersistedForkChoiceStore`.
+    pub fn from_persisted(
+        persisted: PersistedForkChoiceStore,
+        store: Arc<HotColdDB<E, Hot, Cold>>,
+    ) -> Result<Self, Error> {
+        Ok(Self {
+            store,
+            balances_cache: persisted.balances_cache,
+            time: persisted.time,
+            finalized_checkpoint: persisted.finalized_checkpoint,
+            justified_checkpoint: persisted.justified_checkpoint,
+            justified_balances: persisted.justified_balances,
+            best_justified_checkpoint: persisted.best_justified_checkpoint,
+            _phantom: PhantomData,
+        })
+    }
+}
+
+impl<E, Hot, Cold> ForkChoiceStore<E> for BeaconForkChoiceStore<E, Hot, Cold>
+where
+    E: EthSpec,
+    Hot: ItemStore<E>,
+    Cold: ItemStore<E>,
+{
+    type Error = Error;
+
+    fn get_current_slot(&self) -> Slot {
+        self.time
+    }
+
+    fn set_current_slot(&mut self, slot: Slot) {
+        self.time = slot
+    }
+
+    fn on_verified_block(
+        &mut self,
+        _block: &BeaconBlock<E>,
+        block_root: Hash256,
+        state: &BeaconState<E>,
+    ) -> Result<(), Self::Error> {
+        self.balances_cache.process_state(block_root, state)
+    }
+
+    fn justified_checkpoint(&self) -> &Checkpoint {
+        &self.justified_checkpoint
+    }
+
+    fn justified_balances(&self) -> &[u64] {
+        &self.justified_balances
+    }
+
+    fn best_justified_checkpoint(&self) -> &Checkpoint {
+        &self.best_justified_checkpoint
+    }
+
+    fn finalized_checkpoint(&self) -> &Checkpoint {
+        &self.finalized_checkpoint
+    }
+
+    fn set_finalized_checkpoint(&mut self, checkpoint: Checkpoint) {
+        self.finalized_checkpoint = checkpoint
+    }
+
+    fn set_justified_checkpoint(&mut self, checkpoint: Checkpoint) -> Result<(), Error> {
+        self.justified_checkpoint = checkpoint;
+
+        if let Some(balances) = self.balances_cache.get(self.justified_checkpoint.root) {
+            metrics::inc_counter(&metrics::BALANCES_CACHE_HITS);
+            self.justified_balances = balances;
+        } else {
+            metrics::inc_counter(&metrics::BALANCES_CACHE_MISSES);
+            let justified_block = self
+                .store
+                .get_block(&self.justified_checkpoint.root)
+                .map_err(Error::FailedToReadBlock)?
+                .ok_or(Error::MissingBlock(self.justified_checkpoint.root))?
+                .deconstruct()
+                .0;
+
+            self.justified_balances = self
+                .store
+                .get_state(&justified_block.state_root(), Some(justified_block.slot()))
+                .map_err(Error::FailedToReadState)?
+                .ok_or_else(|| Error::MissingState(justified_block.state_root()))?
+                .balances()
+                .clone()
+                .into();
+        }
+
+        Ok(())
+    }
+
+    fn set_best_justified_checkpoint(&mut self, checkpoint: Checkpoint) {
+        self.best_justified_checkpoint = checkpoint
+    }
+}
+
+/// A container which allows persisting the `BeaconForkChoiceStore` to the on-disk database.
+#[derive(Encode, Decode)]
+pub struct PersistedForkChoiceStore {
+    balances_cache: BalancesCache,
+    time: Slot,
+    finalized_checkpoint: Checkpoint,
+    justified_checkpoint: Checkpoint,
+    justified_balances: Vec<u64>,
+    best_justified_checkpoint: Checkpoint,
+}

beacon_node/beacon_chain/src/beacon_proposer_cache.rs

@@ -0,0 +1,127 @@
+//! The `BeaconProposer` cache stores the proposer indices for some epoch.
+//!
+//! This cache is keyed by `(epoch, block_root)` where `block_root` is the block root at
+//! `end_slot(epoch - 1)`. We make the assertion that the proposer shuffling is identical for all
+//! blocks in `epoch` which share the common ancestor of `block_root`.
+//!
+//! The cache is a fairly unintelligent LRU cache that is not pruned after finality. This makes it
+//! very simple to reason about, but it might store values that are useless due to finalization. The
+//! values it stores are very small, so this should not be an issue.
+
+use lru::LruCache;
+use smallvec::SmallVec;
+use types::{BeaconStateError, Epoch, EthSpec, Fork, Hash256, Slot, Unsigned};
+
+/// The number of sets of proposer indices that should be cached.
+const CACHE_SIZE: usize = 16;
+
+/// This value is fairly unimportant, it's used to avoid heap allocations. The result of it being
+/// incorrect is non-substantial from a consensus perspective (and probably also from a
+/// performance perspective).
+const TYPICAL_SLOTS_PER_EPOCH: usize = 32;
+
+/// For some given slot, this contains the proposer index (`index`) and the `fork` that should be
+/// used to verify their signature.
+pub struct Proposer {
+    pub index: usize,
+    pub fork: Fork,
+}
+
+/// The list of proposers for some given `epoch`, alongside the `fork` that should be used to verify
+/// their signatures.
+pub struct EpochBlockProposers {
+    /// The epoch to which the proposers pertain.
+    epoch: Epoch,
+    /// The fork that should be used to verify proposer signatures.
+    fork: Fork,
+    /// A list of length `T::EthSpec::slots_per_epoch()`, representing the proposers for each slot
+    /// in that epoch.
+    ///
+    /// E.g., if `self.epoch == 1`, then `self.proposers[0]` contains the proposer for slot `32`.
+    proposers: SmallVec<[usize; TYPICAL_SLOTS_PER_EPOCH]>,
+}
+
+/// A cache to store the proposers for some epoch.
+///
+/// See the module-level documentation for more information.
+pub struct BeaconProposerCache {
+    cache: LruCache<(Epoch, Hash256), EpochBlockProposers>,
+}
+
+impl Default for BeaconProposerCache {
+    fn default() -> Self {
+        Self {
+            cache: LruCache::new(CACHE_SIZE),
+        }
+    }
+}
+
+impl BeaconProposerCache {
+    /// If it is cached, returns the proposer for the block at `slot` where the block has the
+    /// ancestor block root of `shuffling_decision_block` at `end_slot(slot.epoch() - 1)`.
+    pub fn get_slot<T: EthSpec>(
+        &mut self,
+        shuffling_decision_block: Hash256,
+        slot: Slot,
+    ) -> Option<Proposer> {
+        let epoch = slot.epoch(T::slots_per_epoch());
+        let key = (epoch, shuffling_decision_block);
+        if let Some(cache) = self.cache.get(&key) {
+            // This `if` statement is likely unnecessary, but it feels like good practice.
+            if epoch == cache.epoch {
+                cache
+                    .proposers
+                    .get(slot.as_usize() % T::SlotsPerEpoch::to_usize())
+                    .map(|&index| Proposer {
+                        index,
+                        fork: cache.fork,
+                    })
+            } else {
+                None
+            }
+        } else {
+            None
+        }
+    }
+
+    /// As per `Self::get_slot`, but returns all proposers in all slots for the given `epoch`.
+    ///
+    /// The nth slot in the returned `SmallVec` will be equal to the nth slot in the given `epoch`.
+    /// E.g., if `epoch == 1` then `smallvec[0]` refers to slot 32 (assuming `SLOTS_PER_EPOCH ==
+    /// 32`).
+    pub fn get_epoch<T: EthSpec>(
+        &mut self,
+        shuffling_decision_block: Hash256,
+        epoch: Epoch,
+    ) -> Option<&SmallVec<[usize; TYPICAL_SLOTS_PER_EPOCH]>> {
+        let key = (epoch, shuffling_decision_block);
+        self.cache.get(&key).map(|cache| &cache.proposers)
+    }
+
+    /// Insert the proposers into the cache.
+    ///
+    /// See `Self::get` for a description of `shuffling_decision_block`.
+    ///
+    /// The `fork` value must be valid to verify proposer signatures in `epoch`.
+    pub fn insert(
+        &mut self,
+        epoch: Epoch,
+        shuffling_decision_block: Hash256,
+        proposers: Vec<usize>,
+        fork: Fork,
+    ) -> Result<(), BeaconStateError> {
+        let key = (epoch, shuffling_decision_block);
+        if !self.cache.contains(&key) {
+            self.cache.put(
+                key,
+                EpochBlockProposers {
+                    epoch,
+                    fork,
+                    proposers: proposers.into(),
+                },
+            );
+        }
+
+        Ok(())
+    }
+}

beacon_node/beacon_chain/src/beacon_snapshot.rs

@@ -0,0 +1,55 @@
+use serde_derive::Serialize;
+use types::{beacon_state::CloneConfig, BeaconState, EthSpec, Hash256, SignedBeaconBlock};
+
+/// Represents some block and its associated state. Generally, this will be used for tracking the
+/// head, justified head and finalized head.
+#[derive(Clone, Serialize, PartialEq, Debug)]
+pub struct BeaconSnapshot<E: EthSpec> {
+    pub beacon_block: SignedBeaconBlock<E>,
+    pub beacon_block_root: Hash256,
+    pub beacon_state: BeaconState<E>,
+}
+
+impl<E: EthSpec> BeaconSnapshot<E> {
+    /// Create a new checkpoint.
+    pub fn new(
+        beacon_block: SignedBeaconBlock<E>,
+        beacon_block_root: Hash256,
+        beacon_state: BeaconState<E>,
+    ) -> Self {
+        Self {
+            beacon_block,
+            beacon_block_root,
+            beacon_state,
+        }
+    }
+
+    /// Returns the state root from `self.beacon_block`.
+    ///
+    /// ## Caution
+    ///
+    /// It is not strictly enforced that `root(self.beacon_state) == self.beacon_state_root()`.
+    pub fn beacon_state_root(&self) -> Hash256 {
+        self.beacon_block.message().state_root()
+    }
+
+    /// Update all fields of the checkpoint.
+    pub fn update(
+        &mut self,
+        beacon_block: SignedBeaconBlock<E>,
+        beacon_block_root: Hash256,
+        beacon_state: BeaconState<E>,
+    ) {
+        self.beacon_block = beacon_block;
+        self.beacon_block_root = beacon_block_root;
+        self.beacon_state = beacon_state;
+    }
+
+    pub fn clone_with(&self, clone_config: CloneConfig) -> Self {
+        Self {
+            beacon_block: self.beacon_block.clone(),
+            beacon_block_root: self.beacon_block_root,
+            beacon_state: self.beacon_state.clone_with(clone_config),
+        }
+    }
+}

beacon_node/beacon_chain/src/chain_config.rs

@@ -0,0 +1,24 @@
+use serde_derive::{Deserialize, Serialize};
+use types::Checkpoint;
+
+#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]
+pub struct ChainConfig {
+    /// Maximum number of slots to skip when importing a consensus message (e.g., block,
+    /// attestation, etc).
+    ///
+    /// If `None`, there is no limit.
+    pub import_max_skip_slots: Option<u64>,
+    /// A user-input `Checkpoint` that must exist in the beacon chain's sync path.
+    ///
+    /// If `None`, there is no weak subjectivity verification.
+    pub weak_subjectivity_checkpoint: Option<Checkpoint>,
+}
+
+impl Default for ChainConfig {
+    fn default() -> Self {
+        Self {
+            import_max_skip_slots: None,
+            weak_subjectivity_checkpoint: None,
+        }
+    }
+}

beacon_node/beacon_chain/src/checkpoint.rs

@@ -1,44 +0,0 @@
-use serde_derive::Serialize;
-use ssz_derive::{Decode, Encode};
-use types::{BeaconBlock, BeaconState, EthSpec, Hash256};
-
-/// Represents some block and it's associated state. Generally, this will be used for tracking the
-/// head, justified head and finalized head.
-#[derive(Clone, Serialize, PartialEq, Debug, Encode, Decode)]
-pub struct CheckPoint<E: EthSpec> {
-    pub beacon_block: BeaconBlock<E>,
-    pub beacon_block_root: Hash256,
-    pub beacon_state: BeaconState<E>,
-    pub beacon_state_root: Hash256,
-}
-
-impl<E: EthSpec> CheckPoint<E> {
-    /// Create a new checkpoint.
-    pub fn new(
-        beacon_block: BeaconBlock<E>,
-        beacon_block_root: Hash256,
-        beacon_state: BeaconState<E>,
-        beacon_state_root: Hash256,
-    ) -> Self {
-        Self {
-            beacon_block,
-            beacon_block_root,
-            beacon_state,
-            beacon_state_root,
-        }
-    }
-
-    /// Update all fields of the checkpoint.
-    pub fn update(
-        &mut self,
-        beacon_block: BeaconBlock<E>,
-        beacon_block_root: Hash256,
-        beacon_state: BeaconState<E>,
-        beacon_state_root: Hash256,
-    ) {
-        self.beacon_block = beacon_block;
-        self.beacon_block_root = beacon_block_root;
-        self.beacon_state = beacon_state;
-        self.beacon_state_root = beacon_state_root;
-    }
-}

beacon_node/beacon_chain/src/checkpoint_cache.rs

@@ -1,124 +0,0 @@
-use crate::checkpoint::CheckPoint;
-use crate::metrics;
-use parking_lot::RwLock;
-use types::{BeaconBlock, BeaconState, EthSpec, Hash256};
-
-const CACHE_SIZE: usize = 4;
-
-struct Inner<T: EthSpec> {
-    oldest: usize,
-    limit: usize,
-    checkpoints: Vec<CheckPoint<T>>,
-}
-
-impl<T: EthSpec> Default for Inner<T> {
-    fn default() -> Self {
-        Self {
-            oldest: 0,
-            limit: CACHE_SIZE,
-            checkpoints: vec![],
-        }
-    }
-}
-
-pub struct CheckPointCache<T: EthSpec> {
-    inner: RwLock<Inner<T>>,
-}
-
-impl<T: EthSpec> Default for CheckPointCache<T> {
-    fn default() -> Self {
-        Self {
-            inner: RwLock::new(Inner::default()),
-        }
-    }
-}
-
-impl<T: EthSpec> CheckPointCache<T> {
-    pub fn insert(&self, checkpoint: &CheckPoint<T>) {
-        if self
-            .inner
-            .read()
-            .checkpoints
-            .iter()
-            // This is `O(n)` but whilst `n == 4` it ain't no thing.
-            .any(|local| local.beacon_state_root == checkpoint.beacon_state_root)
-        {
-            // Adding a known checkpoint to the cache should be a no-op.
-            return;
-        }
-
-        let mut inner = self.inner.write();
-
-        if inner.checkpoints.len() < inner.limit {
-            inner.checkpoints.push(checkpoint.clone())
-        } else {
-            let i = inner.oldest; // to satisfy the borrow checker.
-            inner.checkpoints[i] = checkpoint.clone();
-            inner.oldest += 1;
-            inner.oldest %= inner.limit;
-        }
-    }
-
-    pub fn get_state(&self, state_root: &Hash256) -> Option<BeaconState<T>> {
-        self.inner
-            .read()
-            .checkpoints
-            .iter()
-            // Also `O(n)`.
-            .find(|checkpoint| checkpoint.beacon_state_root == *state_root)
-            .map(|checkpoint| {
-                metrics::inc_counter(&metrics::CHECKPOINT_CACHE_HITS);
-
-                checkpoint.beacon_state.clone()
-            })
-            .or_else(|| {
-                metrics::inc_counter(&metrics::CHECKPOINT_CACHE_MISSES);
-
-                None
-            })
-    }
-
-    pub fn get_state_only_with_committee_cache(
-        &self,
-        state_root: &Hash256,
-    ) -> Option<BeaconState<T>> {
-        self.inner
-            .read()
-            .checkpoints
-            .iter()
-            // Also `O(n)`.
-            .find(|checkpoint| checkpoint.beacon_state_root == *state_root)
-            .map(|checkpoint| {
-                metrics::inc_counter(&metrics::CHECKPOINT_CACHE_HITS);
-
-                let mut state = checkpoint.beacon_state.clone_without_caches();
-                state.committee_caches = checkpoint.beacon_state.committee_caches.clone();
-
-                state
-            })
-            .or_else(|| {
-                metrics::inc_counter(&metrics::CHECKPOINT_CACHE_MISSES);
-
-                None
-            })
-    }
-
-    pub fn get_block(&self, block_root: &Hash256) -> Option<BeaconBlock<T>> {
-        self.inner
-            .read()
-            .checkpoints
-            .iter()
-            // Also `O(n)`.
-            .find(|checkpoint| checkpoint.beacon_block_root == *block_root)
-            .map(|checkpoint| {
-                metrics::inc_counter(&metrics::CHECKPOINT_CACHE_HITS);
-
-                checkpoint.beacon_block.clone()
-            })
-            .or_else(|| {
-                metrics::inc_counter(&metrics::CHECKPOINT_CACHE_MISSES);
-
-                None
-            })
-    }
-}

beacon_node/beacon_chain/src/errors.rs

@@ -1,10 +1,28 @@
+use crate::attester_cache::Error as AttesterCacheError;
+use crate::beacon_chain::ForkChoiceError;
+use crate::beacon_fork_choice_store::Error as ForkChoiceStoreError;
 use crate::eth1_chain::Error as Eth1ChainError;
-use crate::fork_choice::Error as ForkChoiceError;
+use crate::migrate::PruningError;
+use crate::naive_aggregation_pool::Error as NaiveAggregationError;
+use crate::observed_aggregates::Error as ObservedAttestationsError;
+use crate::observed_attesters::Error as ObservedAttestersError;
+use crate::observed_block_producers::Error as ObservedBlockProducersError;
+use futures::channel::mpsc::TrySendError;
+use operation_pool::OpPoolError;
+use safe_arith::ArithError;
 use ssz_types::Error as SszTypesError;
-use state_processing::per_block_processing::errors::AttestationValidationError;
-use state_processing::BlockProcessingError;
-use state_processing::SlotProcessingError;
+use state_processing::{
+    block_signature_verifier::Error as BlockSignatureVerifierError,
+    per_block_processing::errors::{
+        AttestationValidationError, AttesterSlashingValidationError, ExitValidationError,
+        ProposerSlashingValidationError, SyncCommitteeMessageValidationError,
+    },
+    signature_sets::Error as SignatureSetError,
+    state_advance::Error as StateAdvanceError,
+    BlockProcessingError, SlotProcessingError,
+};
 use std::time::Duration;
+use task_executor::ShutdownReason;
 use types::*;
 
 macro_rules! easy_from_to {
@@ -17,7 +35,7 @@ macro_rules! easy_from_to {
     };
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug)]
 pub enum BeaconChainError {
     InsufficientValidators,
     UnableToReadSlot,
@@ -32,30 +50,111 @@ pub enum BeaconChainError {
     DBInconsistent(String),
     DBError(store::Error),
     ForkChoiceError(ForkChoiceError),
+    ForkChoiceStoreError(ForkChoiceStoreError),
     MissingBeaconBlock(Hash256),
     MissingBeaconState(Hash256),
     SlotProcessingError(SlotProcessingError),
+    StateAdvanceError(StateAdvanceError),
     UnableToAdvanceState(String),
     NoStateForAttestation {
         beacon_block_root: Hash256,
     },
+    CannotAttestToFutureState,
     AttestationValidationError(AttestationValidationError),
+    SyncCommitteeMessageValidationError(SyncCommitteeMessageValidationError),
+    ExitValidationError(ExitValidationError),
+    ProposerSlashingValidationError(ProposerSlashingValidationError),
+    AttesterSlashingValidationError(AttesterSlashingValidationError),
     StateSkipTooLarge {
         start_slot: Slot,
         requested_slot: Slot,
         max_task_runtime: Duration,
     },
+    MissingFinalizedStateRoot(Slot),
     /// Returned when an internal check fails, indicating corrupt data.
     InvariantViolated(String),
     SszTypesError(SszTypesError),
+    NoProposerForSlot(Slot),
+    CanonicalHeadLockTimeout,
+    AttestationCacheLockTimeout,
+    ValidatorPubkeyCacheLockTimeout,
+    SnapshotCacheLockTimeout,
+    IncorrectStateForAttestation(RelativeEpochError),
+    InvalidValidatorPubkeyBytes(bls::Error),
+    ValidatorPubkeyCacheIncomplete(usize),
+    SignatureSetError(SignatureSetError),
+    BlockSignatureVerifierError(state_processing::block_signature_verifier::Error),
+    DuplicateValidatorPublicKey,
+    ValidatorPubkeyCacheFileError(String),
+    ValidatorIndexUnknown(usize),
+    ValidatorPubkeyUnknown(PublicKeyBytes),
+    OpPoolError(OpPoolError),
+    NaiveAggregationError(NaiveAggregationError),
+    ObservedAttestationsError(ObservedAttestationsError),
+    ObservedAttestersError(ObservedAttestersError),
+    ObservedBlockProducersError(ObservedBlockProducersError),
+    AttesterCacheError(AttesterCacheError),
+    PruningError(PruningError),
+    ArithError(ArithError),
+    InvalidShufflingId {
+        shuffling_epoch: Epoch,
+        head_block_epoch: Epoch,
+    },
+    WeakSubjectivtyVerificationFailure,
+    WeakSubjectivtyShutdownError(TrySendError<ShutdownReason>),
+    AttestingToFinalizedSlot {
+        finalized_slot: Slot,
+        request_slot: Slot,
+    },
+    AttestingToAncientSlot {
+        lowest_permissible_slot: Slot,
+        request_slot: Slot,
+    },
+    BadPreState {
+        parent_root: Hash256,
+        parent_slot: Slot,
+        block_root: Hash256,
+        block_slot: Slot,
+        state_slot: Slot,
+    },
+    InvalidStateForShuffling {
+        state_epoch: Epoch,
+        shuffling_epoch: Epoch,
+    },
+    SyncDutiesError(BeaconStateError),
+    InconsistentForwardsIter {
+        request_slot: Slot,
+        slot: Slot,
+    },
+    InvalidReorgSlotIter {
+        old_slot: Slot,
+        new_slot: Slot,
+    },
+    AltairForkDisabled,
 }
 
 easy_from_to!(SlotProcessingError, BeaconChainError);
 easy_from_to!(AttestationValidationError, BeaconChainError);
+easy_from_to!(SyncCommitteeMessageValidationError, BeaconChainError);
+easy_from_to!(ExitValidationError, BeaconChainError);
+easy_from_to!(ProposerSlashingValidationError, BeaconChainError);
+easy_from_to!(AttesterSlashingValidationError, BeaconChainError);
 easy_from_to!(SszTypesError, BeaconChainError);
+easy_from_to!(OpPoolError, BeaconChainError);
+easy_from_to!(NaiveAggregationError, BeaconChainError);
+easy_from_to!(ObservedAttestationsError, BeaconChainError);
+easy_from_to!(ObservedAttestersError, BeaconChainError);
+easy_from_to!(ObservedBlockProducersError, BeaconChainError);
+easy_from_to!(AttesterCacheError, BeaconChainError);
+easy_from_to!(BlockSignatureVerifierError, BeaconChainError);
+easy_from_to!(PruningError, BeaconChainError);
+easy_from_to!(ArithError, BeaconChainError);
+easy_from_to!(ForkChoiceStoreError, BeaconChainError);
+easy_from_to!(StateAdvanceError, BeaconChainError);
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug)]
 pub enum BlockProductionError {
+    UnableToGetHeadInfo(BeaconChainError),
     UnableToGetBlockRootFromState,
     UnableToReadSlot,
     UnableToProduceAtSlot(Slot),
@@ -63,12 +162,19 @@ pub enum BlockProductionError {
     BlockProcessingError(BlockProcessingError),
     Eth1ChainError(Eth1ChainError),
     BeaconStateError(BeaconStateError),
+    StateAdvanceError(StateAdvanceError),
+    OpPoolError(OpPoolError),
     /// The `BeaconChain` was explicitly configured _without_ a connection to eth1, therefore it
     /// cannot produce blocks.
     NoEth1ChainConnection,
+    StateSlotTooHigh {
+        produce_at_slot: Slot,
+        state_slot: Slot,
+    },
 }
 
 easy_from_to!(BlockProcessingError, BlockProductionError);
 easy_from_to!(BeaconStateError, BlockProductionError);
 easy_from_to!(SlotProcessingError, BlockProductionError);
 easy_from_to!(Eth1ChainError, BlockProductionError);
+easy_from_to!(StateAdvanceError, BlockProductionError);

beacon_node/beacon_chain/src/events.rs

@@ -1,65 +1,128 @@
-use serde_derive::{Deserialize, Serialize};
-use std::marker::PhantomData;
-use types::{Attestation, BeaconBlock, Epoch, EthSpec, Hash256};
-pub use websocket_server::WebSocketSender;
+pub use eth2::types::{EventKind, SseBlock, SseFinalizedCheckpoint, SseHead};
+use slog::{trace, Logger};
+use tokio::sync::broadcast;
+use tokio::sync::broadcast::{error::SendError, Receiver, Sender};
+use types::EthSpec;
 
-pub trait EventHandler<T: EthSpec>: Sized + Send + Sync {
-    fn register(&self, kind: EventKind<T>) -> Result<(), String>;
+const DEFAULT_CHANNEL_CAPACITY: usize = 16;
+
+pub struct ServerSentEventHandler<T: EthSpec> {
+    attestation_tx: Sender<EventKind<T>>,
+    block_tx: Sender<EventKind<T>>,
+    finalized_tx: Sender<EventKind<T>>,
+    head_tx: Sender<EventKind<T>>,
+    exit_tx: Sender<EventKind<T>>,
+    chain_reorg: Sender<EventKind<T>>,
+    log: Logger,
 }
 
-pub struct NullEventHandler<T: EthSpec>(PhantomData<T>);
+impl<T: EthSpec> ServerSentEventHandler<T> {
+    pub fn new(log: Logger) -> Self {
+        let (attestation_tx, _) = broadcast::channel(DEFAULT_CHANNEL_CAPACITY);
+        let (block_tx, _) = broadcast::channel(DEFAULT_CHANNEL_CAPACITY);
+        let (finalized_tx, _) = broadcast::channel(DEFAULT_CHANNEL_CAPACITY);
+        let (head_tx, _) = broadcast::channel(DEFAULT_CHANNEL_CAPACITY);
+        let (exit_tx, _) = broadcast::channel(DEFAULT_CHANNEL_CAPACITY);
+        let (chain_reorg, _) = broadcast::channel(DEFAULT_CHANNEL_CAPACITY);
 
-impl<T: EthSpec> EventHandler<T> for WebSocketSender<T> {
-    fn register(&self, kind: EventKind<T>) -> Result<(), String> {
-        self.send_string(
-            serde_json::to_string(&kind)
-                .map_err(|e| format!("Unable to serialize event: {:?}", e))?,
-        )
+        Self {
+            attestation_tx,
+            block_tx,
+            finalized_tx,
+            head_tx,
+            exit_tx,
+            chain_reorg,
+            log,
+        }
+    }
+
+    pub fn new_with_capacity(log: Logger, capacity: usize) -> Self {
+        let (attestation_tx, _) = broadcast::channel(capacity);
+        let (block_tx, _) = broadcast::channel(capacity);
+        let (finalized_tx, _) = broadcast::channel(capacity);
+        let (head_tx, _) = broadcast::channel(capacity);
+        let (exit_tx, _) = broadcast::channel(capacity);
+        let (chain_reorg, _) = broadcast::channel(capacity);
+
+        Self {
+            attestation_tx,
+            block_tx,
+            finalized_tx,
+            head_tx,
+            exit_tx,
+            chain_reorg,
+            log,
+        }
+    }
+
+    pub fn register(&self, kind: EventKind<T>) {
+        let result = match kind {
+            EventKind::Attestation(attestation) => self
+                .attestation_tx
+                .send(EventKind::Attestation(attestation))
+                .map(|count| trace!(self.log, "Registering server-sent attestation event"; "receiver_count" => count)),
+            EventKind::Block(block) => self.block_tx.send(EventKind::Block(block))
+                .map(|count| trace!(self.log, "Registering server-sent block event"; "receiver_count" => count)),
+            EventKind::FinalizedCheckpoint(checkpoint) => self.finalized_tx
+                .send(EventKind::FinalizedCheckpoint(checkpoint))
+                .map(|count| trace!(self.log, "Registering server-sent finalized checkpoint event"; "receiver_count" => count)),
+            EventKind::Head(head) => self.head_tx.send(EventKind::Head(head))
+                .map(|count| trace!(self.log, "Registering server-sent head event"; "receiver_count" => count)),
+            EventKind::VoluntaryExit(exit) => self.exit_tx.send(EventKind::VoluntaryExit(exit))
+                .map(|count| trace!(self.log, "Registering server-sent voluntary exit event"; "receiver_count" => count)),
+            EventKind::ChainReorg(reorg) => self.chain_reorg.send(EventKind::ChainReorg(reorg))
+                .map(|count| trace!(self.log, "Registering server-sent chain reorg event"; "receiver_count" => count)),
+        };
+        if let Err(SendError(event)) = result {
+            trace!(self.log, "No receivers registered to listen for event"; "event" => ?event);
+        }
+    }
+
+    pub fn subscribe_attestation(&self) -> Receiver<EventKind<T>> {
+        self.attestation_tx.subscribe()
+    }
+
+    pub fn subscribe_block(&self) -> Receiver<EventKind<T>> {
+        self.block_tx.subscribe()
+    }
+
+    pub fn subscribe_finalized(&self) -> Receiver<EventKind<T>> {
+        self.finalized_tx.subscribe()
+    }
+
+    pub fn subscribe_head(&self) -> Receiver<EventKind<T>> {
+        self.head_tx.subscribe()
+    }
+
+    pub fn subscribe_exit(&self) -> Receiver<EventKind<T>> {
+        self.exit_tx.subscribe()
+    }
+
+    pub fn subscribe_reorgs(&self) -> Receiver<EventKind<T>> {
+        self.chain_reorg.subscribe()
+    }
+
+    pub fn has_attestation_subscribers(&self) -> bool {
+        self.attestation_tx.receiver_count() > 0
+    }
+
+    pub fn has_block_subscribers(&self) -> bool {
+        self.block_tx.receiver_count() > 0
+    }
+
+    pub fn has_finalized_subscribers(&self) -> bool {
+        self.finalized_tx.receiver_count() > 0
+    }
+
+    pub fn has_head_subscribers(&self) -> bool {
+        self.head_tx.receiver_count() > 0
+    }
+
+    pub fn has_exit_subscribers(&self) -> bool {
+        self.exit_tx.receiver_count() > 0
+    }
+
+    pub fn has_reorg_subscribers(&self) -> bool {
+        self.chain_reorg.receiver_count() > 0
     }
 }
-
-impl<T: EthSpec> EventHandler<T> for NullEventHandler<T> {
-    fn register(&self, _kind: EventKind<T>) -> Result<(), String> {
-        Ok(())
-    }
-}
-
-impl<T: EthSpec> Default for NullEventHandler<T> {
-    fn default() -> Self {
-        NullEventHandler(PhantomData)
-    }
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-#[serde(
-    bound = "T: EthSpec",
-    rename_all = "snake_case",
-    tag = "event",
-    content = "data"
-)]
-pub enum EventKind<T: EthSpec> {
-    BeaconHeadChanged {
-        reorg: bool,
-        current_head_beacon_block_root: Hash256,
-        previous_head_beacon_block_root: Hash256,
-    },
-    BeaconFinalization {
-        epoch: Epoch,
-        root: Hash256,
-    },
-    BeaconBlockImported {
-        block_root: Hash256,
-        block: Box<BeaconBlock<T>>,
-    },
-    BeaconBlockRejected {
-        reason: String,
-        block: Box<BeaconBlock<T>>,
-    },
-    BeaconAttestationImported {
-        attestation: Box<Attestation<T>>,
-    },
-    BeaconAttestationRejected {
-        reason: String,
-        attestation: Box<Attestation<T>>,
-    },
-}

beacon_node/beacon_chain/src/fork_choice.rs

@@ -1,365 +0,0 @@
-use crate::{errors::BeaconChainError, metrics, BeaconChain, BeaconChainTypes};
-use lmd_ghost::LmdGhost;
-use parking_lot::RwLock;
-use ssz_derive::{Decode, Encode};
-use state_processing::{common::get_attesting_indices, per_slot_processing};
-use std::sync::Arc;
-use store::{Error as StoreError, Store};
-use types::{
-    Attestation, BeaconBlock, BeaconState, BeaconStateError, Checkpoint, EthSpec, Hash256, Slot,
-};
-
-type Result<T> = std::result::Result<T, Error>;
-
-#[derive(Debug, PartialEq)]
-pub enum Error {
-    MissingBlock(Hash256),
-    MissingState(Hash256),
-    BackendError(String),
-    BeaconStateError(BeaconStateError),
-    StoreError(StoreError),
-    BeaconChainError(Box<BeaconChainError>),
-}
-
-pub struct ForkChoice<T: BeaconChainTypes> {
-    store: Arc<T::Store>,
-    backend: T::LmdGhost,
-    /// Used for resolving the `0x00..00` alias back to genesis.
-    ///
-    /// Does not necessarily need to be the _actual_ genesis, it suffices to be the finalized root
-    /// whenever the struct was instantiated.
-    genesis_block_root: Hash256,
-    /// The fork choice rule's current view of the justified checkpoint.
-    justified_checkpoint: RwLock<Checkpoint>,
-    /// The best justified checkpoint we've seen, which may be ahead of `justified_checkpoint`.
-    best_justified_checkpoint: RwLock<Checkpoint>,
-}
-
-impl<T: BeaconChainTypes> PartialEq for ForkChoice<T> {
-    /// This implementation ignores the `store`.
-    fn eq(&self, other: &Self) -> bool {
-        self.backend == other.backend
-            && self.genesis_block_root == other.genesis_block_root
-            && *self.justified_checkpoint.read() == *other.justified_checkpoint.read()
-            && *self.best_justified_checkpoint.read() == *other.best_justified_checkpoint.read()
-    }
-}
-
-impl<T: BeaconChainTypes> ForkChoice<T> {
-    /// Instantiate a new fork chooser.
-    ///
-    /// "Genesis" does not necessarily need to be the absolute genesis, it can be some finalized
-    /// block.
-    pub fn new(
-        store: Arc<T::Store>,
-        backend: T::LmdGhost,
-        genesis_block_root: Hash256,
-        genesis_slot: Slot,
-    ) -> Self {
-        let justified_checkpoint = Checkpoint {
-            epoch: genesis_slot.epoch(T::EthSpec::slots_per_epoch()),
-            root: genesis_block_root,
-        };
-        Self {
-            store: store.clone(),
-            backend,
-            genesis_block_root,
-            justified_checkpoint: RwLock::new(justified_checkpoint.clone()),
-            best_justified_checkpoint: RwLock::new(justified_checkpoint),
-        }
-    }
-
-    /// Determine whether the fork choice's view of the justified checkpoint should be updated.
-    ///
-    /// To prevent the bouncing attack, an update is allowed only in these conditions:
-    ///
-    /// * We're in the first SAFE_SLOTS_TO_UPDATE_JUSTIFIED slots of the epoch, or
-    /// * The new justified checkpoint is a descendant of the current justified checkpoint
-    fn should_update_justified_checkpoint(
-        &self,
-        chain: &BeaconChain<T>,
-        new_justified_checkpoint: &Checkpoint,
-    ) -> Result<bool> {
-        if Self::compute_slots_since_epoch_start(chain.slot()?)
-            < chain.spec.safe_slots_to_update_justified
-        {
-            return Ok(true);
-        }
-
-        let justified_checkpoint = self.justified_checkpoint.read().clone();
-
-        let current_justified_block = chain
-            .get_block(&justified_checkpoint.root)?
-            .ok_or_else(|| Error::MissingBlock(justified_checkpoint.root))?;
-
-        let new_justified_block = chain
-            .get_block(&new_justified_checkpoint.root)?
-            .ok_or_else(|| Error::MissingBlock(new_justified_checkpoint.root))?;
-
-        let slots_per_epoch = T::EthSpec::slots_per_epoch();
-
-        Ok(
-            new_justified_block.slot > justified_checkpoint.epoch.start_slot(slots_per_epoch)
-                && chain.get_ancestor_block_root(
-                    new_justified_checkpoint.root,
-                    current_justified_block.slot,
-                )? == Some(justified_checkpoint.root),
-        )
-    }
-
-    /// Calculate how far `slot` lies from the start of its epoch.
-    fn compute_slots_since_epoch_start(slot: Slot) -> u64 {
-        let slots_per_epoch = T::EthSpec::slots_per_epoch();
-        (slot - slot.epoch(slots_per_epoch).start_slot(slots_per_epoch)).as_u64()
-    }
-
-    /// Run the fork choice rule to determine the head.
-    pub fn find_head(&self, chain: &BeaconChain<T>) -> Result<Hash256> {
-        let timer = metrics::start_timer(&metrics::FORK_CHOICE_FIND_HEAD_TIMES);
-
-        let (start_state, start_block_root, start_block_slot) = {
-            // Check if we should update our view of the justified checkpoint.
-            // Doing this check here should be quasi-equivalent to the update in the `on_tick`
-            // function of the spec, so long as `find_head` is called at least once during the first
-            // SAFE_SLOTS_TO_UPDATE_JUSTIFIED slots.
-            let best_justified_checkpoint = self.best_justified_checkpoint.read();
-            if self.should_update_justified_checkpoint(chain, &best_justified_checkpoint)? {
-                *self.justified_checkpoint.write() = best_justified_checkpoint.clone();
-            }
-
-            let current_justified_checkpoint = self.justified_checkpoint.read().clone();
-
-            let (block_root, block_justified_slot) = (
-                current_justified_checkpoint.root,
-                current_justified_checkpoint
-                    .epoch
-                    .start_slot(T::EthSpec::slots_per_epoch()),
-            );
-
-            let block = chain
-                .store
-                .get::<BeaconBlock<T::EthSpec>>(&block_root)?
-                .ok_or_else(|| Error::MissingBlock(block_root))?;
-
-            // Resolve the `0x00.. 00` alias back to genesis
-            let block_root = if block_root == Hash256::zero() {
-                self.genesis_block_root
-            } else {
-                block_root
-            };
-
-            let mut state: BeaconState<T::EthSpec> = chain
-                .store
-                .get_state(&block.state_root, Some(block.slot))?
-                .ok_or_else(|| Error::MissingState(block.state_root))?;
-
-            // Fast-forward the state to the start slot of the epoch where it was justified.
-            for _ in block.slot.as_u64()..block_justified_slot.as_u64() {
-                per_slot_processing(&mut state, &chain.spec)
-                    .map_err(BeaconChainError::SlotProcessingError)?
-            }
-
-            (state, block_root, block_justified_slot)
-        };
-
-        // A function that returns the weight for some validator index.
-        let weight = |validator_index: usize| -> Option<u64> {
-            start_state
-                .validators
-                .get(validator_index)
-                .map(|v| v.effective_balance)
-        };
-
-        let result = self
-            .backend
-            .find_head(start_block_slot, start_block_root, weight)
-            .map_err(Into::into);
-
-        metrics::stop_timer(timer);
-
-        result
-    }
-
-    /// Process all attestations in the given `block`.
-    ///
-    /// Assumes the block (and therefore its attestations) are valid. It is a logic error to
-    /// provide an invalid block.
-    pub fn process_block(
-        &self,
-        chain: &BeaconChain<T>,
-        state: &BeaconState<T::EthSpec>,
-        block: &BeaconBlock<T::EthSpec>,
-        block_root: Hash256,
-    ) -> Result<()> {
-        let timer = metrics::start_timer(&metrics::FORK_CHOICE_PROCESS_BLOCK_TIMES);
-        // Note: we never count the block as a latest message, only attestations.
-        //
-        // I (Paul H) do not have an explicit reference to this, but I derive it from this
-        // document:
-        //
-        // https://github.com/ethereum/eth2.0-specs/blob/v0.7.0/specs/core/0_fork-choice.md
-        for attestation in &block.body.attestations {
-            // If the `data.beacon_block_root` block is not known to us, simply ignore the latest
-            // vote.
-            if let Some(block) = self
-                .store
-                .get::<BeaconBlock<T::EthSpec>>(&attestation.data.beacon_block_root)?
-            {
-                self.process_attestation(state, attestation, &block)?;
-            }
-        }
-
-        // Check if we should update our view of the justified checkpoint
-        if state.current_justified_checkpoint.epoch > self.justified_checkpoint.read().epoch {
-            *self.best_justified_checkpoint.write() = state.current_justified_checkpoint.clone();
-            if self
-                .should_update_justified_checkpoint(chain, &state.current_justified_checkpoint)?
-            {
-                *self.justified_checkpoint.write() = state.current_justified_checkpoint.clone();
-            }
-        }
-
-        // This does not apply a vote to the block, it just makes fork choice aware of the block so
-        // it can still be identified as the head even if it doesn't have any votes.
-        //
-        // A case where a block without any votes can be the head is where it is the only child of
-        // a block that has the majority of votes applied to it.
-        self.backend.process_block(block, block_root)?;
-
-        metrics::stop_timer(timer);
-
-        Ok(())
-    }
-
-    /// Process an attestation which references `block` in `attestation.data.beacon_block_root`.
-    ///
-    /// Assumes the attestation is valid.
-    pub fn process_attestation(
-        &self,
-        state: &BeaconState<T::EthSpec>,
-        attestation: &Attestation<T::EthSpec>,
-        block: &BeaconBlock<T::EthSpec>,
-    ) -> Result<()> {
-        let timer = metrics::start_timer(&metrics::FORK_CHOICE_PROCESS_ATTESTATION_TIMES);
-
-        let block_hash = attestation.data.beacon_block_root;
-
-        // Ignore any attestations to the zero hash.
-        //
-        // This is an edge case that results from the spec aliasing the zero hash to the genesis
-        // block. Attesters may attest to the zero hash if they have never seen a block.
-        //
-        // We have two options here:
-        //
-        //  1. Apply all zero-hash attestations to the zero hash.
-        //  2. Ignore all attestations to the zero hash.
-        //
-        // (1) becomes weird once we hit finality and fork choice drops the genesis block. (2) is
-        // fine because votes to the genesis block are not useful; all validators implicitly attest
-        // to genesis just by being present in the chain.
-        //
-        // Additionally, don't add any block hash to fork choice unless we have imported the block.
-        if block_hash != Hash256::zero() {
-            let validator_indices =
-                get_attesting_indices(state, &attestation.data, &attestation.aggregation_bits)?;
-
-            for validator_index in validator_indices {
-                self.backend
-                    .process_attestation(validator_index, block_hash, block.slot)?;
-            }
-        }
-
-        metrics::stop_timer(timer);
-
-        Ok(())
-    }
-
-    /// Returns the latest message for a given validator, if any.
-    ///
-    /// Returns `(block_root, block_slot)`.
-    pub fn latest_message(&self, validator_index: usize) -> Option<(Hash256, Slot)> {
-        self.backend.latest_message(validator_index)
-    }
-
-    /// Runs an integrity verification function on the underlying fork choice algorithm.
-    ///
-    /// Returns `Ok(())` if the underlying fork choice has maintained it's integrity,
-    /// `Err(description)` otherwise.
-    pub fn verify_integrity(&self) -> core::result::Result<(), String> {
-        self.backend.verify_integrity()
-    }
-
-    /// Inform the fork choice that the given block (and corresponding root) have been finalized so
-    /// it may prune it's storage.
-    ///
-    /// `finalized_block_root` must be the root of `finalized_block`.
-    pub fn process_finalization(
-        &self,
-        finalized_block: &BeaconBlock<T::EthSpec>,
-        finalized_block_root: Hash256,
-    ) -> Result<()> {
-        self.backend
-            .update_finalized_root(finalized_block, finalized_block_root)
-            .map_err(Into::into)
-    }
-
-    /// Returns a `SszForkChoice` which contains the current state of `Self`.
-    pub fn as_ssz_container(&self) -> SszForkChoice {
-        SszForkChoice {
-            genesis_block_root: self.genesis_block_root.clone(),
-            justified_checkpoint: self.justified_checkpoint.read().clone(),
-            best_justified_checkpoint: self.best_justified_checkpoint.read().clone(),
-            backend_bytes: self.backend.as_bytes(),
-        }
-    }
-
-    /// Instantiates `Self` from a prior `SszForkChoice`.
-    ///
-    /// The created `Self` will have the same state as the `Self` that created the `SszForkChoice`.
-    pub fn from_ssz_container(ssz_container: SszForkChoice, store: Arc<T::Store>) -> Result<Self> {
-        let backend = LmdGhost::from_bytes(&ssz_container.backend_bytes, store.clone())?;
-
-        Ok(Self {
-            store,
-            backend,
-            genesis_block_root: ssz_container.genesis_block_root,
-            justified_checkpoint: RwLock::new(ssz_container.justified_checkpoint),
-            best_justified_checkpoint: RwLock::new(ssz_container.best_justified_checkpoint),
-        })
-    }
-}
-
-/// Helper struct that is used to encode/decode the state of the `ForkChoice` as SSZ bytes.
-///
-/// This is used when persisting the state of the `BeaconChain` to disk.
-#[derive(Encode, Decode, Clone)]
-pub struct SszForkChoice {
-    genesis_block_root: Hash256,
-    justified_checkpoint: Checkpoint,
-    best_justified_checkpoint: Checkpoint,
-    backend_bytes: Vec<u8>,
-}
-
-impl From<BeaconStateError> for Error {
-    fn from(e: BeaconStateError) -> Error {
-        Error::BeaconStateError(e)
-    }
-}
-
-impl From<BeaconChainError> for Error {
-    fn from(e: BeaconChainError) -> Error {
-        Error::BeaconChainError(Box::new(e))
-    }
-}
-
-impl From<StoreError> for Error {
-    fn from(e: StoreError) -> Error {
-        Error::StoreError(e)
-    }
-}
-
-impl From<String> for Error {
-    fn from(e: String) -> Error {
-        Error::BackendError(e)
-    }
-}

beacon_node/beacon_chain/src/head_tracker.rs

@@ -1,8 +1,7 @@
 use parking_lot::RwLock;
 use ssz_derive::{Decode, Encode};
 use std::collections::HashMap;
-use std::iter::FromIterator;
-use types::{BeaconBlock, EthSpec, Hash256, Slot};
+use types::{Hash256, Slot};
 
 #[derive(Debug, PartialEq)]
 pub enum Error {
@@ -15,7 +14,7 @@ pub enum Error {
 /// In order for this struct to be effective, every single block that is imported must be
 /// registered here.
 #[derive(Default, Debug)]
-pub struct HeadTracker(RwLock<HashMap<Hash256, Slot>>);
+pub struct HeadTracker(pub RwLock<HashMap<Hash256, Slot>>);
 
 impl HeadTracker {
     /// Register a block with `Self`, so it may or may not be included in a `Self::heads` call.
@@ -23,11 +22,15 @@ impl HeadTracker {
     /// This function assumes that no block is imported without its parent having already been
     /// imported. It cannot detect an error if this is not the case, it is the responsibility of
     /// the upstream user.
-    pub fn register_block<E: EthSpec>(&self, block_root: Hash256, block: &BeaconBlock<E>) {
+    pub fn register_block(&self, block_root: Hash256, parent_root: Hash256, slot: Slot) {
         let mut map = self.0.write();
+        map.remove(&parent_root);
+        map.insert(block_root, slot);
+    }
 
-        map.remove(&block.parent_root);
-        map.insert(block_root, block.slot);
+    /// Returns true iff `block_root` is a recognized head.
+    pub fn contains_head(&self, block_root: Hash256) -> bool {
+        self.0.read().contains_key(&block_root)
     }
 
     /// Returns the list of heads in the chain.
@@ -42,14 +45,7 @@ impl HeadTracker {
     /// Returns a `SszHeadTracker`, which contains all necessary information to restore the state
     /// of `Self` at some later point.
     pub fn to_ssz_container(&self) -> SszHeadTracker {
-        let (roots, slots) = self
-            .0
-            .read()
-            .iter()
-            .map(|(hash, slot)| (*hash, *slot))
-            .unzip();
-
-        SszHeadTracker { roots, slots }
+        SszHeadTracker::from_map(&*self.0.read())
     }
 
     /// Creates a new `Self` from the given `SszHeadTracker`, restoring `Self` to the same state of
@@ -59,18 +55,17 @@ impl HeadTracker {
         let slots_len = ssz_container.slots.len();
 
         if roots_len != slots_len {
-            return Err(Error::MismatchingLengths {
+            Err(Error::MismatchingLengths {
                 roots_len,
                 slots_len,
-            });
+            })
         } else {
-            let map = HashMap::from_iter(
-                ssz_container
-                    .roots
-                    .iter()
-                    .zip(ssz_container.slots.iter())
-                    .map(|(root, slot)| (*root, *slot)),
-            );
+            let map = ssz_container
+                .roots
+                .iter()
+                .zip(ssz_container.slots.iter())
+                .map(|(root, slot)| (*root, *slot))
+                .collect::<HashMap<_, _>>();
 
             Ok(Self(RwLock::new(map)))
         }
@@ -92,11 +87,18 @@ pub struct SszHeadTracker {
     slots: Vec<Slot>,
 }
 
+impl SszHeadTracker {
+    pub fn from_map(map: &HashMap<Hash256, Slot>) -> Self {
+        let (roots, slots) = map.iter().map(|(hash, slot)| (*hash, *slot)).unzip();
+        SszHeadTracker { roots, slots }
+    }
+}
+
 #[cfg(test)]
 mod test {
     use super::*;
     use ssz::{Decode, Encode};
-    use types::MainnetEthSpec;
+    use types::{BeaconBlock, EthSpec, MainnetEthSpec};
 
     type E = MainnetEthSpec;
 
@@ -107,17 +109,17 @@ mod test {
         let head_tracker = HeadTracker::default();
 
         for i in 0..16 {
-            let mut block = BeaconBlock::empty(spec);
+            let mut block: BeaconBlock<E> = BeaconBlock::empty(spec);
             let block_root = Hash256::from_low_u64_be(i);
 
-            block.slot = Slot::new(i);
-            block.parent_root = if i == 0 {
+            *block.slot_mut() = Slot::new(i);
+            *block.parent_root_mut() = if i == 0 {
                 Hash256::random()
             } else {
                 Hash256::from_low_u64_be(i - 1)
             };
 
-            head_tracker.register_block::<E>(block_root, &block);
+            head_tracker.register_block(block_root, block.parent_root(), block.slot());
         }
 
         assert_eq!(
@@ -126,11 +128,11 @@ mod test {
             "should only have one head"
         );
 
-        let mut block = BeaconBlock::empty(spec);
+        let mut block: BeaconBlock<E> = BeaconBlock::empty(spec);
         let block_root = Hash256::from_low_u64_be(42);
-        block.slot = Slot::new(15);
-        block.parent_root = Hash256::from_low_u64_be(14);
-        head_tracker.register_block::<E>(block_root, &block);
+        *block.slot_mut() = Slot::new(15);
+        *block.parent_root_mut() = Hash256::from_low_u64_be(14);
+        head_tracker.register_block(block_root, block.parent_root(), block.slot());
 
         let heads = head_tracker.heads();
 

beacon_node/beacon_chain/src/lib.rs

@@ -1,29 +1,48 @@
 #![recursion_limit = "128"] // For lazy-static
-#[macro_use]
-extern crate lazy_static;
-
+pub mod attestation_verification;
+mod attester_cache;
 mod beacon_chain;
+mod beacon_fork_choice_store;
+mod beacon_proposer_cache;
+mod beacon_snapshot;
+mod block_verification;
 pub mod builder;
-mod checkpoint;
-mod checkpoint_cache;
+pub mod chain_config;
 mod errors;
 pub mod eth1_chain;
 pub mod events;
-mod fork_choice;
 mod head_tracker;
 mod metrics;
+pub mod migrate;
+mod naive_aggregation_pool;
+mod observed_aggregates;
+mod observed_attesters;
+mod observed_block_producers;
+pub mod observed_operations;
 mod persisted_beacon_chain;
+mod persisted_fork_choice;
+pub mod schema_change;
+mod shuffling_cache;
+mod snapshot_cache;
+pub mod state_advance_timer;
+pub mod sync_committee_verification;
 pub mod test_utils;
+mod timeout_rw_lock;
+pub mod validator_monitor;
+mod validator_pubkey_cache;
 
 pub use self::beacon_chain::{
-    AttestationProcessingOutcome, BeaconChain, BeaconChainTypes, BlockProcessingOutcome,
+    AttestationProcessingOutcome, BeaconChain, BeaconChainTypes, BeaconStore, ChainSegmentResult,
+    ForkChoiceError, StateSkipConfig, WhenSlotSkipped, MAXIMUM_GOSSIP_CLOCK_DISPARITY,
 };
-pub use self::checkpoint::CheckPoint;
+pub use self::beacon_snapshot::BeaconSnapshot;
+pub use self::chain_config::ChainConfig;
 pub use self::errors::{BeaconChainError, BlockProductionError};
+pub use attestation_verification::Error as AttestationError;
+pub use beacon_fork_choice_store::{BeaconForkChoiceStore, Error as ForkChoiceStoreError};
+pub use block_verification::{BlockError, GossipVerifiedBlock};
 pub use eth1_chain::{Eth1Chain, Eth1ChainBackend};
-pub use events::EventHandler;
-pub use fork_choice::ForkChoice;
-pub use lmd_ghost;
+pub use events::ServerSentEventHandler;
 pub use metrics::scrape_for_metrics;
 pub use parking_lot;
 pub use slot_clock;

beacon_node/beacon_chain/src/metrics.rs

@@ -1,6 +1,10 @@
-use crate::{BeaconChain, BeaconChainTypes};
+use crate::observed_attesters::SlotSubcommitteeIndex;
+use crate::types::consts::altair::SYNC_COMMITTEE_SUBNET_COUNT;
+use crate::{BeaconChain, BeaconChainError, BeaconChainTypes};
+use lazy_static::lazy_static;
 pub use lighthouse_metrics::*;
-use types::{BeaconState, Epoch, Hash256, Slot};
+use slot_clock::SlotClock;
+use types::{BeaconState, Epoch, EthSpec, Hash256, Slot};
 
 lazy_static! {
     /*
@@ -32,6 +36,10 @@ lazy_static! {
         "beacon_block_processing_committee_building_seconds",
         "Time spent building/obtaining committees for block processing."
     );
+    pub static ref BLOCK_PROCESSING_SIGNATURE: Result<Histogram> = try_create_histogram(
+        "beacon_block_processing_signature_seconds",
+        "Time spent doing signature verification for a block."
+    );
     pub static ref BLOCK_PROCESSING_CORE: Result<Histogram> = try_create_histogram(
         "beacon_block_processing_core_seconds",
         "Time spent doing the core per_block_processing state processing."
@@ -44,9 +52,13 @@ lazy_static! {
         "beacon_block_processing_db_write_seconds",
         "Time spent writing a newly processed block and state to DB"
     );
-    pub static ref BLOCK_PROCESSING_FORK_CHOICE_REGISTER: Result<Histogram> = try_create_histogram(
-        "beacon_block_processing_fork_choice_register_seconds",
-        "Time spent registering the new block with fork choice (but not finding head)"
+    pub static ref BLOCK_PROCESSING_ATTESTATION_OBSERVATION: Result<Histogram> = try_create_histogram(
+        "beacon_block_processing_attestation_observation_seconds",
+        "Time spent hashing and remembering all the attestations in the block"
+    );
+    pub static ref BLOCK_SYNC_AGGREGATE_SET_BITS: Result<IntGauge> = try_create_int_gauge(
+        "block_sync_aggregate_set_bits",
+        "The number of true bits in the last sync aggregate in a block"
     );
 
     /*
@@ -62,6 +74,30 @@ lazy_static! {
     );
     pub static ref BLOCK_PRODUCTION_TIMES: Result<Histogram> =
         try_create_histogram("beacon_block_production_seconds", "Full runtime of block production");
+    pub static ref BLOCK_PRODUCTION_STATE_LOAD_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_block_production_state_load_seconds",
+        "Time taken to load the base state for block production"
+    );
+    pub static ref BLOCK_PRODUCTION_SLOT_PROCESS_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_block_production_slot_process_seconds",
+        "Time taken to advance the state to the block production slot"
+    );
+    pub static ref BLOCK_PRODUCTION_UNAGGREGATED_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_block_production_unaggregated_seconds",
+        "Time taken to import the naive aggregation pool for block production"
+    );
+    pub static ref BLOCK_PRODUCTION_ATTESTATION_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_block_production_attestation_seconds",
+        "Time taken to pack attestations into a block"
+    );
+    pub static ref BLOCK_PRODUCTION_PROCESS_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_block_production_process_seconds",
+        "Time taken to process the block produced"
+    );
+    pub static ref BLOCK_PRODUCTION_STATE_ROOT_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_block_production_state_root_seconds",
+        "Time taken to calculate the block's state root"
+    );
 
     /*
      * Block Statistics
@@ -71,42 +107,129 @@ lazy_static! {
         "Number of attestations in a block"
     );
 
+    /*
+     * Unaggregated Attestation Verification
+     */
+    pub static ref UNAGGREGATED_ATTESTATION_PROCESSING_REQUESTS: Result<IntCounter> = try_create_int_counter(
+        "beacon_unaggregated_attestation_processing_requests_total",
+        "Count of all unaggregated attestations submitted for processing"
+    );
+    pub static ref UNAGGREGATED_ATTESTATION_PROCESSING_SUCCESSES: Result<IntCounter> = try_create_int_counter(
+        "beacon_unaggregated_attestation_processing_successes_total",
+        "Number of unaggregated attestations verified for gossip"
+    );
+    pub static ref UNAGGREGATED_ATTESTATION_GOSSIP_VERIFICATION_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_unaggregated_attestation_gossip_verification_seconds",
+        "Full runtime of aggregated attestation gossip verification"
+    );
+
+    /*
+     * Aggregated Attestation Verification
+     */
+    pub static ref AGGREGATED_ATTESTATION_PROCESSING_REQUESTS: Result<IntCounter> = try_create_int_counter(
+        "beacon_aggregated_attestation_processing_requests_total",
+        "Count of all aggregated attestations submitted for processing"
+    );
+    pub static ref AGGREGATED_ATTESTATION_PROCESSING_SUCCESSES: Result<IntCounter> = try_create_int_counter(
+        "beacon_aggregated_attestation_processing_successes_total",
+        "Number of aggregated attestations verified for gossip"
+    );
+    pub static ref AGGREGATED_ATTESTATION_GOSSIP_VERIFICATION_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_aggregated_attestation_gossip_verification_seconds",
+        "Full runtime of aggregated attestation gossip verification"
+    );
+
+    /*
+     * General Attestation Processing
+     */
+    pub static ref ATTESTATION_PROCESSING_APPLY_TO_AGG_POOL: Result<Histogram> = try_create_histogram(
+        "beacon_attestation_processing_apply_to_agg_pool",
+        "Time spent applying an attestation to the naive aggregation pool"
+    );
+    pub static ref ATTESTATION_PROCESSING_AGG_POOL_PRUNE: Result<Histogram> = try_create_histogram(
+        "beacon_attestation_processing_agg_pool_prune",
+        "Time spent for the agg pool to prune"
+    );
+    pub static ref ATTESTATION_PROCESSING_AGG_POOL_INSERT: Result<Histogram> = try_create_histogram(
+        "beacon_attestation_processing_agg_pool_insert",
+        "Time spent for the outer pool.insert() function of agg pool"
+    );
+    pub static ref ATTESTATION_PROCESSING_AGG_POOL_CORE_INSERT: Result<Histogram> = try_create_histogram(
+        "beacon_attestation_processing_agg_pool_core_insert",
+        "Time spent for the core map.insert() function of agg pool"
+    );
+    pub static ref ATTESTATION_PROCESSING_AGG_POOL_AGGREGATION: Result<Histogram> = try_create_histogram(
+        "beacon_attestation_processing_agg_pool_aggregation",
+        "Time spent doing signature aggregation when adding to the agg poll"
+    );
+    pub static ref ATTESTATION_PROCESSING_AGG_POOL_CREATE_MAP: Result<Histogram> = try_create_histogram(
+        "beacon_attestation_processing_agg_pool_create_map",
+        "Time spent for creating a map for a new slot"
+    );
+    pub static ref ATTESTATION_PROCESSING_APPLY_TO_OP_POOL: Result<Histogram> = try_create_histogram(
+        "beacon_attestation_processing_apply_to_op_pool",
+        "Time spent applying an attestation to the block inclusion pool"
+    );
+
     /*
      * Attestation Processing
      */
-    pub static ref ATTESTATION_PROCESSING_REQUESTS: Result<IntCounter> = try_create_int_counter(
-        "beacon_attestation_processing_requests_total",
-        "Count of all attestations submitted for processing"
+    pub static ref ATTESTATION_PROCESSING_SHUFFLING_CACHE_WAIT_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_attestation_processing_shuffling_cache_wait_seconds",
+        "Time spent on waiting for the shuffling cache lock during attestation processing"
     );
-    pub static ref ATTESTATION_PROCESSING_SUCCESSES: Result<IntCounter> = try_create_int_counter(
-        "beacon_attestation_processing_successes_total",
-        "total_attestation_processing_successes"
+    pub static ref ATTESTATION_PROCESSING_COMMITTEE_BUILDING_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_attestation_processing_committee_building_seconds",
+        "Time spent on building committees during attestation processing"
     );
-    pub static ref ATTESTATION_PROCESSING_TIMES: Result<Histogram> = try_create_histogram(
-        "beacon_attestation_processing_seconds",
-        "Full runtime of attestation processing"
+    pub static ref ATTESTATION_PROCESSING_STATE_READ_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_attestation_processing_state_read_seconds",
+        "Time spent on reading the state during attestation processing"
     );
-    pub static ref ATTESTATION_PROCESSING_CORE: Result<Histogram> = try_create_histogram(
-        "beacon_attestation_processing_core_seconds",
-        "Time spent on the core spec processing of attestation processing"
+    pub static ref ATTESTATION_PROCESSING_STATE_SKIP_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_attestation_processing_state_skip_seconds",
+        "Time spent on reading the state during attestation processing"
     );
+    pub static ref ATTESTATION_PROCESSING_SIGNATURE_SETUP_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_attestation_processing_signature_setup_seconds",
+        "Time spent on setting up for the signature verification of attestation processing"
+    );
+    pub static ref ATTESTATION_PROCESSING_SIGNATURE_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_attestation_processing_signature_seconds",
+        "Time spent on the signature verification of attestation processing"
+    );
+
+    /*
+     * Shuffling cache
+     */
+    pub static ref SHUFFLING_CACHE_HITS: Result<IntCounter> =
+        try_create_int_counter("beacon_shuffling_cache_hits_total", "Count of times shuffling cache fulfils request");
+    pub static ref SHUFFLING_CACHE_MISSES: Result<IntCounter> =
+        try_create_int_counter("beacon_shuffling_cache_misses_total", "Count of times shuffling cache fulfils request");
 
     /*
      * Attestation Production
      */
-    pub static ref ATTESTATION_PRODUCTION_REQUESTS: Result<IntCounter> = try_create_int_counter(
-        "beacon_attestation_production_requests_total",
-        "Count of all attestation production requests"
-    );
-    pub static ref ATTESTATION_PRODUCTION_SUCCESSES: Result<IntCounter> = try_create_int_counter(
-        "beacon_attestation_production_successes_total",
-        "Count of attestations processed without error"
-    );
-    pub static ref ATTESTATION_PRODUCTION_TIMES: Result<Histogram> = try_create_histogram(
+    pub static ref ATTESTATION_PRODUCTION_SECONDS: Result<Histogram> = try_create_histogram(
         "beacon_attestation_production_seconds",
         "Full runtime of attestation production"
     );
+    pub static ref ATTESTATION_PRODUCTION_HEAD_SCRAPE_SECONDS: Result<Histogram> = try_create_histogram(
+        "attestation_production_head_scrape_seconds",
+        "Time taken to read the head state"
+    );
+    pub static ref ATTESTATION_PRODUCTION_CACHE_INTERACTION_SECONDS: Result<Histogram> = try_create_histogram(
+        "attestation_production_cache_interaction_seconds",
+        "Time spent interacting with the attester cache"
+    );
+    pub static ref ATTESTATION_PRODUCTION_CACHE_PRIME_SECONDS: Result<Histogram> = try_create_histogram(
+        "attestation_production_cache_prime_seconds",
+        "Time spent loading a new state from the disk due to a cache miss"
+    );
+}
 
+// Second lazy-static block is used to account for macro recursion limit.
+lazy_static! {
     /*
      * Fork Choice
      */
@@ -138,20 +261,28 @@ lazy_static! {
         "beacon_fork_choice_process_attestation_seconds",
         "Time taken to add an attestation to fork choice"
     );
+    pub static ref BALANCES_CACHE_HITS: Result<IntCounter> =
+        try_create_int_counter("beacon_balances_cache_hits_total", "Count of times balances cache fulfils request");
+    pub static ref BALANCES_CACHE_MISSES: Result<IntCounter> =
+        try_create_int_counter("beacon_balances_cache_misses_total", "Count of times balances cache fulfils request");
 
     /*
-     * Persisting BeaconChain to disk
+     * Persisting BeaconChain components to disk
      */
-    pub static ref PERSIST_CHAIN: Result<Histogram> =
-        try_create_histogram("beacon_persist_chain", "Time taken to update the canonical head");
+    pub static ref PERSIST_HEAD: Result<Histogram> =
+        try_create_histogram("beacon_persist_head", "Time taken to persist the canonical head");
+    pub static ref PERSIST_OP_POOL: Result<Histogram> =
+        try_create_histogram("beacon_persist_op_pool", "Time taken to persist the operations pool");
+    pub static ref PERSIST_ETH1_CACHE: Result<Histogram> =
+        try_create_histogram("beacon_persist_eth1_cache", "Time taken to persist the eth1 caches");
+    pub static ref PERSIST_FORK_CHOICE: Result<Histogram> =
+        try_create_histogram("beacon_persist_fork_choice", "Time taken to persist the fork choice struct");
 
     /*
-     * Checkpoint cache
+     * Eth1
      */
-    pub static ref CHECKPOINT_CACHE_HITS: Result<IntCounter> =
-        try_create_int_counter("beacon_checkpoint_cache_hits_total", "Count of times checkpoint cache fulfils request");
-    pub static ref CHECKPOINT_CACHE_MISSES: Result<IntCounter> =
-        try_create_int_counter("beacon_checkpoint_cache_misses_total", "Count of times checkpoint cache fulfils request");
+    pub static ref DEFAULT_ETH1_VOTES: Result<IntCounter> =
+        try_create_int_counter("beacon_eth1_default_votes", "Count of times we have voted default value for eth1 data");
 
     /*
      * Chain Head
@@ -188,69 +319,577 @@ lazy_static! {
         try_create_int_gauge("beacon_head_state_withdrawn_validators_total", "Sum of all validator balances at the head of the chain");
     pub static ref HEAD_STATE_ETH1_DEPOSIT_INDEX: Result<IntGauge> =
         try_create_int_gauge("beacon_head_state_eth1_deposit_index", "Eth1 deposit index at the head of the chain");
+
+    /*
+     * Operation Pool
+     */
+    pub static ref OP_POOL_NUM_ATTESTATIONS: Result<IntGauge> =
+        try_create_int_gauge("beacon_op_pool_attestations_total", "Count of attestations in the op pool");
+    pub static ref OP_POOL_NUM_ATTESTER_SLASHINGS: Result<IntGauge> =
+        try_create_int_gauge("beacon_op_pool_attester_slashings_total", "Count of attester slashings in the op pool");
+    pub static ref OP_POOL_NUM_PROPOSER_SLASHINGS: Result<IntGauge> =
+        try_create_int_gauge("beacon_op_pool_proposer_slashings_total", "Count of proposer slashings in the op pool");
+    pub static ref OP_POOL_NUM_VOLUNTARY_EXITS: Result<IntGauge> =
+        try_create_int_gauge("beacon_op_pool_voluntary_exits_total", "Count of voluntary exits in the op pool");
+    pub static ref OP_POOL_NUM_SYNC_CONTRIBUTIONS: Result<IntGauge> =
+        try_create_int_gauge("beacon_op_pool_sync_contributions_total", "Count of sync contributions in the op pool");
+
+
+    /*
+     * Attestation Observation Metrics
+     */
+    pub static ref ATTN_OBSERVATION_PREV_EPOCH_ATTESTERS: Result<IntGauge> = try_create_int_gauge(
+        "beacon_attn_observation_epoch_attesters",
+        "Count of attesters that have been seen by the beacon chain in the previous epoch"
+    );
+    pub static ref ATTN_OBSERVATION_PREV_EPOCH_AGGREGATORS: Result<IntGauge> = try_create_int_gauge(
+        "beacon_attn_observation_epoch_aggregators",
+        "Count of aggregators that have been seen by the beacon chain in the previous epoch"
+    );
+
+    /*
+     * Sync Committee Observation Metrics
+     */
+    pub static ref SYNC_COMM_OBSERVATION_PREV_SLOT_SIGNERS: Result<IntGauge> = try_create_int_gauge(
+        "beacon_sync_comm_observation_slot_signers",
+        "Count of sync committee contributors that have been seen by the beacon chain in the previous slot"
+    );
+    pub static ref SYNC_COMM_OBSERVATION_PREV_SLOT_AGGREGATORS: Result<IntGauge> = try_create_int_gauge(
+        "beacon_sync_comm_observation_slot_aggregators",
+        "Count of sync committee aggregators that have been seen by the beacon chain in the previous slot"
+    );
+}
+
+// Third lazy-static block is used to account for macro recursion limit.
+lazy_static! {
+    /*
+     * Validator Monitor Metrics (balances, etc)
+     */
+    pub static ref VALIDATOR_MONITOR_BALANCE_GWEI: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_balance_gwei",
+            "The validator's balance in gwei.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_EFFECTIVE_BALANCE_GWEI: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_effective_balance_gwei",
+            "The validator's effective balance in gwei.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_SLASHED: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_slashed",
+            "Set to 1 if the validator is slashed.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_ACTIVE: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_active",
+            "Set to 1 if the validator is active.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_EXITED: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_exited",
+            "Set to 1 if the validator is exited.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_WITHDRAWABLE: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_withdrawable",
+            "Set to 1 if the validator is withdrawable.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_ACTIVATION_ELIGIBILITY_EPOCH: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_activation_eligibility_epoch",
+            "Set to the epoch where the validator will be eligible for activation.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_ACTIVATION_EPOCH: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_activation_epoch",
+            "Set to the epoch where the validator will activate.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_EXIT_EPOCH: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_exit_epoch",
+            "Set to the epoch where the validator will exit.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_WITHDRAWABLE_EPOCH: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_withdrawable_epoch",
+            "Set to the epoch where the validator will be withdrawable.",
+            &["validator"]
+        );
+
+    /*
+     * Validator Monitor Metrics (per-epoch summaries)
+     */
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_ATTESTER_HIT: Result<IntCounterVec> =
+        try_create_int_counter_vec(
+            "validator_monitor_prev_epoch_on_chain_attester_hit",
+            "Incremented if the validator is flagged as a previous epoch attester \
+            during per epoch processing",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_ATTESTER_MISS: Result<IntCounterVec> =
+        try_create_int_counter_vec(
+            "validator_monitor_prev_epoch_on_chain_attester_miss",
+            "Incremented if the validator is not flagged as a previous epoch attester \
+            during per epoch processing",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_HEAD_ATTESTER_HIT: Result<IntCounterVec> =
+        try_create_int_counter_vec(
+            "validator_monitor_prev_epoch_on_chain_head_attester_hit",
+            "Incremented if the validator is flagged as a previous epoch head attester \
+            during per epoch processing",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_HEAD_ATTESTER_MISS: Result<IntCounterVec> =
+        try_create_int_counter_vec(
+            "validator_monitor_prev_epoch_on_chain_head_attester_miss",
+            "Incremented if the validator is not flagged as a previous epoch head attester \
+            during per epoch processing",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_TARGET_ATTESTER_HIT: Result<IntCounterVec> =
+        try_create_int_counter_vec(
+            "validator_monitor_prev_epoch_on_chain_target_attester_hit",
+            "Incremented if the validator is flagged as a previous epoch target attester \
+            during per epoch processing",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_TARGET_ATTESTER_MISS: Result<IntCounterVec> =
+        try_create_int_counter_vec(
+            "validator_monitor_prev_epoch_on_chain_target_attester_miss",
+            "Incremented if the validator is not flagged as a previous epoch target attester \
+            during per epoch processing",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_INCLUSION_DISTANCE: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_prev_epoch_on_chain_inclusion_distance",
+            "The attestation inclusion distance calculated during per epoch processing",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATIONS_TOTAL: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_prev_epoch_attestations_total",
+            "The number of unagg. attestations seen in the previous epoch.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATIONS_MIN_DELAY_SECONDS: Result<HistogramVec> =
+        try_create_histogram_vec(
+            "validator_monitor_prev_epoch_attestations_min_delay_seconds",
+            "The min delay between when the validator should send the attestation and when it was received.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATION_AGGREGATE_INCLUSIONS: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_prev_epoch_attestation_aggregate_inclusions",
+            "The count of times an attestation was seen inside an aggregate.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATION_BLOCK_INCLUSIONS: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_prev_epoch_attestation_block_inclusions",
+            "The count of times an attestation was seen inside a block.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATION_BLOCK_MIN_INCLUSION_DISTANCE: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_prev_epoch_attestation_block_min_inclusion_distance",
+            "The minimum inclusion distance observed for the inclusion of an attestation in a block.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_BEACON_BLOCKS_TOTAL: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_prev_epoch_beacon_blocks_total",
+            "The number of beacon_blocks seen in the previous epoch.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_BEACON_BLOCKS_MIN_DELAY_SECONDS: Result<HistogramVec> =
+        try_create_histogram_vec(
+            "validator_monitor_prev_epoch_beacon_blocks_min_delay_seconds",
+            "The min delay between when the validator should send the block and when it was received.",
+            &["validator"]
+       );
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_AGGREGATES_TOTAL: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_prev_epoch_aggregates_total",
+            "The number of aggregates seen in the previous epoch.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_AGGREGATES_MIN_DELAY_SECONDS: Result<HistogramVec> =
+        try_create_histogram_vec(
+            "validator_monitor_prev_epoch_aggregates_min_delay_seconds",
+            "The min delay between when the validator should send the aggregate and when it was received.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_EXITS_TOTAL: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_prev_epoch_exits_total",
+            "The number of exits seen in the previous epoch.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_PROPOSER_SLASHINGS_TOTAL: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_prev_epoch_proposer_slashings_total",
+            "The number of proposer slashings seen in the previous epoch.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_ATTESTER_SLASHINGS_TOTAL: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_prev_epoch_attester_slashings_total",
+            "The number of attester slashings seen in the previous epoch.",
+            &["validator"]
+        );
+
+    /*
+     * Validator Monitor Metrics (real-time)
+     */
+    pub static ref VALIDATOR_MONITOR_VALIDATORS_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "validator_monitor_validators_total",
+        "Count of validators that are specifically monitored by this beacon node"
+    );
+    pub static ref VALIDATOR_MONITOR_UNAGGREGATED_ATTESTATION_TOTAL: Result<IntCounterVec> = try_create_int_counter_vec(
+        "validator_monitor_unaggregated_attestation_total",
+        "Number of unaggregated attestations seen",
+        &["src", "validator"]
+    );
+    pub static ref VALIDATOR_MONITOR_UNAGGREGATED_ATTESTATION_DELAY_SECONDS: Result<HistogramVec> = try_create_histogram_vec(
+        "validator_monitor_unaggregated_attestation_delay_seconds",
+        "The delay between when the validator should send the attestation and when it was received.",
+        &["src", "validator"]
+    );
+    pub static ref VALIDATOR_MONITOR_AGGREGATED_ATTESTATION_TOTAL: Result<IntCounterVec> = try_create_int_counter_vec(
+        "validator_monitor_aggregated_attestation_total",
+        "Number of aggregated attestations seen",
+        &["src", "validator"]
+    );
+    pub static ref VALIDATOR_MONITOR_AGGREGATED_ATTESTATION_DELAY_SECONDS: Result<HistogramVec> = try_create_histogram_vec(
+        "validator_monitor_aggregated_attestation_delay_seconds",
+        "The delay between then the validator should send the aggregate and when it was received.",
+        &["src", "validator"]
+    );
+    pub static ref VALIDATOR_MONITOR_ATTESTATION_IN_AGGREGATE_TOTAL: Result<IntCounterVec> = try_create_int_counter_vec(
+        "validator_monitor_attestation_in_aggregate_total",
+        "Number of times an attestation has been seen in an aggregate",
+        &["src", "validator"]
+    );
+    pub static ref VALIDATOR_MONITOR_ATTESTATION_IN_AGGREGATE_DELAY_SECONDS: Result<HistogramVec> = try_create_histogram_vec(
+        "validator_monitor_attestation_in_aggregate_delay_seconds",
+        "The delay between when the validator should send the aggregate and when it was received.",
+        &["src", "validator"]
+    );
+    pub static ref VALIDATOR_MONITOR_ATTESTATION_IN_BLOCK_TOTAL: Result<IntCounterVec> = try_create_int_counter_vec(
+        "validator_monitor_attestation_in_block_total",
+        "Number of times an attestation has been seen in a block",
+        &["src", "validator"]
+    );
+    pub static ref VALIDATOR_MONITOR_ATTESTATION_IN_BLOCK_DELAY_SLOTS: Result<IntGaugeVec> = try_create_int_gauge_vec(
+        "validator_monitor_attestation_in_block_delay_slots",
+        "The excess slots (beyond the minimum delay) between the attestation slot and the block slot.",
+        &["src", "validator"]
+    );
+    pub static ref VALIDATOR_MONITOR_BEACON_BLOCK_TOTAL: Result<IntCounterVec> = try_create_int_counter_vec(
+        "validator_monitor_beacon_block_total",
+        "Number of beacon blocks seen",
+        &["src", "validator"]
+    );
+    pub static ref VALIDATOR_MONITOR_BEACON_BLOCK_DELAY_SECONDS: Result<HistogramVec> = try_create_histogram_vec(
+        "validator_monitor_beacon_block_delay_seconds",
+        "The delay between when the validator should send the block and when it was received.",
+        &["src", "validator"]
+    );
+    pub static ref VALIDATOR_MONITOR_EXIT_TOTAL: Result<IntCounterVec> = try_create_int_counter_vec(
+        "validator_monitor_exit_total",
+        "Number of beacon exits seen",
+        &["src", "validator"]
+    );
+    pub static ref VALIDATOR_MONITOR_PROPOSER_SLASHING_TOTAL: Result<IntCounterVec> = try_create_int_counter_vec(
+        "validator_monitor_proposer_slashing_total",
+        "Number of proposer slashings seen",
+        &["src", "validator"]
+    );
+    pub static ref VALIDATOR_MONITOR_ATTESTER_SLASHING_TOTAL: Result<IntCounterVec> = try_create_int_counter_vec(
+        "validator_monitor_attester_slashing_total",
+        "Number of attester slashings seen",
+        &["src", "validator"]
+    );
+
+    /*
+     * Block Delay Metrics
+     */
+    pub static ref BEACON_BLOCK_IMPORTED_SLOT_START_DELAY_TIME: Result<Histogram> = try_create_histogram(
+        "beacon_block_imported_slot_start_delay_time",
+        "Duration between the start of the blocks slot and the current time when it was imported.",
+    );
+    pub static ref BEACON_BLOCK_HEAD_SLOT_START_DELAY_TIME: Result<Histogram> = try_create_histogram(
+        "beacon_block_head_slot_start_delay_time",
+        "Duration between the start of the blocks slot and the current time when it was as head.",
+    );
+    pub static ref BEACON_BLOCK_HEAD_SLOT_START_DELAY_EXCEEDED_TOTAL: Result<IntCounter> = try_create_int_counter(
+        "beacon_block_head_slot_start_delay_exceeded_total",
+        "Triggered when the duration between the start of the blocks slot and the current time \
+        will result in failed attestations.",
+    );
+
+    /*
+     * General block metrics
+     */
+    pub static ref GOSSIP_BEACON_BLOCK_SKIPPED_SLOTS: Result<IntGauge> =
+        try_create_int_gauge(
+            "gossip_beacon_block_skipped_slots",
+            "For each gossip blocks, the number of skip slots between it and its parent"
+        );
+}
+
+// Fourth lazy-static block is used to account for macro recursion limit.
+lazy_static! {
+    /*
+     * Sync Committee Message Verification
+     */
+    pub static ref SYNC_MESSAGE_PROCESSING_REQUESTS: Result<IntCounter> = try_create_int_counter(
+        "beacon_sync_committee_message_processing_requests_total",
+        "Count of all sync messages submitted for processing"
+    );
+    pub static ref SYNC_MESSAGE_PROCESSING_SUCCESSES: Result<IntCounter> = try_create_int_counter(
+        "beacon_sync_committee_message_processing_successes_total",
+        "Number of sync messages verified for gossip"
+    );
+    pub static ref SYNC_MESSAGE_GOSSIP_VERIFICATION_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_sync_committee_message_gossip_verification_seconds",
+        "Full runtime of sync contribution gossip verification"
+    );
+
+    /*
+     * Sync Committee Contribution Verification
+     */
+    pub static ref SYNC_CONTRIBUTION_PROCESSING_REQUESTS: Result<IntCounter> = try_create_int_counter(
+        "beacon_sync_contribution_processing_requests_total",
+        "Count of all sync contributions submitted for processing"
+    );
+    pub static ref SYNC_CONTRIBUTION_PROCESSING_SUCCESSES: Result<IntCounter> = try_create_int_counter(
+        "beacon_sync_contribution_processing_successes_total",
+        "Number of sync contributions verified for gossip"
+    );
+    pub static ref SYNC_CONTRIBUTION_GOSSIP_VERIFICATION_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_sync_contribution_gossip_verification_seconds",
+        "Full runtime of sync contribution gossip verification"
+    );
+
+    /*
+     * General Sync Committee Contribution Processing
+     */
+    pub static ref SYNC_CONTRIBUTION_PROCESSING_APPLY_TO_AGG_POOL: Result<Histogram> = try_create_histogram(
+        "beacon_sync_contribution_processing_apply_to_agg_pool",
+        "Time spent applying a sync contribution to the naive aggregation pool"
+    );
+    pub static ref SYNC_CONTRIBUTION_PROCESSING_AGG_POOL_PRUNE: Result<Histogram> = try_create_histogram(
+        "beacon_sync_contribution_processing_agg_pool_prune",
+        "Time spent for the agg pool to prune"
+    );
+    pub static ref SYNC_CONTRIBUTION_PROCESSING_AGG_POOL_INSERT: Result<Histogram> = try_create_histogram(
+        "beacon_sync_contribution_processing_agg_pool_insert",
+        "Time spent for the outer pool.insert() function of agg pool"
+    );
+    pub static ref SYNC_CONTRIBUTION_PROCESSING_AGG_POOL_CORE_INSERT: Result<Histogram> = try_create_histogram(
+        "beacon_sync_contribution_processing_agg_pool_core_insert",
+        "Time spent for the core map.insert() function of agg pool"
+    );
+    pub static ref SYNC_CONTRIBUTION_PROCESSING_AGG_POOL_AGGREGATION: Result<Histogram> = try_create_histogram(
+        "beacon_sync_contribution_processing_agg_pool_aggregation",
+        "Time spent doing signature aggregation when adding to the agg poll"
+    );
+    pub static ref SYNC_CONTRIBUTION_PROCESSING_AGG_POOL_CREATE_MAP: Result<Histogram> = try_create_histogram(
+        "beacon_sync_contribution_processing_agg_pool_create_map",
+        "Time spent for creating a map for a new slot"
+    );
+    pub static ref SYNC_CONTRIBUTION_PROCESSING_APPLY_TO_OP_POOL: Result<Histogram> = try_create_histogram(
+        "beacon_sync_contribution_processing_apply_to_op_pool",
+        "Time spent applying a sync contribution to the block inclusion pool"
+    );
+    pub static ref SYNC_CONTRIBUTION_PROCESSING_SIGNATURE_SETUP_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_sync_contribution_processing_signature_setup_seconds",
+        "Time spent on setting up for the signature verification of sync contribution processing"
+    );
+    pub static ref SYNC_CONTRIBUTION_PROCESSING_SIGNATURE_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_sync_contribution_processing_signature_seconds",
+        "Time spent on the signature verification of sync contribution processing"
+    );
+
+        /*
+     * General Sync Committee Contribution Processing
+     */
+    pub static ref SYNC_MESSAGE_PROCESSING_SIGNATURE_SETUP_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_sync_committee_message_processing_signature_setup_seconds",
+        "Time spent on setting up for the signature verification of sync message processing"
+    );
+    pub static ref SYNC_MESSAGE_PROCESSING_SIGNATURE_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_sync_committee_message_processing_signature_seconds",
+        "Time spent on the signature verification of sync message processing"
+    );
 }
 
 /// Scrape the `beacon_chain` for metrics that are not constantly updated (e.g., the present slot,
 /// head state info, etc) and update the Prometheus `DEFAULT_REGISTRY`.
 pub fn scrape_for_metrics<T: BeaconChainTypes>(beacon_chain: &BeaconChain<T>) {
-    scrape_head_state::<T>(
-        &beacon_chain.head().beacon_state,
-        beacon_chain.head().beacon_state_root,
+    let _ = beacon_chain.with_head(|head| {
+        scrape_head_state(&head.beacon_state, head.beacon_state_root());
+        Ok::<_, BeaconChainError>(())
+    });
+
+    if let Some(slot) = beacon_chain.slot_clock.now() {
+        scrape_attestation_observation(slot, beacon_chain);
+        scrape_sync_committee_observation(slot, beacon_chain);
+    }
+
+    set_gauge_by_usize(
+        &OP_POOL_NUM_ATTESTATIONS,
+        beacon_chain.op_pool.num_attestations(),
     );
+    set_gauge_by_usize(
+        &OP_POOL_NUM_ATTESTER_SLASHINGS,
+        beacon_chain.op_pool.num_attester_slashings(),
+    );
+    set_gauge_by_usize(
+        &OP_POOL_NUM_PROPOSER_SLASHINGS,
+        beacon_chain.op_pool.num_proposer_slashings(),
+    );
+    set_gauge_by_usize(
+        &OP_POOL_NUM_VOLUNTARY_EXITS,
+        beacon_chain.op_pool.num_voluntary_exits(),
+    );
+    set_gauge_by_usize(
+        &OP_POOL_NUM_SYNC_CONTRIBUTIONS,
+        beacon_chain.op_pool.num_sync_contributions(),
+    );
+
+    beacon_chain
+        .validator_monitor
+        .read()
+        .scrape_metrics(&beacon_chain.slot_clock, &beacon_chain.spec);
 }
 
 /// Scrape the given `state` assuming it's the head state, updating the `DEFAULT_REGISTRY`.
-fn scrape_head_state<T: BeaconChainTypes>(state: &BeaconState<T::EthSpec>, state_root: Hash256) {
-    set_gauge_by_slot(&HEAD_STATE_SLOT, state.slot);
+fn scrape_head_state<T: EthSpec>(state: &BeaconState<T>, state_root: Hash256) {
+    set_gauge_by_slot(&HEAD_STATE_SLOT, state.slot());
     set_gauge_by_hash(&HEAD_STATE_ROOT, state_root);
     set_gauge_by_slot(
         &HEAD_STATE_LATEST_BLOCK_SLOT,
-        state.latest_block_header.slot,
+        state.latest_block_header().slot,
     );
     set_gauge_by_hash(
         &HEAD_STATE_CURRENT_JUSTIFIED_ROOT,
-        state.current_justified_checkpoint.root,
+        state.current_justified_checkpoint().root,
     );
     set_gauge_by_epoch(
         &HEAD_STATE_CURRENT_JUSTIFIED_EPOCH,
-        state.current_justified_checkpoint.epoch,
+        state.current_justified_checkpoint().epoch,
     );
     set_gauge_by_hash(
         &HEAD_STATE_PREVIOUS_JUSTIFIED_ROOT,
-        state.previous_justified_checkpoint.root,
+        state.previous_justified_checkpoint().root,
     );
     set_gauge_by_epoch(
         &HEAD_STATE_PREVIOUS_JUSTIFIED_EPOCH,
-        state.previous_justified_checkpoint.epoch,
+        state.previous_justified_checkpoint().epoch,
+    );
+    set_gauge_by_hash(
+        &HEAD_STATE_FINALIZED_ROOT,
+        state.finalized_checkpoint().root,
     );
-    set_gauge_by_hash(&HEAD_STATE_FINALIZED_ROOT, state.finalized_checkpoint.root);
     set_gauge_by_epoch(
         &HEAD_STATE_FINALIZED_EPOCH,
-        state.finalized_checkpoint.epoch,
+        state.finalized_checkpoint().epoch,
     );
-    set_gauge_by_usize(&HEAD_STATE_TOTAL_VALIDATORS, state.validators.len());
-    set_gauge_by_u64(&HEAD_STATE_VALIDATOR_BALANCES, state.balances.iter().sum());
-    set_gauge_by_usize(
-        &HEAD_STATE_ACTIVE_VALIDATORS,
-        state
-            .validators
-            .iter()
-            .filter(|v| v.is_active_at(state.current_epoch()))
-            .count(),
+    set_gauge_by_usize(&HEAD_STATE_TOTAL_VALIDATORS, state.validators().len());
+    set_gauge_by_u64(
+        &HEAD_STATE_VALIDATOR_BALANCES,
+        state.balances().iter().sum(),
     );
-    set_gauge_by_usize(
-        &HEAD_STATE_SLASHED_VALIDATORS,
-        state.validators.iter().filter(|v| v.slashed).count(),
+    set_gauge_by_u64(&HEAD_STATE_ETH1_DEPOSIT_INDEX, state.eth1_deposit_index());
+    set_gauge_by_usize(&HEAD_STATE_TOTAL_VALIDATORS, state.validators().len());
+    set_gauge_by_u64(
+        &HEAD_STATE_VALIDATOR_BALANCES,
+        state.balances().iter().sum(),
     );
-    set_gauge_by_usize(
-        &HEAD_STATE_WITHDRAWN_VALIDATORS,
-        state
-            .validators
-            .iter()
-            .filter(|v| v.is_withdrawable_at(state.current_epoch()))
-            .count(),
-    );
-    set_gauge_by_u64(&HEAD_STATE_ETH1_DEPOSIT_INDEX, state.eth1_deposit_index);
+
+    let mut num_active: usize = 0;
+    let mut num_slashed: usize = 0;
+    let mut num_withdrawn: usize = 0;
+
+    for v in state.validators() {
+        if v.is_active_at(state.current_epoch()) {
+            num_active += 1;
+        }
+
+        if v.slashed {
+            num_slashed += 1;
+        }
+
+        if v.is_withdrawable_at(state.current_epoch()) {
+            num_withdrawn += 1;
+        }
+    }
+
+    set_gauge_by_usize(&HEAD_STATE_ACTIVE_VALIDATORS, num_active);
+    set_gauge_by_usize(&HEAD_STATE_SLASHED_VALIDATORS, num_slashed);
+    set_gauge_by_usize(&HEAD_STATE_WITHDRAWN_VALIDATORS, num_withdrawn);
+}
+
+fn scrape_attestation_observation<T: BeaconChainTypes>(slot_now: Slot, chain: &BeaconChain<T>) {
+    let prev_epoch = slot_now.epoch(T::EthSpec::slots_per_epoch()) - 1;
+
+    if let Some(count) = chain
+        .observed_gossip_attesters
+        .read()
+        .observed_validator_count(prev_epoch)
+    {
+        set_gauge_by_usize(&ATTN_OBSERVATION_PREV_EPOCH_ATTESTERS, count);
+    }
+
+    if let Some(count) = chain
+        .observed_aggregators
+        .read()
+        .observed_validator_count(prev_epoch)
+    {
+        set_gauge_by_usize(&ATTN_OBSERVATION_PREV_EPOCH_AGGREGATORS, count);
+    }
+}
+
+fn scrape_sync_committee_observation<T: BeaconChainTypes>(slot_now: Slot, chain: &BeaconChain<T>) {
+    let prev_slot = slot_now - 1;
+
+    let contributors = chain.observed_sync_contributors.read();
+    let mut contributor_sum = 0;
+    for i in 0..SYNC_COMMITTEE_SUBNET_COUNT {
+        if let Some(count) =
+            contributors.observed_validator_count(SlotSubcommitteeIndex::new(prev_slot, i))
+        {
+            contributor_sum += count;
+        }
+    }
+    drop(contributors);
+    set_gauge_by_usize(&SYNC_COMM_OBSERVATION_PREV_SLOT_SIGNERS, contributor_sum);
+
+    let sync_aggregators = chain.observed_sync_aggregators.read();
+    let mut aggregator_sum = 0;
+    for i in 0..SYNC_COMMITTEE_SUBNET_COUNT {
+        if let Some(count) =
+            sync_aggregators.observed_validator_count(SlotSubcommitteeIndex::new(prev_slot, i))
+        {
+            aggregator_sum += count;
+        }
+    }
+    drop(sync_aggregators);
+    set_gauge_by_usize(&SYNC_COMM_OBSERVATION_PREV_SLOT_AGGREGATORS, aggregator_sum);
 }
 
 fn set_gauge_by_slot(gauge: &Result<IntGauge>, value: Slot) {

beacon_node/beacon_chain/src/migrate.rs

@@ -0,0 +1,543 @@
+use crate::beacon_chain::BEACON_CHAIN_DB_KEY;
+use crate::errors::BeaconChainError;
+use crate::head_tracker::{HeadTracker, SszHeadTracker};
+use crate::persisted_beacon_chain::{PersistedBeaconChain, DUMMY_CANONICAL_HEAD_BLOCK_ROOT};
+use parking_lot::Mutex;
+use slog::{debug, error, info, warn, Logger};
+use std::collections::{HashMap, HashSet};
+use std::mem;
+use std::sync::{mpsc, Arc};
+use std::thread;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use store::hot_cold_store::{migrate_database, HotColdDBError};
+use store::iter::RootsIterator;
+use store::{Error, ItemStore, StoreItem, StoreOp};
+pub use store::{HotColdDB, MemoryStore};
+use types::{
+    BeaconState, BeaconStateError, BeaconStateHash, Checkpoint, Epoch, EthSpec, Hash256,
+    SignedBeaconBlockHash, Slot,
+};
+
+/// Compact at least this frequently, finalization permitting (7 days).
+const MAX_COMPACTION_PERIOD_SECONDS: u64 = 604800;
+/// Compact at *most* this frequently, to prevent over-compaction during sync (2 hours).
+const MIN_COMPACTION_PERIOD_SECONDS: u64 = 7200;
+/// Compact after a large finality gap, if we respect `MIN_COMPACTION_PERIOD_SECONDS`.
+const COMPACTION_FINALITY_DISTANCE: u64 = 1024;
+
+/// The background migrator runs a thread to perform pruning and migrate state from the hot
+/// to the cold database.
+pub struct BackgroundMigrator<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> {
+    db: Arc<HotColdDB<E, Hot, Cold>>,
+    #[allow(clippy::type_complexity)]
+    tx_thread: Option<Mutex<(mpsc::Sender<MigrationNotification>, thread::JoinHandle<()>)>>,
+    /// Genesis block root, for persisting the `PersistedBeaconChain`.
+    genesis_block_root: Hash256,
+    log: Logger,
+}
+
+#[derive(Debug, Default, Clone, PartialEq, Eq)]
+pub struct MigratorConfig {
+    pub blocking: bool,
+}
+
+impl MigratorConfig {
+    pub fn blocking(mut self) -> Self {
+        self.blocking = true;
+        self
+    }
+}
+
+/// Pruning can be successful, or in rare cases deferred to a later point.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum PruningOutcome {
+    /// The pruning succeeded and updated the pruning checkpoint from `old_finalized_checkpoint`.
+    Successful {
+        old_finalized_checkpoint: Checkpoint,
+    },
+    DeferredConcurrentMutation,
+}
+
+/// Logic errors that can occur during pruning, none of these should ever happen.
+#[derive(Debug)]
+pub enum PruningError {
+    IncorrectFinalizedState {
+        state_slot: Slot,
+        new_finalized_slot: Slot,
+    },
+    MissingInfoForCanonicalChain {
+        slot: Slot,
+    },
+    UnexpectedEqualStateRoots,
+    UnexpectedUnequalStateRoots,
+}
+
+/// Message sent to the migration thread containing the information it needs to run.
+pub struct MigrationNotification {
+    finalized_state_root: BeaconStateHash,
+    finalized_checkpoint: Checkpoint,
+    head_tracker: Arc<HeadTracker>,
+    genesis_block_root: Hash256,
+}
+
+impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Hot, Cold> {
+    /// Create a new `BackgroundMigrator` and spawn its thread if necessary.
+    pub fn new(
+        db: Arc<HotColdDB<E, Hot, Cold>>,
+        config: MigratorConfig,
+        genesis_block_root: Hash256,
+        log: Logger,
+    ) -> Self {
+        let tx_thread = if config.blocking {
+            None
+        } else {
+            Some(Mutex::new(Self::spawn_thread(db.clone(), log.clone())))
+        };
+        Self {
+            db,
+            tx_thread,
+            genesis_block_root,
+            log,
+        }
+    }
+
+    /// Process a finalized checkpoint from the `BeaconChain`.
+    ///
+    /// If successful, all forks descending from before the `finalized_checkpoint` will be
+    /// pruned, and the split point of the database will be advanced to the slot of the finalized
+    /// checkpoint.
+    pub fn process_finalization(
+        &self,
+        finalized_state_root: BeaconStateHash,
+        finalized_checkpoint: Checkpoint,
+        head_tracker: Arc<HeadTracker>,
+    ) -> Result<(), BeaconChainError> {
+        let notif = MigrationNotification {
+            finalized_state_root,
+            finalized_checkpoint,
+            head_tracker,
+            genesis_block_root: self.genesis_block_root,
+        };
+
+        // Async path, on the background thread.
+        if let Some(tx_thread) = &self.tx_thread {
+            let (ref mut tx, ref mut thread) = *tx_thread.lock();
+
+            // Restart the background thread if it has crashed.
+            if let Err(tx_err) = tx.send(notif) {
+                let (new_tx, new_thread) = Self::spawn_thread(self.db.clone(), self.log.clone());
+
+                *tx = new_tx;
+                let old_thread = mem::replace(thread, new_thread);
+
+                // Join the old thread, which will probably have panicked, or may have
+                // halted normally just now as a result of us dropping the old `mpsc::Sender`.
+                if let Err(thread_err) = old_thread.join() {
+                    warn!(
+                        self.log,
+                        "Migration thread died, so it was restarted";
+                        "reason" => format!("{:?}", thread_err)
+                    );
+                }
+
+                // Retry at most once, we could recurse but that would risk overflowing the stack.
+                let _ = tx.send(tx_err.0);
+            }
+        }
+        // Synchronous path, on the current thread.
+        else {
+            Self::run_migration(self.db.clone(), notif, &self.log)
+        }
+
+        Ok(())
+    }
+
+    /// Perform the actual work of `process_finalization`.
+    fn run_migration(db: Arc<HotColdDB<E, Hot, Cold>>, notif: MigrationNotification, log: &Logger) {
+        let finalized_state_root = notif.finalized_state_root;
+
+        let finalized_state = match db.get_state(&finalized_state_root.into(), None) {
+            Ok(Some(state)) => state,
+            other => {
+                error!(
+                    log,
+                    "Migrator failed to load state";
+                    "state_root" => ?finalized_state_root,
+                    "error" => ?other
+                );
+                return;
+            }
+        };
+
+        let old_finalized_checkpoint = match Self::prune_abandoned_forks(
+            db.clone(),
+            notif.head_tracker,
+            finalized_state_root,
+            &finalized_state,
+            notif.finalized_checkpoint,
+            notif.genesis_block_root,
+            log,
+        ) {
+            Ok(PruningOutcome::Successful {
+                old_finalized_checkpoint,
+            }) => old_finalized_checkpoint,
+            Ok(PruningOutcome::DeferredConcurrentMutation) => {
+                warn!(
+                    log,
+                    "Pruning deferred because of a concurrent mutation";
+                    "message" => "this is expected only very rarely!"
+                );
+                return;
+            }
+            Err(e) => {
+                warn!(log, "Block pruning failed"; "error" => format!("{:?}", e));
+                return;
+            }
+        };
+
+        match migrate_database(db.clone(), finalized_state_root.into(), &finalized_state) {
+            Ok(()) => {}
+            Err(Error::HotColdDBError(HotColdDBError::FreezeSlotUnaligned(slot))) => {
+                debug!(
+                    log,
+                    "Database migration postponed, unaligned finalized block";
+                    "slot" => slot.as_u64()
+                );
+            }
+            Err(e) => {
+                warn!(
+                    log,
+                    "Database migration failed";
+                    "error" => format!("{:?}", e)
+                );
+                return;
+            }
+        };
+
+        // Finally, compact the database so that new free space is properly reclaimed.
+        if let Err(e) = Self::run_compaction(
+            db,
+            old_finalized_checkpoint.epoch,
+            notif.finalized_checkpoint.epoch,
+            log,
+        ) {
+            warn!(log, "Database compaction failed"; "error" => format!("{:?}", e));
+        }
+    }
+
+    /// Spawn a new child thread to run the migration process.
+    ///
+    /// Return a channel handle for sending new finalized states to the thread.
+    fn spawn_thread(
+        db: Arc<HotColdDB<E, Hot, Cold>>,
+        log: Logger,
+    ) -> (mpsc::Sender<MigrationNotification>, thread::JoinHandle<()>) {
+        let (tx, rx) = mpsc::channel();
+        let thread = thread::spawn(move || {
+            while let Ok(notif) = rx.recv() {
+                // Read the rest of the messages in the channel, ultimately choosing the `notif`
+                // with the highest finalized epoch.
+                let notif = rx
+                    .try_iter()
+                    .fold(notif, |best, other: MigrationNotification| {
+                        if other.finalized_checkpoint.epoch > best.finalized_checkpoint.epoch {
+                            other
+                        } else {
+                            best
+                        }
+                    });
+
+                Self::run_migration(db.clone(), notif, &log);
+            }
+        });
+        (tx, thread)
+    }
+
+    /// Traverses live heads and prunes blocks and states of chains that we know can't be built
+    /// upon because finalization would prohibit it. This is an optimisation intended to save disk
+    /// space.
+    #[allow(clippy::too_many_arguments)]
+    fn prune_abandoned_forks(
+        store: Arc<HotColdDB<E, Hot, Cold>>,
+        head_tracker: Arc<HeadTracker>,
+        new_finalized_state_hash: BeaconStateHash,
+        new_finalized_state: &BeaconState<E>,
+        new_finalized_checkpoint: Checkpoint,
+        genesis_block_root: Hash256,
+        log: &Logger,
+    ) -> Result<PruningOutcome, BeaconChainError> {
+        let old_finalized_checkpoint =
+            store
+                .load_pruning_checkpoint()?
+                .unwrap_or_else(|| Checkpoint {
+                    epoch: Epoch::new(0),
+                    root: Hash256::zero(),
+                });
+
+        let old_finalized_slot = old_finalized_checkpoint
+            .epoch
+            .start_slot(E::slots_per_epoch());
+        let new_finalized_slot = new_finalized_checkpoint
+            .epoch
+            .start_slot(E::slots_per_epoch());
+        let new_finalized_block_hash = new_finalized_checkpoint.root.into();
+
+        // The finalized state must be for the epoch boundary slot, not the slot of the finalized
+        // block.
+        if new_finalized_state.slot() != new_finalized_slot {
+            return Err(PruningError::IncorrectFinalizedState {
+                state_slot: new_finalized_state.slot(),
+                new_finalized_slot,
+            }
+            .into());
+        }
+
+        debug!(
+            log,
+            "Starting database pruning";
+            "old_finalized_epoch" => old_finalized_checkpoint.epoch,
+            "new_finalized_epoch" => new_finalized_checkpoint.epoch,
+        );
+        // For each slot between the new finalized checkpoint and the old finalized checkpoint,
+        // collect the beacon block root and state root of the canonical chain.
+        let newly_finalized_chain: HashMap<Slot, (SignedBeaconBlockHash, BeaconStateHash)> =
+            std::iter::once(Ok((
+                new_finalized_slot,
+                (new_finalized_block_hash, new_finalized_state_hash),
+            )))
+            .chain(
+                RootsIterator::new(store.clone(), new_finalized_state).map(|res| {
+                    res.map(|(block_root, state_root, slot)| {
+                        (slot, (block_root.into(), state_root.into()))
+                    })
+                }),
+            )
+            .take_while(|res| {
+                res.as_ref()
+                    .map_or(true, |(slot, _)| *slot >= old_finalized_slot)
+            })
+            .collect::<Result<_, _>>()?;
+
+        // We don't know which blocks are shared among abandoned chains, so we buffer and delete
+        // everything in one fell swoop.
+        let mut abandoned_blocks: HashSet<SignedBeaconBlockHash> = HashSet::new();
+        let mut abandoned_states: HashSet<(Slot, BeaconStateHash)> = HashSet::new();
+        let mut abandoned_heads: HashSet<Hash256> = HashSet::new();
+
+        let heads = head_tracker.heads();
+        debug!(
+            log,
+            "Extra pruning information";
+            "old_finalized_root" => format!("{:?}", old_finalized_checkpoint.root),
+            "new_finalized_root" => format!("{:?}", new_finalized_checkpoint.root),
+            "head_count" => heads.len(),
+        );
+
+        for (head_hash, head_slot) in heads {
+            let mut potentially_abandoned_head = Some(head_hash);
+            let mut potentially_abandoned_blocks = vec![];
+
+            let head_state_hash = store
+                .get_block(&head_hash)?
+                .ok_or_else(|| BeaconStateError::MissingBeaconBlock(head_hash.into()))?
+                .state_root();
+
+            // Iterate backwards from this head, staging blocks and states for deletion.
+            let iter = std::iter::once(Ok((head_hash, head_state_hash, head_slot)))
+                .chain(RootsIterator::from_block(store.clone(), head_hash)?);
+
+            for maybe_tuple in iter {
+                let (block_root, state_root, slot) = maybe_tuple?;
+                let block_root = SignedBeaconBlockHash::from(block_root);
+                let state_root = BeaconStateHash::from(state_root);
+
+                match newly_finalized_chain.get(&slot) {
+                    // If there's no information about a slot on the finalized chain, then
+                    // it should be because it's ahead of the new finalized slot. Stage
+                    // the fork's block and state for possible deletion.
+                    None => {
+                        if slot > new_finalized_slot {
+                            potentially_abandoned_blocks.push((
+                                slot,
+                                Some(block_root),
+                                Some(state_root),
+                            ));
+                        } else if slot >= old_finalized_slot {
+                            return Err(PruningError::MissingInfoForCanonicalChain { slot }.into());
+                        } else {
+                            // We must assume here any candidate chains include the old finalized
+                            // checkpoint, i.e. there aren't any forks starting at a block that is a
+                            // strict ancestor of old_finalized_checkpoint.
+                            warn!(
+                                log,
+                                "Found a chain that should already have been pruned";
+                                "head_block_root" => format!("{:?}", head_hash),
+                                "head_slot" => head_slot,
+                            );
+                            potentially_abandoned_head.take();
+                            break;
+                        }
+                    }
+                    Some((finalized_block_root, finalized_state_root)) => {
+                        // This fork descends from a newly finalized block, we can stop.
+                        if block_root == *finalized_block_root {
+                            // Sanity check: if the slot and block root match, then the
+                            // state roots should match too.
+                            if state_root != *finalized_state_root {
+                                return Err(PruningError::UnexpectedUnequalStateRoots.into());
+                            }
+
+                            // If the fork descends from the whole finalized chain,
+                            // do not prune it. Otherwise continue to delete all
+                            // of the blocks and states that have been staged for
+                            // deletion so far.
+                            if slot == new_finalized_slot {
+                                potentially_abandoned_blocks.clear();
+                                potentially_abandoned_head.take();
+                            }
+                            // If there are skipped slots on the fork to be pruned, then
+                            // we will have just staged the common block for deletion.
+                            // Unstage it.
+                            else {
+                                for (_, block_root, _) in
+                                    potentially_abandoned_blocks.iter_mut().rev()
+                                {
+                                    if block_root.as_ref() == Some(finalized_block_root) {
+                                        *block_root = None;
+                                    } else {
+                                        break;
+                                    }
+                                }
+                            }
+                            break;
+                        } else {
+                            if state_root == *finalized_state_root {
+                                return Err(PruningError::UnexpectedEqualStateRoots.into());
+                            }
+                            potentially_abandoned_blocks.push((
+                                slot,
+                                Some(block_root),
+                                Some(state_root),
+                            ));
+                        }
+                    }
+                }
+            }
+
+            if let Some(abandoned_head) = potentially_abandoned_head {
+                debug!(
+                    log,
+                    "Pruning head";
+                    "head_block_root" => format!("{:?}", abandoned_head),
+                    "head_slot" => head_slot,
+                );
+                abandoned_heads.insert(abandoned_head);
+                abandoned_blocks.extend(
+                    potentially_abandoned_blocks
+                        .iter()
+                        .filter_map(|(_, maybe_block_hash, _)| *maybe_block_hash),
+                );
+                abandoned_states.extend(potentially_abandoned_blocks.iter().filter_map(
+                    |(slot, _, maybe_state_hash)| maybe_state_hash.map(|sr| (*slot, sr)),
+                ));
+            }
+        }
+
+        // Update the head tracker before the database, so that we maintain the invariant
+        // that a block present in the head tracker is present in the database.
+        // See https://github.com/sigp/lighthouse/issues/1557
+        let mut head_tracker_lock = head_tracker.0.write();
+
+        // Check that all the heads to be deleted are still present. The absence of any
+        // head indicates a race, that will likely resolve itself, so we defer pruning until
+        // later.
+        for head_hash in &abandoned_heads {
+            if !head_tracker_lock.contains_key(head_hash) {
+                return Ok(PruningOutcome::DeferredConcurrentMutation);
+            }
+        }
+
+        // Then remove them for real.
+        for head_hash in abandoned_heads {
+            head_tracker_lock.remove(&head_hash);
+        }
+
+        let batch: Vec<StoreOp<E>> = abandoned_blocks
+            .into_iter()
+            .map(Into::into)
+            .map(StoreOp::DeleteBlock)
+            .chain(
+                abandoned_states
+                    .into_iter()
+                    .map(|(slot, state_hash)| StoreOp::DeleteState(state_hash.into(), Some(slot))),
+            )
+            .collect();
+
+        let mut kv_batch = store.convert_to_kv_batch(&batch)?;
+
+        // Persist the head in case the process is killed or crashes here. This prevents
+        // the head tracker reverting after our mutation above.
+        let persisted_head = PersistedBeaconChain {
+            _canonical_head_block_root: DUMMY_CANONICAL_HEAD_BLOCK_ROOT,
+            genesis_block_root,
+            ssz_head_tracker: SszHeadTracker::from_map(&*head_tracker_lock),
+        };
+        drop(head_tracker_lock);
+        kv_batch.push(persisted_head.as_kv_store_op(BEACON_CHAIN_DB_KEY));
+
+        // Persist the new finalized checkpoint as the pruning checkpoint.
+        kv_batch.push(store.pruning_checkpoint_store_op(new_finalized_checkpoint));
+
+        store.hot_db.do_atomically(kv_batch)?;
+        debug!(log, "Database pruning complete");
+
+        Ok(PruningOutcome::Successful {
+            old_finalized_checkpoint,
+        })
+    }
+
+    /// Compact the database if it has been more than `COMPACTION_PERIOD_SECONDS` since it
+    /// was last compacted.
+    pub fn run_compaction(
+        db: Arc<HotColdDB<E, Hot, Cold>>,
+        old_finalized_epoch: Epoch,
+        new_finalized_epoch: Epoch,
+        log: &Logger,
+    ) -> Result<(), Error> {
+        if !db.compact_on_prune() {
+            return Ok(());
+        }
+
+        let last_compaction_timestamp = db
+            .load_compaction_timestamp()?
+            .unwrap_or_else(|| Duration::from_secs(0));
+        let start_time = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .unwrap_or(last_compaction_timestamp);
+        let seconds_since_last_compaction = start_time
+            .checked_sub(last_compaction_timestamp)
+            .as_ref()
+            .map_or(0, Duration::as_secs);
+
+        if seconds_since_last_compaction > MAX_COMPACTION_PERIOD_SECONDS
+            || (new_finalized_epoch - old_finalized_epoch > COMPACTION_FINALITY_DISTANCE
+                && seconds_since_last_compaction > MIN_COMPACTION_PERIOD_SECONDS)
+        {
+            info!(
+                log,
+                "Starting database compaction";
+                "old_finalized_epoch" => old_finalized_epoch,
+                "new_finalized_epoch" => new_finalized_epoch,
+            );
+            db.compact()?;
+
+            let finish_time = SystemTime::now()
+                .duration_since(UNIX_EPOCH)
+                .unwrap_or(start_time);
+            db.store_compaction_timestamp(finish_time)?;
+
+            info!(log, "Database compaction complete");
+        }
+        Ok(())
+    }
+}

beacon_node/beacon_chain/src/naive_aggregation_pool.rs

@@ -0,0 +1,799 @@
+use crate::metrics;
+use std::collections::HashMap;
+use tree_hash::TreeHash;
+use types::consts::altair::SYNC_COMMITTEE_SUBNET_COUNT;
+use types::slot_data::SlotData;
+use types::sync_committee_contribution::SyncContributionData;
+use types::{Attestation, AttestationData, EthSpec, Hash256, Slot, SyncCommitteeContribution};
+
+type AttestationDataRoot = Hash256;
+type SyncDataRoot = Hash256;
+
+/// The number of slots that will be stored in the pool.
+///
+/// For example, if `SLOTS_RETAINED == 3` and the pool is pruned at slot `6`, then all items
+/// at slots less than `4` will be dropped and any future item with a slot less than `4`
+/// will be refused.
+const SLOTS_RETAINED: usize = 3;
+
+/// The maximum number of distinct `AttestationData` that will be stored in each slot.
+///
+/// This is a DoS protection measure.
+const MAX_ATTESTATIONS_PER_SLOT: usize = 16_384;
+
+/// Returned upon successfully inserting an item into the pool.
+#[derive(Debug, PartialEq)]
+pub enum InsertOutcome {
+    /// The item had not been seen before and was added to the pool.
+    NewItemInserted { committee_index: usize },
+    /// A validator signature for the given item's `Data` was already known. No changes were
+    /// made.
+    SignatureAlreadyKnown { committee_index: usize },
+    /// The item's `Data` was known, but a signature for the given validator was not yet
+    /// known. The signature was aggregated into the pool.
+    SignatureAggregated { committee_index: usize },
+}
+
+#[derive(Debug, PartialEq)]
+pub enum Error {
+    /// The given `data.slot` was too low to be stored. No changes were made.
+    SlotTooLow {
+        slot: Slot,
+        lowest_permissible_slot: Slot,
+    },
+    /// The given `aggregation_bits` field was empty.
+    NoAggregationBitsSet,
+    /// The given `aggregation_bits` field had more than one signature. The number of
+    /// signatures found is included.
+    MoreThanOneAggregationBitSet(usize),
+    /// We have reached the maximum number of unique items that can be stored in a
+    /// slot. This is a DoS protection function.
+    ReachedMaxItemsPerSlot(usize),
+    /// The given `aggregation_bits` field had a different length to the one currently
+    /// stored. This indicates a fairly serious error somewhere in the code that called this
+    /// function.
+    InconsistentBitfieldLengths,
+    /// The given item was for the incorrect slot. This is an internal error.
+    IncorrectSlot { expected: Slot, actual: Slot },
+}
+
+/// Implemented for items in the `NaiveAggregationPool`. Requires that items implement `SlotData`,
+/// which means they have an associated slot. This handles aggregation of items that are inserted.
+pub trait AggregateMap {
+    /// `Key` should be a hash of `Data`.
+    type Key;
+
+    /// The item stored in the map
+    type Value: Clone + SlotData;
+
+    /// The unique fields of `Value`, hashed to create `Key`.
+    type Data: SlotData;
+
+    /// Create a new `AggregateMap` with capacity `initial_capacity`.
+    fn new(initial_capacity: usize) -> Self;
+
+    /// Insert a `Value` into `Self`, returning a result.
+    fn insert(&mut self, value: &Self::Value) -> Result<InsertOutcome, Error>;
+
+    /// Get a `Value` from `Self` based on `Data`.
+    fn get(&self, data: &Self::Data) -> Option<Self::Value>;
+
+    /// Get a reference to the inner `HashMap`.
+    fn get_map(&self) -> &HashMap<Self::Key, Self::Value>;
+
+    /// Get a `Value` from `Self` based on `Key`, which is a hash of `Data`.
+    fn get_by_root(&self, root: &Self::Key) -> Option<&Self::Value>;
+
+    /// The number of items store in `Self`.
+    fn len(&self) -> usize;
+
+    /// Start a timer observing inserts.
+    fn start_insert_timer() -> Option<metrics::HistogramTimer>;
+
+    /// Start a timer observing the time it takes to create a new map for a new slot.
+    fn start_create_map_timer() -> Option<metrics::HistogramTimer>;
+
+    /// Start a timer observing the time it takes to prune the pool.
+    fn start_prune_timer() -> Option<metrics::HistogramTimer>;
+
+    /// The default capacity of `Self`.
+    fn default_capacity() -> usize;
+}
+
+/// A collection of `Attestation` objects, keyed by their `attestation.data`. Enforces that all
+/// `attestation` are from the same slot.
+pub struct AggregatedAttestationMap<E: EthSpec> {
+    map: HashMap<AttestationDataRoot, Attestation<E>>,
+}
+
+impl<E: EthSpec> AggregateMap for AggregatedAttestationMap<E> {
+    type Key = AttestationDataRoot;
+    type Value = Attestation<E>;
+    type Data = AttestationData;
+
+    /// Create an empty collection with the given `initial_capacity`.
+    fn new(initial_capacity: usize) -> Self {
+        Self {
+            map: HashMap::with_capacity(initial_capacity),
+        }
+    }
+
+    /// Insert an attestation into `self`, aggregating it into the pool.
+    ///
+    /// The given attestation (`a`) must only have one signature.
+    fn insert(&mut self, a: &Self::Value) -> Result<InsertOutcome, Error> {
+        let _timer = metrics::start_timer(&metrics::ATTESTATION_PROCESSING_AGG_POOL_CORE_INSERT);
+
+        let set_bits = a
+            .aggregation_bits
+            .iter()
+            .enumerate()
+            .filter(|(_i, bit)| *bit)
+            .map(|(i, _bit)| i)
+            .collect::<Vec<_>>();
+
+        let committee_index = set_bits
+            .first()
+            .copied()
+            .ok_or(Error::NoAggregationBitsSet)?;
+
+        if set_bits.len() > 1 {
+            return Err(Error::MoreThanOneAggregationBitSet(set_bits.len()));
+        }
+
+        let attestation_data_root = a.data.tree_hash_root();
+
+        if let Some(existing_attestation) = self.map.get_mut(&attestation_data_root) {
+            if existing_attestation
+                .aggregation_bits
+                .get(committee_index)
+                .map_err(|_| Error::InconsistentBitfieldLengths)?
+            {
+                Ok(InsertOutcome::SignatureAlreadyKnown { committee_index })
+            } else {
+                let _timer =
+                    metrics::start_timer(&metrics::ATTESTATION_PROCESSING_AGG_POOL_AGGREGATION);
+                existing_attestation.aggregate(a);
+                Ok(InsertOutcome::SignatureAggregated { committee_index })
+            }
+        } else {
+            if self.map.len() >= MAX_ATTESTATIONS_PER_SLOT {
+                return Err(Error::ReachedMaxItemsPerSlot(MAX_ATTESTATIONS_PER_SLOT));
+            }
+
+            self.map.insert(attestation_data_root, a.clone());
+            Ok(InsertOutcome::NewItemInserted { committee_index })
+        }
+    }
+
+    /// Returns an aggregated `Attestation` with the given `data`, if any.
+    ///
+    /// The given `a.data.slot` must match the slot that `self` was initialized with.
+    fn get(&self, data: &Self::Data) -> Option<Self::Value> {
+        self.map.get(&data.tree_hash_root()).cloned()
+    }
+
+    fn get_map(&self) -> &HashMap<Self::Key, Self::Value> {
+        &self.map
+    }
+
+    /// Returns an aggregated `Attestation` with the given `root`, if any.
+    fn get_by_root(&self, root: &Self::Key) -> Option<&Self::Value> {
+        self.map.get(root)
+    }
+
+    fn len(&self) -> usize {
+        self.map.len()
+    }
+
+    fn start_insert_timer() -> Option<metrics::HistogramTimer> {
+        metrics::start_timer(&metrics::ATTESTATION_PROCESSING_AGG_POOL_INSERT)
+    }
+
+    fn start_create_map_timer() -> Option<metrics::HistogramTimer> {
+        metrics::start_timer(&metrics::ATTESTATION_PROCESSING_AGG_POOL_CREATE_MAP)
+    }
+
+    fn start_prune_timer() -> Option<metrics::HistogramTimer> {
+        metrics::start_timer(&metrics::ATTESTATION_PROCESSING_AGG_POOL_PRUNE)
+    }
+
+    /// Use the `TARGET_COMMITTEE_SIZE`.
+    ///
+    /// Note: hard-coded until `TARGET_COMMITTEE_SIZE` is available via `EthSpec`.
+    fn default_capacity() -> usize {
+        128
+    }
+}
+
+/// A collection of `SyncCommitteeContribution`, keyed by their `SyncContributionData`. Enforces that all
+/// contributions are from the same slot.
+pub struct SyncContributionAggregateMap<E: EthSpec> {
+    map: HashMap<SyncDataRoot, SyncCommitteeContribution<E>>,
+}
+
+impl<E: EthSpec> AggregateMap for SyncContributionAggregateMap<E> {
+    type Key = SyncDataRoot;
+    type Value = SyncCommitteeContribution<E>;
+    type Data = SyncContributionData;
+
+    /// Create an empty collection with the given `initial_capacity`.
+    fn new(initial_capacity: usize) -> Self {
+        Self {
+            map: HashMap::with_capacity(initial_capacity),
+        }
+    }
+
+    /// Insert a sync committee contribution into `self`, aggregating it into the pool.
+    ///
+    /// The given sync contribution must only have one signature.
+    fn insert(
+        &mut self,
+        contribution: &SyncCommitteeContribution<E>,
+    ) -> Result<InsertOutcome, Error> {
+        let _timer =
+            metrics::start_timer(&metrics::SYNC_CONTRIBUTION_PROCESSING_AGG_POOL_CORE_INSERT);
+
+        let set_bits = contribution
+            .aggregation_bits
+            .iter()
+            .enumerate()
+            .filter(|(_i, bit)| *bit)
+            .map(|(i, _bit)| i)
+            .collect::<Vec<_>>();
+
+        let committee_index = set_bits
+            .first()
+            .copied()
+            .ok_or(Error::NoAggregationBitsSet)?;
+
+        if set_bits.len() > 1 {
+            return Err(Error::MoreThanOneAggregationBitSet(set_bits.len()));
+        }
+
+        let sync_data_root = SyncContributionData::from_contribution(contribution).tree_hash_root();
+
+        if let Some(existing_contribution) = self.map.get_mut(&sync_data_root) {
+            if existing_contribution
+                .aggregation_bits
+                .get(committee_index)
+                .map_err(|_| Error::InconsistentBitfieldLengths)?
+            {
+                Ok(InsertOutcome::SignatureAlreadyKnown { committee_index })
+            } else {
+                let _timer = metrics::start_timer(
+                    &metrics::SYNC_CONTRIBUTION_PROCESSING_AGG_POOL_AGGREGATION,
+                );
+                existing_contribution.aggregate(contribution);
+                Ok(InsertOutcome::SignatureAggregated { committee_index })
+            }
+        } else {
+            if self.map.len() >= E::sync_committee_size() {
+                return Err(Error::ReachedMaxItemsPerSlot(E::sync_committee_size()));
+            }
+
+            self.map.insert(sync_data_root, contribution.clone());
+            Ok(InsertOutcome::NewItemInserted { committee_index })
+        }
+    }
+
+    /// Returns an aggregated `SyncCommitteeContribution` with the given `data`, if any.
+    ///
+    /// The given `data.slot` must match the slot that `self` was initialized with.
+    fn get(&self, data: &SyncContributionData) -> Option<SyncCommitteeContribution<E>> {
+        self.map.get(&data.tree_hash_root()).cloned()
+    }
+
+    fn get_map(&self) -> &HashMap<SyncDataRoot, SyncCommitteeContribution<E>> {
+        &self.map
+    }
+
+    /// Returns an aggregated `SyncCommitteeContribution` with the given `root`, if any.
+    fn get_by_root(&self, root: &SyncDataRoot) -> Option<&SyncCommitteeContribution<E>> {
+        self.map.get(root)
+    }
+
+    fn len(&self) -> usize {
+        self.map.len()
+    }
+
+    fn start_insert_timer() -> Option<metrics::HistogramTimer> {
+        metrics::start_timer(&metrics::SYNC_CONTRIBUTION_PROCESSING_AGG_POOL_INSERT)
+    }
+
+    fn start_create_map_timer() -> Option<metrics::HistogramTimer> {
+        metrics::start_timer(&metrics::SYNC_CONTRIBUTION_PROCESSING_AGG_POOL_CREATE_MAP)
+    }
+
+    fn start_prune_timer() -> Option<metrics::HistogramTimer> {
+        metrics::start_timer(&metrics::SYNC_CONTRIBUTION_PROCESSING_AGG_POOL_PRUNE)
+    }
+
+    /// Default to `SYNC_COMMITTEE_SUBNET_COUNT`.
+    fn default_capacity() -> usize {
+        SYNC_COMMITTEE_SUBNET_COUNT as usize
+    }
+}
+
+/// A pool of `Attestation` or `SyncCommitteeContribution` that is specially designed to store
+/// "unaggregated" messages from the native aggregation scheme.
+///
+/// **The `NaiveAggregationPool` does not do any verification. It assumes that all `Attestation`
+/// or `SyncCommitteeContribution` objects provided are valid.**
+///
+/// ## Details
+///
+/// The pool sorts the items by `slot`, then by `Data`.
+///
+/// As each item is added it is aggregated with any existing item with the same `Data`. Considering
+/// that the pool only accepts attestations or sync contributions with a single
+/// signature, there should only ever be a single aggregated `Attestation` for any given
+/// `AttestationData` or a single `SyncCommitteeContribution` for any given `SyncContributionData`.
+///
+/// The pool has a capacity for `SLOTS_RETAINED` slots, when a new `slot` is
+/// provided, the oldest slot is dropped and replaced with the new slot. The pool can also be
+/// pruned by supplying a `current_slot`; all existing items with a slot lower than
+/// `current_slot - SLOTS_RETAINED` will be removed and any future item with a slot lower
+/// than that will also be refused. Pruning is done automatically based upon the items it
+/// receives and it can be triggered manually.
+pub struct NaiveAggregationPool<T: AggregateMap> {
+    lowest_permissible_slot: Slot,
+    maps: HashMap<Slot, T>,
+}
+
+impl<T: AggregateMap> Default for NaiveAggregationPool<T> {
+    fn default() -> Self {
+        Self {
+            lowest_permissible_slot: Slot::new(0),
+            maps: HashMap::new(),
+        }
+    }
+}
+
+impl<T: AggregateMap> NaiveAggregationPool<T> {
+    /// Insert an item into `self`, aggregating it into the pool.
+    ///
+    /// The given item must only have one signature and have an
+    /// `slot` that is not lower than `self.lowest_permissible_slot`.
+    ///
+    /// The pool may be pruned if the given item has a slot higher than any
+    /// previously seen.
+    pub fn insert(&mut self, item: &T::Value) -> Result<InsertOutcome, Error> {
+        let _timer = T::start_insert_timer();
+        let slot = item.get_slot();
+        let lowest_permissible_slot = self.lowest_permissible_slot;
+
+        // Reject any items that are too old.
+        if slot < lowest_permissible_slot {
+            return Err(Error::SlotTooLow {
+                slot,
+                lowest_permissible_slot,
+            });
+        }
+
+        let outcome = if let Some(map) = self.maps.get_mut(&slot) {
+            map.insert(item)
+        } else {
+            let _timer = T::start_create_map_timer();
+            // To avoid re-allocations, try and determine a rough initial capacity for the new item
+            // by obtaining the mean size of all items in earlier epoch.
+            let (count, sum) = self
+                .maps
+                .iter()
+                // Only include epochs that are less than the given slot in the average. This should
+                // generally avoid including recent epochs that are still "filling up".
+                .filter(|(map_slot, _item)| **map_slot < slot)
+                .map(|(_slot, map)| map.len())
+                .fold((0, 0), |(count, sum), len| (count + 1, sum + len));
+
+            let initial_capacity = sum.checked_div(count).unwrap_or_else(T::default_capacity);
+
+            let mut aggregate_map = T::new(initial_capacity);
+            let outcome = aggregate_map.insert(item);
+            self.maps.insert(slot, aggregate_map);
+
+            outcome
+        };
+
+        self.prune(slot);
+
+        outcome
+    }
+
+    /// Returns the total number of items stored in `self`.
+    pub fn num_items(&self) -> usize {
+        self.maps.iter().map(|(_, map)| map.len()).sum()
+    }
+
+    /// Returns an aggregated `T::Value` with the given `T::Data`, if any.
+    pub fn get(&self, data: &T::Data) -> Option<T::Value> {
+        self.maps
+            .get(&data.get_slot())
+            .and_then(|map| map.get(data))
+    }
+
+    /// Returns an aggregated `T::Value` with the given `slot` and `root`, if any.
+    pub fn get_by_slot_and_root(&self, slot: Slot, root: &T::Key) -> Option<T::Value> {
+        self.maps
+            .get(&slot)
+            .and_then(|map| map.get_by_root(root).cloned())
+    }
+
+    /// Iterate all items in all slots of `self`.
+    pub fn iter(&self) -> impl Iterator<Item = &T::Value> {
+        self.maps
+            .iter()
+            .map(|(_slot, map)| map.get_map().iter().map(|(_key, value)| value))
+            .flatten()
+    }
+
+    /// Removes any items with a slot lower than `current_slot` and bars any future
+    /// items with a slot lower than `current_slot - SLOTS_RETAINED`.
+    pub fn prune(&mut self, current_slot: Slot) {
+        let _timer = T::start_prune_timer();
+
+        let lowest_permissible_slot = current_slot.saturating_sub(Slot::from(SLOTS_RETAINED));
+
+        // No need to prune if the lowest permissible slot has not changed and the queue length is
+        // less than the maximum
+        if self.lowest_permissible_slot == lowest_permissible_slot
+            && self.maps.len() <= SLOTS_RETAINED
+        {
+            return;
+        }
+
+        self.lowest_permissible_slot = lowest_permissible_slot;
+
+        // Remove any maps that are definitely expired.
+        self.maps
+            .retain(|slot, _map| *slot >= lowest_permissible_slot);
+
+        // If we have too many maps, remove the lowest amount to ensure we only have
+        // `SLOTS_RETAINED` left.
+        if self.maps.len() > SLOTS_RETAINED {
+            let mut slots = self
+                .maps
+                .iter()
+                .map(|(slot, _map)| *slot)
+                .collect::<Vec<_>>();
+            // Sort is generally pretty slow, however `SLOTS_RETAINED` is quite low so it should be
+            // negligible.
+            slots.sort_unstable();
+            slots
+                .into_iter()
+                .take(self.maps.len().saturating_sub(SLOTS_RETAINED))
+                .for_each(|slot| {
+                    self.maps.remove(&slot);
+                })
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use ssz_types::BitList;
+    use store::BitVector;
+    use types::{
+        test_utils::{generate_deterministic_keypair, test_random_instance},
+        Fork, Hash256, SyncCommitteeMessage,
+    };
+
+    type E = types::MainnetEthSpec;
+
+    fn get_attestation(slot: Slot) -> Attestation<E> {
+        let mut a: Attestation<E> = test_random_instance();
+        a.data.slot = slot;
+        a.aggregation_bits = BitList::with_capacity(4).expect("should create bitlist");
+        a
+    }
+
+    fn get_sync_contribution(slot: Slot) -> SyncCommitteeContribution<E> {
+        let mut a: SyncCommitteeContribution<E> = test_random_instance();
+        a.slot = slot;
+        a.aggregation_bits = BitVector::new();
+        a
+    }
+
+    fn sign_attestation(a: &mut Attestation<E>, i: usize, genesis_validators_root: Hash256) {
+        a.sign(
+            &generate_deterministic_keypair(i).sk,
+            i,
+            &Fork::default(),
+            genesis_validators_root,
+            &E::default_spec(),
+        )
+        .expect("should sign attestation");
+    }
+
+    fn sign_sync_contribution(
+        a: &mut SyncCommitteeContribution<E>,
+        i: usize,
+        genesis_validators_root: Hash256,
+    ) {
+        let sync_message = SyncCommitteeMessage::new::<E>(
+            a.slot,
+            a.beacon_block_root,
+            i as u64,
+            &generate_deterministic_keypair(i).sk,
+            &Fork::default(),
+            genesis_validators_root,
+            &E::default_spec(),
+        );
+        let signed_contribution: SyncCommitteeContribution<E> =
+            SyncCommitteeContribution::from_message(&sync_message, a.subcommittee_index, i)
+                .unwrap();
+
+        a.aggregate(&signed_contribution);
+    }
+
+    fn unset_attestation_bit(a: &mut Attestation<E>, i: usize) {
+        a.aggregation_bits
+            .set(i, false)
+            .expect("should unset aggregation bit")
+    }
+
+    fn unset_sync_contribution_bit(a: &mut SyncCommitteeContribution<E>, i: usize) {
+        a.aggregation_bits
+            .set(i, false)
+            .expect("should unset aggregation bit")
+    }
+
+    fn mutate_attestation_block_root(a: &mut Attestation<E>, block_root: Hash256) {
+        a.data.beacon_block_root = block_root
+    }
+
+    fn mutate_attestation_slot(a: &mut Attestation<E>, slot: Slot) {
+        a.data.slot = slot
+    }
+
+    fn attestation_block_root_comparator(a: &Attestation<E>, block_root: Hash256) -> bool {
+        a.data.beacon_block_root == block_root
+    }
+
+    fn key_from_attestation(a: &Attestation<E>) -> AttestationData {
+        a.data.clone()
+    }
+
+    fn mutate_sync_contribution_block_root(
+        a: &mut SyncCommitteeContribution<E>,
+        block_root: Hash256,
+    ) {
+        a.beacon_block_root = block_root
+    }
+
+    fn mutate_sync_contribution_slot(a: &mut SyncCommitteeContribution<E>, slot: Slot) {
+        a.slot = slot
+    }
+
+    fn sync_contribution_block_root_comparator(
+        a: &SyncCommitteeContribution<E>,
+        block_root: Hash256,
+    ) -> bool {
+        a.beacon_block_root == block_root
+    }
+
+    fn key_from_sync_contribution(a: &SyncCommitteeContribution<E>) -> SyncContributionData {
+        SyncContributionData::from_contribution(a)
+    }
+
+    macro_rules! test_suite {
+        (
+            $mod_name: ident,
+            $get_method_name: ident,
+            $sign_method_name: ident,
+            $unset_method_name: ident,
+            $block_root_mutator: ident,
+            $slot_mutator: ident,
+            $block_root_comparator: ident,
+            $key_getter: ident,
+            $map_type: ident,
+            $item_limit: expr
+        ) => {
+            #[cfg(test)]
+            mod $mod_name {
+                use super::*;
+
+                #[test]
+                fn single_item() {
+                    let mut a = $get_method_name(Slot::new(0));
+
+                    let mut pool: NaiveAggregationPool<$map_type<E>> =
+                        NaiveAggregationPool::default();
+
+                    assert_eq!(
+                        pool.insert(&a),
+                        Err(Error::NoAggregationBitsSet),
+                        "should not accept item without any signatures"
+                    );
+
+                    $sign_method_name(&mut a, 0, Hash256::random());
+
+                    assert_eq!(
+                        pool.insert(&a),
+                        Ok(InsertOutcome::NewItemInserted { committee_index: 0 }),
+                        "should accept new item"
+                    );
+                    assert_eq!(
+                        pool.insert(&a),
+                        Ok(InsertOutcome::SignatureAlreadyKnown { committee_index: 0 }),
+                        "should acknowledge duplicate signature"
+                    );
+
+                    let retrieved = pool
+                        .get(&$key_getter(&a))
+                        .expect("should not error while getting item");
+                    assert_eq!(retrieved, a, "retrieved item should equal the one inserted");
+
+                    $sign_method_name(&mut a, 1, Hash256::random());
+
+                    assert_eq!(
+                        pool.insert(&a),
+                        Err(Error::MoreThanOneAggregationBitSet(2)),
+                        "should not accept item with multiple signatures"
+                    );
+                }
+
+                #[test]
+                fn multiple_items() {
+                    let mut a_0 = $get_method_name(Slot::new(0));
+                    let mut a_1 = a_0.clone();
+
+                    let genesis_validators_root = Hash256::random();
+                    $sign_method_name(&mut a_0, 0, genesis_validators_root);
+                    $sign_method_name(&mut a_1, 1, genesis_validators_root);
+
+                    let mut pool: NaiveAggregationPool<$map_type<E>> =
+                        NaiveAggregationPool::default();
+
+                    assert_eq!(
+                        pool.insert(&a_0),
+                        Ok(InsertOutcome::NewItemInserted { committee_index: 0 }),
+                        "should accept a_0"
+                    );
+                    assert_eq!(
+                        pool.insert(&a_1),
+                        Ok(InsertOutcome::SignatureAggregated { committee_index: 1 }),
+                        "should accept a_1"
+                    );
+
+                    let retrieved = pool
+                        .get(&$key_getter(&a_0))
+                        .expect("should not error while getting attestation");
+
+                    let mut a_01 = a_0.clone();
+                    a_01.aggregate(&a_1);
+
+                    assert_eq!(retrieved, a_01, "retrieved item should be aggregated");
+
+                    /*
+                     * Throw different data in there and ensure it isn't aggregated
+                     */
+
+                    let mut a_different = a_0.clone();
+                    let different_root = Hash256::from_low_u64_be(1337);
+                    $unset_method_name(&mut a_different, 0);
+                    $sign_method_name(&mut a_different, 2, genesis_validators_root);
+                    assert!(!$block_root_comparator(&a_different, different_root));
+                    $block_root_mutator(&mut a_different, different_root);
+
+                    assert_eq!(
+                        pool.insert(&a_different),
+                        Ok(InsertOutcome::NewItemInserted { committee_index: 2 }),
+                        "should accept a_different"
+                    );
+
+                    assert_eq!(
+                        pool.get(&$key_getter(&a_0))
+                            .expect("should not error while getting item"),
+                        retrieved,
+                        "should not have aggregated different items with different data"
+                    );
+                }
+
+                #[test]
+                fn auto_pruning_item() {
+                    let mut base = $get_method_name(Slot::new(0));
+                    $sign_method_name(&mut base, 0, Hash256::random());
+
+                    let mut pool: NaiveAggregationPool<$map_type<E>> =
+                        NaiveAggregationPool::default();
+
+                    for i in 0..SLOTS_RETAINED * 2 {
+                        let slot = Slot::from(i);
+                        let mut a = base.clone();
+                        $slot_mutator(&mut a, slot);
+
+                        assert_eq!(
+                            pool.insert(&a),
+                            Ok(InsertOutcome::NewItemInserted { committee_index: 0 }),
+                            "should accept new item"
+                        );
+
+                        if i < SLOTS_RETAINED {
+                            let len = i + 1;
+                            assert_eq!(pool.maps.len(), len, "the pool should have length {}", len);
+                        } else {
+                            assert_eq!(
+                                pool.maps.len(),
+                                SLOTS_RETAINED,
+                                "the pool should have length SLOTS_RETAINED"
+                            );
+
+                            let mut pool_slots = pool
+                                .maps
+                                .iter()
+                                .map(|(slot, _map)| *slot)
+                                .collect::<Vec<_>>();
+
+                            pool_slots.sort_unstable();
+
+                            for (j, pool_slot) in pool_slots.iter().enumerate() {
+                                let expected_slot = slot - (SLOTS_RETAINED - 1 - j) as u64;
+                                assert_eq!(
+                                    *pool_slot, expected_slot,
+                                    "the slot of the map should be {}",
+                                    expected_slot
+                                )
+                            }
+                        }
+                    }
+                }
+
+                #[test]
+                fn max_items() {
+                    let mut base = $get_method_name(Slot::new(0));
+                    $sign_method_name(&mut base, 0, Hash256::random());
+
+                    let mut pool: NaiveAggregationPool<$map_type<E>> =
+                        NaiveAggregationPool::default();
+
+                    for i in 0..=$item_limit {
+                        let mut a = base.clone();
+                        $block_root_mutator(&mut a, Hash256::from_low_u64_be(i as u64));
+
+                        if i < $item_limit {
+                            assert_eq!(
+                                pool.insert(&a),
+                                Ok(InsertOutcome::NewItemInserted { committee_index: 0 }),
+                                "should accept item below limit"
+                            );
+                        } else {
+                            assert_eq!(
+                                pool.insert(&a),
+                                Err(Error::ReachedMaxItemsPerSlot($item_limit)),
+                                "should not accept item above limit"
+                            );
+                        }
+                    }
+                }
+            }
+        };
+    }
+
+    test_suite! {
+        attestation_tests,
+        get_attestation,
+        sign_attestation,
+        unset_attestation_bit,
+        mutate_attestation_block_root,
+        mutate_attestation_slot,
+        attestation_block_root_comparator,
+        key_from_attestation,
+        AggregatedAttestationMap,
+        MAX_ATTESTATIONS_PER_SLOT
+    }
+
+    test_suite! {
+        sync_contribution_tests,
+        get_sync_contribution,
+        sign_sync_contribution,
+        unset_sync_contribution_bit,
+        mutate_sync_contribution_block_root,
+        mutate_sync_contribution_slot,
+        sync_contribution_block_root_comparator,
+        key_from_sync_contribution,
+        SyncContributionAggregateMap,
+        E::sync_committee_size()
+    }
+}

beacon_node/beacon_chain/src/observed_aggregates.rs

@@ -0,0 +1,507 @@
+//! Provides an `ObservedAggregates` struct which allows us to reject aggregated attestations or
+//! sync committee contributions if we've already seen them.
+
+use std::collections::HashSet;
+use std::marker::PhantomData;
+use tree_hash::TreeHash;
+use types::consts::altair::{
+    SYNC_COMMITTEE_SUBNET_COUNT, TARGET_AGGREGATORS_PER_SYNC_SUBCOMMITTEE,
+};
+use types::slot_data::SlotData;
+use types::{Attestation, EthSpec, Hash256, Slot, SyncCommitteeContribution};
+
+pub type ObservedSyncContributions<E> = ObservedAggregates<SyncCommitteeContribution<E>, E>;
+pub type ObservedAggregateAttestations<E> = ObservedAggregates<Attestation<E>, E>;
+
+/// A trait use to associate capacity constants with the type being stored in `ObservedAggregates`.
+pub trait Consts {
+    /// The default capacity of items stored per slot, in a single `SlotHashSet`.
+    const DEFAULT_PER_SLOT_CAPACITY: usize;
+
+    /// The maximum number of slots
+    fn max_slot_capacity() -> usize;
+
+    /// The maximum number of items stored per slot, in a single `SlotHashSet`.
+    fn max_per_slot_capacity() -> usize;
+}
+
+impl<T: EthSpec> Consts for Attestation<T> {
+    /// Use 128 as it's the target committee size for the mainnet spec. This is perhaps a little
+    /// wasteful for the minimal spec, but considering it's approx. 128 * 32 bytes we're not wasting
+    /// much.
+    const DEFAULT_PER_SLOT_CAPACITY: usize = 128;
+
+    /// We need to keep attestations for each slot of the current epoch.
+    fn max_slot_capacity() -> usize {
+        T::slots_per_epoch() as usize
+    }
+
+    /// As a DoS protection measure, the maximum number of distinct `Attestations` or
+    /// `SyncCommitteeContributions` that will be recorded for each slot.
+    ///
+    /// Currently this is set to ~524k. If we say that each entry is 40 bytes (Hash256 (32 bytes) + an
+    /// 8 byte hash) then this comes to about 20mb per slot. If we're storing 34 of these slots, then
+    /// we're at 680mb. This is a lot of memory usage, but probably not a show-stopper for most
+    /// reasonable hardware.
+    ///
+    /// Upstream conditions should strongly restrict the amount of attestations that can show up in
+    /// this pool. The maximum size with respect to upstream restrictions is more likely on the order
+    /// of the number of validators.
+    fn max_per_slot_capacity() -> usize {
+        1 << 19 // 524,288
+    }
+}
+
+impl<T: EthSpec> Consts for SyncCommitteeContribution<T> {
+    /// Set to `TARGET_AGGREGATORS_PER_SYNC_SUBCOMMITTEE * SYNC_COMMITTEE_SUBNET_COUNT`. This is the
+    /// expected number of aggregators per slot across all subcommittees.
+    const DEFAULT_PER_SLOT_CAPACITY: usize =
+        (SYNC_COMMITTEE_SUBNET_COUNT * TARGET_AGGREGATORS_PER_SYNC_SUBCOMMITTEE) as usize;
+
+    /// We only need to keep contributions related to the current slot.
+    fn max_slot_capacity() -> usize {
+        1
+    }
+
+    /// We should never receive more aggregates than there are sync committee participants.
+    fn max_per_slot_capacity() -> usize {
+        T::sync_committee_size()
+    }
+}
+
+#[derive(Debug, PartialEq)]
+pub enum ObserveOutcome {
+    /// This item was already known.
+    AlreadyKnown,
+    /// This was the first time this item was observed.
+    New,
+}
+
+#[derive(Debug, PartialEq)]
+pub enum Error {
+    SlotTooLow {
+        slot: Slot,
+        lowest_permissible_slot: Slot,
+    },
+    /// The function to obtain a set index failed, this is an internal error.
+    InvalidSetIndex(usize),
+    /// We have reached the maximum number of unique items that can be observed in a slot.
+    /// This is a DoS protection function.
+    ReachedMaxObservationsPerSlot(usize),
+    IncorrectSlot {
+        expected: Slot,
+        attestation: Slot,
+    },
+}
+
+/// A `HashSet` that contains entries related to some `Slot`.
+struct SlotHashSet {
+    set: HashSet<Hash256>,
+    slot: Slot,
+    max_capacity: usize,
+}
+
+impl SlotHashSet {
+    pub fn new(slot: Slot, initial_capacity: usize, max_capacity: usize) -> Self {
+        Self {
+            slot,
+            set: HashSet::with_capacity(initial_capacity),
+            max_capacity,
+        }
+    }
+
+    /// Store the items in self so future observations recognise its existence.
+    pub fn observe_item<T: SlotData>(
+        &mut self,
+        item: &T,
+        root: Hash256,
+    ) -> Result<ObserveOutcome, Error> {
+        if item.get_slot() != self.slot {
+            return Err(Error::IncorrectSlot {
+                expected: self.slot,
+                attestation: item.get_slot(),
+            });
+        }
+
+        if self.set.contains(&root) {
+            Ok(ObserveOutcome::AlreadyKnown)
+        } else {
+            // Here we check to see if this slot has reached the maximum observation count.
+            //
+            // The resulting behaviour is that we are no longer able to successfully observe new
+            // items, however we will continue to return `is_known` values. We could also
+            // disable `is_known`, however then we would stop forwarding items across the
+            // gossip network and I think that this is a worse case than sending some invalid ones.
+            // The underlying libp2p network is responsible for removing duplicate messages, so
+            // this doesn't risk a broadcast loop.
+            if self.set.len() >= self.max_capacity {
+                return Err(Error::ReachedMaxObservationsPerSlot(self.max_capacity));
+            }
+
+            self.set.insert(root);
+
+            Ok(ObserveOutcome::New)
+        }
+    }
+
+    /// Indicates if `item` has been observed before.
+    pub fn is_known<T: SlotData>(&self, item: &T, root: Hash256) -> Result<bool, Error> {
+        if item.get_slot() != self.slot {
+            return Err(Error::IncorrectSlot {
+                expected: self.slot,
+                attestation: item.get_slot(),
+            });
+        }
+
+        Ok(self.set.contains(&root))
+    }
+
+    /// The number of observed items in `self`.
+    pub fn len(&self) -> usize {
+        self.set.len()
+    }
+}
+
+/// Stores the roots of objects for some number of `Slots`, so we can determine if
+/// these have previously been seen on the network.
+pub struct ObservedAggregates<T: TreeHash + SlotData + Consts, E: EthSpec> {
+    lowest_permissible_slot: Slot,
+    sets: Vec<SlotHashSet>,
+    _phantom_spec: PhantomData<E>,
+    _phantom_tree_hash: PhantomData<T>,
+}
+
+impl<T: TreeHash + SlotData + Consts, E: EthSpec> Default for ObservedAggregates<T, E> {
+    fn default() -> Self {
+        Self {
+            lowest_permissible_slot: Slot::new(0),
+            sets: vec![],
+            _phantom_spec: PhantomData,
+            _phantom_tree_hash: PhantomData,
+        }
+    }
+}
+
+impl<T: TreeHash + SlotData + Consts, E: EthSpec> ObservedAggregates<T, E> {
+    /// Store the root of `item` in `self`.
+    ///
+    /// `root` must equal `item.tree_hash_root()`.
+    pub fn observe_item(
+        &mut self,
+        item: &T,
+        root_opt: Option<Hash256>,
+    ) -> Result<ObserveOutcome, Error> {
+        let index = self.get_set_index(item.get_slot())?;
+        let root = root_opt.unwrap_or_else(|| item.tree_hash_root());
+
+        self.sets
+            .get_mut(index)
+            .ok_or(Error::InvalidSetIndex(index))
+            .and_then(|set| set.observe_item(item, root))
+    }
+
+    /// Check to see if the `root` of `item` is in self.
+    ///
+    /// `root` must equal `a.tree_hash_root()`.
+    pub fn is_known(&mut self, item: &T, root: Hash256) -> Result<bool, Error> {
+        let index = self.get_set_index(item.get_slot())?;
+
+        self.sets
+            .get(index)
+            .ok_or(Error::InvalidSetIndex(index))
+            .and_then(|set| set.is_known(item, root))
+    }
+
+    /// The maximum number of slots that items are stored for.
+    fn max_capacity(&self) -> u64 {
+        // We add `2` in order to account for one slot either side of the range due to
+        // `MAXIMUM_GOSSIP_CLOCK_DISPARITY`.
+        (T::max_slot_capacity() + 2) as u64
+    }
+
+    /// Removes any items with a slot lower than `current_slot` and bars any future
+    /// item with a slot lower than `current_slot - SLOTS_RETAINED`.
+    pub fn prune(&mut self, current_slot: Slot) {
+        let lowest_permissible_slot = current_slot.saturating_sub(self.max_capacity() - 1);
+
+        self.sets.retain(|set| set.slot >= lowest_permissible_slot);
+
+        self.lowest_permissible_slot = lowest_permissible_slot;
+    }
+
+    /// Returns the index of `self.set` that matches `slot`.
+    ///
+    /// If there is no existing set for this slot one will be created. If `self.sets.len() >=
+    /// Self::max_capacity()`, the set with the lowest slot will be replaced.
+    fn get_set_index(&mut self, slot: Slot) -> Result<usize, Error> {
+        let lowest_permissible_slot = self.lowest_permissible_slot;
+
+        if slot < lowest_permissible_slot {
+            return Err(Error::SlotTooLow {
+                slot,
+                lowest_permissible_slot,
+            });
+        }
+
+        // Prune the pool if this item indicates that the current slot has advanced.
+        if lowest_permissible_slot + self.max_capacity() < slot + 1 {
+            self.prune(slot)
+        }
+
+        if let Some(index) = self.sets.iter().position(|set| set.slot == slot) {
+            return Ok(index);
+        }
+
+        // To avoid re-allocations, try and determine a rough initial capacity for the new set
+        // by obtaining the mean size of all items in earlier epoch.
+        let (count, sum) = self
+            .sets
+            .iter()
+            // Only include slots that are less than the given slot in the average. This should
+            // generally avoid including recent slots that are still "filling up".
+            .filter(|set| set.slot < slot)
+            .map(|set| set.len())
+            .fold((0, 0), |(count, sum), len| (count + 1, sum + len));
+        // If we are unable to determine an average, just use the `self.default_per_slot_capacity`.
+        let initial_capacity = sum
+            .checked_div(count)
+            .unwrap_or(T::DEFAULT_PER_SLOT_CAPACITY);
+
+        if self.sets.len() < self.max_capacity() as usize || self.sets.is_empty() {
+            let index = self.sets.len();
+            self.sets.push(SlotHashSet::new(
+                slot,
+                initial_capacity,
+                T::max_per_slot_capacity(),
+            ));
+            return Ok(index);
+        }
+
+        let index = self
+            .sets
+            .iter()
+            .enumerate()
+            .min_by_key(|(_i, set)| set.slot)
+            .map(|(i, _set)| i)
+            .expect("sets cannot be empty due to previous .is_empty() check");
+
+        self.sets[index] = SlotHashSet::new(slot, initial_capacity, T::max_per_slot_capacity());
+
+        Ok(index)
+    }
+}
+
+#[cfg(test)]
+#[cfg(not(debug_assertions))]
+mod tests {
+    use super::*;
+    use tree_hash::TreeHash;
+    use types::{test_utils::test_random_instance, Hash256};
+
+    type E = types::MainnetEthSpec;
+
+    fn get_attestation(slot: Slot, beacon_block_root: u64) -> Attestation<E> {
+        let mut a: Attestation<E> = test_random_instance();
+        a.data.slot = slot;
+        a.data.beacon_block_root = Hash256::from_low_u64_be(beacon_block_root);
+        a
+    }
+
+    fn get_sync_contribution(slot: Slot, beacon_block_root: u64) -> SyncCommitteeContribution<E> {
+        let mut a: SyncCommitteeContribution<E> = test_random_instance();
+        a.slot = slot;
+        a.beacon_block_root = Hash256::from_low_u64_be(beacon_block_root);
+        a
+    }
+
+    macro_rules! test_suite {
+        ($mod_name: ident, $type: ident, $method_name: ident) => {
+            #[cfg(test)]
+            mod $mod_name {
+                use super::*;
+
+                const NUM_ELEMENTS: usize = 8;
+
+                fn single_slot_test(store: &mut $type<E>, slot: Slot) {
+                    let items = (0..NUM_ELEMENTS as u64)
+                        .map(|i| $method_name(slot, i))
+                        .collect::<Vec<_>>();
+
+                    for a in &items {
+                        assert_eq!(
+                            store.is_known(a, a.tree_hash_root()),
+                            Ok(false),
+                            "should indicate an unknown attestation is unknown"
+                        );
+                        assert_eq!(
+                            store.observe_item(a, None),
+                            Ok(ObserveOutcome::New),
+                            "should observe new attestation"
+                        );
+                    }
+
+                    for a in &items {
+                        assert_eq!(
+                            store.is_known(a, a.tree_hash_root()),
+                            Ok(true),
+                            "should indicate a known attestation is known"
+                        );
+                        assert_eq!(
+                            store.observe_item(a, Some(a.tree_hash_root())),
+                            Ok(ObserveOutcome::AlreadyKnown),
+                            "should acknowledge an existing attestation"
+                        );
+                    }
+                }
+
+                #[test]
+                fn single_slot() {
+                    let mut store = $type::default();
+
+                    single_slot_test(&mut store, Slot::new(0));
+
+                    assert_eq!(store.sets.len(), 1, "should have a single set stored");
+                    assert_eq!(
+                        store.sets[0].len(),
+                        NUM_ELEMENTS,
+                        "set should have NUM_ELEMENTS elements"
+                    );
+                }
+
+                #[test]
+                fn mulitple_contiguous_slots() {
+                    let mut store = $type::default();
+                    let max_cap = store.max_capacity();
+
+                    for i in 0..max_cap * 3 {
+                        let slot = Slot::new(i);
+
+                        single_slot_test(&mut store, slot);
+
+                        /*
+                         * Ensure that the number of sets is correct.
+                         */
+
+                        if i < max_cap {
+                            assert_eq!(
+                                store.sets.len(),
+                                i as usize + 1,
+                                "should have a {} sets stored",
+                                i + 1
+                            );
+                        } else {
+                            assert_eq!(
+                                store.sets.len(),
+                                max_cap as usize,
+                                "should have max_capacity sets stored"
+                            );
+                        }
+
+                        /*
+                         * Ensure that each set contains the correct number of elements.
+                         */
+
+                        for set in &store.sets[..] {
+                            assert_eq!(
+                                set.len(),
+                                NUM_ELEMENTS,
+                                "each store should have NUM_ELEMENTS elements"
+                            )
+                        }
+
+                        /*
+                         *  Ensure that all the sets have the expected slots
+                         */
+
+                        let mut store_slots =
+                            store.sets.iter().map(|set| set.slot).collect::<Vec<_>>();
+
+                        assert!(
+                            store_slots.len() <= store.max_capacity() as usize,
+                            "store size should not exceed max"
+                        );
+
+                        store_slots.sort_unstable();
+
+                        let expected_slots = (i.saturating_sub(max_cap - 1)..=i)
+                            .map(Slot::new)
+                            .collect::<Vec<_>>();
+
+                        assert_eq!(expected_slots, store_slots, "should have expected slots");
+                    }
+                }
+
+                #[test]
+                fn mulitple_non_contiguous_slots() {
+                    let mut store = $type::default();
+                    let max_cap = store.max_capacity();
+
+                    let to_skip = vec![1_u64, 2, 3, 5, 6, 29, 30, 31, 32, 64];
+                    let slots = (0..max_cap * 3)
+                        .into_iter()
+                        .filter(|i| !to_skip.contains(i))
+                        .collect::<Vec<_>>();
+
+                    for &i in &slots {
+                        if to_skip.contains(&i) {
+                            continue;
+                        }
+
+                        let slot = Slot::from(i);
+
+                        single_slot_test(&mut store, slot);
+
+                        /*
+                         * Ensure that each set contains the correct number of elements.
+                         */
+
+                        for set in &store.sets[..] {
+                            assert_eq!(
+                                set.len(),
+                                NUM_ELEMENTS,
+                                "each store should have NUM_ELEMENTS elements"
+                            )
+                        }
+
+                        /*
+                         *  Ensure that all the sets have the expected slots
+                         */
+
+                        let mut store_slots =
+                            store.sets.iter().map(|set| set.slot).collect::<Vec<_>>();
+
+                        store_slots.sort_unstable();
+
+                        assert!(
+                            store_slots.len() <= store.max_capacity() as usize,
+                            "store size should not exceed max"
+                        );
+
+                        let lowest = store.lowest_permissible_slot.as_u64();
+                        let highest = slot.as_u64();
+                        let expected_slots = (lowest..=highest)
+                            .filter(|i| !to_skip.contains(i))
+                            .map(Slot::new)
+                            .collect::<Vec<_>>();
+
+                        assert_eq!(
+                            expected_slots,
+                            &store_slots[..],
+                            "should have expected slots"
+                        );
+                    }
+                }
+            }
+        };
+    }
+    test_suite!(
+        observed_sync_aggregates,
+        ObservedSyncContributions,
+        get_sync_contribution
+    );
+    test_suite!(
+        observed_aggregate_attestations,
+        ObservedAggregateAttestations,
+        get_attestation
+    );
+}

beacon_node/beacon_chain/src/observed_block_producers.rs

@@ -0,0 +1,403 @@
+//! Provides the `ObservedBlockProducers` struct which allows for rejecting gossip blocks from
+//! validators that have already produced a block.
+
+use std::collections::{HashMap, HashSet};
+use std::marker::PhantomData;
+use types::{BeaconBlockRef, Epoch, EthSpec, Slot, Unsigned};
+
+#[derive(Debug, PartialEq)]
+pub enum Error {
+    /// The slot of the provided block is prior to finalization and should not have been provided
+    /// to this function. This is an internal error.
+    FinalizedBlock { slot: Slot, finalized_slot: Slot },
+    /// The function to obtain a set index failed, this is an internal error.
+    ValidatorIndexTooHigh(u64),
+}
+
+/// Maintains a cache of observed `(block.slot, block.proposer)`.
+///
+/// The cache supports pruning based upon the finalized epoch. It does not automatically prune, you
+/// must call `Self::prune` manually.
+///
+/// The maximum size of the cache is determined by `slots_since_finality *
+/// VALIDATOR_REGISTRY_LIMIT`. This is quite a large size, so it's important that upstream
+/// functions only use this cache for blocks with a valid signature. Only allowing valid signed
+/// blocks reduces the theoretical maximum size of this cache to `slots_since_finality *
+/// active_validator_count`, however in reality that is more like `slots_since_finality *
+/// known_distinct_shufflings` which is much smaller.
+pub struct ObservedBlockProducers<E: EthSpec> {
+    finalized_slot: Slot,
+    items: HashMap<Slot, HashSet<u64>>,
+    _phantom: PhantomData<E>,
+}
+
+impl<E: EthSpec> Default for ObservedBlockProducers<E> {
+    /// Instantiates `Self` with `finalized_slot == 0`.
+    fn default() -> Self {
+        Self {
+            finalized_slot: Slot::new(0),
+            items: HashMap::new(),
+            _phantom: PhantomData,
+        }
+    }
+}
+
+impl<E: EthSpec> ObservedBlockProducers<E> {
+    /// Observe that the `block` was produced by `block.proposer_index` at `block.slot`. This will
+    /// update `self` so future calls to it indicate that this block is known.
+    ///
+    /// The supplied `block` **MUST** be signature verified (see struct-level documentation).
+    ///
+    /// ## Errors
+    ///
+    /// - `block.proposer_index` is greater than `VALIDATOR_REGISTRY_LIMIT`.
+    /// - `block.slot` is equal to or less than the latest pruned `finalized_slot`.
+    pub fn observe_proposer(&mut self, block: BeaconBlockRef<'_, E>) -> Result<bool, Error> {
+        self.sanitize_block(block)?;
+
+        let did_not_exist = self
+            .items
+            .entry(block.slot())
+            .or_insert_with(|| HashSet::with_capacity(E::SlotsPerEpoch::to_usize()))
+            .insert(block.proposer_index());
+
+        Ok(!did_not_exist)
+    }
+
+    /// Returns `Ok(true)` if the `block` has been observed before, `Ok(false)` if not. Does not
+    /// update the cache, so calling this function multiple times will continue to return
+    /// `Ok(false)`, until `Self::observe_proposer` is called.
+    ///
+    /// ## Errors
+    ///
+    /// - `block.proposer_index` is greater than `VALIDATOR_REGISTRY_LIMIT`.
+    /// - `block.slot` is equal to or less than the latest pruned `finalized_slot`.
+    pub fn proposer_has_been_observed(&self, block: BeaconBlockRef<'_, E>) -> Result<bool, Error> {
+        self.sanitize_block(block)?;
+
+        let exists = self
+            .items
+            .get(&block.slot())
+            .map_or(false, |set| set.contains(&block.proposer_index()));
+
+        Ok(exists)
+    }
+
+    /// Returns `Ok(())` if the given `block` is sane.
+    fn sanitize_block(&self, block: BeaconBlockRef<'_, E>) -> Result<(), Error> {
+        if block.proposer_index() >= E::ValidatorRegistryLimit::to_u64() {
+            return Err(Error::ValidatorIndexTooHigh(block.proposer_index()));
+        }
+
+        let finalized_slot = self.finalized_slot;
+        if finalized_slot > 0 && block.slot() <= finalized_slot {
+            return Err(Error::FinalizedBlock {
+                slot: block.slot(),
+                finalized_slot,
+            });
+        }
+
+        Ok(())
+    }
+
+    /// Removes all observations of blocks equal to or earlier than `finalized_slot`.
+    ///
+    /// Stores `finalized_slot` in `self`, so that `self` will reject any block that has a slot
+    /// equal to or less than `finalized_slot`.
+    ///
+    /// No-op if `finalized_slot == 0`.
+    pub fn prune(&mut self, finalized_slot: Slot) {
+        if finalized_slot == 0 {
+            return;
+        }
+
+        self.finalized_slot = finalized_slot;
+        self.items.retain(|slot, _set| *slot > finalized_slot);
+    }
+
+    /// Returns `true` if the given `validator_index` has been stored in `self` at `epoch`.
+    ///
+    /// This is useful for doppelganger detection.
+    pub fn index_seen_at_epoch(&self, validator_index: u64, epoch: Epoch) -> bool {
+        self.items.iter().any(|(slot, producers)| {
+            slot.epoch(E::slots_per_epoch()) == epoch && producers.contains(&validator_index)
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use types::{BeaconBlock, MainnetEthSpec};
+
+    type E = MainnetEthSpec;
+
+    fn get_block(slot: u64, proposer: u64) -> BeaconBlock<E> {
+        let mut block = BeaconBlock::empty(&E::default_spec());
+        *block.slot_mut() = slot.into();
+        *block.proposer_index_mut() = proposer;
+        block
+    }
+
+    #[test]
+    fn pruning() {
+        let mut cache = ObservedBlockProducers::default();
+
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 0, "no slots should be present");
+
+        // Slot 0, proposer 0
+        let block_a = get_block(0, 0);
+
+        assert_eq!(
+            cache.observe_proposer(block_a.to_ref()),
+            Ok(false),
+            "can observe proposer, indicates proposer unobserved"
+        );
+
+        /*
+         * Preconditions.
+         */
+
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 1, "only one slot should be present");
+        assert_eq!(
+            cache
+                .items
+                .get(&Slot::new(0))
+                .expect("slot zero should be present")
+                .len(),
+            1,
+            "only one proposer should be present"
+        );
+
+        /*
+         * Check that a prune at the genesis slot does nothing.
+         */
+
+        cache.prune(Slot::new(0));
+
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 1, "only one slot should be present");
+        assert_eq!(
+            cache
+                .items
+                .get(&Slot::new(0))
+                .expect("slot zero should be present")
+                .len(),
+            1,
+            "only one proposer should be present"
+        );
+
+        /*
+         * Check that a prune empties the cache
+         */
+
+        cache.prune(E::slots_per_epoch().into());
+        assert_eq!(
+            cache.finalized_slot,
+            Slot::from(E::slots_per_epoch()),
+            "finalized slot is updated"
+        );
+        assert_eq!(cache.items.len(), 0, "no items left");
+
+        /*
+         * Check that we can't insert a finalized block
+         */
+
+        // First slot of finalized epoch, proposer 0
+        let block_b = get_block(E::slots_per_epoch(), 0);
+
+        assert_eq!(
+            cache.observe_proposer(block_b.to_ref()),
+            Err(Error::FinalizedBlock {
+                slot: E::slots_per_epoch().into(),
+                finalized_slot: E::slots_per_epoch().into(),
+            }),
+            "cant insert finalized block"
+        );
+
+        assert_eq!(cache.items.len(), 0, "block was not added");
+
+        /*
+         * Check that we _can_ insert a non-finalized block
+         */
+
+        let three_epochs = E::slots_per_epoch() * 3;
+
+        // First slot of finalized epoch, proposer 0
+        let block_b = get_block(three_epochs, 0);
+
+        assert_eq!(
+            cache.observe_proposer(block_b.to_ref()),
+            Ok(false),
+            "can insert non-finalized block"
+        );
+
+        assert_eq!(cache.items.len(), 1, "only one slot should be present");
+        assert_eq!(
+            cache
+                .items
+                .get(&Slot::new(three_epochs))
+                .expect("the three epochs slot should be present")
+                .len(),
+            1,
+            "only one proposer should be present"
+        );
+
+        /*
+         * Check that a prune doesnt wipe later blocks
+         */
+
+        let two_epochs = E::slots_per_epoch() * 2;
+        cache.prune(two_epochs.into());
+
+        assert_eq!(
+            cache.finalized_slot,
+            Slot::from(two_epochs),
+            "finalized slot is updated"
+        );
+
+        assert_eq!(cache.items.len(), 1, "only one slot should be present");
+        assert_eq!(
+            cache
+                .items
+                .get(&Slot::new(three_epochs))
+                .expect("the three epochs slot should be present")
+                .len(),
+            1,
+            "only one proposer should be present"
+        );
+    }
+
+    #[test]
+    fn simple_observations() {
+        let mut cache = ObservedBlockProducers::default();
+
+        // Slot 0, proposer 0
+        let block_a = get_block(0, 0);
+
+        assert_eq!(
+            cache.proposer_has_been_observed(block_a.to_ref()),
+            Ok(false),
+            "no observation in empty cache"
+        );
+        assert_eq!(
+            cache.observe_proposer(block_a.to_ref()),
+            Ok(false),
+            "can observe proposer, indicates proposer unobserved"
+        );
+        assert_eq!(
+            cache.proposer_has_been_observed(block_a.to_ref()),
+            Ok(true),
+            "observed block is indicated as true"
+        );
+        assert_eq!(
+            cache.observe_proposer(block_a.to_ref()),
+            Ok(true),
+            "observing again indicates true"
+        );
+
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 1, "only one slot should be present");
+        assert_eq!(
+            cache
+                .items
+                .get(&Slot::new(0))
+                .expect("slot zero should be present")
+                .len(),
+            1,
+            "only one proposer should be present"
+        );
+
+        // Slot 1, proposer 0
+        let block_b = get_block(1, 0);
+
+        assert_eq!(
+            cache.proposer_has_been_observed(block_b.to_ref()),
+            Ok(false),
+            "no observation for new slot"
+        );
+        assert_eq!(
+            cache.observe_proposer(block_b.to_ref()),
+            Ok(false),
+            "can observe proposer for new slot, indicates proposer unobserved"
+        );
+        assert_eq!(
+            cache.proposer_has_been_observed(block_b.to_ref()),
+            Ok(true),
+            "observed block in slot 1 is indicated as true"
+        );
+        assert_eq!(
+            cache.observe_proposer(block_b.to_ref()),
+            Ok(true),
+            "observing slot 1 again indicates true"
+        );
+
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 2, "two slots should be present");
+        assert_eq!(
+            cache
+                .items
+                .get(&Slot::new(0))
+                .expect("slot zero should be present")
+                .len(),
+            1,
+            "only one proposer should be present in slot 0"
+        );
+        assert_eq!(
+            cache
+                .items
+                .get(&Slot::new(1))
+                .expect("slot zero should be present")
+                .len(),
+            1,
+            "only one proposer should be present in slot 1"
+        );
+
+        // Slot 0, proposer 1
+        let block_c = get_block(0, 1);
+
+        assert_eq!(
+            cache.proposer_has_been_observed(block_c.to_ref()),
+            Ok(false),
+            "no observation for new proposer"
+        );
+        assert_eq!(
+            cache.observe_proposer(block_c.to_ref()),
+            Ok(false),
+            "can observe new proposer, indicates proposer unobserved"
+        );
+        assert_eq!(
+            cache.proposer_has_been_observed(block_c.to_ref()),
+            Ok(true),
+            "observed new proposer block is indicated as true"
+        );
+        assert_eq!(
+            cache.observe_proposer(block_c.to_ref()),
+            Ok(true),
+            "observing new proposer again indicates true"
+        );
+
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 2, "two slots should be present");
+        assert_eq!(
+            cache
+                .items
+                .get(&Slot::new(0))
+                .expect("slot zero should be present")
+                .len(),
+            2,
+            "two proposers should be present in slot 0"
+        );
+        assert_eq!(
+            cache
+                .items
+                .get(&Slot::new(1))
+                .expect("slot zero should be present")
+                .len(),
+            1,
+            "only one proposer should be present in slot 1"
+        );
+    }
+}

beacon_node/beacon_chain/src/observed_operations.rs

@@ -0,0 +1,110 @@
+use derivative::Derivative;
+use smallvec::SmallVec;
+use state_processing::{SigVerifiedOp, VerifyOperation};
+use std::collections::HashSet;
+use std::marker::PhantomData;
+use types::{
+    AttesterSlashing, BeaconState, ChainSpec, EthSpec, ProposerSlashing, SignedVoluntaryExit,
+};
+
+/// Number of validator indices to store on the stack in `observed_validators`.
+pub const SMALL_VEC_SIZE: usize = 8;
+
+/// Stateful tracker for exit/slashing operations seen on the network.
+///
+/// Implements the conditions for gossip verification of exits and slashings from the P2P spec.
+#[derive(Debug, Derivative)]
+#[derivative(Default(bound = "T: ObservableOperation<E>, E: EthSpec"))]
+pub struct ObservedOperations<T: ObservableOperation<E>, E: EthSpec> {
+    /// Indices of validators for whom we have already seen an instance of an operation `T`.
+    ///
+    /// For voluntary exits, this is the set of all `signed_voluntary_exit.message.validator_index`.
+    /// For proposer slashings, this is the set of all `proposer_slashing.index`.
+    /// For attester slashings, this is the set of all validators who would be slashed by
+    /// previously seen attester slashings, i.e. those validators in the intersection of
+    /// `attestation_1.attester_indices` and `attestation_2.attester_indices`.
+    observed_validator_indices: HashSet<u64>,
+    _phantom: PhantomData<(T, E)>,
+}
+
+/// Was the observed operation new and valid for further processing, or a useless duplicate?
+#[derive(Debug, PartialEq, Eq, Clone)]
+pub enum ObservationOutcome<T> {
+    New(SigVerifiedOp<T>),
+    AlreadyKnown,
+}
+
+/// Trait for exits and slashings which can be observed using `ObservedOperations`.
+pub trait ObservableOperation<E: EthSpec>: VerifyOperation<E> + Sized {
+    /// The set of validator indices involved in this operation.
+    ///
+    /// See the comment on `observed_validator_indices` above for detail.
+    fn observed_validators(&self) -> SmallVec<[u64; SMALL_VEC_SIZE]>;
+}
+
+impl<E: EthSpec> ObservableOperation<E> for SignedVoluntaryExit {
+    fn observed_validators(&self) -> SmallVec<[u64; SMALL_VEC_SIZE]> {
+        std::iter::once(self.message.validator_index).collect()
+    }
+}
+
+impl<E: EthSpec> ObservableOperation<E> for ProposerSlashing {
+    fn observed_validators(&self) -> SmallVec<[u64; SMALL_VEC_SIZE]> {
+        std::iter::once(self.signed_header_1.message.proposer_index).collect()
+    }
+}
+
+impl<E: EthSpec> ObservableOperation<E> for AttesterSlashing<E> {
+    fn observed_validators(&self) -> SmallVec<[u64; SMALL_VEC_SIZE]> {
+        let attestation_1_indices = self
+            .attestation_1
+            .attesting_indices
+            .iter()
+            .copied()
+            .collect::<HashSet<u64>>();
+        let attestation_2_indices = self
+            .attestation_2
+            .attesting_indices
+            .iter()
+            .copied()
+            .collect::<HashSet<u64>>();
+        attestation_1_indices
+            .intersection(&attestation_2_indices)
+            .copied()
+            .collect()
+    }
+}
+
+impl<T: ObservableOperation<E>, E: EthSpec> ObservedOperations<T, E> {
+    pub fn verify_and_observe(
+        &mut self,
+        op: T,
+        head_state: &BeaconState<E>,
+        spec: &ChainSpec,
+    ) -> Result<ObservationOutcome<T>, T::Error> {
+        let observed_validator_indices = &mut self.observed_validator_indices;
+        let new_validator_indices = op.observed_validators();
+
+        // If all of the new validator indices have been previously observed, short-circuit
+        // the validation. This implements the uniqueness check part of the spec, which for attester
+        // slashings reads:
+        //
+        // At least one index in the intersection of the attesting indices of each attestation has
+        // not yet been seen in any prior attester_slashing.
+        if new_validator_indices
+            .iter()
+            .all(|index| observed_validator_indices.contains(index))
+        {
+            return Ok(ObservationOutcome::AlreadyKnown);
+        }
+
+        // Validate the op using operation-specific logic (`verify_attester_slashing`, etc).
+        let verified_op = op.validate(head_state, spec)?;
+
+        // Add the relevant indices to the set of known indices to prevent processing of duplicates
+        // in the future.
+        observed_validator_indices.extend(new_validator_indices);
+
+        Ok(ObservationOutcome::New(verified_op))
+    }
+}

beacon_node/beacon_chain/src/persisted_beacon_chain.rs

@@ -1,26 +1,27 @@
-use crate::fork_choice::SszForkChoice;
 use crate::head_tracker::SszHeadTracker;
-use crate::{BeaconChainTypes, CheckPoint};
-use operation_pool::PersistedOperationPool;
 use ssz::{Decode, Encode};
 use ssz_derive::{Decode, Encode};
-use store::{DBColumn, Error as StoreError, SimpleStoreItem};
+use store::{DBColumn, Error as StoreError, StoreItem};
 use types::Hash256;
 
-/// 32-byte key for accessing the `PersistedBeaconChain`.
-pub const BEACON_CHAIN_DB_KEY: &str = "PERSISTEDBEACONCHAINPERSISTEDBEA";
+/// Dummy value to use for the canonical head block root, see below.
+pub const DUMMY_CANONICAL_HEAD_BLOCK_ROOT: Hash256 = Hash256::repeat_byte(0xff);
 
 #[derive(Clone, Encode, Decode)]
-pub struct PersistedBeaconChain<T: BeaconChainTypes> {
-    pub canonical_head: CheckPoint<T::EthSpec>,
-    pub finalized_checkpoint: CheckPoint<T::EthSpec>,
-    pub op_pool: PersistedOperationPool<T::EthSpec>,
+pub struct PersistedBeaconChain {
+    /// This value is ignored to resolve the issue described here:
+    ///
+    /// https://github.com/sigp/lighthouse/pull/1639
+    ///
+    /// Its removal is tracked here:
+    ///
+    /// https://github.com/sigp/lighthouse/issues/1784
+    pub _canonical_head_block_root: Hash256,
     pub genesis_block_root: Hash256,
     pub ssz_head_tracker: SszHeadTracker,
-    pub fork_choice: SszForkChoice,
 }
 
-impl<T: BeaconChainTypes> SimpleStoreItem for PersistedBeaconChain<T> {
+impl StoreItem for PersistedBeaconChain {
     fn db_column() -> DBColumn {
         DBColumn::BeaconChain
     }

beacon_node/beacon_chain/src/persisted_fork_choice.rs

@@ -0,0 +1,25 @@
+use crate::beacon_fork_choice_store::PersistedForkChoiceStore as ForkChoiceStore;
+use fork_choice::PersistedForkChoice as ForkChoice;
+use ssz::{Decode, Encode};
+use ssz_derive::{Decode, Encode};
+use store::{DBColumn, Error, StoreItem};
+
+#[derive(Encode, Decode)]
+pub struct PersistedForkChoice {
+    pub fork_choice: ForkChoice,
+    pub fork_choice_store: ForkChoiceStore,
+}
+
+impl StoreItem for PersistedForkChoice {
+    fn db_column() -> DBColumn {
+        DBColumn::ForkChoice
+    }
+
+    fn as_store_bytes(&self) -> Vec<u8> {
+        self.as_ssz_bytes()
+    }
+
+    fn from_store_bytes(bytes: &[u8]) -> std::result::Result<Self, Error> {
+        Self::from_ssz_bytes(bytes).map_err(Into::into)
+    }
+}

beacon_node/beacon_chain/src/schema_change.rs

@@ -0,0 +1,83 @@
+//! Utilities for managing database schema changes.
+use crate::beacon_chain::{BeaconChainTypes, OP_POOL_DB_KEY};
+use crate::validator_pubkey_cache::ValidatorPubkeyCache;
+use operation_pool::{PersistedOperationPool, PersistedOperationPoolBase};
+use std::fs;
+use std::path::Path;
+use std::sync::Arc;
+use store::hot_cold_store::{HotColdDB, HotColdDBError};
+use store::metadata::{SchemaVersion, CURRENT_SCHEMA_VERSION};
+use store::Error as StoreError;
+
+const PUBKEY_CACHE_FILENAME: &str = "pubkey_cache.ssz";
+
+/// Migrate the database from one schema version to another, applying all requisite mutations.
+pub fn migrate_schema<T: BeaconChainTypes>(
+    db: Arc<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>,
+    datadir: &Path,
+    from: SchemaVersion,
+    to: SchemaVersion,
+) -> Result<(), StoreError> {
+    match (from, to) {
+        // Migrating from the current schema version to iself is always OK, a no-op.
+        (_, _) if from == to && to == CURRENT_SCHEMA_VERSION => Ok(()),
+        // Migrate across multiple versions by recursively migrating one step at a time.
+        (_, _) if from.as_u64() + 1 < to.as_u64() => {
+            let next = SchemaVersion(from.as_u64() + 1);
+            migrate_schema::<T>(db.clone(), datadir, from, next)?;
+            migrate_schema::<T>(db, datadir, next, to)
+        }
+        // Migration from v0.3.0 to v0.3.x, adding the temporary states column.
+        // Nothing actually needs to be done, but once a DB uses v2 it shouldn't go back.
+        (SchemaVersion(1), SchemaVersion(2)) => {
+            db.store_schema_version(to)?;
+            Ok(())
+        }
+        // Migration for removing the pubkey cache.
+        (SchemaVersion(2), SchemaVersion(3)) => {
+            let pk_cache_path = datadir.join(PUBKEY_CACHE_FILENAME);
+
+            // Load from file, store to DB.
+            ValidatorPubkeyCache::<T>::load_from_file(&pk_cache_path)
+                .and_then(|cache| ValidatorPubkeyCache::convert(cache, db.clone()))
+                .map_err(|e| StoreError::SchemaMigrationError(format!("{:?}", e)))?;
+
+            db.store_schema_version(to)?;
+
+            // Delete cache file now that keys are stored in the DB.
+            fs::remove_file(&pk_cache_path).map_err(|e| {
+                StoreError::SchemaMigrationError(format!(
+                    "unable to delete {}: {:?}",
+                    pk_cache_path.display(),
+                    e
+                ))
+            })?;
+
+            Ok(())
+        }
+        // Migration for adding sync committee contributions to the persisted op pool.
+        (SchemaVersion(3), SchemaVersion(4)) => {
+            // Deserialize from what exists in the database using the `PersistedOperationPoolBase`
+            // variant and convert it to the Altair variant.
+            let pool_opt = db
+                .get_item::<PersistedOperationPoolBase<T::EthSpec>>(&OP_POOL_DB_KEY)?
+                .map(PersistedOperationPool::Base)
+                .map(PersistedOperationPool::base_to_altair);
+
+            if let Some(pool) = pool_opt {
+                // Store the converted pool under the same key.
+                db.put_item::<PersistedOperationPool<T::EthSpec>>(&OP_POOL_DB_KEY, &pool)?;
+            }
+
+            db.store_schema_version(to)?;
+
+            Ok(())
+        }
+        // Anything else is an error.
+        (_, _) => Err(HotColdDBError::UnsupportedSchemaVersion {
+            target_version: to,
+            current_version: from,
+        }
+        .into()),
+    }
+}

beacon_node/beacon_chain/src/shuffling_cache.rs

@@ -0,0 +1,75 @@
+use crate::metrics;
+use lru::LruCache;
+use types::{beacon_state::CommitteeCache, AttestationShufflingId, Epoch, Hash256};
+
+/// The size of the LRU cache that stores committee caches for quicker verification.
+///
+/// Each entry should be `8 + 800,000 = 800,008` bytes in size with 100k validators. (8-byte hash +
+/// 100k indices). Therefore, this cache should be approx `16 * 800,008 = 12.8 MB`. (Note: this
+/// ignores a few extra bytes in the caches that should be insignificant compared to the indices).
+const CACHE_SIZE: usize = 16;
+
+/// Provides an LRU cache for `CommitteeCache`.
+///
+/// It has been named `ShufflingCache` because `CommitteeCacheCache` is a bit weird and looks like
+/// a find/replace error.
+pub struct ShufflingCache {
+    cache: LruCache<AttestationShufflingId, CommitteeCache>,
+}
+
+impl ShufflingCache {
+    pub fn new() -> Self {
+        Self {
+            cache: LruCache::new(CACHE_SIZE),
+        }
+    }
+
+    pub fn get(&mut self, key: &AttestationShufflingId) -> Option<&CommitteeCache> {
+        let opt = self.cache.get(key);
+
+        if opt.is_some() {
+            metrics::inc_counter(&metrics::SHUFFLING_CACHE_HITS);
+        } else {
+            metrics::inc_counter(&metrics::SHUFFLING_CACHE_MISSES);
+        }
+
+        opt
+    }
+
+    pub fn contains(&self, key: &AttestationShufflingId) -> bool {
+        self.cache.contains(key)
+    }
+
+    pub fn insert(&mut self, key: AttestationShufflingId, committee_cache: &CommitteeCache) {
+        if !self.cache.contains(&key) {
+            self.cache.put(key, committee_cache.clone());
+        }
+    }
+}
+
+/// Contains the shuffling IDs for a beacon block.
+pub struct BlockShufflingIds {
+    pub current: AttestationShufflingId,
+    pub next: AttestationShufflingId,
+    pub block_root: Hash256,
+}
+
+impl BlockShufflingIds {
+    /// Returns the shuffling ID for the given epoch.
+    ///
+    /// Returns `None` if `epoch` is prior to `self.current.shuffling_epoch`.
+    pub fn id_for_epoch(&self, epoch: Epoch) -> Option<AttestationShufflingId> {
+        if epoch == self.current.shuffling_epoch {
+            Some(self.current.clone())
+        } else if epoch == self.next.shuffling_epoch {
+            Some(self.next.clone())
+        } else if epoch > self.next.shuffling_epoch {
+            Some(AttestationShufflingId::from_components(
+                epoch,
+                self.block_root,
+            ))
+        } else {
+            None
+        }
+    }
+}

beacon_node/beacon_chain/src/snapshot_cache.rs

@@ -0,0 +1,421 @@
+use crate::BeaconSnapshot;
+use std::cmp;
+use types::{
+    beacon_state::CloneConfig, BeaconState, Epoch, EthSpec, Hash256, SignedBeaconBlock, Slot,
+};
+
+/// The default size of the cache.
+pub const DEFAULT_SNAPSHOT_CACHE_SIZE: usize = 4;
+
+/// This snapshot is to be used for verifying a child of `self.beacon_block`.
+#[derive(Debug)]
+pub struct PreProcessingSnapshot<T: EthSpec> {
+    /// This state is equivalent to the `self.beacon_block.state_root()` state that has been
+    /// advanced forward one slot using `per_slot_processing`. This state is "primed and ready" for
+    /// the application of another block.
+    pub pre_state: BeaconState<T>,
+    /// This value is only set to `Some` if the `pre_state` was *not* advanced forward.
+    pub beacon_state_root: Option<Hash256>,
+    pub beacon_block: SignedBeaconBlock<T>,
+    pub beacon_block_root: Hash256,
+}
+
+impl<T: EthSpec> From<BeaconSnapshot<T>> for PreProcessingSnapshot<T> {
+    fn from(snapshot: BeaconSnapshot<T>) -> Self {
+        let beacon_state_root = Some(snapshot.beacon_state_root());
+        Self {
+            pre_state: snapshot.beacon_state,
+            beacon_state_root,
+            beacon_block: snapshot.beacon_block,
+            beacon_block_root: snapshot.beacon_block_root,
+        }
+    }
+}
+
+impl<T: EthSpec> CacheItem<T> {
+    pub fn new_without_pre_state(snapshot: BeaconSnapshot<T>) -> Self {
+        Self {
+            beacon_block: snapshot.beacon_block,
+            beacon_block_root: snapshot.beacon_block_root,
+            beacon_state: snapshot.beacon_state,
+            pre_state: None,
+        }
+    }
+
+    fn clone_to_snapshot_with(&self, clone_config: CloneConfig) -> BeaconSnapshot<T> {
+        BeaconSnapshot {
+            beacon_state: self.beacon_state.clone_with(clone_config),
+            beacon_block: self.beacon_block.clone(),
+            beacon_block_root: self.beacon_block_root,
+        }
+    }
+
+    pub fn into_pre_state(self) -> PreProcessingSnapshot<T> {
+        // Do not include the beacon state root if the state has been advanced.
+        let beacon_state_root =
+            Some(self.beacon_block.state_root()).filter(|_| self.pre_state.is_none());
+
+        PreProcessingSnapshot {
+            beacon_block: self.beacon_block,
+            beacon_block_root: self.beacon_block_root,
+            pre_state: self.pre_state.unwrap_or(self.beacon_state),
+            beacon_state_root,
+        }
+    }
+}
+
+/// The information required for block production.
+pub struct BlockProductionPreState<T: EthSpec> {
+    /// This state may or may not have been advanced forward a single slot.
+    ///
+    /// See the documentation in the `crate::state_advance_timer` module for more information.
+    pub pre_state: BeaconState<T>,
+    /// This value will only be `Some` if `self.pre_state` was **not** advanced forward a single
+    /// slot.
+    ///
+    /// This value can be used to avoid tree-hashing the state during the first call to
+    /// `per_slot_processing`.
+    pub state_root: Option<Hash256>,
+}
+
+pub enum StateAdvance<T: EthSpec> {
+    /// The cache does not contain the supplied block root.
+    BlockNotFound,
+    /// The cache contains the supplied block root but the state has already been advanced.
+    AlreadyAdvanced,
+    /// The cache contains the supplied block root and the state has not yet been advanced.
+    State {
+        state: Box<BeaconState<T>>,
+        state_root: Hash256,
+        block_slot: Slot,
+    },
+}
+
+/// The item stored in the `SnapshotCache`.
+pub struct CacheItem<T: EthSpec> {
+    beacon_block: SignedBeaconBlock<T>,
+    beacon_block_root: Hash256,
+    /// This state is equivalent to `self.beacon_block.state_root()`.
+    beacon_state: BeaconState<T>,
+    /// This state is equivalent to `self.beacon_state` that has had `per_slot_processing` applied
+    /// to it. This state assists in optimizing block processing.
+    pre_state: Option<BeaconState<T>>,
+}
+
+impl<T: EthSpec> Into<BeaconSnapshot<T>> for CacheItem<T> {
+    fn into(self) -> BeaconSnapshot<T> {
+        BeaconSnapshot {
+            beacon_state: self.beacon_state,
+            beacon_block: self.beacon_block,
+            beacon_block_root: self.beacon_block_root,
+        }
+    }
+}
+
+/// Provides a cache of `BeaconSnapshot` that is intended primarily for block processing.
+///
+/// ## Cache Queuing
+///
+/// The cache has a non-standard queue mechanism (specifically, it is not LRU).
+///
+/// The cache has a max number of elements (`max_len`). Until `max_len` is achieved, all snapshots
+/// are simply added to the queue. Once `max_len` is achieved, adding a new snapshot will cause an
+/// existing snapshot to be ejected. The ejected snapshot will:
+///
+/// - Never be the `head_block_root`.
+/// - Be the snapshot with the lowest `state.slot` (ties broken arbitrarily).
+pub struct SnapshotCache<T: EthSpec> {
+    max_len: usize,
+    head_block_root: Hash256,
+    snapshots: Vec<CacheItem<T>>,
+}
+
+impl<T: EthSpec> SnapshotCache<T> {
+    /// Instantiate a new cache which contains the `head` snapshot.
+    ///
+    /// Setting `max_len = 0` is equivalent to setting `max_len = 1`.
+    pub fn new(max_len: usize, head: BeaconSnapshot<T>) -> Self {
+        Self {
+            max_len: cmp::max(max_len, 1),
+            head_block_root: head.beacon_block_root,
+            snapshots: vec![CacheItem::new_without_pre_state(head)],
+        }
+    }
+
+    /// Insert a snapshot, potentially removing an existing snapshot if `self` is at capacity (see
+    /// struct-level documentation for more info).
+    pub fn insert(&mut self, snapshot: BeaconSnapshot<T>, pre_state: Option<BeaconState<T>>) {
+        let item = CacheItem {
+            beacon_block: snapshot.beacon_block,
+            beacon_block_root: snapshot.beacon_block_root,
+            beacon_state: snapshot.beacon_state,
+            pre_state,
+        };
+
+        if self.snapshots.len() < self.max_len {
+            self.snapshots.push(item);
+        } else {
+            let insert_at = self
+                .snapshots
+                .iter()
+                .enumerate()
+                .filter_map(|(i, snapshot)| {
+                    if snapshot.beacon_block_root != self.head_block_root {
+                        Some((i, snapshot.beacon_state.slot()))
+                    } else {
+                        None
+                    }
+                })
+                .min_by_key(|(_i, slot)| *slot)
+                .map(|(i, _slot)| i);
+
+            if let Some(i) = insert_at {
+                self.snapshots[i] = item;
+            }
+        }
+    }
+
+    /// If available, returns a `CacheItem` that should be used for importing/processing a block.
+    /// The method will remove the block from `self`, carrying across any caches that may or may not
+    /// be built.
+    pub fn get_state_for_block_processing(&mut self, block_root: Hash256) -> Option<CacheItem<T>> {
+        self.snapshots
+            .iter()
+            .position(|snapshot| snapshot.beacon_block_root == block_root)
+            .map(|i| self.snapshots.remove(i))
+    }
+
+    /// If available, obtains a clone of a `BeaconState` that should be used for block production.
+    /// The clone will use `CloneConfig:all()`, ensuring any tree-hash cache is cloned too.
+    ///
+    /// ## Note
+    ///
+    /// This method clones the `BeaconState` (instead of removing it) since we assume that any block
+    /// we produce will soon be pushed to the `BeaconChain` for importing/processing. Keeping a copy
+    /// of that `BeaconState` in `self` will greatly help with import times.
+    pub fn get_state_for_block_production(
+        &self,
+        block_root: Hash256,
+    ) -> Option<BlockProductionPreState<T>> {
+        self.snapshots
+            .iter()
+            .find(|snapshot| snapshot.beacon_block_root == block_root)
+            .map(|snapshot| {
+                if let Some(pre_state) = &snapshot.pre_state {
+                    BlockProductionPreState {
+                        pre_state: pre_state.clone_with(CloneConfig::all()),
+                        state_root: None,
+                    }
+                } else {
+                    BlockProductionPreState {
+                        pre_state: snapshot.beacon_state.clone_with(CloneConfig::all()),
+                        state_root: Some(snapshot.beacon_block.state_root()),
+                    }
+                }
+            })
+    }
+
+    /// If there is a snapshot with `block_root`, clone it and return the clone.
+    pub fn get_cloned(
+        &self,
+        block_root: Hash256,
+        clone_config: CloneConfig,
+    ) -> Option<BeaconSnapshot<T>> {
+        self.snapshots
+            .iter()
+            .find(|snapshot| snapshot.beacon_block_root == block_root)
+            .map(|snapshot| snapshot.clone_to_snapshot_with(clone_config))
+    }
+
+    pub fn get_for_state_advance(&mut self, block_root: Hash256) -> StateAdvance<T> {
+        if let Some(snapshot) = self
+            .snapshots
+            .iter_mut()
+            .find(|snapshot| snapshot.beacon_block_root == block_root)
+        {
+            if snapshot.pre_state.is_some() {
+                StateAdvance::AlreadyAdvanced
+            } else {
+                let cloned = snapshot
+                    .beacon_state
+                    .clone_with(CloneConfig::committee_caches_only());
+
+                StateAdvance::State {
+                    state: Box::new(std::mem::replace(&mut snapshot.beacon_state, cloned)),
+                    state_root: snapshot.beacon_block.state_root(),
+                    block_slot: snapshot.beacon_block.slot(),
+                }
+            }
+        } else {
+            StateAdvance::BlockNotFound
+        }
+    }
+
+    pub fn update_pre_state(&mut self, block_root: Hash256, state: BeaconState<T>) -> Option<()> {
+        self.snapshots
+            .iter_mut()
+            .find(|snapshot| snapshot.beacon_block_root == block_root)
+            .map(|snapshot| {
+                snapshot.pre_state = Some(state);
+            })
+    }
+
+    /// Removes all snapshots from the queue that are less than or equal to the finalized epoch.
+    pub fn prune(&mut self, finalized_epoch: Epoch) {
+        self.snapshots.retain(|snapshot| {
+            snapshot.beacon_state.slot() > finalized_epoch.start_slot(T::slots_per_epoch())
+        })
+    }
+
+    /// Inform the cache that the head of the beacon chain has changed.
+    ///
+    /// The snapshot that matches this `head_block_root` will never be ejected from the cache
+    /// during `Self::insert`.
+    pub fn update_head(&mut self, head_block_root: Hash256) {
+        self.head_block_root = head_block_root
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::test_utils::{BeaconChainHarness, EphemeralHarnessType};
+    use store::StoreConfig;
+    use types::{
+        test_utils::generate_deterministic_keypair, BeaconBlock, Epoch, MainnetEthSpec,
+        SignedBeaconBlock, Slot,
+    };
+
+    fn get_harness() -> BeaconChainHarness<EphemeralHarnessType<MainnetEthSpec>> {
+        let harness = BeaconChainHarness::new_with_store_config(
+            MainnetEthSpec,
+            None,
+            types::test_utils::generate_deterministic_keypairs(1),
+            StoreConfig::default(),
+        );
+
+        harness.advance_slot();
+
+        harness
+    }
+
+    const CACHE_SIZE: usize = 4;
+
+    fn get_snapshot(i: u64) -> BeaconSnapshot<MainnetEthSpec> {
+        let spec = MainnetEthSpec::default_spec();
+
+        let beacon_state = get_harness().chain.head_beacon_state().unwrap();
+
+        let signed_beacon_block = SignedBeaconBlock::from_block(
+            BeaconBlock::empty(&spec),
+            generate_deterministic_keypair(0)
+                .sk
+                .sign(Hash256::from_low_u64_be(42)),
+        );
+
+        BeaconSnapshot {
+            beacon_state,
+            beacon_block: signed_beacon_block,
+            beacon_block_root: Hash256::from_low_u64_be(i),
+        }
+    }
+
+    #[test]
+    fn insert_get_prune_update() {
+        let mut cache = SnapshotCache::new(CACHE_SIZE, get_snapshot(0));
+
+        // Insert a bunch of entries in the cache. It should look like this:
+        //
+        // Index    Root
+        // 0        0     <--head
+        // 1        1
+        // 2        2
+        // 3        3
+        for i in 1..CACHE_SIZE as u64 {
+            let mut snapshot = get_snapshot(i);
+
+            // Each snapshot should be one slot into an epoch, with each snapshot one epoch apart.
+            *snapshot.beacon_state.slot_mut() =
+                Slot::from(i * MainnetEthSpec::slots_per_epoch() + 1);
+
+            cache.insert(snapshot, None);
+
+            assert_eq!(
+                cache.snapshots.len(),
+                i as usize + 1,
+                "cache length should be as expected"
+            );
+            assert_eq!(cache.head_block_root, Hash256::from_low_u64_be(0));
+        }
+
+        // Insert a new value in the cache. Afterwards it should look like:
+        //
+        // Index    Root
+        // 0        0     <--head
+        // 1        42
+        // 2        2
+        // 3        3
+        assert_eq!(cache.snapshots.len(), CACHE_SIZE);
+        cache.insert(get_snapshot(42), None);
+        assert_eq!(cache.snapshots.len(), CACHE_SIZE);
+
+        assert!(
+            cache
+                .get_state_for_block_processing(Hash256::from_low_u64_be(1))
+                .is_none(),
+            "the snapshot with the lowest slot should have been removed during the insert function"
+        );
+        assert!(cache
+            .get_cloned(Hash256::from_low_u64_be(1), CloneConfig::none())
+            .is_none());
+
+        assert_eq!(
+            cache
+                .get_cloned(Hash256::from_low_u64_be(0), CloneConfig::none())
+                .expect("the head should still be in the cache")
+                .beacon_block_root,
+            Hash256::from_low_u64_be(0),
+            "get_cloned should get the correct snapshot"
+        );
+        assert_eq!(
+            cache
+                .get_state_for_block_processing(Hash256::from_low_u64_be(0))
+                .expect("the head should still be in the cache")
+                .beacon_block_root,
+            Hash256::from_low_u64_be(0),
+            "get_state_for_block_processing should get the correct snapshot"
+        );
+
+        assert_eq!(
+            cache.snapshots.len(),
+            CACHE_SIZE - 1,
+            "get_state_for_block_processing should shorten the cache"
+        );
+
+        // Prune the cache. Afterwards it should look like:
+        //
+        // Index    Root
+        // 0        2
+        // 1        3
+        cache.prune(Epoch::new(2));
+
+        assert_eq!(cache.snapshots.len(), 2);
+
+        cache.update_head(Hash256::from_low_u64_be(2));
+
+        // Over-fill the cache so it needs to eject some old values on insert.
+        for i in 0..CACHE_SIZE as u64 {
+            cache.insert(get_snapshot(u64::max_value() - i), None);
+        }
+
+        // Ensure that the new head value was not removed from the cache.
+        assert_eq!(
+            cache
+                .get_state_for_block_processing(Hash256::from_low_u64_be(2))
+                .expect("the new head should still be in the cache")
+                .beacon_block_root,
+            Hash256::from_low_u64_be(2),
+            "get_state_for_block_processing should get the correct snapshot"
+        );
+    }
+}

beacon_node/beacon_chain/src/state_advance_timer.rs

@@ -0,0 +1,386 @@
+//! Provides a timer which runs in the tail-end of each slot and maybe advances the state of the
+//! head block forward a single slot.
+//!
+//! This provides an optimization with the following benefits:
+//!
+//! 1. Removes the burden of a single, mandatory `per_slot_processing` call from the leading-edge of
+//!    block processing. This helps import blocks faster.
+//! 2. Allows the node to learn of the shuffling for the next epoch, before the first block from
+//!    that epoch has arrived. This helps reduce gossip block propagation times.
+//!
+//! The downsides to this optimization are:
+//!
+//! 1. We are required to store an additional `BeaconState` for the head block. This consumes
+//!    memory.
+//! 2. There's a possibility that the head block is never built upon, causing wasted CPU cycles.
+use crate::validator_monitor::HISTORIC_EPOCHS as VALIDATOR_MONITOR_HISTORIC_EPOCHS;
+use crate::{
+    beacon_chain::{ATTESTATION_CACHE_LOCK_TIMEOUT, BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT},
+    snapshot_cache::StateAdvance,
+    BeaconChain, BeaconChainError, BeaconChainTypes,
+};
+use slog::{debug, error, warn, Logger};
+use slot_clock::SlotClock;
+use state_processing::per_slot_processing;
+use std::sync::{
+    atomic::{AtomicBool, Ordering},
+    Arc,
+};
+use task_executor::TaskExecutor;
+use tokio::time::sleep;
+use types::{AttestationShufflingId, EthSpec, Hash256, RelativeEpoch, Slot};
+
+/// If the head slot is more than `MAX_ADVANCE_DISTANCE` from the current slot, then don't perform
+/// the state advancement.
+///
+/// This avoids doing unnecessary work whilst the node is syncing or has perhaps been put to sleep
+/// for some period of time.
+const MAX_ADVANCE_DISTANCE: u64 = 4;
+
+#[derive(Debug)]
+enum Error {
+    BeaconChain(BeaconChainError),
+    HeadMissingFromSnapshotCache(Hash256),
+    MaxDistanceExceeded { current_slot: Slot, head_slot: Slot },
+    StateAlreadyAdvanced { block_root: Hash256 },
+    BadStateSlot { state_slot: Slot, block_slot: Slot },
+}
+
+impl From<BeaconChainError> for Error {
+    fn from(e: BeaconChainError) -> Self {
+        Self::BeaconChain(e)
+    }
+}
+
+/// Provides a simple thread-safe lock to be used for task co-ordination. Practically equivalent to
+/// `Mutex<()>`.
+#[derive(Clone)]
+struct Lock(Arc<AtomicBool>);
+
+impl Lock {
+    /// Instantiate an unlocked self.
+    pub fn new() -> Self {
+        Self(Arc::new(AtomicBool::new(false)))
+    }
+
+    /// Lock self, returning `true` if the lock was already set.
+    pub fn lock(&self) -> bool {
+        self.0.fetch_or(true, Ordering::SeqCst)
+    }
+
+    /// Unlock self.
+    pub fn unlock(&self) {
+        self.0.store(false, Ordering::SeqCst);
+    }
+}
+
+/// Spawns the timer described in the module-level documentation.
+pub fn spawn_state_advance_timer<T: BeaconChainTypes>(
+    executor: TaskExecutor,
+    beacon_chain: Arc<BeaconChain<T>>,
+    log: Logger,
+) {
+    executor.spawn(
+        state_advance_timer(executor.clone(), beacon_chain, log),
+        "state_advance_timer",
+    );
+}
+
+/// Provides the timer described in the module-level documentation.
+async fn state_advance_timer<T: BeaconChainTypes>(
+    executor: TaskExecutor,
+    beacon_chain: Arc<BeaconChain<T>>,
+    log: Logger,
+) {
+    let is_running = Lock::new();
+    let slot_clock = &beacon_chain.slot_clock;
+    let slot_duration = slot_clock.slot_duration();
+
+    loop {
+        match beacon_chain.slot_clock.duration_to_next_slot() {
+            Some(duration) => sleep(duration + (slot_duration / 4) * 3).await,
+            None => {
+                error!(log, "Failed to read slot clock");
+                // If we can't read the slot clock, just wait another slot.
+                sleep(slot_duration).await;
+                continue;
+            }
+        };
+
+        // Only start spawn the state advance task if the lock was previously free.
+        if !is_running.lock() {
+            let log = log.clone();
+            let beacon_chain = beacon_chain.clone();
+            let is_running = is_running.clone();
+
+            executor.spawn_blocking(
+                move || {
+                    match advance_head(&beacon_chain, &log) {
+                        Ok(()) => (),
+                        Err(Error::BeaconChain(e)) => error!(
+                            log,
+                            "Failed to advance head state";
+                            "error" => ?e
+                        ),
+                        Err(Error::StateAlreadyAdvanced { block_root }) => debug!(
+                            log,
+                            "State already advanced on slot";
+                            "block_root" => ?block_root
+                        ),
+                        Err(Error::MaxDistanceExceeded {
+                            current_slot,
+                            head_slot,
+                        }) => debug!(
+                            log,
+                            "Refused to advance head state";
+                            "head_slot" => head_slot,
+                            "current_slot" => current_slot,
+                        ),
+                        other => warn!(
+                            log,
+                            "Did not advance head state";
+                            "reason" => ?other
+                        ),
+                    };
+
+                    // Permit this blocking task to spawn again, next time the timer fires.
+                    is_running.unlock();
+                },
+                "state_advance_blocking",
+            );
+        } else {
+            warn!(
+                log,
+                "State advance routine overloaded";
+                "msg" => "system resources may be overloaded"
+            )
+        }
+    }
+}
+
+/// Reads the `snapshot_cache` from the `beacon_chain` and attempts to take a clone of the
+/// `BeaconState` of the head block. If it obtains this clone, the state will be advanced a single
+/// slot then placed back in the `snapshot_cache` to be used for block verification.
+///
+/// See the module-level documentation for rationale.
+fn advance_head<T: BeaconChainTypes>(
+    beacon_chain: &BeaconChain<T>,
+    log: &Logger,
+) -> Result<(), Error> {
+    let current_slot = beacon_chain.slot()?;
+
+    // These brackets ensure that the `head_slot` value is dropped before we run fork choice and
+    // potentially invalidate it.
+    //
+    // Fork-choice is not run *before* this function to avoid unnecessary calls whilst syncing.
+    {
+        let head_slot = beacon_chain.head_info()?.slot;
+
+        // Don't run this when syncing or if lagging too far behind.
+        if head_slot + MAX_ADVANCE_DISTANCE < current_slot {
+            return Err(Error::MaxDistanceExceeded {
+                current_slot,
+                head_slot,
+            });
+        }
+    }
+
+    // Run fork choice so we get the latest view of the head.
+    //
+    // This is useful since it's quite likely that the last time we ran fork choice was shortly
+    // after receiving the latest gossip block, but not necessarily after we've received the
+    // majority of attestations.
+    beacon_chain.fork_choice()?;
+
+    let head_root = beacon_chain.head_info()?.block_root;
+
+    let (head_slot, head_state_root, mut state) = match beacon_chain
+        .snapshot_cache
+        .try_write_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT)
+        .ok_or(BeaconChainError::SnapshotCacheLockTimeout)?
+        .get_for_state_advance(head_root)
+    {
+        StateAdvance::AlreadyAdvanced => {
+            return Err(Error::StateAlreadyAdvanced {
+                block_root: head_root,
+            })
+        }
+        StateAdvance::BlockNotFound => return Err(Error::HeadMissingFromSnapshotCache(head_root)),
+        StateAdvance::State {
+            state,
+            state_root,
+            block_slot,
+        } => (block_slot, state_root, *state),
+    };
+
+    let initial_slot = state.slot();
+    let initial_epoch = state.current_epoch();
+
+    let state_root = if state.slot() == head_slot {
+        Some(head_state_root)
+    } else {
+        // Protect against advancing a state more than a single slot.
+        //
+        // Advancing more than one slot without storing the intermediate state would corrupt the
+        // database. Future works might store temporary, intermediate states inside this function.
+        return Err(Error::BadStateSlot {
+            block_slot: head_slot,
+            state_slot: state.slot(),
+        });
+    };
+
+    // Advance the state a single slot.
+    if let Some(summary) = per_slot_processing(&mut state, state_root, &beacon_chain.spec)
+        .map_err(BeaconChainError::from)?
+    {
+        // Expose Prometheus metrics.
+        if let Err(e) = summary.observe_metrics() {
+            error!(
+                log,
+                "Failed to observe epoch summary metrics";
+                "src" => "state_advance_timer",
+                "error" => ?e
+            );
+        }
+
+        // Only notify the validator monitor for recent blocks.
+        if state.current_epoch() + VALIDATOR_MONITOR_HISTORIC_EPOCHS as u64
+            >= current_slot.epoch(T::EthSpec::slots_per_epoch())
+        {
+            // Potentially create logs/metrics for locally monitored validators.
+            if let Err(e) = beacon_chain
+                .validator_monitor
+                .read()
+                .process_validator_statuses(state.current_epoch(), &summary, &beacon_chain.spec)
+            {
+                error!(
+                    log,
+                    "Unable to process validator statuses";
+                    "error" => ?e
+                );
+            }
+        }
+    }
+
+    debug!(
+        log,
+        "Advanced head state one slot";
+        "head_root" => ?head_root,
+        "state_slot" => state.slot(),
+        "current_slot" => current_slot,
+    );
+
+    // Build the current epoch cache, to prepare to compute proposer duties.
+    state
+        .build_committee_cache(RelativeEpoch::Current, &beacon_chain.spec)
+        .map_err(BeaconChainError::from)?;
+    // Build the next epoch cache, to prepare to compute attester duties.
+    state
+        .build_committee_cache(RelativeEpoch::Next, &beacon_chain.spec)
+        .map_err(BeaconChainError::from)?;
+
+    // If the `pre_state` is in a later epoch than `state`, pre-emptively add the proposer shuffling
+    // for the state's current epoch and the committee cache for the state's next epoch.
+    if initial_epoch < state.current_epoch() {
+        // Update the proposer cache.
+        //
+        // We supply the `head_root` as the decision block since the prior `if` statement guarantees
+        // the head root is the latest block from the prior epoch.
+        beacon_chain
+            .beacon_proposer_cache
+            .lock()
+            .insert(
+                state.current_epoch(),
+                head_root,
+                state
+                    .get_beacon_proposer_indices(&beacon_chain.spec)
+                    .map_err(BeaconChainError::from)?,
+                state.fork(),
+            )
+            .map_err(BeaconChainError::from)?;
+
+        // Update the attester cache.
+        let shuffling_id = AttestationShufflingId::new(head_root, &state, RelativeEpoch::Next)
+            .map_err(BeaconChainError::from)?;
+        let committee_cache = state
+            .committee_cache(RelativeEpoch::Next)
+            .map_err(BeaconChainError::from)?;
+        beacon_chain
+            .shuffling_cache
+            .try_write_for(ATTESTATION_CACHE_LOCK_TIMEOUT)
+            .ok_or(BeaconChainError::AttestationCacheLockTimeout)?
+            .insert(shuffling_id.clone(), committee_cache);
+
+        debug!(
+            log,
+            "Primed proposer and attester caches";
+            "head_root" => ?head_root,
+            "next_epoch_shuffling_root" => ?shuffling_id.shuffling_decision_block,
+            "state_epoch" => state.current_epoch(),
+            "current_epoch" => current_slot.epoch(T::EthSpec::slots_per_epoch()),
+        );
+    }
+
+    // Apply the state to the attester cache, if the cache deems it interesting.
+    beacon_chain
+        .attester_cache
+        .maybe_cache_state(&state, head_root, &beacon_chain.spec)
+        .map_err(BeaconChainError::from)?;
+
+    let final_slot = state.slot();
+
+    // Insert the advanced state back into the snapshot cache.
+    beacon_chain
+        .snapshot_cache
+        .try_write_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT)
+        .ok_or(BeaconChainError::SnapshotCacheLockTimeout)?
+        .update_pre_state(head_root, state)
+        .ok_or(Error::HeadMissingFromSnapshotCache(head_root))?;
+
+    // If we have moved into the next slot whilst processing the state then this function is going
+    // to become ineffective and likely become a hindrance as we're stealing the tree hash cache
+    // from the snapshot cache (which may force the next block to rebuild a new one).
+    //
+    // If this warning occurs very frequently on well-resourced machines then we should consider
+    // starting it earlier in the slot. Otherwise, it's a good indication that the machine is too
+    // slow/overloaded and will be useful information for the user.
+    let starting_slot = current_slot;
+    let current_slot = beacon_chain.slot()?;
+    if starting_slot < current_slot {
+        warn!(
+            log,
+            "State advance too slow";
+            "head_root" => %head_root,
+            "advanced_slot" => final_slot,
+            "current_slot" => current_slot,
+            "starting_slot" => starting_slot,
+            "msg" => "system resources may be overloaded",
+        );
+    }
+
+    debug!(
+        log,
+        "Completed state advance";
+        "head_root" => ?head_root,
+        "advanced_slot" => final_slot,
+        "initial_slot" => initial_slot,
+    );
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn lock() {
+        let lock = Lock::new();
+        assert!(!lock.lock());
+        assert!(lock.lock());
+        assert!(lock.lock());
+        lock.unlock();
+        assert!(!lock.lock());
+        assert!(lock.lock());
+    }
+}

beacon_node/beacon_chain/src/sync_committee_verification.rs

@@ -0,0 +1,658 @@
+//! Provides verification for the following sync committee messages:
+//!
+//! - "Unaggregated" `SyncCommitteeMessage` received from either gossip or the HTTP API.
+//! - "Aggregated" `SignedContributionAndProof` received from gossip or the HTTP API.
+//!
+//! For clarity, we define:
+//!
+//! - Unaggregated: a `SyncCommitteeMessage` object.
+//! - Aggregated: a `SignedContributionAndProof` which has zero or more signatures.
+//!   - Note: "zero or more" may soon change to "one or more".
+//!
+//! Similar to the `crate::block_verification` module, we try to avoid doing duplicate verification
+//! work as a sync committee message passes through different stages of verification. We represent these
+//! different stages of verification with wrapper types. These wrapper-types flow in a particular
+//! pattern:
+//!
+//! ```ignore
+//!      types::SyncCommitteeMessage      types::SignedContributionAndProof
+//!              |                                    |
+//!              ▼                                    ▼
+//!      VerifiedSyncCommitteeMessage               VerifiedSyncContribution
+//!              |                                    |
+//!              -------------------------------------
+//!                                |
+//!                                ▼
+//!                  impl SignatureVerifiedSyncContribution
+//! ```
+
+use crate::observed_attesters::SlotSubcommitteeIndex;
+use crate::{
+    beacon_chain::{MAXIMUM_GOSSIP_CLOCK_DISPARITY, VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT},
+    metrics,
+    observed_aggregates::ObserveOutcome,
+    BeaconChain, BeaconChainError, BeaconChainTypes,
+};
+use bls::{verify_signature_sets, PublicKeyBytes};
+use derivative::Derivative;
+use safe_arith::ArithError;
+use slot_clock::SlotClock;
+use state_processing::per_block_processing::errors::SyncCommitteeMessageValidationError;
+use state_processing::signature_sets::{
+    signed_sync_aggregate_selection_proof_signature_set, signed_sync_aggregate_signature_set,
+    sync_committee_contribution_signature_set_from_pubkeys,
+    sync_committee_message_set_from_pubkeys,
+};
+use std::borrow::Cow;
+use std::collections::HashMap;
+use strum::AsRefStr;
+use tree_hash::TreeHash;
+use types::consts::altair::SYNC_COMMITTEE_SUBNET_COUNT;
+use types::slot_data::SlotData;
+use types::sync_committee::Error as SyncCommitteeError;
+use types::{
+    sync_committee_contribution::Error as ContributionError, AggregateSignature, BeaconStateError,
+    EthSpec, Hash256, SignedContributionAndProof, Slot, SyncCommitteeContribution,
+    SyncCommitteeMessage, SyncSelectionProof, SyncSubnetId,
+};
+
+/// Returned when a sync committee contribution was not successfully verified. It might not have been verified for
+/// two reasons:
+///
+/// - The sync committee message is malformed or inappropriate for the context (indicated by all variants
+///   other than `BeaconChainError`).
+/// - The application encountered an internal error whilst attempting to determine validity
+///   (the `BeaconChainError` variant)
+#[derive(Debug, AsRefStr)]
+pub enum Error {
+    /// The sync committee message is from a slot that is later than the current slot (with respect to the
+    /// gossip clock disparity).
+    ///
+    /// ## Peer scoring
+    ///
+    /// Assuming the local clock is correct, the peer has sent an invalid message.
+    FutureSlot {
+        message_slot: Slot,
+        latest_permissible_slot: Slot,
+    },
+    /// The sync committee message is from a slot that is prior to the earliest permissible slot (with
+    /// respect to the gossip clock disparity).
+    ///
+    /// ## Peer scoring
+    ///
+    /// Assuming the local clock is correct, the peer has sent an invalid message.
+    PastSlot {
+        message_slot: Slot,
+        earliest_permissible_slot: Slot,
+    },
+    /// The sync committee message's aggregation bits were empty when they shouldn't be.
+    ///
+    /// ## Peer scoring
+    ///
+    /// The peer has sent an invalid message.
+    EmptyAggregationBitfield,
+    /// The `selection_proof` on the sync contribution does not elect it as an aggregator.
+    ///
+    /// ## Peer scoring
+    ///
+    /// The peer has sent an invalid message.
+    InvalidSelectionProof { aggregator_index: u64 },
+    /// The `selection_proof` on the sync committee contribution selects it as a validator, however the
+    /// aggregator index is not in the committee for that sync contribution.
+    ///
+    /// ## Peer scoring
+    ///
+    /// The peer has sent an invalid message.
+    AggregatorNotInCommittee { aggregator_index: u64 },
+    /// The aggregator index refers to a validator index that we have not seen.
+    ///
+    /// ## Peer scoring
+    ///
+    /// The peer has sent an invalid message.
+    AggregatorPubkeyUnknown(u64),
+    /// The sync contribution has been seen before; either in a block, on the gossip network or from a
+    /// local validator.
+    ///
+    /// ## Peer scoring
+    ///
+    /// It's unclear if this sync contribution is valid, however we have already observed it and do not
+    /// need to observe it again.
+    SyncContributionAlreadyKnown(Hash256),
+    /// There has already been an aggregation observed for this validator, we refuse to process a
+    /// second.
+    ///
+    /// ## Peer scoring
+    ///
+    /// It's unclear if this sync committee message is valid, however we have already observed an aggregate
+    /// sync committee message from this validator for this epoch and should not observe another.
+    AggregatorAlreadyKnown(u64),
+    /// The aggregator index is higher than the maximum possible validator count.
+    ///
+    /// ## Peer scoring
+    ///
+    /// The peer has sent an invalid message.
+    UnknownValidatorIndex(usize),
+    /// The public key of the validator has not been seen locally.
+    ///
+    /// ## Peer scoring
+    ///
+    /// It's unclear if this sync committee message is valid, however we have already observed an aggregate
+    /// sync committee message from this validator for this epoch and should not observe another.
+    UnknownValidatorPubkey(PublicKeyBytes),
+    /// A signature on the sync committee message is invalid.
+    ///
+    /// ## Peer scoring
+    ///
+    /// The peer has sent an invalid message.
+    InvalidSignature,
+    /// We have already observed a signature for the `validator_index` and refuse to process
+    /// another.
+    ///
+    /// ## Peer scoring
+    ///
+    /// It's unclear if this sync message is valid, however we have already observed a
+    /// signature from this validator for this slot and should not observe
+    /// another.
+    PriorSyncCommitteeMessageKnown { validator_index: u64, slot: Slot },
+    /// The sync committee message was received on an invalid sync committee message subnet.
+    ///
+    /// ## Peer scoring
+    ///
+    /// The peer has sent an invalid message.
+    InvalidSubnetId {
+        received: SyncSubnetId,
+        expected: Vec<SyncSubnetId>,
+    },
+    /// The sync message failed the `state_processing` verification stage.
+    ///
+    /// ## Peer scoring
+    ///
+    /// The peer has sent an invalid message.
+    Invalid(SyncCommitteeMessageValidationError),
+    /// There was an error whilst processing the sync contribution. It is not known if it is valid or invalid.
+    ///
+    /// ## Peer scoring
+    ///
+    /// We were unable to process this sync committee message due to an internal error. It's unclear if the
+    /// sync committee message is valid.
+    BeaconChainError(BeaconChainError),
+    /// There was an error whilst processing the sync contribution. It is not known if it is valid or invalid.
+    ///
+    /// ## Peer scoring
+    ///
+    /// We were unable to process this sync committee message due to an internal error. It's unclear if the
+    /// sync committee message is valid.
+    BeaconStateError(BeaconStateError),
+    /// There was an error whilst processing the sync contribution. It is not known if it is valid or invalid.
+    ///
+    /// ## Peer scoring
+    ///
+    /// We were unable to process this sync committee message due to an internal error. It's unclear if the
+    /// sync committee message is valid.
+    InvalidSubcommittee {
+        subcommittee_index: u64,
+        subcommittee_size: u64,
+    },
+    /// There was an error whilst processing the sync contribution. It is not known if it is valid or invalid.
+    ///
+    /// ## Peer scoring
+    ///
+    /// We were unable to process this sync committee message due to an internal error. It's unclear if the
+    /// sync committee message is valid.
+    ArithError(ArithError),
+    /// There was an error whilst processing the sync contribution. It is not known if it is valid or invalid.
+    ///
+    /// ## Peer scoring
+    ///
+    /// We were unable to process this sync committee message due to an internal error. It's unclear if the
+    /// sync committee message is valid.
+    ContributionError(ContributionError),
+    /// There was an error whilst processing the sync contribution. It is not known if it is valid or invalid.
+    ///
+    /// ## Peer scoring
+    ///
+    /// We were unable to process this sync committee message due to an internal error. It's unclear if the
+    /// sync committee message is valid.
+    SyncCommitteeError(SyncCommitteeError),
+}
+
+impl From<BeaconChainError> for Error {
+    fn from(e: BeaconChainError) -> Self {
+        Error::BeaconChainError(e)
+    }
+}
+
+impl From<BeaconStateError> for Error {
+    fn from(e: BeaconStateError) -> Self {
+        Error::BeaconStateError(e)
+    }
+}
+
+impl From<SyncCommitteeError> for Error {
+    fn from(e: SyncCommitteeError) -> Self {
+        Error::SyncCommitteeError(e)
+    }
+}
+
+impl From<ArithError> for Error {
+    fn from(e: ArithError) -> Self {
+        Error::ArithError(e)
+    }
+}
+
+impl From<ContributionError> for Error {
+    fn from(e: ContributionError) -> Self {
+        Error::ContributionError(e)
+    }
+}
+
+/// Wraps a `SignedContributionAndProof` that has been verified for propagation on the gossip network.\
+#[derive(Derivative)]
+#[derivative(Clone(bound = "T: BeaconChainTypes"))]
+pub struct VerifiedSyncContribution<T: BeaconChainTypes> {
+    signed_aggregate: SignedContributionAndProof<T::EthSpec>,
+}
+
+/// Wraps a `SyncCommitteeMessage` that has been verified for propagation on the gossip network.
+#[derive(Clone)]
+pub struct VerifiedSyncCommitteeMessage {
+    sync_message: SyncCommitteeMessage,
+    subnet_positions: HashMap<SyncSubnetId, Vec<usize>>,
+}
+
+impl<T: BeaconChainTypes> VerifiedSyncContribution<T> {
+    /// Returns `Ok(Self)` if the `signed_aggregate` is valid to be (re)published on the gossip
+    /// network.
+    pub fn verify(
+        signed_aggregate: SignedContributionAndProof<T::EthSpec>,
+        chain: &BeaconChain<T>,
+    ) -> Result<Self, Error> {
+        let aggregator_index = signed_aggregate.message.aggregator_index;
+        let contribution = &signed_aggregate.message.contribution;
+        let subcommittee_index = contribution.subcommittee_index as usize;
+
+        // Ensure sync committee contribution is within the MAXIMUM_GOSSIP_CLOCK_DISPARITY allowance.
+        verify_propagation_slot_range(chain, contribution)?;
+
+        // Validate subcommittee index.
+        if contribution.subcommittee_index >= SYNC_COMMITTEE_SUBNET_COUNT {
+            return Err(Error::InvalidSubcommittee {
+                subcommittee_index: contribution.subcommittee_index,
+                subcommittee_size: SYNC_COMMITTEE_SUBNET_COUNT,
+            });
+        }
+
+        // Ensure that the sync committee message has participants.
+        if contribution.aggregation_bits.is_zero() {
+            return Err(Error::EmptyAggregationBitfield);
+        }
+
+        // Ensure the aggregator's pubkey is in the declared subcommittee of the current sync committee
+        let pubkey_bytes = chain
+            .validator_pubkey_bytes(aggregator_index as usize)?
+            .ok_or(Error::UnknownValidatorIndex(aggregator_index as usize))?;
+        let sync_subcommittee_pubkeys = chain
+            .sync_committee_at_next_slot(contribution.get_slot())?
+            .get_subcommittee_pubkeys(subcommittee_index)?;
+
+        if !sync_subcommittee_pubkeys.contains(&pubkey_bytes) {
+            return Err(Error::AggregatorNotInCommittee { aggregator_index });
+        };
+
+        // Ensure the valid sync contribution has not already been seen locally.
+        let contribution_root = contribution.tree_hash_root();
+        if chain
+            .observed_sync_contributions
+            .write()
+            .is_known(contribution, contribution_root)
+            .map_err(|e| Error::BeaconChainError(e.into()))?
+        {
+            return Err(Error::SyncContributionAlreadyKnown(contribution_root));
+        }
+
+        // Ensure there has been no other observed aggregate for the given `aggregator_index`.
+        //
+        // Note: do not observe yet, only observe once the sync contribution has been verified.
+        let observed_key =
+            SlotSubcommitteeIndex::new(contribution.slot, contribution.subcommittee_index);
+        match chain
+            .observed_sync_aggregators
+            .read()
+            .validator_has_been_observed(observed_key, aggregator_index as usize)
+        {
+            Ok(true) => Err(Error::AggregatorAlreadyKnown(aggregator_index)),
+            Ok(false) => Ok(()),
+            Err(e) => Err(BeaconChainError::from(e).into()),
+        }?;
+
+        // Note: this clones the signature which is known to be a relatively slow operation.
+        //
+        // Future optimizations should remove this clone.
+        let selection_proof =
+            SyncSelectionProof::from(signed_aggregate.message.selection_proof.clone());
+
+        if !selection_proof
+            .is_aggregator::<T::EthSpec>()
+            .map_err(|e| Error::BeaconChainError(e.into()))?
+        {
+            return Err(Error::InvalidSelectionProof { aggregator_index });
+        }
+
+        // Gather all validator pubkeys that signed this contribution.
+        let participant_pubkeys = sync_subcommittee_pubkeys
+            .into_iter()
+            .zip(contribution.aggregation_bits.iter())
+            .filter_map(|(pubkey, bit)| bit.then(|| pubkey))
+            .collect::<Vec<_>>();
+
+        // Ensure that all signatures are valid.
+        if !verify_signed_aggregate_signatures(
+            chain,
+            &signed_aggregate,
+            participant_pubkeys.as_slice(),
+        )? {
+            return Err(Error::InvalidSignature);
+        }
+
+        let contribution = &signed_aggregate.message.contribution;
+        let aggregator_index = signed_aggregate.message.aggregator_index;
+
+        // Observe the valid sync contribution so we do not re-process it.
+        //
+        // It's important to double check that the contribution is not already known, otherwise two
+        // contribution processed at the same time could be published.
+        if let ObserveOutcome::AlreadyKnown = chain
+            .observed_sync_contributions
+            .write()
+            .observe_item(contribution, Some(contribution_root))
+            .map_err(|e| Error::BeaconChainError(e.into()))?
+        {
+            return Err(Error::SyncContributionAlreadyKnown(contribution_root));
+        }
+
+        // Observe the aggregator so we don't process another aggregate from them.
+        //
+        // It's important to double check that the sync committee message is not already known, otherwise two
+        // sync committee messages processed at the same time could be published.
+        if chain
+            .observed_sync_aggregators
+            .write()
+            .observe_validator(observed_key, aggregator_index as usize)
+            .map_err(BeaconChainError::from)?
+        {
+            return Err(Error::PriorSyncCommitteeMessageKnown {
+                validator_index: aggregator_index,
+                slot: contribution.slot,
+            });
+        }
+        Ok(VerifiedSyncContribution { signed_aggregate })
+    }
+
+    /// A helper function to add this aggregate to `beacon_chain.op_pool`.
+    pub fn add_to_pool(self, chain: &BeaconChain<T>) -> Result<(), Error> {
+        chain.add_contribution_to_block_inclusion_pool(self)
+    }
+
+    /// Returns the underlying `contribution` for the `signed_aggregate`.
+    pub fn contribution(self) -> SyncCommitteeContribution<T::EthSpec> {
+        self.signed_aggregate.message.contribution
+    }
+
+    /// Returns the underlying `signed_aggregate`.
+    pub fn aggregate(&self) -> &SignedContributionAndProof<T::EthSpec> {
+        &self.signed_aggregate
+    }
+}
+
+impl VerifiedSyncCommitteeMessage {
+    /// Returns `Ok(Self)` if the `sync_message` is valid to be (re)published on the gossip
+    /// network.
+    ///
+    /// `subnet_id` is the subnet from which we received this sync message. This function will
+    /// verify that it was received on the correct subnet.
+    pub fn verify<T: BeaconChainTypes>(
+        sync_message: SyncCommitteeMessage,
+        subnet_id: SyncSubnetId,
+        chain: &BeaconChain<T>,
+    ) -> Result<Self, Error> {
+        // Ensure sync committee message is for the current slot (within a
+        // MAXIMUM_GOSSIP_CLOCK_DISPARITY allowance).
+        //
+        // We do not queue future sync committee messages for later processing.
+        verify_propagation_slot_range(chain, &sync_message)?;
+
+        // Ensure the `subnet_id` is valid for the given validator.
+        let pubkey = chain
+            .validator_pubkey_bytes(sync_message.validator_index as usize)?
+            .ok_or(Error::UnknownValidatorIndex(
+                sync_message.validator_index as usize,
+            ))?;
+
+        let sync_committee = chain.sync_committee_at_next_slot(sync_message.get_slot())?;
+        let subnet_positions = sync_committee.subcommittee_positions_for_public_key(&pubkey)?;
+
+        if !subnet_positions.contains_key(&subnet_id) {
+            return Err(Error::InvalidSubnetId {
+                received: subnet_id,
+                expected: subnet_positions.keys().cloned().collect::<Vec<_>>(),
+            });
+        }
+
+        // The sync committee message is the first valid message received for the participating validator
+        // for the slot, sync_message.slot.
+        let validator_index = sync_message.validator_index;
+        if chain
+            .observed_sync_contributors
+            .read()
+            .validator_has_been_observed(
+                SlotSubcommitteeIndex::new(sync_message.slot, subnet_id.into()),
+                validator_index as usize,
+            )
+            .map_err(BeaconChainError::from)?
+        {
+            return Err(Error::PriorSyncCommitteeMessageKnown {
+                validator_index,
+                slot: sync_message.slot,
+            });
+        }
+
+        // The aggregate signature of the sync committee message is valid.
+        verify_sync_committee_message(chain, &sync_message, &pubkey)?;
+
+        // Now that the sync committee message has been fully verified, store that we have received a valid
+        // sync committee message from this validator.
+        //
+        // It's important to double check that the sync committee message still hasn't been observed, since
+        // there can be a race-condition if we receive two sync committee messages at the same time and
+        // process them in different threads.
+        if chain
+            .observed_sync_contributors
+            .write()
+            .observe_validator(
+                SlotSubcommitteeIndex::new(sync_message.slot, subnet_id.into()),
+                validator_index as usize,
+            )
+            .map_err(BeaconChainError::from)?
+        {
+            return Err(Error::PriorSyncCommitteeMessageKnown {
+                validator_index,
+                slot: sync_message.slot,
+            });
+        }
+
+        Ok(Self {
+            sync_message,
+            subnet_positions,
+        })
+    }
+
+    /// A helper function to add this sync committee message to `beacon_chain.naive_sync_aggregation_pool`.
+    pub fn add_to_pool<T: BeaconChainTypes>(self, chain: &BeaconChain<T>) -> Result<Self, Error> {
+        chain.add_to_naive_sync_aggregation_pool(self)
+    }
+
+    /// Returns the subcommittee positions for the sync message, keyed on the `SyncSubnetId` for
+    /// the subnets the signature should be sent on.
+    pub fn subnet_positions(&self) -> &HashMap<SyncSubnetId, Vec<usize>> {
+        &self.subnet_positions
+    }
+
+    /// Returns the wrapped `SyncCommitteeMessage`.
+    pub fn sync_message(&self) -> &SyncCommitteeMessage {
+        &self.sync_message
+    }
+}
+
+/// Verify that the `sync_contribution` is within the acceptable gossip propagation range, with reference
+/// to the current slot of the `chain`.
+///
+/// Accounts for `MAXIMUM_GOSSIP_CLOCK_DISPARITY`.
+pub fn verify_propagation_slot_range<T: BeaconChainTypes, U: SlotData>(
+    chain: &BeaconChain<T>,
+    sync_contribution: &U,
+) -> Result<(), Error> {
+    let message_slot = sync_contribution.get_slot();
+
+    let latest_permissible_slot = chain
+        .slot_clock
+        .now_with_future_tolerance(MAXIMUM_GOSSIP_CLOCK_DISPARITY)
+        .ok_or(BeaconChainError::UnableToReadSlot)?;
+    if message_slot > latest_permissible_slot {
+        return Err(Error::FutureSlot {
+            message_slot,
+            latest_permissible_slot,
+        });
+    }
+
+    let earliest_permissible_slot = chain
+        .slot_clock
+        .now_with_past_tolerance(MAXIMUM_GOSSIP_CLOCK_DISPARITY)
+        .ok_or(BeaconChainError::UnableToReadSlot)?;
+
+    if message_slot < earliest_permissible_slot {
+        return Err(Error::PastSlot {
+            message_slot,
+            earliest_permissible_slot,
+        });
+    }
+
+    Ok(())
+}
+
+/// Verifies all the signatures in a `SignedContributionAndProof` using BLS batch verification. This
+/// includes three signatures:
+///
+/// - `signed_aggregate.signature`
+/// - `signed_aggregate.message.selection_proof`
+/// - `signed_aggregate.message.aggregate.signature`
+///
+/// # Returns
+///
+/// - `Ok(true)`: if all signatures are valid.
+/// - `Ok(false)`: if one or more signatures are invalid.
+/// - `Err(e)`: if there was an error preventing signature verification.
+pub fn verify_signed_aggregate_signatures<T: BeaconChainTypes>(
+    chain: &BeaconChain<T>,
+    signed_aggregate: &SignedContributionAndProof<T::EthSpec>,
+    participant_pubkeys: &[PublicKeyBytes],
+) -> Result<bool, Error> {
+    let pubkey_cache = chain
+        .validator_pubkey_cache
+        .try_read_for(VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT)
+        .ok_or(BeaconChainError::ValidatorPubkeyCacheLockTimeout)?;
+
+    let aggregator_index = signed_aggregate.message.aggregator_index;
+    if aggregator_index >= pubkey_cache.len() as u64 {
+        return Err(Error::AggregatorPubkeyUnknown(aggregator_index));
+    }
+
+    let next_slot_epoch =
+        (signed_aggregate.message.contribution.slot + 1).epoch(T::EthSpec::slots_per_epoch());
+    let fork = chain.spec.fork_at_epoch(next_slot_epoch);
+
+    let signature_sets = vec![
+        signed_sync_aggregate_selection_proof_signature_set(
+            |validator_index| pubkey_cache.get(validator_index).map(Cow::Borrowed),
+            signed_aggregate,
+            &fork,
+            chain.genesis_validators_root,
+            &chain.spec,
+        )
+        .map_err(BeaconChainError::SignatureSetError)?,
+        signed_sync_aggregate_signature_set(
+            |validator_index| pubkey_cache.get(validator_index).map(Cow::Borrowed),
+            signed_aggregate,
+            &fork,
+            chain.genesis_validators_root,
+            &chain.spec,
+        )
+        .map_err(BeaconChainError::SignatureSetError)?,
+        sync_committee_contribution_signature_set_from_pubkeys::<T::EthSpec, _>(
+            |validator_index| {
+                pubkey_cache
+                    .get_pubkey_from_pubkey_bytes(validator_index)
+                    .map(Cow::Borrowed)
+            },
+            participant_pubkeys,
+            &signed_aggregate.message.contribution.signature,
+            signed_aggregate
+                .message
+                .contribution
+                .slot
+                .epoch(T::EthSpec::slots_per_epoch()),
+            signed_aggregate.message.contribution.beacon_block_root,
+            &fork,
+            chain.genesis_validators_root,
+            &chain.spec,
+        )
+        .map_err(BeaconChainError::SignatureSetError)?,
+    ];
+
+    Ok(verify_signature_sets(signature_sets.iter()))
+}
+
+/// Verifies that the signature of the `sync_message` is valid.
+pub fn verify_sync_committee_message<T: BeaconChainTypes>(
+    chain: &BeaconChain<T>,
+    sync_message: &SyncCommitteeMessage,
+    pubkey_bytes: &PublicKeyBytes,
+) -> Result<(), Error> {
+    let signature_setup_timer =
+        metrics::start_timer(&metrics::SYNC_MESSAGE_PROCESSING_SIGNATURE_SETUP_TIMES);
+
+    let pubkey_cache = chain
+        .validator_pubkey_cache
+        .try_read_for(VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT)
+        .ok_or(BeaconChainError::ValidatorPubkeyCacheLockTimeout)?;
+
+    let pubkey = pubkey_cache
+        .get_pubkey_from_pubkey_bytes(pubkey_bytes)
+        .map(Cow::Borrowed)
+        .ok_or_else(|| Error::UnknownValidatorPubkey(*pubkey_bytes))?;
+
+    let next_slot_epoch = (sync_message.get_slot() + 1).epoch(T::EthSpec::slots_per_epoch());
+    let fork = chain.spec.fork_at_epoch(next_slot_epoch);
+
+    let agg_sig = AggregateSignature::from(&sync_message.signature);
+    let signature_set = sync_committee_message_set_from_pubkeys::<T::EthSpec>(
+        pubkey,
+        &agg_sig,
+        sync_message.slot.epoch(T::EthSpec::slots_per_epoch()),
+        sync_message.beacon_block_root,
+        &fork,
+        chain.genesis_validators_root,
+        &chain.spec,
+    )
+    .map_err(BeaconChainError::SignatureSetError)?;
+
+    metrics::stop_timer(signature_setup_timer);
+
+    let _signature_verification_timer =
+        metrics::start_timer(&metrics::SYNC_MESSAGE_PROCESSING_SIGNATURE_TIMES);
+
+    if signature_set.verify() {
+        Ok(())
+    } else {
+        Err(Error::InvalidSignature)
+    }
+}

beacon_node/beacon_chain/src/timeout_rw_lock.rs

@@ -0,0 +1,20 @@
+use parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard};
+use std::time::Duration;
+
+/// A simple wrapper around `parking_lot::RwLock` that only permits read/write access with a
+/// time-out (i.e., no indefinitely-blocking operations).
+pub struct TimeoutRwLock<T>(RwLock<T>);
+
+impl<T> TimeoutRwLock<T> {
+    pub fn new(inner: T) -> Self {
+        Self(RwLock::new(inner))
+    }
+
+    pub fn try_read_for(&self, timeout: Duration) -> Option<RwLockReadGuard<T>> {
+        self.0.try_read_for(timeout)
+    }
+
+    pub fn try_write_for(&self, timeout: Duration) -> Option<RwLockWriteGuard<T>> {
+        self.0.try_write_for(timeout)
+    }
+}

beacon_node/beacon_chain/src/validator_pubkey_cache.rs

@@ -0,0 +1,472 @@
+use crate::errors::BeaconChainError;
+use crate::{BeaconChainTypes, BeaconStore};
+use ssz::{Decode, DecodeError, Encode};
+use std::collections::HashMap;
+use std::convert::TryInto;
+use std::fs::{File, OpenOptions};
+use std::io::{self, Read, Write};
+use std::path::Path;
+use store::{DBColumn, Error as StoreError, StoreItem};
+use types::{BeaconState, Hash256, PublicKey, PublicKeyBytes};
+
+/// Provides a mapping of `validator_index -> validator_publickey`.
+///
+/// This cache exists for two reasons:
+///
+/// 1. To avoid reading a `BeaconState` from disk each time we need a public key.
+/// 2. To reduce the amount of public key _decompression_ required. A `BeaconState` stores public
+///    keys in compressed form and they are needed in decompressed form for signature verification.
+///    Decompression is expensive when many keys are involved.
+///
+/// The cache has a `backing` that it uses to maintain a persistent, on-disk
+/// copy of itself. This allows it to be restored between process invocations.
+pub struct ValidatorPubkeyCache<T: BeaconChainTypes> {
+    pubkeys: Vec<PublicKey>,
+    indices: HashMap<PublicKeyBytes, usize>,
+    pubkey_bytes: Vec<PublicKeyBytes>,
+    backing: PubkeyCacheBacking<T>,
+}
+
+/// Abstraction over on-disk backing.
+///
+/// `File` backing is legacy, `Database` is current.
+enum PubkeyCacheBacking<T: BeaconChainTypes> {
+    File(ValidatorPubkeyCacheFile),
+    Database(BeaconStore<T>),
+}
+
+impl<T: BeaconChainTypes> ValidatorPubkeyCache<T> {
+    /// Create a new public key cache using the keys in `state.validators`.
+    ///
+    /// Also creates a new persistence file, returning an error if there is already a file at
+    /// `persistence_path`.
+    pub fn new(
+        state: &BeaconState<T::EthSpec>,
+        store: BeaconStore<T>,
+    ) -> Result<Self, BeaconChainError> {
+        let mut cache = Self {
+            pubkeys: vec![],
+            indices: HashMap::new(),
+            pubkey_bytes: vec![],
+            backing: PubkeyCacheBacking::Database(store),
+        };
+
+        cache.import_new_pubkeys(state)?;
+
+        Ok(cache)
+    }
+
+    /// Load the pubkey cache from the given on-disk database.
+    pub fn load_from_store(store: BeaconStore<T>) -> Result<Self, BeaconChainError> {
+        let mut pubkeys = vec![];
+        let mut indices = HashMap::new();
+        let mut pubkey_bytes = vec![];
+
+        for validator_index in 0.. {
+            if let Some(DatabasePubkey(pubkey)) =
+                store.get_item(&DatabasePubkey::key_for_index(validator_index))?
+            {
+                pubkeys.push((&pubkey).try_into().map_err(Error::PubkeyDecode)?);
+                pubkey_bytes.push(pubkey);
+                indices.insert(pubkey, validator_index);
+            } else {
+                break;
+            }
+        }
+
+        Ok(ValidatorPubkeyCache {
+            pubkeys,
+            indices,
+            pubkey_bytes,
+            backing: PubkeyCacheBacking::Database(store),
+        })
+    }
+
+    /// DEPRECATED: used only for migration
+    pub fn load_from_file<P: AsRef<Path>>(path: P) -> Result<Self, BeaconChainError> {
+        ValidatorPubkeyCacheFile::open(&path)
+            .and_then(ValidatorPubkeyCacheFile::into_cache)
+            .map_err(Into::into)
+    }
+
+    /// Convert a cache using `File` backing to one using `Database` backing.
+    ///
+    /// This will write all of the keys from `existing_cache` to `store`.
+    pub fn convert(existing_cache: Self, store: BeaconStore<T>) -> Result<Self, BeaconChainError> {
+        let mut result = ValidatorPubkeyCache {
+            pubkeys: Vec::with_capacity(existing_cache.pubkeys.len()),
+            indices: HashMap::with_capacity(existing_cache.indices.len()),
+            pubkey_bytes: Vec::with_capacity(existing_cache.indices.len()),
+            backing: PubkeyCacheBacking::Database(store),
+        };
+        result.import(existing_cache.pubkeys.iter().map(PublicKeyBytes::from))?;
+        Ok(result)
+    }
+
+    /// Scan the given `state` and add any new validator public keys.
+    ///
+    /// Does not delete any keys from `self` if they don't appear in `state`.
+    pub fn import_new_pubkeys(
+        &mut self,
+        state: &BeaconState<T::EthSpec>,
+    ) -> Result<(), BeaconChainError> {
+        if state.validators().len() > self.pubkeys.len() {
+            self.import(
+                state.validators()[self.pubkeys.len()..]
+                    .iter()
+                    .map(|v| v.pubkey),
+            )
+        } else {
+            Ok(())
+        }
+    }
+
+    /// Adds zero or more validators to `self`.
+    fn import<I>(&mut self, validator_keys: I) -> Result<(), BeaconChainError>
+    where
+        I: Iterator<Item = PublicKeyBytes> + ExactSizeIterator,
+    {
+        self.pubkey_bytes.reserve(validator_keys.len());
+        self.pubkeys.reserve(validator_keys.len());
+        self.indices.reserve(validator_keys.len());
+
+        for pubkey in validator_keys {
+            let i = self.pubkeys.len();
+
+            if self.indices.contains_key(&pubkey) {
+                return Err(BeaconChainError::DuplicateValidatorPublicKey);
+            }
+
+            // The item is written to disk _before_ it is written into
+            // the local struct.
+            //
+            // This means that a pubkey cache read from disk will always be equivalent to or
+            // _later than_ the cache that was running in the previous instance of Lighthouse.
+            //
+            // The motivation behind this ordering is that we do not want to have states that
+            // reference a pubkey that is not in our cache. However, it's fine to have pubkeys
+            // that are never referenced in a state.
+            match &mut self.backing {
+                PubkeyCacheBacking::File(persistence_file) => {
+                    persistence_file.append(i, &pubkey)?;
+                }
+                PubkeyCacheBacking::Database(store) => {
+                    store.put_item(&DatabasePubkey::key_for_index(i), &DatabasePubkey(pubkey))?;
+                }
+            }
+
+            self.pubkeys.push(
+                (&pubkey)
+                    .try_into()
+                    .map_err(BeaconChainError::InvalidValidatorPubkeyBytes)?,
+            );
+            self.pubkey_bytes.push(pubkey);
+
+            self.indices.insert(pubkey, i);
+        }
+
+        Ok(())
+    }
+
+    /// Get the public key for a validator with index `i`.
+    pub fn get(&self, i: usize) -> Option<&PublicKey> {
+        self.pubkeys.get(i)
+    }
+
+    /// Get the `PublicKey` for a validator with `PublicKeyBytes`.
+    pub fn get_pubkey_from_pubkey_bytes(&self, pubkey: &PublicKeyBytes) -> Option<&PublicKey> {
+        self.get_index(pubkey)
+            .map(|index| self.get(index))
+            .flatten()
+    }
+
+    /// Get the public key (in bytes form) for a validator with index `i`.
+    pub fn get_pubkey_bytes(&self, i: usize) -> Option<&PublicKeyBytes> {
+        self.pubkey_bytes.get(i)
+    }
+
+    /// Get the index of a validator with `pubkey`.
+    pub fn get_index(&self, pubkey: &PublicKeyBytes) -> Option<usize> {
+        self.indices.get(pubkey).copied()
+    }
+
+    /// Returns the number of validators in the cache.
+    pub fn len(&self) -> usize {
+        self.indices.len()
+    }
+}
+
+/// Wrapper for a public key stored in the database.
+///
+/// Keyed by the validator index as `Hash256::from_low_u64_be(index)`.
+struct DatabasePubkey(PublicKeyBytes);
+
+impl StoreItem for DatabasePubkey {
+    fn db_column() -> DBColumn {
+        DBColumn::PubkeyCache
+    }
+
+    fn as_store_bytes(&self) -> Vec<u8> {
+        self.0.as_ssz_bytes()
+    }
+
+    fn from_store_bytes(bytes: &[u8]) -> Result<Self, StoreError> {
+        Ok(Self(PublicKeyBytes::from_ssz_bytes(bytes)?))
+    }
+}
+
+impl DatabasePubkey {
+    fn key_for_index(index: usize) -> Hash256 {
+        Hash256::from_low_u64_be(index as u64)
+    }
+}
+
+/// Allows for maintaining an on-disk copy of the `ValidatorPubkeyCache`. The file is raw SSZ bytes
+/// (not ASCII encoded).
+///
+/// ## Writes
+///
+/// Each entry is simply appended to the file.
+///
+/// ## Reads
+///
+/// The whole file is parsed as an SSZ "variable list" of objects.
+///
+/// This parsing method is possible because the items in the list are fixed-length SSZ objects.
+struct ValidatorPubkeyCacheFile(File);
+
+#[derive(Debug)]
+enum Error {
+    Io(io::Error),
+    Ssz(DecodeError),
+    PubkeyDecode(bls::Error),
+    /// The file read from disk does not have a contiguous list of validator public keys. The file
+    /// has become corrupted.
+    InconsistentIndex {
+        expected: Option<usize>,
+        found: usize,
+    },
+}
+
+impl From<Error> for BeaconChainError {
+    fn from(e: Error) -> BeaconChainError {
+        BeaconChainError::ValidatorPubkeyCacheFileError(format!("{:?}", e))
+    }
+}
+
+impl ValidatorPubkeyCacheFile {
+    /// Opens an existing file for reading and writing.
+    pub fn open<P: AsRef<Path>>(path: P) -> Result<Self, Error> {
+        OpenOptions::new()
+            .read(true)
+            .write(true)
+            .create(false)
+            .append(true)
+            .open(path)
+            .map(Self)
+            .map_err(Error::Io)
+    }
+
+    /// Append a public key to file.
+    ///
+    /// The provided `index` should each be one greater than the previous and start at 0.
+    /// Otherwise, the file will become corrupted and unable to be converted into a cache .
+    pub fn append(&mut self, index: usize, pubkey: &PublicKeyBytes) -> Result<(), Error> {
+        append_to_file(&mut self.0, index, pubkey)
+    }
+
+    /// Creates a `ValidatorPubkeyCache` by reading and parsing the underlying file.
+    pub fn into_cache<T: BeaconChainTypes>(mut self) -> Result<ValidatorPubkeyCache<T>, Error> {
+        let mut bytes = vec![];
+        self.0.read_to_end(&mut bytes).map_err(Error::Io)?;
+
+        let list: Vec<(usize, PublicKeyBytes)> = Vec::from_ssz_bytes(&bytes).map_err(Error::Ssz)?;
+
+        let mut last = None;
+        let mut pubkeys = Vec::with_capacity(list.len());
+        let mut indices = HashMap::with_capacity(list.len());
+        let mut pubkey_bytes = Vec::with_capacity(list.len());
+
+        for (index, pubkey) in list {
+            let expected = last.map(|n| n + 1);
+            if expected.map_or(true, |expected| index == expected) {
+                last = Some(index);
+                pubkeys.push((&pubkey).try_into().map_err(Error::PubkeyDecode)?);
+                pubkey_bytes.push(pubkey);
+                indices.insert(pubkey, index);
+            } else {
+                return Err(Error::InconsistentIndex {
+                    expected,
+                    found: index,
+                });
+            }
+        }
+
+        Ok(ValidatorPubkeyCache {
+            pubkeys,
+            indices,
+            pubkey_bytes,
+            backing: PubkeyCacheBacking::File(self),
+        })
+    }
+}
+
+fn append_to_file(file: &mut File, index: usize, pubkey: &PublicKeyBytes) -> Result<(), Error> {
+    let mut line = Vec::with_capacity(index.ssz_bytes_len() + pubkey.ssz_bytes_len());
+
+    index.ssz_append(&mut line);
+    pubkey.ssz_append(&mut line);
+
+    file.write_all(&line).map_err(Error::Io)
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::test_utils::{test_logger, BeaconChainHarness, EphemeralHarnessType};
+    use std::sync::Arc;
+    use store::{HotColdDB, StoreConfig};
+    use tempfile::tempdir;
+    use types::{
+        test_utils::generate_deterministic_keypair, BeaconState, EthSpec, Keypair, MainnetEthSpec,
+    };
+
+    type E = MainnetEthSpec;
+    type T = EphemeralHarnessType<E>;
+
+    fn get_state(validator_count: usize) -> (BeaconState<E>, Vec<Keypair>) {
+        let harness = BeaconChainHarness::new_with_store_config(
+            MainnetEthSpec,
+            None,
+            types::test_utils::generate_deterministic_keypairs(validator_count),
+            StoreConfig::default(),
+        );
+
+        harness.advance_slot();
+
+        (harness.get_current_state(), harness.validator_keypairs)
+    }
+
+    fn get_store() -> BeaconStore<T> {
+        Arc::new(
+            HotColdDB::open_ephemeral(<_>::default(), E::default_spec(), test_logger()).unwrap(),
+        )
+    }
+
+    #[allow(clippy::needless_range_loop)]
+    fn check_cache_get(cache: &ValidatorPubkeyCache<T>, keypairs: &[Keypair]) {
+        let validator_count = keypairs.len();
+
+        for i in 0..validator_count + 1 {
+            if i < validator_count {
+                let pubkey = cache.get(i).expect("pubkey should be present");
+                assert_eq!(pubkey, &keypairs[i].pk, "pubkey should match cache");
+
+                let pubkey_bytes: PublicKeyBytes = pubkey.clone().into();
+
+                assert_eq!(
+                    i,
+                    cache
+                        .get_index(&pubkey_bytes)
+                        .expect("should resolve index"),
+                    "index should match cache"
+                );
+            } else {
+                assert_eq!(
+                    cache.get(i),
+                    None,
+                    "should not get pubkey for out of bounds index",
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn basic_operation() {
+        let (state, keypairs) = get_state(8);
+
+        let store = get_store();
+
+        let mut cache = ValidatorPubkeyCache::new(&state, store).expect("should create cache");
+
+        check_cache_get(&cache, &keypairs[..]);
+
+        // Try adding a state with the same number of keypairs.
+        let (state, keypairs) = get_state(8);
+        cache
+            .import_new_pubkeys(&state)
+            .expect("should import pubkeys");
+        check_cache_get(&cache, &keypairs[..]);
+
+        // Try adding a state with less keypairs.
+        let (state, _) = get_state(1);
+        cache
+            .import_new_pubkeys(&state)
+            .expect("should import pubkeys");
+        check_cache_get(&cache, &keypairs[..]);
+
+        // Try adding a state with more keypairs.
+        let (state, keypairs) = get_state(12);
+        cache
+            .import_new_pubkeys(&state)
+            .expect("should import pubkeys");
+        check_cache_get(&cache, &keypairs[..]);
+    }
+
+    #[test]
+    fn persistence() {
+        let (state, keypairs) = get_state(8);
+
+        let store = get_store();
+
+        // Create a new cache.
+        let cache = ValidatorPubkeyCache::new(&state, store.clone()).expect("should create cache");
+        check_cache_get(&cache, &keypairs[..]);
+        drop(cache);
+
+        // Re-init the cache from the file.
+        let mut cache =
+            ValidatorPubkeyCache::load_from_store(store.clone()).expect("should open cache");
+        check_cache_get(&cache, &keypairs[..]);
+
+        // Add some more keypairs.
+        let (state, keypairs) = get_state(12);
+        cache
+            .import_new_pubkeys(&state)
+            .expect("should import pubkeys");
+        check_cache_get(&cache, &keypairs[..]);
+        drop(cache);
+
+        // Re-init the cache from the file.
+        let cache = ValidatorPubkeyCache::load_from_store(store).expect("should open cache");
+        check_cache_get(&cache, &keypairs[..]);
+    }
+
+    #[test]
+    fn invalid_persisted_file() {
+        let dir = tempdir().expect("should create tempdir");
+        let path = dir.path().join("cache.ssz");
+        let pubkey = generate_deterministic_keypair(0).pk.into();
+
+        let mut file = File::create(&path).expect("should create file");
+        append_to_file(&mut file, 0, &pubkey).expect("should write to file");
+        drop(file);
+
+        let cache = ValidatorPubkeyCache::<T>::load_from_file(&path).expect("should open cache");
+        drop(cache);
+
+        let mut file = OpenOptions::new()
+            .write(true)
+            .append(true)
+            .open(&path)
+            .expect("should open file");
+
+        append_to_file(&mut file, 42, &pubkey).expect("should write bad data to file");
+        drop(file);
+
+        assert!(
+            ValidatorPubkeyCache::<T>::load_from_file(&path).is_err(),
+            "should not parse invalid file"
+        );
+    }
+}

beacon_node/beacon_chain/tests/attestation_production.rs

@@ -0,0 +1,131 @@
+#![cfg(not(debug_assertions))]
+
+#[macro_use]
+extern crate lazy_static;
+
+use beacon_chain::test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy};
+use beacon_chain::{StateSkipConfig, WhenSlotSkipped};
+use store::config::StoreConfig;
+use tree_hash::TreeHash;
+use types::{AggregateSignature, EthSpec, Keypair, MainnetEthSpec, RelativeEpoch, Slot};
+
+pub const VALIDATOR_COUNT: usize = 16;
+
+lazy_static! {
+    /// A cached set of keys.
+    static ref KEYPAIRS: Vec<Keypair> = types::test_utils::generate_deterministic_keypairs(VALIDATOR_COUNT);
+}
+
+/// This test builds a chain that is just long enough to finalize an epoch then it produces an
+/// attestation at each slot from genesis through to three epochs past the head.
+///
+/// It checks the produced attestation against some locally computed values.
+#[test]
+fn produces_attestations() {
+    let num_blocks_produced = MainnetEthSpec::slots_per_epoch() * 4;
+    let additional_slots_tested = MainnetEthSpec::slots_per_epoch() * 3;
+
+    let harness = BeaconChainHarness::new_with_store_config(
+        MainnetEthSpec,
+        None,
+        KEYPAIRS[..].to_vec(),
+        StoreConfig::default(),
+    );
+
+    let chain = &harness.chain;
+
+    // Test all valid committee indices for all slots in the chain.
+    // for slot in 0..=current_slot.as_u64() + MainnetEthSpec::slots_per_epoch() * 3 {
+    for slot in 0..=num_blocks_produced + additional_slots_tested {
+        if slot > 0 && slot <= num_blocks_produced {
+            harness.advance_slot();
+
+            harness.extend_chain(
+                1,
+                BlockStrategy::OnCanonicalHead,
+                AttestationStrategy::AllValidators,
+            );
+        }
+
+        let slot = Slot::from(slot);
+        let mut state = chain
+            .state_at_slot(slot, StateSkipConfig::WithStateRoots)
+            .expect("should get state");
+
+        let block_slot = if slot <= num_blocks_produced {
+            slot
+        } else {
+            Slot::from(num_blocks_produced)
+        };
+
+        let block = chain
+            .block_at_slot(block_slot, WhenSlotSkipped::Prev)
+            .expect("should get block")
+            .expect("block should not be skipped");
+        let block_root = block.message().tree_hash_root();
+
+        let epoch_boundary_slot = state
+            .current_epoch()
+            .start_slot(MainnetEthSpec::slots_per_epoch());
+        let target_root = if state.slot() == epoch_boundary_slot {
+            block_root
+        } else {
+            *state
+                .get_block_root(epoch_boundary_slot)
+                .expect("should get target block root")
+        };
+
+        state
+            .build_committee_cache(RelativeEpoch::Current, &harness.chain.spec)
+            .unwrap();
+        let committee_cache = state
+            .committee_cache(RelativeEpoch::Current)
+            .expect("should get committee_cache");
+
+        let committee_count = committee_cache.committees_per_slot();
+
+        for index in 0..committee_count {
+            let committee_len = committee_cache
+                .get_beacon_committee(slot, index)
+                .expect("should get committee for slot")
+                .committee
+                .len();
+
+            let attestation = chain
+                .produce_unaggregated_attestation(slot, index)
+                .expect("should produce attestation");
+
+            let data = &attestation.data;
+
+            assert_eq!(
+                attestation.aggregation_bits.len(),
+                committee_len,
+                "bad committee len"
+            );
+            assert!(
+                attestation.aggregation_bits.is_zero(),
+                "some committee bits are set"
+            );
+            assert_eq!(
+                attestation.signature,
+                AggregateSignature::empty(),
+                "bad signature"
+            );
+            assert_eq!(data.index, index, "bad index");
+            assert_eq!(data.slot, slot, "bad slot");
+            assert_eq!(data.beacon_block_root, block_root, "bad block root");
+            assert_eq!(
+                data.source,
+                state.current_justified_checkpoint(),
+                "bad source"
+            );
+            assert_eq!(
+                data.source,
+                state.current_justified_checkpoint(),
+                "bad source"
+            );
+            assert_eq!(data.target.epoch, state.current_epoch(), "bad target epoch");
+            assert_eq!(data.target.root, target_root, "bad target root");
+        }
+    }
+}

beacon_node/beacon_chain/tests/op_verification.rs

@@ -0,0 +1,234 @@
+//! Tests for gossip verification of voluntary exits, propser slashings and attester slashings.
+
+#![cfg(not(debug_assertions))]
+
+#[macro_use]
+extern crate lazy_static;
+
+use beacon_chain::observed_operations::ObservationOutcome;
+use beacon_chain::test_utils::{
+    test_spec, AttestationStrategy, BeaconChainHarness, BlockStrategy, DiskHarnessType,
+};
+use sloggers::{null::NullLoggerBuilder, Build};
+use std::sync::Arc;
+use store::{LevelDB, StoreConfig};
+use tempfile::{tempdir, TempDir};
+use types::*;
+
+pub const VALIDATOR_COUNT: usize = 24;
+
+lazy_static! {
+    /// A cached set of keys.
+    static ref KEYPAIRS: Vec<Keypair> =
+        types::test_utils::generate_deterministic_keypairs(VALIDATOR_COUNT);
+}
+
+type E = MinimalEthSpec;
+type TestHarness = BeaconChainHarness<DiskHarnessType<E>>;
+type HotColdDB = store::HotColdDB<E, LevelDB<E>, LevelDB<E>>;
+
+fn get_store(db_path: &TempDir) -> Arc<HotColdDB> {
+    let spec = test_spec::<E>();
+    let hot_path = db_path.path().join("hot_db");
+    let cold_path = db_path.path().join("cold_db");
+    let config = StoreConfig::default();
+    let log = NullLoggerBuilder.build().expect("logger should build");
+    HotColdDB::open(&hot_path, &cold_path, |_, _, _| Ok(()), config, spec, log)
+        .expect("disk store should initialize")
+}
+
+fn get_harness(store: Arc<HotColdDB>, validator_count: usize) -> TestHarness {
+    let harness = BeaconChainHarness::new_with_disk_store(
+        MinimalEthSpec,
+        None,
+        store,
+        KEYPAIRS[0..validator_count].to_vec(),
+    );
+    harness.advance_slot();
+    harness
+}
+
+#[test]
+fn voluntary_exit() {
+    let db_path = tempdir().unwrap();
+    let store = get_store(&db_path);
+    let harness = get_harness(store.clone(), VALIDATOR_COUNT);
+    let spec = &harness.chain.spec.clone();
+
+    harness.extend_chain(
+        (E::slots_per_epoch() * (spec.shard_committee_period + 1)) as usize,
+        BlockStrategy::OnCanonicalHead,
+        AttestationStrategy::AllValidators,
+    );
+
+    let validator_index1 = VALIDATOR_COUNT - 1;
+    let validator_index2 = VALIDATOR_COUNT - 2;
+
+    let exit1 = harness.make_voluntary_exit(
+        validator_index1 as u64,
+        Epoch::new(spec.shard_committee_period),
+    );
+
+    // First verification should show it to be fresh.
+    assert!(matches!(
+        harness
+            .chain
+            .verify_voluntary_exit_for_gossip(exit1.clone())
+            .unwrap(),
+        ObservationOutcome::New(_)
+    ));
+
+    // Second should not.
+    assert!(matches!(
+        harness
+            .chain
+            .verify_voluntary_exit_for_gossip(exit1.clone()),
+        Ok(ObservationOutcome::AlreadyKnown)
+    ));
+
+    // A different exit for the same validator should also be detected as a duplicate.
+    let exit2 = harness.make_voluntary_exit(
+        validator_index1 as u64,
+        Epoch::new(spec.shard_committee_period + 1),
+    );
+    assert!(matches!(
+        harness.chain.verify_voluntary_exit_for_gossip(exit2),
+        Ok(ObservationOutcome::AlreadyKnown)
+    ));
+
+    // Exit for a different validator should be fine.
+    let exit3 = harness.make_voluntary_exit(
+        validator_index2 as u64,
+        Epoch::new(spec.shard_committee_period),
+    );
+    assert!(matches!(
+        harness
+            .chain
+            .verify_voluntary_exit_for_gossip(exit3)
+            .unwrap(),
+        ObservationOutcome::New(_)
+    ));
+}
+
+#[test]
+fn proposer_slashing() {
+    let db_path = tempdir().unwrap();
+    let store = get_store(&db_path);
+    let harness = get_harness(store.clone(), VALIDATOR_COUNT);
+
+    let validator_index1 = VALIDATOR_COUNT - 1;
+    let validator_index2 = VALIDATOR_COUNT - 2;
+
+    let slashing1 = harness.make_proposer_slashing(validator_index1 as u64);
+
+    // First slashing for this proposer should be allowed.
+    assert!(matches!(
+        harness
+            .chain
+            .verify_proposer_slashing_for_gossip(slashing1.clone())
+            .unwrap(),
+        ObservationOutcome::New(_)
+    ));
+    // Duplicate slashing should be detected.
+    assert!(matches!(
+        harness
+            .chain
+            .verify_proposer_slashing_for_gossip(slashing1.clone())
+            .unwrap(),
+        ObservationOutcome::AlreadyKnown
+    ));
+
+    // Different slashing for the same index should be rejected
+    let slashing2 = ProposerSlashing {
+        signed_header_1: slashing1.signed_header_2,
+        signed_header_2: slashing1.signed_header_1,
+    };
+    assert!(matches!(
+        harness
+            .chain
+            .verify_proposer_slashing_for_gossip(slashing2)
+            .unwrap(),
+        ObservationOutcome::AlreadyKnown
+    ));
+
+    // Proposer slashing for a different index should be accepted
+    let slashing3 = harness.make_proposer_slashing(validator_index2 as u64);
+    assert!(matches!(
+        harness
+            .chain
+            .verify_proposer_slashing_for_gossip(slashing3)
+            .unwrap(),
+        ObservationOutcome::New(_)
+    ));
+}
+
+#[test]
+fn attester_slashing() {
+    let db_path = tempdir().unwrap();
+    let store = get_store(&db_path);
+    let harness = get_harness(store.clone(), VALIDATOR_COUNT);
+
+    // First third of the validators
+    let first_third = (0..VALIDATOR_COUNT as u64 / 3).collect::<Vec<_>>();
+    // First half of the validators
+    let first_half = (0..VALIDATOR_COUNT as u64 / 2).collect::<Vec<_>>();
+    // Last third of the validators
+    let last_third = (2 * VALIDATOR_COUNT as u64 / 3..VALIDATOR_COUNT as u64).collect::<Vec<_>>();
+    // Last half of the validators
+    let second_half = (VALIDATOR_COUNT as u64 / 2..VALIDATOR_COUNT as u64).collect::<Vec<_>>();
+
+    // Slashing for first third of validators should be accepted.
+    let slashing1 = harness.make_attester_slashing(first_third);
+    assert!(matches!(
+        harness
+            .chain
+            .verify_attester_slashing_for_gossip(slashing1.clone())
+            .unwrap(),
+        ObservationOutcome::New(_)
+    ));
+
+    // Overlapping slashing for first half of validators should also be accepted.
+    let slashing2 = harness.make_attester_slashing(first_half);
+    assert!(matches!(
+        harness
+            .chain
+            .verify_attester_slashing_for_gossip(slashing2.clone())
+            .unwrap(),
+        ObservationOutcome::New(_)
+    ));
+
+    // Repeating slashing1 or slashing2 should be rejected
+    assert!(matches!(
+        harness
+            .chain
+            .verify_attester_slashing_for_gossip(slashing1.clone())
+            .unwrap(),
+        ObservationOutcome::AlreadyKnown
+    ));
+    assert!(matches!(
+        harness
+            .chain
+            .verify_attester_slashing_for_gossip(slashing2.clone())
+            .unwrap(),
+        ObservationOutcome::AlreadyKnown
+    ));
+
+    // Slashing for last half of validators should be accepted (distinct from all existing)
+    let slashing3 = harness.make_attester_slashing(second_half);
+    assert!(matches!(
+        harness
+            .chain
+            .verify_attester_slashing_for_gossip(slashing3)
+            .unwrap(),
+        ObservationOutcome::New(_)
+    ));
+    // Slashing for last third (contained in last half) should be rejected.
+    let slashing4 = harness.make_attester_slashing(last_third);
+    assert!(matches!(
+        harness
+            .chain
+            .verify_attester_slashing_for_gossip(slashing4)
+            .unwrap(),
+        ObservationOutcome::AlreadyKnown
+    ));
+}

beacon_node/beacon_chain/tests/persistence_tests.rs

@@ -1,136 +0,0 @@
-#![cfg(not(debug_assertions))]
-
-#[macro_use]
-extern crate lazy_static;
-
-use beacon_chain::{
-    test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy},
-    BeaconChain, BeaconChainTypes,
-};
-use sloggers::{null::NullLoggerBuilder, Build};
-use std::sync::Arc;
-use store::DiskStore;
-use tempfile::{tempdir, TempDir};
-use types::{EthSpec, Keypair, MinimalEthSpec};
-
-type E = MinimalEthSpec;
-
-// Should ideally be divisible by 3.
-pub const VALIDATOR_COUNT: usize = 24;
-
-lazy_static! {
-    /// A cached set of keys.
-    static ref KEYPAIRS: Vec<Keypair> = types::test_utils::generate_deterministic_keypairs(VALIDATOR_COUNT);
-}
-
-fn get_store(db_path: &TempDir) -> Arc<DiskStore<E>> {
-    let spec = E::default_spec();
-    let hot_path = db_path.path().join("hot_db");
-    let cold_path = db_path.path().join("cold_db");
-    let slots_per_restore_point = MinimalEthSpec::slots_per_historical_root() as u64;
-    let log = NullLoggerBuilder.build().expect("logger should build");
-    Arc::new(
-        DiskStore::open(&hot_path, &cold_path, slots_per_restore_point, spec, log)
-            .expect("disk store should initialize"),
-    )
-}
-
-#[test]
-fn finalizes_after_resuming_from_db() {
-    let validator_count = 16;
-    let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 8;
-    let first_half = num_blocks_produced / 2;
-
-    let db_path = tempdir().unwrap();
-    let store = get_store(&db_path);
-
-    let harness = BeaconChainHarness::new_with_disk_store(
-        MinimalEthSpec,
-        store.clone(),
-        KEYPAIRS[0..validator_count].to_vec(),
-    );
-
-    harness.advance_slot();
-
-    harness.extend_chain(
-        first_half as usize,
-        BlockStrategy::OnCanonicalHead,
-        AttestationStrategy::AllValidators,
-    );
-
-    assert!(
-        harness.chain.head().beacon_state.finalized_checkpoint.epoch > 0,
-        "the chain should have already finalized"
-    );
-
-    let latest_slot = harness.chain.slot().expect("should have a slot");
-
-    harness.chain.persist().expect("should persist the chain");
-
-    let resumed_harness = BeaconChainHarness::resume_from_disk_store(
-        MinimalEthSpec,
-        store,
-        KEYPAIRS[0..validator_count].to_vec(),
-    );
-
-    assert_chains_pretty_much_the_same(&harness.chain, &resumed_harness.chain);
-
-    // Ensures we don't accidentally use it again.
-    //
-    // Note: this will persist the chain again, but that shouldn't matter since nothing has
-    // changed.
-    drop(harness);
-
-    // Set the slot clock of the resumed harness to be in the slot following the previous harness.
-    //
-    // This allows us to produce the block at the next slot.
-    resumed_harness
-        .chain
-        .slot_clock
-        .set_slot(latest_slot.as_u64() + 1);
-
-    resumed_harness.extend_chain(
-        (num_blocks_produced - first_half) as usize,
-        BlockStrategy::OnCanonicalHead,
-        AttestationStrategy::AllValidators,
-    );
-
-    let state = &resumed_harness.chain.head().beacon_state;
-    assert_eq!(
-        state.slot, num_blocks_produced,
-        "head should be at the current slot"
-    );
-    assert_eq!(
-        state.current_epoch(),
-        num_blocks_produced / MinimalEthSpec::slots_per_epoch(),
-        "head should be at the expected epoch"
-    );
-    assert_eq!(
-        state.current_justified_checkpoint.epoch,
-        state.current_epoch() - 1,
-        "the head should be justified one behind the current epoch"
-    );
-    assert_eq!(
-        state.finalized_checkpoint.epoch,
-        state.current_epoch() - 2,
-        "the head should be finalized two behind the current epoch"
-    );
-}
-
-/// Checks that two chains are the same, for the purpose of this tests.
-///
-/// Several fields that are hard/impossible to check are ignored (e.g., the store).
-fn assert_chains_pretty_much_the_same<T: BeaconChainTypes>(a: &BeaconChain<T>, b: &BeaconChain<T>) {
-    assert_eq!(a.spec, b.spec, "spec should be equal");
-    assert_eq!(a.op_pool, b.op_pool, "op_pool should be equal");
-    assert_eq!(a.head(), b.head(), "head() should be equal");
-    assert_eq!(a.heads(), b.heads(), "heads() should be equal");
-    assert_eq!(
-        a.genesis_block_root, b.genesis_block_root,
-        "genesis_block_root should be equal"
-    );
-    assert!(
-        a.fork_choice == b.fork_choice,
-        "fork_choice should be equal"
-    );
-}

beacon_node/beacon_chain/tests/sync_committee_verification.rs

@@ -0,0 +1,668 @@
+#![cfg(not(debug_assertions))]
+
+#[macro_use]
+extern crate lazy_static;
+
+use beacon_chain::sync_committee_verification::Error as SyncCommitteeError;
+use beacon_chain::test_utils::{BeaconChainHarness, EphemeralHarnessType, RelativeSyncCommittee};
+use int_to_bytes::int_to_bytes32;
+use safe_arith::SafeArith;
+use store::{SignedContributionAndProof, SyncCommitteeMessage};
+use tree_hash::TreeHash;
+use types::consts::altair::SYNC_COMMITTEE_SUBNET_COUNT;
+use types::{
+    AggregateSignature, Epoch, EthSpec, Hash256, Keypair, MainnetEthSpec, SecretKey, Slot,
+    SyncSelectionProof, SyncSubnetId, Unsigned,
+};
+
+pub type E = MainnetEthSpec;
+
+pub const VALIDATOR_COUNT: usize = 256;
+
+lazy_static! {
+    /// A cached set of keys.
+    static ref KEYPAIRS: Vec<Keypair> = types::test_utils::generate_deterministic_keypairs(VALIDATOR_COUNT);
+}
+
+/// Returns a beacon chain harness.
+fn get_harness(validator_count: usize) -> BeaconChainHarness<EphemeralHarnessType<E>> {
+    let mut spec = E::default_spec();
+    spec.altair_fork_epoch = Some(Epoch::new(0));
+    let harness = BeaconChainHarness::new(
+        MainnetEthSpec,
+        Some(spec),
+        KEYPAIRS[0..validator_count].to_vec(),
+    );
+
+    harness.advance_slot();
+
+    harness
+}
+
+/// Returns a sync message that is valid for some slot in the given `chain`.
+///
+/// Also returns some info about who created it.
+fn get_valid_sync_committee_message(
+    harness: &BeaconChainHarness<EphemeralHarnessType<E>>,
+    slot: Slot,
+    relative_sync_committee: RelativeSyncCommittee,
+) -> (SyncCommitteeMessage, usize, SecretKey, SyncSubnetId) {
+    let head_state = harness
+        .chain
+        .head_beacon_state()
+        .expect("should get head state");
+    let head_block_root = harness
+        .chain
+        .head()
+        .expect("should get head state")
+        .beacon_block_root;
+    let (signature, _) = harness
+        .make_sync_committee_messages(&head_state, head_block_root, slot, relative_sync_committee)
+        .get(0)
+        .expect("sync messages should exist")
+        .get(0)
+        .expect("first sync message should exist")
+        .clone();
+
+    (
+        signature.clone(),
+        signature.validator_index as usize,
+        harness.validator_keypairs[signature.validator_index as usize]
+            .sk
+            .clone(),
+        SyncSubnetId::new(0),
+    )
+}
+
+fn get_valid_sync_contribution(
+    harness: &BeaconChainHarness<EphemeralHarnessType<E>>,
+    relative_sync_committee: RelativeSyncCommittee,
+) -> (SignedContributionAndProof<E>, usize, SecretKey) {
+    let head_state = harness
+        .chain
+        .head_beacon_state()
+        .expect("should get head state");
+
+    let head_block_root = harness
+        .chain
+        .head()
+        .expect("should get head state")
+        .beacon_block_root;
+    let sync_contributions = harness.make_sync_contributions(
+        &head_state,
+        head_block_root,
+        head_state.slot(),
+        relative_sync_committee,
+    );
+
+    let (_, contribution_opt) = sync_contributions
+        .get(0)
+        .expect("sync contributions should exist");
+    let contribution = contribution_opt
+        .as_ref()
+        .cloned()
+        .expect("signed contribution and proof should exist");
+
+    let aggregator_index = contribution.message.aggregator_index as usize;
+
+    (
+        contribution,
+        aggregator_index,
+        harness.validator_keypairs[aggregator_index].sk.clone(),
+    )
+}
+
+/// Returns a proof and index for a validator that is **not** an aggregator for the current sync period.
+fn get_non_aggregator(
+    harness: &BeaconChainHarness<EphemeralHarnessType<E>>,
+    slot: Slot,
+) -> (usize, SecretKey) {
+    let state = &harness.chain.head().expect("should get head").beacon_state;
+    let sync_subcommittee_size = E::sync_committee_size()
+        .safe_div(SYNC_COMMITTEE_SUBNET_COUNT as usize)
+        .expect("should determine sync subcommittee size");
+    let sync_committee = state
+        .current_sync_committee()
+        .expect("should use altair state")
+        .clone();
+    let non_aggregator_index = sync_committee
+        .pubkeys
+        .chunks(sync_subcommittee_size)
+        .enumerate()
+        .find_map(|(subcommittee_index, subcommittee)| {
+            subcommittee.iter().find_map(|pubkey| {
+                let validator_index = harness
+                    .chain
+                    .validator_index(&pubkey)
+                    .expect("should get validator index")
+                    .expect("pubkey should exist in beacon chain");
+
+                let selection_proof = SyncSelectionProof::new::<E>(
+                    slot,
+                    subcommittee_index as u64,
+                    &harness.validator_keypairs[validator_index].sk,
+                    &state.fork(),
+                    state.genesis_validators_root(),
+                    &harness.spec,
+                );
+
+                if !selection_proof
+                    .is_aggregator::<E>()
+                    .expect("should determine aggregator")
+                {
+                    Some(validator_index)
+                } else {
+                    None
+                }
+            })
+        })
+        .expect("should find at least one non-aggregator");
+
+    let aggregator_sk = harness.validator_keypairs[non_aggregator_index].sk.clone();
+    (non_aggregator_index, aggregator_sk)
+}
+
+/// Tests verification of `SignedContributionAndProof` from the gossip network.
+#[test]
+fn aggregated_gossip_verification() {
+    let harness = get_harness(VALIDATOR_COUNT);
+    let state = harness.get_current_state();
+
+    harness.add_attested_blocks_at_slots(
+        state,
+        Hash256::zero(),
+        &[Slot::new(1), Slot::new(2)],
+        (0..VALIDATOR_COUNT).collect::<Vec<_>>().as_slice(),
+    );
+
+    let current_slot = harness.chain.slot().expect("should get slot");
+
+    let (valid_aggregate, aggregator_index, aggregator_sk) =
+        get_valid_sync_contribution(&harness, RelativeSyncCommittee::Current);
+
+    macro_rules! assert_invalid {
+            ($desc: tt, $attn_getter: expr, $($error: pat) |+ $( if $guard: expr )?) => {
+                assert!(
+                    matches!(
+                        harness
+                            .chain
+                            .verify_sync_contribution_for_gossip($attn_getter)
+                            .err()
+                            .expect(&format!(
+                                "{} should error during verify_sync_contribution_for_gossip",
+                                $desc
+                            )),
+                        $( $error ) |+ $( if $guard )?
+                    ),
+                    "case: {}",
+                    $desc,
+                );
+            };
+        }
+
+    /*
+     * The following two tests ensure:
+     *
+     * The contribution's slot is for the current slot, i.e. contribution.slot == current_slot
+     * (with a MAXIMUM_GOSSIP_CLOCK_DISPARITY allowance).
+     */
+
+    let future_slot = current_slot + 1;
+    assert_invalid!(
+        "aggregate from future slot",
+        {
+            let mut a = valid_aggregate.clone();
+            a.message.contribution.slot = future_slot;
+            a
+        },
+        SyncCommitteeError::FutureSlot { message_slot, latest_permissible_slot }
+        if message_slot == future_slot && latest_permissible_slot == current_slot
+    );
+
+    let early_slot = current_slot
+        .as_u64()
+        // Subtract an additional slot since the harness will be exactly on the start of the
+        // slot and the propagation tolerance will allow an extra slot.
+        .checked_sub(2)
+        .expect("chain is not sufficiently deep for test")
+        .into();
+    assert_invalid!(
+        "aggregate from past slot",
+        {
+            let mut a = valid_aggregate.clone();
+            a.message.contribution.slot = early_slot;
+            a
+        },
+        SyncCommitteeError::PastSlot {
+            message_slot,
+
+            earliest_permissible_slot
+        }
+        if message_slot == early_slot
+            && earliest_permissible_slot == current_slot - 1
+    );
+
+    /*
+     * The following test ensures:
+     *
+     * The subcommittee index is in the allowed range,
+     * i.e. `contribution.subcommittee_index < SYNC_COMMITTEE_SUBNET_COUNT`.
+     */
+
+    assert_invalid!(
+        "subcommittee index out of range",
+        {
+            let mut a = valid_aggregate.clone();
+            a.message.contribution.subcommittee_index = SYNC_COMMITTEE_SUBNET_COUNT;
+            a
+        },
+       SyncCommitteeError::InvalidSubcommittee {
+                subcommittee_index,
+                subcommittee_size,
+            }
+            if subcommittee_index == SYNC_COMMITTEE_SUBNET_COUNT && subcommittee_size == SYNC_COMMITTEE_SUBNET_COUNT
+
+    );
+
+    /*
+     * The following test ensures:
+     *
+     * The sync contribution has participants.
+     */
+
+    assert_invalid!(
+        "aggregate with no participants",
+        {
+            let mut a = valid_aggregate.clone();
+            let aggregation_bits = &mut a.message.contribution.aggregation_bits;
+            aggregation_bits.difference_inplace(&aggregation_bits.clone());
+            assert!(aggregation_bits.is_zero());
+            a.message.contribution.signature = AggregateSignature::infinity();
+            a
+        },
+        SyncCommitteeError::EmptyAggregationBitfield
+    );
+
+    /*
+     * This test ensures:
+     *
+     * The aggregator signature, signed_contribution_and_proof.signature, is valid.
+     */
+
+    assert_invalid!(
+        "aggregate with bad signature",
+        {
+            let mut a = valid_aggregate.clone();
+
+            a.signature = aggregator_sk.sign(Hash256::from_low_u64_be(42));
+
+            a
+        },
+        SyncCommitteeError::InvalidSignature
+    );
+
+    /*
+     * The following test ensures:
+     *
+     * The contribution_and_proof.selection_proof is a valid signature of the `SyncAggregatorSelectionData`
+     * derived from the contribution by the validator with index `contribution_and_proof.aggregator_index`.
+     */
+
+    assert_invalid!(
+        "aggregate with bad selection proof signature",
+        {
+            let mut a = valid_aggregate.clone();
+
+            // Generate some random signature until happens to be a valid selection proof. We need
+            // this in order to reach the signature verification code.
+            //
+            // Could run for ever, but that seems _really_ improbable.
+            let mut i: u64 = 0;
+            a.message.selection_proof = loop {
+                i += 1;
+                let proof: SyncSelectionProof = aggregator_sk
+                    .sign(Hash256::from_slice(&int_to_bytes32(i)))
+                    .into();
+                if proof
+                    .is_aggregator::<E>()
+                    .expect("should determine aggregator")
+                {
+                    break proof.into();
+                }
+            };
+
+            a
+        },
+        SyncCommitteeError::InvalidSignature
+    );
+
+    /*
+     * The following test ensures:
+     *
+     * The aggregate signature is valid for the message `beacon_block_root` and aggregate pubkey
+     * derived from the participation info in `aggregation_bits` for the subcommittee specified by
+     * the `contribution.subcommittee_index`.
+     */
+
+    assert_invalid!(
+        "aggregate with bad aggregate signature",
+        {
+            let mut a = valid_aggregate.clone();
+
+            let mut agg_sig = AggregateSignature::infinity();
+            agg_sig.add_assign(&aggregator_sk.sign(Hash256::from_low_u64_be(42)));
+            a.message.contribution.signature = agg_sig;
+
+            a
+        },
+        SyncCommitteeError::InvalidSignature
+    );
+
+    let too_high_index = <E as EthSpec>::ValidatorRegistryLimit::to_u64() + 1;
+    assert_invalid!(
+        "aggregate with too-high aggregator index",
+        {
+            let mut a = valid_aggregate.clone();
+            a.message.aggregator_index = too_high_index;
+            a
+        },
+        SyncCommitteeError::UnknownValidatorIndex(index)
+        if index == too_high_index as usize
+    );
+
+    /*
+     * The following test ensures:
+     *
+     * The aggregator's validator index is in the declared subcommittee of the current sync
+     * committee -- i.e. state.validators[contribution_and_proof.aggregator_index].pubkey in
+     * get_sync_subcommittee_pubkeys(state, contribution.subcommittee_index).
+     */
+
+    assert_invalid!(
+        "aggregate with unknown aggregator index",
+        {
+            let mut a = valid_aggregate.clone();
+            a.message.contribution.subcommittee_index +=1;
+            a
+        },
+        SyncCommitteeError::AggregatorNotInCommittee {
+            aggregator_index
+        }
+        if aggregator_index == valid_aggregate.message.aggregator_index as u64
+    );
+
+    /*
+     * The following test ensures:
+     *
+     * `contribution_and_proof.selection_proof` selects the validator as an aggregator for the
+     *  slot -- i.e. is_sync_committee_aggregator(contribution_and_proof.selection_proof) returns True.
+     */
+
+    let (non_aggregator_index, non_aggregator_sk) = get_non_aggregator(&harness, current_slot);
+    assert_invalid!(
+        "aggregate from non-aggregator",
+        {
+            SignedContributionAndProof::from_aggregate(
+                non_aggregator_index as u64,
+                valid_aggregate.message.contribution.clone(),
+                None,
+                &non_aggregator_sk,
+                &harness.chain.head_info().expect("should get head info").fork,
+                harness.chain.genesis_validators_root,
+                &harness.chain.spec,
+            )
+        },
+        SyncCommitteeError::InvalidSelectionProof {
+            aggregator_index: index
+        }
+        if index == non_aggregator_index as u64
+    );
+
+    // NOTE: from here on, the tests are stateful, and rely on the valid sync contribution having been
+    // seen. A refactor to give each test case its own state might be nice at some point
+    harness
+        .chain
+        .verify_sync_contribution_for_gossip(valid_aggregate.clone())
+        .expect("should verify sync contribution");
+
+    /*
+     * The following test ensures:
+     *
+     * The sync committee contribution is the first valid contribution received for the aggregator
+     * with index contribution_and_proof.aggregator_index for the slot contribution.slot and
+     * subcommittee index contribution.subcommittee_index.
+     */
+
+    assert_invalid!(
+        "aggregate that has already been seen",
+        valid_aggregate.clone(),
+        SyncCommitteeError::SyncContributionAlreadyKnown(hash)
+        if hash == valid_aggregate.message.contribution.tree_hash_root()
+    );
+
+    /*
+     * The following test ensures:
+     *
+     * The sync committee contribution is the first valid contribution received for the aggregator
+     * with index `contribution_and_proof.aggregator_index` for the slot `contribution.slot` and
+     * subcommittee index `contribution.subcommittee_index`.
+     */
+
+    assert_invalid!(
+        "aggregate from aggregator and subcommittee that has already been seen",
+        {
+            let mut a = valid_aggregate;
+            a.message.contribution.beacon_block_root = Hash256::from_low_u64_le(42);
+            a
+        },
+        SyncCommitteeError::AggregatorAlreadyKnown(index)
+        if index == aggregator_index as u64
+    );
+
+    /*
+     * The following test ensures that:
+     *
+     * A sync committee contribution for the slot before the sync committee period boundary is verified
+     * using the `head_state.next_sync_committee`.
+     */
+
+    // Advance to the slot before the 3rd sync committee period because `current_sync_committee = next_sync_committee`
+    // at genesis.
+    let state = harness.get_current_state();
+    let target_slot = Slot::new(
+        (2 * harness.spec.epochs_per_sync_committee_period.as_u64() * E::slots_per_epoch()) - 1,
+    );
+
+    harness
+        .add_attested_block_at_slot(target_slot, state, Hash256::zero(), &[])
+        .expect("should add block");
+
+    // **Incorrectly** create a sync contribution using the current sync committee
+    let (next_valid_contribution, _, _) =
+        get_valid_sync_contribution(&harness, RelativeSyncCommittee::Current);
+
+    assert_invalid!(
+        "sync contribution created with incorrect sync committee",
+        next_valid_contribution.clone(),
+        SyncCommitteeError::InvalidSignature | SyncCommitteeError::AggregatorNotInCommittee { .. }
+    );
+}
+
+/// Tests the verification conditions for sync committee messages on the gossip network.
+#[test]
+fn unaggregated_gossip_verification() {
+    let harness = get_harness(VALIDATOR_COUNT);
+    let state = harness.get_current_state();
+
+    harness.add_attested_blocks_at_slots(
+        state,
+        Hash256::zero(),
+        &[Slot::new(1), Slot::new(2)],
+        (0..VALIDATOR_COUNT).collect::<Vec<_>>().as_slice(),
+    );
+
+    let current_slot = harness.chain.slot().expect("should get slot");
+
+    let (valid_sync_committee_message, expected_validator_index, validator_sk, subnet_id) =
+        get_valid_sync_committee_message(&harness, current_slot, RelativeSyncCommittee::Current);
+
+    macro_rules! assert_invalid {
+            ($desc: tt, $attn_getter: expr, $subnet_getter: expr, $($error: pat) |+ $( if $guard: expr )?) => {
+                assert!(
+                    matches!(
+                        harness
+                            .chain
+                            .verify_sync_committee_message_for_gossip($attn_getter, $subnet_getter)
+                            .err()
+                            .expect(&format!(
+                                "{} should error during verify_sync_committee_message_for_gossip",
+                                $desc
+                            )),
+                        $( $error ) |+ $( if $guard )?
+                    ),
+                    "case: {}",
+                    $desc,
+                );
+            };
+        }
+
+    /*
+     * The following test ensures:
+     *
+     * The subnet_id is valid for the given validator, i.e. subnet_id in
+     * compute_subnets_for_sync_committee(state, sync_committee_message.validator_index).
+     */
+    let id: u64 = subnet_id.into();
+    let invalid_subnet_id = SyncSubnetId::new(id + 1);
+    assert_invalid!(
+        "invalid subnet id",
+        {
+            valid_sync_committee_message.clone()
+        },
+        invalid_subnet_id,
+        SyncCommitteeError::InvalidSubnetId {
+            received,
+            expected,
+        }
+        if received == invalid_subnet_id && expected.contains(&subnet_id)
+    );
+
+    /*
+     * The following two tests ensure:
+     *
+     * This signature is within a MAXIMUM_GOSSIP_CLOCK_DISPARITY allowance from the current slot.
+     */
+
+    let future_slot = current_slot + 1;
+    assert_invalid!(
+        "sync message from future slot",
+        {
+            let mut signature = valid_sync_committee_message.clone();
+            signature.slot = future_slot;
+            signature
+        },
+        subnet_id,
+        SyncCommitteeError::FutureSlot {
+            message_slot,
+            latest_permissible_slot,
+        }
+        if message_slot == future_slot && latest_permissible_slot == current_slot
+    );
+
+    // Subtract an additional slot since the harness will be exactly on the start of the
+    // slot and the propagation tolerance will allow an extra slot.
+    let early_slot = current_slot
+        .as_u64()
+        .checked_sub(2)
+        .expect("chain is not sufficiently deep for test")
+        .into();
+    assert_invalid!(
+        "sync message from past slot",
+        {
+            let mut signature = valid_sync_committee_message.clone();
+            signature.slot = early_slot;
+            signature
+        },
+        subnet_id,
+        SyncCommitteeError::PastSlot {
+            message_slot,
+
+            earliest_permissible_slot,
+        }
+        if message_slot == early_slot && earliest_permissible_slot == current_slot - 1
+    );
+
+    /*
+     * The following test ensures that:
+     *
+     * The signature is valid for the message beacon_block_root for the validator referenced by
+     * validator_index.
+     */
+    assert_invalid!(
+        "sync message with bad signature",
+        {
+            let mut sync_message = valid_sync_committee_message.clone();
+
+            sync_message.signature = validator_sk.sign(Hash256::from_low_u64_le(424242));
+
+            sync_message
+        },
+        subnet_id,
+        SyncCommitteeError::InvalidSignature
+    );
+
+    harness
+        .chain
+        .verify_sync_committee_message_for_gossip(valid_sync_committee_message.clone(), subnet_id)
+        .expect("valid sync message should be verified");
+
+    /*
+     * The following test ensures that:
+     *
+     * There has been no other valid sync committee message for the declared slot for the
+     * validator referenced by sync_committee_message.validator_index.
+     */
+    assert_invalid!(
+        "sync message that has already been seen",
+        valid_sync_committee_message,
+        subnet_id,
+        SyncCommitteeError::PriorSyncCommitteeMessageKnown {
+            validator_index,
+            slot,
+        }
+        if validator_index == expected_validator_index as u64 && slot == current_slot
+    );
+
+    /*
+     * The following test ensures that:
+     *
+     * A sync committee message for the slot before the sync committee period boundary is verified
+     * using the `head_state.next_sync_committee`.
+     */
+
+    // Advance to the slot before the 3rd sync committee period because `current_sync_committee = next_sync_committee`
+    // at genesis.
+    let state = harness.get_current_state();
+    let target_slot = Slot::new(
+        (2 * harness.spec.epochs_per_sync_committee_period.as_u64() * E::slots_per_epoch()) - 1,
+    );
+
+    harness
+        .add_attested_block_at_slot(target_slot, state, Hash256::zero(), &[])
+        .expect("should add block");
+
+    // **Incorrectly** create a sync message using the current sync committee
+    let (next_valid_sync_committee_message, _, _, next_subnet_id) =
+        get_valid_sync_committee_message(&harness, target_slot, RelativeSyncCommittee::Current);
+
+    assert_invalid!(
+        "sync message on incorrect subnet",
+        next_valid_sync_committee_message.clone(),
+        next_subnet_id,
+        SyncCommitteeError::InvalidSubnetId {
+            received,
+            expected,
+        }
+        if received == subnet_id && !expected.contains(&subnet_id)
+    );
+}

beacon_node/beacon_chain/tests/tests.rs

@@ -3,23 +3,20 @@
 #[macro_use]
 extern crate lazy_static;
 
-use beacon_chain::AttestationProcessingOutcome;
 use beacon_chain::{
+    attestation_verification::Error as AttnError,
     test_utils::{
-        AttestationStrategy, BeaconChainHarness, BlockStrategy, HarnessType, PersistedBeaconChain,
-        BEACON_CHAIN_DB_KEY,
+        AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType,
+        OP_POOL_DB_KEY,
     },
-    BlockProcessingOutcome,
+    StateSkipConfig, WhenSlotSkipped,
 };
-use rand::Rng;
+use operation_pool::PersistedOperationPool;
 use state_processing::{
     per_slot_processing, per_slot_processing::Error as SlotProcessingError, EpochProcessingError,
 };
-use store::Store;
-use types::test_utils::{SeedableRng, TestRandom, XorShiftRng};
-use types::{
-    BeaconStateError, Deposit, EthSpec, Hash256, Keypair, MinimalEthSpec, RelativeEpoch, Slot,
-};
+use store::config::StoreConfig;
+use types::{BeaconStateError, EthSpec, Hash256, Keypair, MinimalEthSpec, RelativeEpoch, Slot};
 
 // Should ideally be divisible by 3.
 pub const VALIDATOR_COUNT: usize = 24;
@@ -29,8 +26,13 @@ lazy_static! {
     static ref KEYPAIRS: Vec<Keypair> = types::test_utils::generate_deterministic_keypairs(VALIDATOR_COUNT);
 }
 
-fn get_harness(validator_count: usize) -> BeaconChainHarness<HarnessType<MinimalEthSpec>> {
-    let harness = BeaconChainHarness::new(MinimalEthSpec, KEYPAIRS[0..validator_count].to_vec());
+fn get_harness(validator_count: usize) -> BeaconChainHarness<EphemeralHarnessType<MinimalEthSpec>> {
+    let harness = BeaconChainHarness::new_with_store_config(
+        MinimalEthSpec,
+        None,
+        KEYPAIRS[0..validator_count].to_vec(),
+        StoreConfig::default(),
+    );
 
     harness.advance_slot();
 
@@ -40,18 +42,18 @@ fn get_harness(validator_count: usize) -> BeaconChainHarness<HarnessType<Minimal
 #[test]
 fn massive_skips() {
     let harness = get_harness(8);
-    let spec = &MinimalEthSpec::default_spec();
-    let mut state = harness.chain.head().beacon_state;
+    let spec = &harness.chain.spec;
+    let mut state = harness.chain.head().expect("should get head").beacon_state;
 
     // Run per_slot_processing until it returns an error.
     let error = loop {
-        match per_slot_processing(&mut state, spec) {
+        match per_slot_processing(&mut state, None, spec) {
             Ok(_) => continue,
             Err(e) => break e,
         }
     };
 
-    assert!(state.slot > 1, "the state should skip at least one slot");
+    assert!(state.slot() > 1, "the state should skip at least one slot");
     assert_eq!(
         error,
         SlotProcessingError::EpochProcessingError(EpochProcessingError::BeaconStateError(
@@ -74,8 +76,18 @@ fn iterators() {
         AttestationStrategy::SomeValidators(vec![]),
     );
 
-    let block_roots: Vec<(Hash256, Slot)> = harness.chain.rev_iter_block_roots().collect();
-    let state_roots: Vec<(Hash256, Slot)> = harness.chain.rev_iter_state_roots().collect();
+    let block_roots: Vec<(Hash256, Slot)> = harness
+        .chain
+        .forwards_iter_block_roots(Slot::new(0))
+        .expect("should get iter")
+        .map(Result::unwrap)
+        .collect();
+    let state_roots: Vec<(Hash256, Slot)> = harness
+        .chain
+        .forwards_iter_state_roots(Slot::new(0))
+        .expect("should get iter")
+        .map(Result::unwrap)
+        .collect();
 
     assert_eq!(
         block_roots.len(),
@@ -101,30 +113,95 @@ fn iterators() {
     block_roots.windows(2).for_each(|x| {
         assert_eq!(
             x[1].1,
-            x[0].1 - 1,
-            "block root slots should be decreasing by one"
+            x[0].1 + 1,
+            "block root slots should be increasing by one"
         )
     });
     state_roots.windows(2).for_each(|x| {
         assert_eq!(
             x[1].1,
-            x[0].1 - 1,
-            "state root slots should be decreasing by one"
+            x[0].1 + 1,
+            "state root slots should be increasing by one"
         )
     });
 
-    let head = &harness.chain.head();
+    let head = &harness.chain.head().expect("should get head");
 
     assert_eq!(
-        *block_roots.first().expect("should have some block roots"),
-        (head.beacon_block_root, head.beacon_block.slot),
-        "first block root and slot should be for the head block"
+        *block_roots.last().expect("should have some block roots"),
+        (head.beacon_block_root, head.beacon_block.slot()),
+        "last block root and slot should be for the head block"
     );
 
     assert_eq!(
-        *state_roots.first().expect("should have some state roots"),
-        (head.beacon_state_root, head.beacon_state.slot),
-        "first state root and slot should be for the head state"
+        *state_roots.last().expect("should have some state roots"),
+        (head.beacon_state_root(), head.beacon_state.slot()),
+        "last state root and slot should be for the head state"
+    );
+}
+
+#[test]
+fn find_reorgs() {
+    let num_blocks_produced = MinimalEthSpec::slots_per_historical_root() + 1;
+
+    let harness = get_harness(VALIDATOR_COUNT);
+
+    harness.extend_chain(
+        num_blocks_produced as usize,
+        BlockStrategy::OnCanonicalHead,
+        // No need to produce attestations for this test.
+        AttestationStrategy::SomeValidators(vec![]),
+    );
+
+    let head_state = harness.chain.head_beacon_state().unwrap();
+    let head_slot = head_state.slot();
+    let genesis_state = harness
+        .chain
+        .state_at_slot(Slot::new(0), StateSkipConfig::WithStateRoots)
+        .unwrap();
+
+    // because genesis is more than `SLOTS_PER_HISTORICAL_ROOT` away, this should return with the
+    // finalized slot.
+    assert_eq!(
+        harness
+            .chain
+            .find_reorg_slot(&genesis_state, harness.chain.genesis_block_root)
+            .unwrap(),
+        head_state
+            .finalized_checkpoint()
+            .epoch
+            .start_slot(MinimalEthSpec::slots_per_epoch())
+    );
+
+    // test head
+    assert_eq!(
+        harness
+            .chain
+            .find_reorg_slot(
+                &head_state,
+                harness.chain.head_beacon_block().unwrap().canonical_root()
+            )
+            .unwrap(),
+        head_slot
+    );
+
+    // Re-org back to the slot prior to the head.
+    let prev_slot = head_slot - Slot::new(1);
+    let prev_state = harness
+        .chain
+        .state_at_slot(prev_slot, StateSkipConfig::WithStateRoots)
+        .unwrap();
+    let prev_block_root = harness
+        .chain
+        .block_root_at_slot(prev_slot, WhenSlotSkipped::None)
+        .unwrap()
+        .unwrap();
+    assert_eq!(
+        harness
+            .chain
+            .find_reorg_slot(&prev_state, prev_block_root)
+            .unwrap(),
+        prev_slot
     );
 }
 
@@ -156,18 +233,22 @@ fn chooses_fork() {
         faulty_fork_blocks,
     );
 
-    assert!(honest_head != faulty_head, "forks should be distinct");
+    assert_ne!(honest_head, faulty_head, "forks should be distinct");
 
-    let state = &harness.chain.head().beacon_state;
+    let state = &harness.chain.head().expect("should get head").beacon_state;
 
     assert_eq!(
-        state.slot,
+        state.slot(),
         Slot::from(initial_blocks + honest_fork_blocks),
         "head should be at the current slot"
     );
 
     assert_eq!(
-        harness.chain.head().beacon_block_root,
+        harness
+            .chain
+            .head()
+            .expect("should get head")
+            .beacon_block_root,
         honest_head,
         "the honest chain should be the canonical chain"
     );
@@ -185,10 +266,11 @@ fn finalizes_with_full_participation() {
         AttestationStrategy::AllValidators,
     );
 
-    let state = &harness.chain.head().beacon_state;
+    let state = &harness.chain.head().expect("should get head").beacon_state;
 
     assert_eq!(
-        state.slot, num_blocks_produced,
+        state.slot(),
+        num_blocks_produced,
         "head should be at the current slot"
     );
     assert_eq!(
@@ -197,12 +279,12 @@ fn finalizes_with_full_participation() {
         "head should be at the expected epoch"
     );
     assert_eq!(
-        state.current_justified_checkpoint.epoch,
+        state.current_justified_checkpoint().epoch,
         state.current_epoch() - 1,
         "the head should be justified one behind the current epoch"
     );
     assert_eq!(
-        state.finalized_checkpoint.epoch,
+        state.finalized_checkpoint().epoch,
         state.current_epoch() - 2,
         "the head should be finalized two behind the current epoch"
     );
@@ -223,10 +305,11 @@ fn finalizes_with_two_thirds_participation() {
         AttestationStrategy::SomeValidators(attesters),
     );
 
-    let state = &harness.chain.head().beacon_state;
+    let state = &harness.chain.head().expect("should get head").beacon_state;
 
     assert_eq!(
-        state.slot, num_blocks_produced,
+        state.slot(),
+        num_blocks_produced,
         "head should be at the current slot"
     );
     assert_eq!(
@@ -240,12 +323,12 @@ fn finalizes_with_two_thirds_participation() {
     // included in blocks during that epoch.
 
     assert_eq!(
-        state.current_justified_checkpoint.epoch,
+        state.current_justified_checkpoint().epoch,
         state.current_epoch() - 2,
         "the head should be justified two behind the current epoch"
     );
     assert_eq!(
-        state.finalized_checkpoint.epoch,
+        state.finalized_checkpoint().epoch,
         state.current_epoch() - 4,
         "the head should be finalized three behind the current epoch"
     );
@@ -267,10 +350,11 @@ fn does_not_finalize_with_less_than_two_thirds_participation() {
         AttestationStrategy::SomeValidators(attesters),
     );
 
-    let state = &harness.chain.head().beacon_state;
+    let state = &harness.chain.head().expect("should get head").beacon_state;
 
     assert_eq!(
-        state.slot, num_blocks_produced,
+        state.slot(),
+        num_blocks_produced,
         "head should be at the current slot"
     );
     assert_eq!(
@@ -279,11 +363,13 @@ fn does_not_finalize_with_less_than_two_thirds_participation() {
         "head should be at the expected epoch"
     );
     assert_eq!(
-        state.current_justified_checkpoint.epoch, 0,
+        state.current_justified_checkpoint().epoch,
+        0,
         "no epoch should have been justified"
     );
     assert_eq!(
-        state.finalized_checkpoint.epoch, 0,
+        state.finalized_checkpoint().epoch,
+        0,
         "no epoch should have been finalized"
     );
 }
@@ -300,10 +386,11 @@ fn does_not_finalize_without_attestation() {
         AttestationStrategy::SomeValidators(vec![]),
     );
 
-    let state = &harness.chain.head().beacon_state;
+    let state = &harness.chain.head().expect("should get head").beacon_state;
 
     assert_eq!(
-        state.slot, num_blocks_produced,
+        state.slot(),
+        num_blocks_produced,
         "head should be at the current slot"
     );
     assert_eq!(
@@ -312,11 +399,13 @@ fn does_not_finalize_without_attestation() {
         "head should be at the expected epoch"
     );
     assert_eq!(
-        state.current_justified_checkpoint.epoch, 0,
+        state.current_justified_checkpoint().epoch,
+        0,
         "no epoch should have been justified"
     );
     assert_eq!(
-        state.finalized_checkpoint.epoch, 0,
+        state.finalized_checkpoint().epoch,
+        0,
         "no epoch should have been finalized"
     );
 }
@@ -335,31 +424,26 @@ fn roundtrip_operation_pool() {
     );
     assert!(harness.chain.op_pool.num_attestations() > 0);
 
-    // Add some deposits
-    let rng = &mut XorShiftRng::from_seed([66; 16]);
-    for i in 0..rng.gen_range(1, VALIDATOR_COUNT) {
-        harness
-            .chain
-            .process_deposit(i as u64, Deposit::random_for_test(rng))
-            .unwrap();
-    }
-
     // TODO: could add some other operations
-    harness.chain.persist().unwrap();
+    harness
+        .chain
+        .persist_op_pool()
+        .expect("should persist op pool");
 
-    let key = Hash256::from_slice(&BEACON_CHAIN_DB_KEY.as_bytes());
-    let p: PersistedBeaconChain<HarnessType<MinimalEthSpec>> =
-        harness.chain.store.get(&key).unwrap().unwrap();
-
-    let restored_op_pool = p
-        .op_pool
-        .into_operation_pool(&p.canonical_head.beacon_state, &harness.spec);
+    let restored_op_pool = harness
+        .chain
+        .store
+        .get_item::<PersistedOperationPool<MinimalEthSpec>>(&OP_POOL_DB_KEY)
+        .expect("should read db")
+        .expect("should find op pool")
+        .into_operation_pool()
+        .unwrap();
 
     assert_eq!(harness.chain.op_pool, restored_op_pool);
 }
 
 #[test]
-fn free_attestations_added_to_fork_choice_some_none() {
+fn unaggregated_attestations_added_to_fork_choice_some_none() {
     let num_blocks_produced = MinimalEthSpec::slots_per_epoch() / 2;
 
     let harness = get_harness(VALIDATOR_COUNT);
@@ -370,8 +454,14 @@ fn free_attestations_added_to_fork_choice_some_none() {
         AttestationStrategy::AllValidators,
     );
 
-    let state = &harness.chain.head().beacon_state;
-    let fork_choice = &harness.chain.fork_choice;
+    let state = &harness.chain.head().expect("should get head").beacon_state;
+    let mut fork_choice = harness.chain.fork_choice.write();
+
+    // Move forward a slot so all queued attestations can be processed.
+    harness.advance_slot();
+    fork_choice
+        .update_time(harness.chain.slot().unwrap())
+        .unwrap();
 
     let validator_slots: Vec<(usize, Slot)> = (0..VALIDATOR_COUNT)
         .into_iter()
@@ -392,8 +482,8 @@ fn free_attestations_added_to_fork_choice_some_none() {
         if slot <= num_blocks_produced && slot != 0 {
             assert_eq!(
                 latest_message.unwrap().1,
-                slot,
-                "Latest message slot for {} should be equal to slot {}.",
+                slot.epoch(MinimalEthSpec::slots_per_epoch()),
+                "Latest message epoch for {} should be equal to epoch {}.",
                 validator,
                 slot
             )
@@ -422,26 +512,47 @@ fn attestations_with_increasing_slots() {
             AttestationStrategy::SomeValidators(vec![]),
         );
 
-        attestations.append(&mut harness.get_free_attestations(
+        let head = harness.chain.head().unwrap();
+        let head_state_root = head.beacon_state_root();
+
+        attestations.extend(harness.get_unaggregated_attestations(
             &AttestationStrategy::AllValidators,
-            &harness.chain.head().beacon_state,
-            harness.chain.head().beacon_block_root,
-            harness.chain.head().beacon_block.slot,
+            &head.beacon_state,
+            head_state_root,
+            head.beacon_block_root,
+            head.beacon_block.slot(),
         ));
 
         harness.advance_slot();
     }
 
-    for attestation in attestations {
-        assert_eq!(
-            harness.chain.process_attestation(attestation),
-            Ok(AttestationProcessingOutcome::Processed)
-        )
+    for (attestation, subnet_id) in attestations.into_iter().flatten() {
+        let res = harness
+            .chain
+            .verify_unaggregated_attestation_for_gossip(attestation.clone(), Some(subnet_id));
+
+        let current_slot = harness.chain.slot().expect("should get slot");
+        let expected_attestation_slot = attestation.data.slot;
+        let expected_earliest_permissible_slot =
+            current_slot - MinimalEthSpec::slots_per_epoch() - 1;
+
+        if expected_attestation_slot < expected_earliest_permissible_slot {
+            assert!(matches!(
+                res.err().unwrap().0,
+                AttnError::PastSlot {
+                    attestation_slot,
+                    earliest_permissible_slot,
+                }
+                if attestation_slot == expected_attestation_slot && earliest_permissible_slot == expected_earliest_permissible_slot
+            ))
+        } else {
+            res.expect("should process attestation");
+        }
     }
 }
 
 #[test]
-fn free_attestations_added_to_fork_choice_all_updated() {
+fn unaggregated_attestations_added_to_fork_choice_all_updated() {
     let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 2 - 1;
 
     let harness = get_harness(VALIDATOR_COUNT);
@@ -452,8 +563,14 @@ fn free_attestations_added_to_fork_choice_all_updated() {
         AttestationStrategy::AllValidators,
     );
 
-    let state = &harness.chain.head().beacon_state;
-    let fork_choice = &harness.chain.fork_choice;
+    let state = &harness.chain.head().expect("should get head").beacon_state;
+    let mut fork_choice = harness.chain.fork_choice.write();
+
+    // Move forward a slot so all queued attestations can be processed.
+    harness.advance_slot();
+    fork_choice
+        .update_time(harness.chain.slot().unwrap())
+        .unwrap();
 
     let validators: Vec<usize> = (0..VALIDATOR_COUNT).collect();
     let slots: Vec<Slot> = validators
@@ -473,7 +590,7 @@ fn free_attestations_added_to_fork_choice_all_updated() {
 
         assert_eq!(
             latest_message.unwrap().1,
-            slot,
+            slot.epoch(MinimalEthSpec::slots_per_epoch()),
             "Latest message slot should be equal to attester duty."
         );
 
@@ -509,18 +626,41 @@ fn run_skip_slot_test(skip_slots: u64) {
     );
 
     assert_eq!(
-        harness_a.chain.head().beacon_block.slot,
+        harness_a
+            .chain
+            .head()
+            .expect("should get head")
+            .beacon_block
+            .slot(),
         Slot::new(skip_slots + 1)
     );
-    assert_eq!(harness_b.chain.head().beacon_block.slot, Slot::new(0));
+    assert_eq!(
+        harness_b
+            .chain
+            .head()
+            .expect("should get head")
+            .beacon_block
+            .slot(),
+        Slot::new(0)
+    );
 
     assert_eq!(
         harness_b
             .chain
-            .process_block(harness_a.chain.head().beacon_block.clone()),
-        Ok(BlockProcessingOutcome::Processed {
-            block_root: harness_a.chain.head().beacon_block_root
-        })
+            .process_block(
+                harness_a
+                    .chain
+                    .head()
+                    .expect("should get head")
+                    .beacon_block
+                    .clone(),
+            )
+            .unwrap(),
+        harness_a
+            .chain
+            .head()
+            .expect("should get head")
+            .beacon_block_root
     );
 
     harness_b
@@ -529,7 +669,12 @@ fn run_skip_slot_test(skip_slots: u64) {
         .expect("should run fork choice");
 
     assert_eq!(
-        harness_b.chain.head().beacon_block.slot,
+        harness_b
+            .chain
+            .head()
+            .expect("should get head")
+            .beacon_block
+            .slot(),
         Slot::new(skip_slots + 1)
     );
 }
@@ -540,3 +685,154 @@ fn produces_and_processes_with_genesis_skip_slots() {
         run_skip_slot_test(i)
     }
 }
+
+#[test]
+fn block_roots_skip_slot_behaviour() {
+    let harness = get_harness(VALIDATOR_COUNT);
+
+    // Test should be longer than the block roots to ensure a DB lookup is triggered.
+    let chain_length = harness
+        .chain
+        .head()
+        .unwrap()
+        .beacon_state
+        .block_roots()
+        .len() as u64
+        * 3;
+
+    let skipped_slots = [1, 6, 7, 10, chain_length];
+
+    // Build a chain with some skip slots.
+    for i in 1..=chain_length {
+        if i > 1 {
+            harness.advance_slot();
+        }
+
+        let slot = harness.chain.slot().unwrap().as_u64();
+
+        if !skipped_slots.contains(&slot) {
+            harness.extend_chain(
+                1,
+                BlockStrategy::OnCanonicalHead,
+                AttestationStrategy::AllValidators,
+            );
+        }
+    }
+
+    let mut prev_unskipped_root = None;
+
+    for target_slot in 0..=chain_length {
+        if skipped_slots.contains(&target_slot) {
+            /*
+             * A skip slot
+             */
+            assert!(
+                harness
+                    .chain
+                    .block_root_at_slot(target_slot.into(), WhenSlotSkipped::None)
+                    .unwrap()
+                    .is_none(),
+                "WhenSlotSkipped::None should return None on a skip slot"
+            );
+
+            let skipped_root = harness
+                .chain
+                .block_root_at_slot(target_slot.into(), WhenSlotSkipped::Prev)
+                .unwrap()
+                .expect("WhenSlotSkipped::Prev should always return Some");
+
+            assert_eq!(
+                skipped_root,
+                prev_unskipped_root.expect("test is badly formed"),
+                "WhenSlotSkipped::Prev should accurately return the prior skipped block"
+            );
+
+            let expected_block = harness.chain.get_block(&skipped_root).unwrap().unwrap();
+
+            assert_eq!(
+                harness
+                    .chain
+                    .block_at_slot(target_slot.into(), WhenSlotSkipped::Prev)
+                    .unwrap()
+                    .unwrap(),
+                expected_block,
+            );
+
+            assert!(
+                harness
+                    .chain
+                    .block_at_slot(target_slot.into(), WhenSlotSkipped::None)
+                    .unwrap()
+                    .is_none(),
+                "WhenSlotSkipped::None should return None on a skip slot"
+            );
+        } else {
+            /*
+             * Not a skip slot
+             */
+            let skips_none = harness
+                .chain
+                .block_root_at_slot(target_slot.into(), WhenSlotSkipped::None)
+                .unwrap()
+                .expect("WhenSlotSkipped::None should return Some for non-skipped block");
+            let skips_prev = harness
+                .chain
+                .block_root_at_slot(target_slot.into(), WhenSlotSkipped::Prev)
+                .unwrap()
+                .expect("WhenSlotSkipped::Prev should always return Some");
+            assert_eq!(
+                skips_none, skips_prev,
+                "WhenSlotSkipped::None and WhenSlotSkipped::Prev should be equal on non-skipped slot"
+            );
+
+            let expected_block = harness.chain.get_block(&skips_prev).unwrap().unwrap();
+
+            assert_eq!(
+                harness
+                    .chain
+                    .block_at_slot(target_slot.into(), WhenSlotSkipped::Prev)
+                    .unwrap()
+                    .unwrap(),
+                expected_block
+            );
+
+            assert_eq!(
+                harness
+                    .chain
+                    .block_at_slot(target_slot.into(), WhenSlotSkipped::None)
+                    .unwrap()
+                    .unwrap(),
+                expected_block
+            );
+
+            prev_unskipped_root = Some(skips_prev);
+        }
+    }
+
+    /*
+     * A future, non-existent slot.
+     */
+
+    let future_slot = harness.chain.slot().unwrap() + 1;
+    assert_eq!(
+        harness.chain.head().unwrap().beacon_block.slot(),
+        future_slot - 2,
+        "test precondition"
+    );
+    assert!(
+        harness
+            .chain
+            .block_root_at_slot(future_slot, WhenSlotSkipped::None)
+            .unwrap()
+            .is_none(),
+        "WhenSlotSkipped::None should return None on a future slot"
+    );
+    assert!(
+        harness
+            .chain
+            .block_root_at_slot(future_slot, WhenSlotSkipped::Prev)
+            .unwrap()
+            .is_none(),
+        "WhenSlotSkipped::Prev should return None on a future slot"
+    );
+}

beacon_node/client/Cargo.toml

@@ -1,41 +1,47 @@
 [package]
 name = "client"
-version = "0.1.0"
-authors = ["Age Manning <Age@AgeManning.com>"]
+version = "0.2.0"
+authors = ["Sigma Prime <contact@sigmaprime.io>"]
 edition = "2018"
 
 [dev-dependencies]
-sloggers = "0.3.4"
-toml = "^0.5"
+sloggers = "1.0.1"
+toml = "0.5.6"
 
 [dependencies]
 beacon_chain = { path = "../beacon_chain" }
 store = { path = "../store" }
 network = { path = "../network" }
-eth2-libp2p = { path = "../eth2-libp2p" }
-rest_api = { path = "../rest_api" }
-parking_lot = "0.9.0"
-websocket_server = { path = "../websocket_server" }
-prometheus = "0.7.0"
-types = { path = "../../eth2/types" }
-tree_hash = "0.1.0"
-eth2_config = { path = "../../eth2/utils/eth2_config" }
-slot_clock = { path = "../../eth2/utils/slot_clock" }
-serde = "1.0.102"
-serde_derive = "1.0.102"
-error-chain = "0.12.1"
-serde_yaml = "0.8.11"
+timer = { path = "../timer" }
+eth2_libp2p = { path = "../eth2_libp2p" }
+parking_lot = "0.11.0"
+prometheus = "0.11.0"
+types = { path = "../../consensus/types" }
+tree_hash = "0.1.1"
+eth2_config = { path = "../../common/eth2_config" }
+slot_clock = { path = "../../common/slot_clock" }
+serde = "1.0.116"
+serde_derive = "1.0.116"
+error-chain = "0.12.4"
+serde_yaml = "0.8.13"
 slog = { version = "2.5.2", features = ["max_level_trace"] }
-slog-async = "2.3.0"
-tokio = "0.1.22"
-dirs = "2.0.2"
-exit-future = "0.1.4"
-futures = "0.1.29"
-reqwest = "0.9.22"
-url = "2.1.0"
-lmd_ghost = { path = "../../eth2/lmd_ghost" }
+slog-async = "2.5.0"
+tokio = "1.10.0"
+dirs = "3.0.1"
+futures = "0.3.7"
+reqwest = { version = "0.11.0", features = ["native-tls-vendored"] }
+url = "2.1.1"
 eth1 = { path = "../eth1" }
 genesis = { path = "../genesis" }
+task_executor = { path = "../../common/task_executor" }
 environment = { path = "../../lighthouse/environment" }
-lighthouse_bootstrap = { path = "../../eth2/utils/lighthouse_bootstrap" }
-eth2_ssz = { path = "../../eth2/utils/ssz" }
+eth2_ssz = "0.1.2"
+lazy_static = "1.4.0"
+lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
+time = "0.2.22"
+directory = {path = "../../common/directory"}
+http_api = { path = "../http_api" }
+http_metrics = { path = "../http_metrics" }
+slasher = { path = "../../slasher" }
+slasher_service = { path = "../../slasher/service" }
+monitoring_api = {path = "../../common/monitoring_api"}

beacon_node/client/src/config.rs

@@ -1,34 +1,31 @@
+use directory::DEFAULT_ROOT_DIR;
 use network::NetworkConfig;
 use serde_derive::{Deserialize, Serialize};
 use std::fs;
 use std::path::PathBuf;
+use types::{Graffiti, PublicKeyBytes};
 
-/// The number initial validators when starting the `Minimal`.
-const TESTNET_SPEC_CONSTANTS: &str = "minimal";
+/// Default directory name for the freezer database under the top-level data dir.
+const DEFAULT_FREEZER_DB_DIR: &str = "freezer_db";
 
 /// Defines how the client should initialize the `BeaconChain` and other components.
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(PartialEq, Debug, Clone, Serialize, Deserialize)]
 pub enum ClientGenesis {
-    /// Reads the genesis state and other persisted data from the `Store`.
-    Resume,
     /// Creates a genesis state as per the 2019 Canada interop specifications.
     Interop {
         validator_count: usize,
         genesis_time: u64,
     },
+    /// Reads the genesis state and other persisted data from the `Store`.
+    FromStore,
     /// Connects to an eth1 node and waits until it can create the genesis state from the deposit
     /// contract.
     DepositContract,
-    /// Loads the genesis state from a SSZ-encoded `BeaconState` file.
-    SszFile { path: PathBuf },
     /// Loads the genesis state from SSZ-encoded `BeaconState` bytes.
     ///
     /// We include the bytes instead of the `BeaconState<E>` because the `EthSpec` type
     /// parameter would be very annoying.
     SszBytes { genesis_state_bytes: Vec<u8> },
-    /// Connects to another Lighthouse instance and reads the genesis state and other data via the
-    /// HTTP API.
-    RemoteNode { server: String, port: Option<u16> },
 }
 
 impl Default for ClientGenesis {
@@ -41,40 +38,60 @@ impl Default for ClientGenesis {
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Config {
     pub data_dir: PathBuf,
-    pub testnet_dir: Option<PathBuf>,
+    /// Name of the directory inside the data directory where the main "hot" DB is located.
+    pub db_name: String,
+    /// Path where the freezer database will be located.
+    pub freezer_db_path: Option<PathBuf>,
     pub log_file: PathBuf,
-    pub spec_constants: String,
     /// If true, the node will use co-ordinated junk for eth1 values.
     ///
     /// This is the method used for the 2019 client interop in Canada.
     pub dummy_eth1_backend: bool,
     pub sync_eth1_chain: bool,
+    /// A list of hard-coded forks that will be disabled.
+    pub disabled_forks: Vec<String>,
+    /// Graffiti to be inserted everytime we create a block.
+    pub graffiti: Graffiti,
+    /// When true, automatically monitor validators using the HTTP API.
+    pub validator_monitor_auto: bool,
+    /// A list of validator pubkeys to monitor.
+    pub validator_monitor_pubkeys: Vec<PublicKeyBytes>,
     #[serde(skip)]
     /// The `genesis` field is not serialized or deserialized by `serde` to ensure it is defined
     /// via the CLI at runtime, instead of from a configuration file saved to disk.
     pub genesis: ClientGenesis,
     pub store: store::StoreConfig,
     pub network: network::NetworkConfig,
-    pub rest_api: rest_api::Config,
-    pub websocket_server: websocket_server::Config,
+    pub chain: beacon_chain::ChainConfig,
     pub eth1: eth1::Config,
+    pub http_api: http_api::Config,
+    pub http_metrics: http_metrics::Config,
+    pub monitoring_api: Option<monitoring_api::Config>,
+    pub slasher: Option<slasher::Config>,
 }
 
 impl Default for Config {
     fn default() -> Self {
         Self {
-            data_dir: PathBuf::from(".lighthouse"),
-            testnet_dir: None,
+            data_dir: PathBuf::from(DEFAULT_ROOT_DIR),
+            db_name: "chain_db".to_string(),
+            freezer_db_path: None,
             log_file: PathBuf::from(""),
             genesis: <_>::default(),
             store: <_>::default(),
             network: NetworkConfig::default(),
-            rest_api: <_>::default(),
-            websocket_server: <_>::default(),
-            spec_constants: TESTNET_SPEC_CONSTANTS.into(),
+            chain: <_>::default(),
             dummy_eth1_backend: false,
             sync_eth1_chain: false,
             eth1: <_>::default(),
+            disabled_forks: Vec::new(),
+            graffiti: Graffiti::default(),
+            http_api: <_>::default(),
+            http_metrics: <_>::default(),
+            monitoring_api: None,
+            slasher: None,
+            validator_monitor_auto: false,
+            validator_monitor_pubkeys: vec![],
         }
     }
 }
@@ -83,21 +100,21 @@ impl Config {
     /// Get the database path without initialising it.
     pub fn get_db_path(&self) -> Option<PathBuf> {
         self.get_data_dir()
-            .map(|data_dir| data_dir.join(&self.store.db_name))
+            .map(|data_dir| data_dir.join(&self.db_name))
     }
 
     /// Get the database path, creating it if necessary.
     pub fn create_db_path(&self) -> Result<PathBuf, String> {
         let db_path = self
             .get_db_path()
-            .ok_or_else(|| "Unable to locate user home directory")?;
+            .ok_or("Unable to locate user home directory")?;
         ensure_dir_exists(db_path)
     }
 
     /// Fetch default path to use for the freezer database.
     fn default_freezer_db_path(&self) -> Option<PathBuf> {
         self.get_data_dir()
-            .map(|data_dir| data_dir.join(self.store.default_freezer_db_dir()))
+            .map(|data_dir| data_dir.join(DEFAULT_FREEZER_DB_DIR))
     }
 
     /// Returns the path to which the client may initialize the on-disk freezer database.
@@ -105,8 +122,7 @@ impl Config {
     /// Will attempt to use the user-supplied path from e.g. the CLI, or will default
     /// to a directory in the data_dir if no path is provided.
     pub fn get_freezer_db_path(&self) -> Option<PathBuf> {
-        self.store
-            .freezer_db_path
+        self.freezer_db_path
             .clone()
             .or_else(|| self.default_freezer_db_path())
     }
@@ -115,7 +131,7 @@ impl Config {
     pub fn create_freezer_db_path(&self) -> Result<PathBuf, String> {
         let freezer_db_path = self
             .get_freezer_db_path()
-            .ok_or_else(|| "Unable to locate user home directory")?;
+            .ok_or("Unable to locate user home directory")?;
         ensure_dir_exists(freezer_db_path)
     }
 
@@ -132,7 +148,7 @@ impl Config {
     pub fn create_data_dir(&self) -> Result<PathBuf, String> {
         let path = self
             .get_data_dir()
-            .ok_or_else(|| "Unable to locate user home directory".to_string())?;
+            .ok_or("Unable to locate user home directory")?;
         ensure_dir_exists(path)
     }
 }
@@ -146,7 +162,6 @@ fn ensure_dir_exists(path: PathBuf) -> Result<PathBuf, String> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use toml;
 
     #[test]
     fn serde() {

beacon_node/client/src/error.rs

@@ -1,5 +1,3 @@
-use network;
-
 use error_chain::error_chain;
 
 error_chain! {

beacon_node/client/src/lib.rs

@@ -1,15 +1,14 @@
 extern crate slog;
 
-mod config;
+pub mod config;
+mod metrics;
 mod notifier;
 
 pub mod builder;
 pub mod error;
 
 use beacon_chain::BeaconChain;
-use eth2_libp2p::{Enr, Multiaddr};
-use exit_future::Signal;
-use network::Service as NetworkService;
+use eth2_libp2p::{Enr, Multiaddr, NetworkGlobals};
 use std::net::SocketAddr;
 use std::sync::Arc;
 
@@ -23,11 +22,11 @@ pub use eth2_config::Eth2Config;
 /// Holds references to running services, cleanly shutting them down when dropped.
 pub struct Client<T: BeaconChainTypes> {
     beacon_chain: Option<Arc<BeaconChain<T>>>,
-    libp2p_network: Option<Arc<NetworkService<T>>>,
-    http_listen_addr: Option<SocketAddr>,
-    websocket_listen_addr: Option<SocketAddr>,
-    /// Exit signals will "fire" when dropped, causing each service to exit gracefully.
-    _exit_signals: Vec<Signal>,
+    network_globals: Option<Arc<NetworkGlobals<T::EthSpec>>>,
+    /// Listen address for the standard eth2.0 API, if the service was started.
+    http_api_listen_addr: Option<SocketAddr>,
+    /// Listen address for the HTTP server which serves Prometheus metrics.
+    http_metrics_listen_addr: Option<SocketAddr>,
 }
 
 impl<T: BeaconChainTypes> Client<T> {
@@ -36,36 +35,28 @@ impl<T: BeaconChainTypes> Client<T> {
         self.beacon_chain.clone()
     }
 
-    /// Returns the address of the client's HTTP API server, if it was started.
-    pub fn http_listen_addr(&self) -> Option<SocketAddr> {
-        self.http_listen_addr
+    /// Returns the address of the client's standard eth2.0 API server, if it was started.
+    pub fn http_api_listen_addr(&self) -> Option<SocketAddr> {
+        self.http_api_listen_addr
     }
 
-    /// Returns the address of the client's WebSocket API server, if it was started.
-    pub fn websocket_listen_addr(&self) -> Option<SocketAddr> {
-        self.websocket_listen_addr
+    /// Returns the address of the client's HTTP Prometheus metrics server, if it was started.
+    pub fn http_metrics_listen_addr(&self) -> Option<SocketAddr> {
+        self.http_metrics_listen_addr
     }
 
     /// Returns the port of the client's libp2p stack, if it was started.
     pub fn libp2p_listen_port(&self) -> Option<u16> {
-        self.libp2p_network.as_ref().map(|n| n.listen_port())
+        self.network_globals.as_ref().map(|n| n.listen_port_tcp())
     }
 
     /// Returns the list of libp2p addresses the client is listening to.
     pub fn libp2p_listen_addresses(&self) -> Option<Vec<Multiaddr>> {
-        self.libp2p_network.as_ref().map(|n| n.listen_multiaddrs())
+        self.network_globals.as_ref().map(|n| n.listen_multiaddrs())
     }
 
     /// Returns the local libp2p ENR of this node, for network discovery.
     pub fn enr(&self) -> Option<Enr> {
-        self.libp2p_network.as_ref().map(|n| n.local_enr())
-    }
-}
-
-impl<T: BeaconChainTypes> Drop for Client<T> {
-    fn drop(&mut self) {
-        if let Some(beacon_chain) = &self.beacon_chain {
-            let _result = beacon_chain.persist();
-        }
+        self.network_globals.as_ref().map(|n| n.local_enr())
     }
 }

beacon_node/client/src/metrics.rs

@@ -0,0 +1,19 @@
+use lazy_static::lazy_static;
+pub use lighthouse_metrics::*;
+
+lazy_static! {
+    pub static ref SYNC_SLOTS_PER_SECOND: Result<IntGauge> = try_create_int_gauge(
+        "sync_slots_per_second",
+        "The number of blocks being imported per second"
+    );
+
+    pub static ref IS_SYNCED: Result<IntGauge> = try_create_int_gauge(
+        "sync_eth2_synced",
+        "Metric to check if the beacon chain is synced to head. 0 if not synced and non-zero if synced"
+    );
+
+    pub static ref NOTIFIER_HEAD_SLOT: Result<IntGauge> = try_create_int_gauge(
+        "notifier_head_slot",
+        "The head slot sourced from the beacon chain notifier"
+    );
+}

beacon_node/client/src/notifier.rs

@@ -1,93 +1,110 @@
+use crate::metrics;
 use beacon_chain::{BeaconChain, BeaconChainTypes};
-use environment::RuntimeContext;
-use exit_future::Signal;
-use futures::{Future, Stream};
-use network::Service as NetworkService;
+use eth2_libp2p::NetworkGlobals;
 use parking_lot::Mutex;
-use slog::{debug, error, info, warn};
+use slog::{debug, error, info, warn, Logger};
 use slot_clock::SlotClock;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
-use tokio::timer::Interval;
+use tokio::time::sleep;
 use types::{EthSpec, Slot};
 
 /// Create a warning log whenever the peer count is at or below this value.
 pub const WARN_PEER_COUNT: usize = 1;
 
-const SECS_PER_MINUTE: f64 = 60.0;
-const SECS_PER_HOUR: f64 = 3600.0;
-const SECS_PER_DAY: f64 = 86400.0; // non-leap
-const SECS_PER_WEEK: f64 = 604800.0; // non-leap
-const DAYS_PER_WEEK: f64 = 7.0;
-const HOURS_PER_DAY: f64 = 24.0;
-const MINUTES_PER_HOUR: f64 = 60.0;
-
-/// How long to wait for the lock on `network.libp2p_service()` before we give up.
-const LIBP2P_LOCK_TIMEOUT: Duration = Duration::from_millis(50);
+const DAYS_PER_WEEK: i64 = 7;
+const HOURS_PER_DAY: i64 = 24;
+const MINUTES_PER_HOUR: i64 = 60;
 
 /// The number of historical observations that should be used to determine the average sync time.
 const SPEEDO_OBSERVATIONS: usize = 4;
 
 /// Spawns a notifier service which periodically logs information about the node.
 pub fn spawn_notifier<T: BeaconChainTypes>(
-    context: RuntimeContext<T::EthSpec>,
+    executor: task_executor::TaskExecutor,
     beacon_chain: Arc<BeaconChain<T>>,
-    network: Arc<NetworkService<T>>,
-    milliseconds_per_slot: u64,
-) -> Result<Signal, String> {
-    let log_1 = context.log.clone();
-    let log_2 = context.log.clone();
-
-    let slot_duration = Duration::from_millis(milliseconds_per_slot);
+    network: Arc<NetworkGlobals<T::EthSpec>>,
+    seconds_per_slot: u64,
+) -> Result<(), String> {
+    let slot_duration = Duration::from_secs(seconds_per_slot);
     let duration_to_next_slot = beacon_chain
         .slot_clock
         .duration_to_next_slot()
-        .ok_or_else(|| "slot_notifier unable to determine time to next slot")?;
+        .ok_or("slot_notifier unable to determine time to next slot")?;
 
     // Run this half way through each slot.
-    let start_instant = Instant::now() + duration_to_next_slot + (slot_duration / 2);
+    let start_instant = tokio::time::Instant::now() + duration_to_next_slot + (slot_duration / 2);
 
     // Run this each slot.
     let interval_duration = slot_duration;
 
     let speedo = Mutex::new(Speedo::default());
+    let log = executor.log().clone();
+    let mut interval = tokio::time::interval_at(start_instant, interval_duration);
 
-    let interval_future = Interval::new(start_instant, interval_duration)
-        .map_err(
-            move |e| error!(log_1, "Slot notifier timer failed"; "error" => format!("{:?}", e)),
-        )
-        .for_each(move |_| {
-            let log = log_2.clone();
+    let interval_future = async move {
+        // Perform pre-genesis logging.
+        loop {
+            match beacon_chain.slot_clock.duration_to_next_slot() {
+                // If the duration to the next slot is greater than the slot duration, then we are
+                // waiting for genesis.
+                Some(next_slot) if next_slot > slot_duration => {
+                    info!(
+                        log,
+                        "Waiting for genesis";
+                        "peers" => peer_count_pretty(network.connected_peers()),
+                        "wait_time" => estimated_time_pretty(Some(next_slot.as_secs() as f64)),
+                    );
+                    eth1_logging(&beacon_chain, &log);
+                    sleep(slot_duration).await;
+                }
+                _ => break,
+            }
+        }
 
-            let connected_peer_count = if let Some(libp2p) = network
-                .libp2p_service()
-                .try_lock_until(Instant::now() + LIBP2P_LOCK_TIMEOUT)
-            {
-                libp2p.swarm.connected_peers()
-            } else {
-                // Use max_value here and we'll print something pretty later.
-                usize::max_value()
+        // Perform post-genesis logging.
+        loop {
+            interval.tick().await;
+            let connected_peer_count = network.connected_peers();
+            let sync_state = network.sync_state();
+
+            let head_info = match beacon_chain.head_info() {
+                Ok(head_info) => head_info,
+                Err(e) => {
+                    error!(log, "Failed to get beacon chain head info"; "error" => format!("{:?}", e));
+                    break;
+                }
             };
 
-            let head = beacon_chain.head();
+            let head_slot = head_info.slot;
+
+            metrics::set_gauge(&metrics::NOTIFIER_HEAD_SLOT, head_slot.as_u64() as i64);
+
+            let current_slot = match beacon_chain.slot() {
+                Ok(slot) => slot,
+                Err(e) => {
+                    error!(
+                        log,
+                        "Unable to read current slot";
+                        "error" => format!("{:?}", e)
+                    );
+                    break;
+                }
+            };
 
-            let head_slot = head.beacon_block.slot;
-            let head_epoch = head_slot.epoch(T::EthSpec::slots_per_epoch());
-            let current_slot = beacon_chain.slot().map_err(|e| {
-                error!(
-                    log,
-                    "Unable to read current slot";
-                    "error" => format!("{:?}", e)
-                )
-            })?;
             let current_epoch = current_slot.epoch(T::EthSpec::slots_per_epoch());
-            let finalized_epoch = head.beacon_state.finalized_checkpoint.epoch;
-            let finalized_root = head.beacon_state.finalized_checkpoint.root;
-            let head_root = head.beacon_block_root;
+            let finalized_epoch = head_info.finalized_checkpoint.epoch;
+            let finalized_root = head_info.finalized_checkpoint.root;
+            let head_root = head_info.block_root;
 
             let mut speedo = speedo.lock();
             speedo.observe(head_slot, Instant::now());
 
+            metrics::set_gauge(
+                &metrics::SYNC_SLOTS_PER_SECOND,
+                speedo.slots_per_second().unwrap_or(0_f64) as i64,
+            );
+
             // The next two lines take advantage of saturating subtraction on `Slot`.
             let head_distance = current_slot - head_slot;
 
@@ -104,66 +121,129 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
                 "head_block" => format!("{}", head_root),
                 "head_slot" => head_slot,
                 "current_slot" => current_slot,
+                "sync_state" =>format!("{}", sync_state)
             );
 
-            if head_epoch + 1 < current_epoch {
+            // Log if we are syncing
+            if sync_state.is_syncing() {
+                metrics::set_gauge(&metrics::IS_SYNCED, 0);
                 let distance = format!(
                     "{} slots ({})",
                     head_distance.as_u64(),
                     slot_distance_pretty(head_distance, slot_duration)
                 );
 
-                info!(
-                    log,
-                    "Syncing";
-                    "peers" => peer_count_pretty(connected_peer_count),
-                    "est_time" => estimated_time_pretty(speedo.estimated_time_till_slot(current_slot)),
-                    "speed" => sync_speed_pretty(speedo.slots_per_second()),
-                    "distance" => distance
-                );
+                let speed = speedo.slots_per_second();
+                let display_speed = speed.map_or(false, |speed| speed != 0.0);
 
-                return Ok(());
-            };
-
-            macro_rules! not_quite_synced_log {
-                ($message: expr) => {
+                if display_speed {
                     info!(
-                        log_2,
-                        $message;
+                        log,
+                        "Syncing";
                         "peers" => peer_count_pretty(connected_peer_count),
-                        "finalized_root" => format!("{}", finalized_root),
-                        "finalized_epoch" => finalized_epoch,
-                        "head_slot" => head_slot,
-                        "current_slot" => current_slot,
+                        "distance" => distance,
+                        "speed" => sync_speed_pretty(speed),
+                        "est_time" => estimated_time_pretty(speedo.estimated_time_till_slot(current_slot)),
+                    );
+                } else {
+                    info!(
+                        log,
+                        "Syncing";
+                        "peers" => peer_count_pretty(connected_peer_count),
+                        "distance" => distance,
+                        "est_time" => estimated_time_pretty(speedo.estimated_time_till_slot(current_slot)),
                     );
                 }
-            }
-
-            if head_epoch + 1 == current_epoch {
-                not_quite_synced_log!("Synced to previous epoch")
-            } else if head_slot != current_slot {
-                not_quite_synced_log!("Synced to current epoch")
-            } else {
+            } else if sync_state.is_synced() {
+                metrics::set_gauge(&metrics::IS_SYNCED, 1);
+                let block_info = if current_slot > head_slot {
+                    "   …  empty".to_string()
+                } else {
+                    head_root.to_string()
+                };
                 info!(
-                    log_2,
+                    log,
                     "Synced";
                     "peers" => peer_count_pretty(connected_peer_count),
                     "finalized_root" => format!("{}", finalized_root),
                     "finalized_epoch" => finalized_epoch,
                     "epoch" => current_epoch,
+                    "block" => block_info,
                     "slot" => current_slot,
                 );
-            };
+            } else {
+                metrics::set_gauge(&metrics::IS_SYNCED, 0);
+                info!(
+                    log,
+                    "Searching for peers";
+                    "peers" => peer_count_pretty(connected_peer_count),
+                    "finalized_root" => format!("{}", finalized_root),
+                    "finalized_epoch" => finalized_epoch,
+                    "head_slot" => head_slot,
+                    "current_slot" => current_slot,
+                );
+            }
 
-            Ok(())
-        });
+            eth1_logging(&beacon_chain, &log);
+        }
+    };
 
-    let (exit_signal, exit) = exit_future::signal();
-    context
-        .executor
-        .spawn(exit.until(interval_future).map(|_| ()));
+    // run the notifier on the current executor
+    executor.spawn(interval_future, "notifier");
 
-    Ok(exit_signal)
+    Ok(())
+}
+
+fn eth1_logging<T: BeaconChainTypes>(beacon_chain: &BeaconChain<T>, log: &Logger) {
+    let current_slot_opt = beacon_chain.slot().ok();
+
+    if let Ok(head_info) = beacon_chain.head_info() {
+        // Perform some logging about the eth1 chain
+        if let Some(eth1_chain) = beacon_chain.eth1_chain.as_ref() {
+            if let Some(status) =
+                eth1_chain.sync_status(head_info.genesis_time, current_slot_opt, &beacon_chain.spec)
+            {
+                debug!(
+                    log,
+                    "Eth1 cache sync status";
+                    "eth1_head_block" => status.head_block_number,
+                    "latest_cached_block_number" => status.latest_cached_block_number,
+                    "latest_cached_timestamp" => status.latest_cached_block_timestamp,
+                    "voting_target_timestamp" => status.voting_target_timestamp,
+                    "ready" => status.lighthouse_is_cached_and_ready
+                );
+
+                if !status.lighthouse_is_cached_and_ready {
+                    let voting_target_timestamp = status.voting_target_timestamp;
+
+                    let distance = status
+                        .latest_cached_block_timestamp
+                        .map(|latest| {
+                            voting_target_timestamp.saturating_sub(latest)
+                                / beacon_chain.spec.seconds_per_eth1_block
+                        })
+                        .map(|distance| distance.to_string())
+                        .unwrap_or_else(|| "initializing deposits".to_string());
+
+                    warn!(
+                        log,
+                        "Syncing eth1 block cache";
+                        "est_blocks_remaining" => distance,
+                    );
+                }
+            } else {
+                error!(
+                    log,
+                    "Unable to determine eth1 sync status";
+                );
+            }
+        }
+    } else {
+        error!(
+            log,
+            "Unable to get head info";
+        );
+    }
 }
 
 /// Returns the peer count, returning something helpful if it's `usize::max_value` (effectively a
@@ -212,31 +292,44 @@ fn seconds_pretty(secs: f64) -> String {
         return "--".into();
     }
 
-    let weeks = secs / SECS_PER_WEEK;
-    let days = secs / SECS_PER_DAY;
-    let hours = secs / SECS_PER_HOUR;
-    let minutes = secs / SECS_PER_MINUTE;
+    let d = time::Duration::seconds_f64(secs);
 
-    if weeks.floor() > 0.0 {
+    let weeks = d.whole_weeks();
+    let days = d.whole_days();
+    let hours = d.whole_hours();
+    let minutes = d.whole_minutes();
+
+    let week_string = if weeks == 1 { "week" } else { "weeks" };
+    let day_string = if days == 1 { "day" } else { "days" };
+    let hour_string = if hours == 1 { "hr" } else { "hrs" };
+    let min_string = if minutes == 1 { "min" } else { "mins" };
+
+    if weeks > 0 {
         format!(
-            "{:.0} weeks {:.0} days",
+            "{:.0} {} {:.0} {}",
             weeks,
-            (days % DAYS_PER_WEEK).round()
+            week_string,
+            days % DAYS_PER_WEEK,
+            day_string
         )
-    } else if days.floor() > 0.0 {
+    } else if days > 0 {
         format!(
-            "{:.0} days {:.0} hrs",
+            "{:.0} {} {:.0} {}",
             days,
-            (hours % HOURS_PER_DAY).round()
+            day_string,
+            hours % HOURS_PER_DAY,
+            hour_string
         )
-    } else if hours.floor() > 0.0 {
+    } else if hours > 0 {
         format!(
-            "{:.0} hrs {:.0} mins",
+            "{:.0} {} {:.0} {}",
             hours,
-            (minutes % MINUTES_PER_HOUR).round()
+            hour_string,
+            minutes % MINUTES_PER_HOUR,
+            min_string
         )
     } else {
-        format!("{:.0} mins", minutes.round())
+        format!("{:.0} {}", minutes, min_string)
     }
 }
 
@@ -297,7 +390,7 @@ impl Speedo {
         let (prev_slot, _) = self.0.last()?;
         let slots_per_second = self.slots_per_second()?;
 
-        if target_slot > *prev_slot {
+        if target_slot > *prev_slot && slots_per_second > 0.0 {
             let distance = (target_slot - *prev_slot).as_u64() as f64;
             Some(distance / slots_per_second)
         } else {

beacon_node/eth1/Cargo.toml

@@ -1,29 +1,36 @@
 [package]
 name = "eth1"
-version = "0.1.0"
+version = "0.2.0"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = "2018"
 
 [dev-dependencies]
-eth1_test_rig = { path = "../../tests/eth1_test_rig" }
+eth1_test_rig = { path = "../../testing/eth1_test_rig" }
+toml = "0.5.6"
+web3 = { version = "0.17.0", default-features = false, features = ["http-tls", "signing", "ws-tls-tokio"] }
+sloggers = "1.0.1"
 environment = { path = "../../lighthouse/environment" }
-toml = "^0.5"
-web3 = "0.8.0"
 
 [dependencies]
-reqwest = "0.9"
-futures = "0.1.25"
-serde_json = "1.0"
-serde = { version = "1.0", features = ["derive"] }
-hex = "0.3"
-types = { path = "../../eth2/types"}
-merkle_proof = { path = "../../eth2/utils/merkle_proof"}
-eth2_ssz = { path = "../../eth2/utils/ssz"}
-tree_hash = { path = "../../eth2/utils/tree_hash"}
-eth2_hashing = { path = "../../eth2/utils/eth2_hashing"}
-parking_lot = "0.7"
-slog = "^2.2.3"
-tokio = "0.1.22"
-state_processing = { path = "../../eth2/state_processing" }
-exit-future = "0.1.4"
-libflate = "0.1"
+reqwest = { version = "0.11.0", features = ["native-tls-vendored"] }
+futures = "0.3.7"
+serde_json = "1.0.58"
+serde = { version = "1.0.116", features = ["derive"] }
+hex = "0.4.2"
+types = { path = "../../consensus/types"}
+merkle_proof = { path = "../../consensus/merkle_proof"}
+eth2_ssz = "0.1.2"
+eth2_ssz_derive = "0.1.0"
+tree_hash = "0.1.1"
+eth2_hashing = "0.1.0"
+parking_lot = "0.11.0"
+slog = "2.5.2"
+tokio = { version = "1.10.0", features = ["full"] }
+state_processing = { path = "../../consensus/state_processing" }
+libflate = "1.0.2"
+lighthouse_metrics = { path = "../../common/lighthouse_metrics"}
+lazy_static = "1.4.0"
+task_executor = { path = "../../common/task_executor" }
+eth2 = { path = "../../common/eth2" }
+fallback = { path = "../../common/fallback" }
+sensitive_url = { path = "../../common/sensitive_url" }

beacon_node/eth1/src/block_cache.rs

@@ -1,5 +1,7 @@
+use ssz_derive::{Decode, Encode};
 use std::ops::RangeInclusive;
-use types::{Eth1Data, Hash256};
+
+pub use eth2::lighthouse::Eth1Block;
 
 #[derive(Debug, PartialEq, Clone)]
 pub enum Error {
@@ -14,31 +16,9 @@ pub enum Error {
     Internal(String),
 }
 
-/// A block of the eth1 chain.
-///
-/// Contains all information required to add a `BlockCache` entry.
-#[derive(Debug, PartialEq, Clone, Eq, Hash)]
-pub struct Eth1Block {
-    pub hash: Hash256,
-    pub timestamp: u64,
-    pub number: u64,
-    pub deposit_root: Option<Hash256>,
-    pub deposit_count: Option<u64>,
-}
-
-impl Eth1Block {
-    pub fn eth1_data(self) -> Option<Eth1Data> {
-        Some(Eth1Data {
-            deposit_root: self.deposit_root?,
-            deposit_count: self.deposit_count?,
-            block_hash: self.hash,
-        })
-    }
-}
-
 /// Stores block and deposit contract information and provides queries based upon the block
 /// timestamp.
-#[derive(Debug, PartialEq, Clone, Default)]
+#[derive(Debug, PartialEq, Clone, Default, Encode, Decode)]
 pub struct BlockCache {
     blocks: Vec<Eth1Block>,
 }
@@ -54,6 +34,16 @@ impl BlockCache {
         self.blocks.is_empty()
     }
 
+    /// Returns the earliest (lowest timestamp) block, if any.
+    pub fn earliest_block(&self) -> Option<&Eth1Block> {
+        self.blocks.first()
+    }
+
+    /// Returns the latest (highest timestamp) block, if any.
+    pub fn latest_block(&self) -> Option<&Eth1Block> {
+        self.blocks.last()
+    }
+
     /// Returns the timestamp of the earliest block in the cache (if any).
     pub fn earliest_block_timestamp(&self) -> Option<u64> {
         self.blocks.first().map(|block| block.timestamp)
@@ -180,6 +170,7 @@ impl BlockCache {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use types::Hash256;
 
     fn get_block(i: u64, interval_secs: u64) -> Eth1Block {
         Eth1Block {
@@ -192,10 +183,7 @@ mod tests {
     }
 
     fn get_blocks(n: usize, interval_secs: u64) -> Vec<Eth1Block> {
-        (0..n as u64)
-            .into_iter()
-            .map(|i| get_block(i, interval_secs))
-            .collect()
+        (0..n as u64).map(|i| get_block(i, interval_secs)).collect()
     }
 
     fn insert(cache: &mut BlockCache, s: Eth1Block) -> Result<(), Error> {
@@ -213,20 +201,20 @@ mod tests {
             insert(&mut cache, block.clone()).expect("should add consecutive blocks");
         }
 
-        for len in vec![0, 1, 2, 3, 4, 8, 15, 16] {
+        for len in &[0, 1, 2, 3, 4, 8, 15, 16] {
             let mut cache = cache.clone();
 
-            cache.truncate(len);
+            cache.truncate(*len);
 
             assert_eq!(
                 cache.blocks.len(),
-                len,
+                *len,
                 "should truncate to length: {}",
-                len
+                *len
             );
         }
 
-        let mut cache_2 = cache.clone();
+        let mut cache_2 = cache;
         cache_2.truncate(17);
         assert_eq!(
             cache_2.blocks.len(),

beacon_node/eth1/src/deposit_cache.rs

@@ -1,17 +1,18 @@
 use crate::DepositLog;
-use eth2_hashing::hash;
-use std::ops::Range;
+use ssz_derive::{Decode, Encode};
+use state_processing::common::DepositDataTree;
+use std::cmp::Ordering;
 use tree_hash::TreeHash;
-use types::{Deposit, Hash256};
+use types::{Deposit, Hash256, DEPOSIT_TREE_DEPTH};
 
-#[derive(Debug, PartialEq, Clone)]
+#[derive(Debug, PartialEq)]
 pub enum Error {
     /// A deposit log was added when a prior deposit was not already in the cache.
     ///
     /// Logs have to be added with monotonically-increasing block numbers.
     NonConsecutive { log_index: u64, expected: usize },
     /// The eth1 event log data was unable to be parsed.
-    LogParseError(String),
+    LogParse(String),
     /// There are insufficient deposits in the cache to fulfil the request.
     InsufficientDeposits {
         known_deposits: usize,
@@ -24,61 +25,99 @@ pub enum Error {
     ///
     /// E.g., you cannot request deposit 10 when the deposit count is 9.
     DepositCountInvalid { deposit_count: u64, range_end: u64 },
+    /// Error with the merkle tree for deposits.
+    DepositTree(merkle_proof::MerkleTreeError),
     /// An unexpected condition was encountered.
-    InternalError(String),
+    Internal(String),
 }
 
-/// Emulates the eth1 deposit contract merkle tree.
-pub struct DepositDataTree {
-    tree: merkle_proof::MerkleTree,
-    mix_in_length: usize,
-    depth: usize,
+#[derive(Encode, Decode, Clone)]
+pub struct SszDepositCache {
+    logs: Vec<DepositLog>,
+    leaves: Vec<Hash256>,
+    deposit_contract_deploy_block: u64,
+    deposit_roots: Vec<Hash256>,
 }
 
-impl DepositDataTree {
-    /// Create a new Merkle tree from a list of leaves (`DepositData::tree_hash_root`) and a fixed depth.
-    pub fn create(leaves: &[Hash256], mix_in_length: usize, depth: usize) -> Self {
+impl SszDepositCache {
+    pub fn from_deposit_cache(cache: &DepositCache) -> Self {
         Self {
-            tree: merkle_proof::MerkleTree::create(leaves, depth),
-            mix_in_length,
-            depth,
+            logs: cache.logs.clone(),
+            leaves: cache.leaves.clone(),
+            deposit_contract_deploy_block: cache.deposit_contract_deploy_block,
+            deposit_roots: cache.deposit_roots.clone(),
         }
     }
 
-    /// Returns 32 bytes representing the "mix in length" for the merkle root of this tree.
-    fn length_bytes(&self) -> Vec<u8> {
-        int_to_bytes32(self.mix_in_length)
-    }
-
-    /// Retrieve the root hash of this Merkle tree with the length mixed in.
-    pub fn root(&self) -> Hash256 {
-        let mut preimage = [0; 64];
-        preimage[0..32].copy_from_slice(&self.tree.hash()[..]);
-        preimage[32..64].copy_from_slice(&self.length_bytes());
-        Hash256::from_slice(&hash(&preimage))
-    }
-
-    /// Return the leaf at `index` and a Merkle proof of its inclusion.
-    ///
-    /// The Merkle proof is in "bottom-up" order, starting with a leaf node
-    /// and moving up the tree. Its length will be exactly equal to `depth + 1`.
-    pub fn generate_proof(&self, index: usize) -> (Hash256, Vec<Hash256>) {
-        let (root, mut proof) = self.tree.generate_proof(index, self.depth);
-        proof.push(Hash256::from_slice(&self.length_bytes()));
-        (root, proof)
+    pub fn to_deposit_cache(&self) -> Result<DepositCache, String> {
+        let deposit_tree =
+            DepositDataTree::create(&self.leaves, self.leaves.len(), DEPOSIT_TREE_DEPTH);
+        // Check for invalid SszDepositCache conditions
+        if self.leaves.len() != self.logs.len() {
+            return Err("Invalid SszDepositCache: logs and leaves should have equal length".into());
+        }
+        // `deposit_roots` also includes the zero root
+        if self.leaves.len() + 1 != self.deposit_roots.len() {
+            return Err(
+                "Invalid SszDepositCache: deposit_roots length must be only one more than leaves"
+                    .into(),
+            );
+        }
+        Ok(DepositCache {
+            logs: self.logs.clone(),
+            leaves: self.leaves.clone(),
+            deposit_contract_deploy_block: self.deposit_contract_deploy_block,
+            deposit_tree,
+            deposit_roots: self.deposit_roots.clone(),
+        })
     }
 }
 
 /// Mirrors the merkle tree of deposits in the eth1 deposit contract.
 ///
 /// Provides `Deposit` objects with merkle proofs included.
-#[derive(Default)]
 pub struct DepositCache {
     logs: Vec<DepositLog>,
-    roots: Vec<Hash256>,
+    leaves: Vec<Hash256>,
+    deposit_contract_deploy_block: u64,
+    /// An incremental merkle tree which represents the current state of the
+    /// deposit contract tree.
+    deposit_tree: DepositDataTree,
+    /// Vector of deposit roots. `deposit_roots[i]` denotes `deposit_root` at
+    /// `deposit_index` `i`.
+    deposit_roots: Vec<Hash256>,
+}
+
+impl Default for DepositCache {
+    fn default() -> Self {
+        let deposit_tree = DepositDataTree::create(&[], 0, DEPOSIT_TREE_DEPTH);
+        let deposit_roots = vec![deposit_tree.root()];
+        DepositCache {
+            logs: Vec::new(),
+            leaves: Vec::new(),
+            deposit_contract_deploy_block: 1,
+            deposit_tree,
+            deposit_roots,
+        }
+    }
+}
+
+#[derive(Debug, PartialEq)]
+pub enum DepositCacheInsertOutcome {
+    Inserted,
+    Duplicate,
 }
 
 impl DepositCache {
+    /// Create new `DepositCache` given block number at which deposit
+    /// contract was deployed.
+    pub fn new(deposit_contract_deploy_block: u64) -> Self {
+        DepositCache {
+            deposit_contract_deploy_block,
+            ..Self::default()
+        }
+    }
+
     /// Returns the number of deposits available in the cache.
     pub fn len(&self) -> usize {
         self.logs.len()
@@ -113,24 +152,29 @@ impl DepositCache {
     ///
     /// - If a log with index `log.index - 1` is not already present in `self` (ignored when empty).
     /// - If a log with `log.index` is already known, but the given `log` is distinct to it.
-    pub fn insert_log(&mut self, log: DepositLog) -> Result<(), Error> {
-        if log.index == self.logs.len() as u64 {
-            self.roots
-                .push(Hash256::from_slice(&log.deposit_data.tree_hash_root()));
-            self.logs.push(log);
-
-            Ok(())
-        } else if log.index < self.logs.len() as u64 {
-            if self.logs[log.index as usize] == log {
-                Ok(())
-            } else {
-                Err(Error::DuplicateDistinctLog(log.index))
+    pub fn insert_log(&mut self, log: DepositLog) -> Result<DepositCacheInsertOutcome, Error> {
+        match log.index.cmp(&(self.logs.len() as u64)) {
+            Ordering::Equal => {
+                let deposit = log.deposit_data.tree_hash_root();
+                self.leaves.push(deposit);
+                self.logs.push(log);
+                self.deposit_tree
+                    .push_leaf(deposit)
+                    .map_err(Error::DepositTree)?;
+                self.deposit_roots.push(self.deposit_tree.root());
+                Ok(DepositCacheInsertOutcome::Inserted)
             }
-        } else {
-            Err(Error::NonConsecutive {
+            Ordering::Less => {
+                if self.logs[log.index as usize] == log {
+                    Ok(DepositCacheInsertOutcome::Duplicate)
+                } else {
+                    Err(Error::DuplicateDistinctLog(log.index))
+                }
+            }
+            Ordering::Greater => Err(Error::NonConsecutive {
                 log_index: log.index,
                 expected: self.logs.len(),
-            })
+            }),
         }
     }
 
@@ -143,27 +187,28 @@ impl DepositCache {
     ///
     /// ## Errors
     ///
-    /// - If `deposit_count` is larger than `range.end`.
+    /// - If `deposit_count` is larger than `end`.
     /// - There are not sufficient deposits in the tree to generate the proof.
     pub fn get_deposits(
         &self,
-        range: Range<u64>,
+        start: u64,
+        end: u64,
         deposit_count: u64,
         tree_depth: usize,
     ) -> Result<(Hash256, Vec<Deposit>), Error> {
-        if deposit_count < range.end {
+        if deposit_count < end {
             // It's invalid to ask for more deposits than should exist.
             Err(Error::DepositCountInvalid {
                 deposit_count,
-                range_end: range.end,
+                range_end: end,
             })
-        } else if range.end > self.logs.len() as u64 {
+        } else if end > self.logs.len() as u64 {
             // The range of requested deposits exceeds the deposits stored locally.
             Err(Error::InsufficientDeposits {
-                requested: range.end,
+                requested: end,
                 known_deposits: self.logs.len(),
             })
-        } else if deposit_count > self.roots.len() as u64 {
+        } else if deposit_count > self.leaves.len() as u64 {
             // There are not `deposit_count` known deposit roots, so we can't build the merkle tree
             // to prove into.
             Err(Error::InsufficientDeposits {
@@ -171,10 +216,10 @@ impl DepositCache {
                 known_deposits: self.logs.len(),
             })
         } else {
-            let roots = self
-                .roots
+            let leaves = self
+                .leaves
                 .get(0..deposit_count as usize)
-                .ok_or_else(|| Error::InternalError("Unable to get known root".into()))?;
+                .ok_or_else(|| Error::Internal("Unable to get known leaves".into()))?;
 
             // Note: there is likely a more optimal solution than recreating the `DepositDataTree`
             // each time this function is called.
@@ -183,12 +228,12 @@ impl DepositCache {
             // last finalized eth1 deposit count. Then, that tree could be cloned and extended for
             // each of these calls.
 
-            let tree = DepositDataTree::create(roots, deposit_count as usize, tree_depth);
+            let tree = DepositDataTree::create(leaves, deposit_count as usize, tree_depth);
 
             let deposits = self
                 .logs
-                .get(range.start as usize..range.end as usize)
-                .ok_or_else(|| Error::InternalError("Unable to get known log".into()))?
+                .get(start as usize..end as usize)
+                .ok_or_else(|| Error::Internal("Unable to get known log".into()))?
                 .iter()
                 .map(|deposit_log| {
                     let (_leaf, proof) = tree.generate_proof(deposit_log.index as usize);
@@ -203,13 +248,50 @@ impl DepositCache {
             Ok((tree.root(), deposits))
         }
     }
-}
 
-/// Returns `int` as little-endian bytes with a length of 32.
-fn int_to_bytes32(int: usize) -> Vec<u8> {
-    let mut vec = int.to_le_bytes().to_vec();
-    vec.resize(32, 0);
-    vec
+    /// Returns the number of deposits with valid signatures that have been observed up to and
+    /// including the block at `block_number`.
+    ///
+    /// Returns `None` if the `block_number` is zero or prior to contract deployment.
+    pub fn get_valid_signature_count(&self, block_number: u64) -> Option<usize> {
+        if block_number == 0 || block_number < self.deposit_contract_deploy_block {
+            None
+        } else {
+            Some(
+                self.logs
+                    .iter()
+                    .take_while(|deposit| deposit.block_number <= block_number)
+                    .filter(|deposit| deposit.signature_is_valid)
+                    .count(),
+            )
+        }
+    }
+
+    /// Returns the number of deposits that have been observed up to and
+    /// including the block at `block_number`.
+    ///
+    /// Returns `None` if the `block_number` is zero or prior to contract deployment.
+    pub fn get_deposit_count_from_cache(&self, block_number: u64) -> Option<u64> {
+        if block_number == 0 || block_number < self.deposit_contract_deploy_block {
+            None
+        } else {
+            Some(
+                self.logs
+                    .iter()
+                    .take_while(|deposit| deposit.block_number <= block_number)
+                    .count() as u64,
+            )
+        }
+    }
+
+    /// Gets the deposit root at block height = block_number.
+    ///
+    /// Fetches the `deposit_count` on or just before the queried `block_number`
+    /// and queries the `deposit_roots` map to get the corresponding `deposit_root`.
+    pub fn get_deposit_root_from_cache(&self, block_number: u64) -> Option<Hash256> {
+        let index = self.get_deposit_count_from_cache(block_number)?;
+        Some(*self.deposit_roots.get(index as usize)?)
+    }
 }
 
 #[cfg(test)]
@@ -217,15 +299,18 @@ pub mod tests {
     use super::*;
     use crate::deposit_log::tests::EXAMPLE_LOG;
     use crate::http::Log;
+    use types::{EthSpec, MainnetEthSpec};
 
     pub const TREE_DEPTH: usize = 32;
 
     fn example_log() -> DepositLog {
+        let spec = MainnetEthSpec::default_spec();
+
         let log = Log {
             block_number: 42,
             data: EXAMPLE_LOG.to_vec(),
         };
-        DepositLog::from_log(&log).expect("should decode log")
+        log.to_deposit_log(&spec).expect("should decode log")
     }
 
     #[test]
@@ -235,7 +320,7 @@ pub mod tests {
         for i in 0..16 {
             let mut log = example_log();
             log.index = i;
-            tree.insert_log(log).expect("should add consecutive logs")
+            tree.insert_log(log).expect("should add consecutive logs");
         }
     }
 
@@ -246,13 +331,16 @@ pub mod tests {
         for i in 0..4 {
             let mut log = example_log();
             log.index = i;
-            tree.insert_log(log).expect("should add consecutive logs")
+            tree.insert_log(log).expect("should add consecutive logs");
         }
 
         // Add duplicate, when given is the same as the one known.
         let mut log = example_log();
         log.index = 3;
-        assert!(tree.insert_log(log).is_ok());
+        assert_eq!(
+            tree.insert_log(log).unwrap(),
+            DepositCacheInsertOutcome::Duplicate
+        );
 
         // Add duplicate, when given is different to the one known.
         let mut log = example_log();
@@ -276,40 +364,40 @@ pub mod tests {
             log.index = i;
             log.block_number = i;
             log.deposit_data.withdrawal_credentials = Hash256::from_low_u64_be(i);
-            tree.insert_log(log).expect("should add consecutive logs")
+            tree.insert_log(log).expect("should add consecutive logs");
         }
 
         // Get 0 deposits, with max deposit count.
         let (_, deposits) = tree
-            .get_deposits(0..0, n, TREE_DEPTH)
+            .get_deposits(0, 0, n, TREE_DEPTH)
             .expect("should get the full tree");
         assert_eq!(deposits.len(), 0, "should return no deposits");
 
         // Get 0 deposits, with 0 deposit count.
         let (_, deposits) = tree
-            .get_deposits(0..0, 0, TREE_DEPTH)
+            .get_deposits(0, 0, 0, TREE_DEPTH)
             .expect("should get the full tree");
         assert_eq!(deposits.len(), 0, "should return no deposits");
 
         // Get 0 deposits, with 0 deposit count, tree depth 0.
         let (_, deposits) = tree
-            .get_deposits(0..0, 0, 0)
+            .get_deposits(0, 0, 0, 0)
             .expect("should get the full tree");
         assert_eq!(deposits.len(), 0, "should return no deposits");
 
         // Get all deposits, with max deposit count.
         let (full_root, deposits) = tree
-            .get_deposits(0..n, n, TREE_DEPTH)
+            .get_deposits(0, n, n, TREE_DEPTH)
             .expect("should get the full tree");
         assert_eq!(deposits.len(), n as usize, "should return all deposits");
 
         // Get 4 deposits, with max deposit count.
         let (root, deposits) = tree
-            .get_deposits(0..4, n, TREE_DEPTH)
+            .get_deposits(0, 4, n, TREE_DEPTH)
             .expect("should get the four from the full tree");
         assert_eq!(
             deposits.len(),
-            4 as usize,
+            4_usize,
             "should get 4 deposits from full tree"
         );
         assert_eq!(
@@ -318,22 +406,19 @@ pub mod tests {
         );
 
         // Get half of the deposits, with half deposit count.
+        let half = n / 2;
         let (half_root, deposits) = tree
-            .get_deposits(0..n / 2, n / 2, TREE_DEPTH)
+            .get_deposits(0, half, half, TREE_DEPTH)
             .expect("should get the half tree");
-        assert_eq!(
-            deposits.len(),
-            n as usize / 2,
-            "should return half deposits"
-        );
+        assert_eq!(deposits.len(), half as usize, "should return half deposits");
 
         // Get 4 deposits, with half deposit count.
         let (root, deposits) = tree
-            .get_deposits(0..4, n / 2, TREE_DEPTH)
+            .get_deposits(0, 4, n / 2, TREE_DEPTH)
             .expect("should get the half tree");
         assert_eq!(
             deposits.len(),
-            4 as usize,
+            4_usize,
             "should get 4 deposits from half tree"
         );
         assert_eq!(
@@ -356,16 +441,16 @@ pub mod tests {
             log.index = i;
             log.block_number = i;
             log.deposit_data.withdrawal_credentials = Hash256::from_low_u64_be(i);
-            tree.insert_log(log).expect("should add consecutive logs")
+            tree.insert_log(log).expect("should add consecutive logs");
         }
 
         // Range too high.
-        assert!(tree.get_deposits(0..n + 1, n, TREE_DEPTH).is_err());
+        assert!(tree.get_deposits(0, n + 1, n, TREE_DEPTH).is_err());
 
         // Count too high.
-        assert!(tree.get_deposits(0..n, n + 1, TREE_DEPTH).is_err());
+        assert!(tree.get_deposits(0, n, n + 1, TREE_DEPTH).is_err());
 
         // Range higher than count.
-        assert!(tree.get_deposits(0..4, 2, TREE_DEPTH).is_err());
+        assert!(tree.get_deposits(0, 4, 2, TREE_DEPTH).is_err());
     }
 }

beacon_node/eth1/src/deposit_log.rs

@@ -1,6 +1,9 @@
 use super::http::Log;
 use ssz::Decode;
-use types::{DepositData, Hash256, PublicKeyBytes, SignatureBytes};
+use state_processing::per_block_processing::signature_sets::deposit_pubkey_signature_message;
+use types::{ChainSpec, DepositData, Hash256, PublicKeyBytes, SignatureBytes};
+
+pub use eth2::lighthouse::DepositLog;
 
 /// The following constants define the layout of bytes in the deposit contract `DepositEvent`. The
 /// event bytes are formatted according to the  Ethereum ABI.
@@ -15,36 +18,26 @@ const SIG_LEN: usize = 96;
 const INDEX_START: usize = SIG_START + 96 + 32;
 const INDEX_LEN: usize = 8;
 
-/// A fully parsed eth1 deposit contract log.
-#[derive(Debug, PartialEq, Clone)]
-pub struct DepositLog {
-    pub deposit_data: DepositData,
-    /// The block number of the log that included this `DepositData`.
-    pub block_number: u64,
-    /// The index included with the deposit log.
-    pub index: u64,
-}
-
-impl DepositLog {
+impl Log {
     /// Attempts to parse a raw `Log` from the deposit contract into a `DepositLog`.
-    pub fn from_log(log: &Log) -> Result<Self, String> {
-        let bytes = &log.data;
+    pub fn to_deposit_log(&self, spec: &ChainSpec) -> Result<DepositLog, String> {
+        let bytes = &self.data;
 
         let pubkey = bytes
             .get(PUBKEY_START..PUBKEY_START + PUBKEY_LEN)
-            .ok_or_else(|| "Insufficient bytes for pubkey".to_string())?;
+            .ok_or("Insufficient bytes for pubkey")?;
         let withdrawal_credentials = bytes
             .get(CREDS_START..CREDS_START + CREDS_LEN)
-            .ok_or_else(|| "Insufficient bytes for withdrawal credential".to_string())?;
+            .ok_or("Insufficient bytes for withdrawal credential")?;
         let amount = bytes
             .get(AMOUNT_START..AMOUNT_START + AMOUNT_LEN)
-            .ok_or_else(|| "Insufficient bytes for amount".to_string())?;
+            .ok_or("Insufficient bytes for amount")?;
         let signature = bytes
             .get(SIG_START..SIG_START + SIG_LEN)
-            .ok_or_else(|| "Insufficient bytes for signature".to_string())?;
+            .ok_or("Insufficient bytes for signature")?;
         let index = bytes
             .get(INDEX_START..INDEX_START + INDEX_LEN)
-            .ok_or_else(|| "Insufficient bytes for index".to_string())?;
+            .ok_or("Insufficient bytes for index")?;
 
         let deposit_data = DepositData {
             pubkey: PublicKeyBytes::from_ssz_bytes(pubkey)
@@ -57,18 +50,24 @@ impl DepositLog {
                 .map_err(|e| format!("Invalid signature ssz: {:?}", e))?,
         };
 
+        let signature_is_valid = deposit_pubkey_signature_message(&deposit_data, spec)
+            .map_or(false, |(public_key, signature, msg)| {
+                signature.verify(&public_key, msg)
+            });
+
         Ok(DepositLog {
             deposit_data,
-            block_number: log.block_number,
+            block_number: self.block_number,
             index: u64::from_ssz_bytes(index).map_err(|e| format!("Invalid index ssz: {:?}", e))?,
+            signature_is_valid,
         })
     }
 }
 
 #[cfg(test)]
 pub mod tests {
-    use super::*;
     use crate::http::Log;
+    use types::{EthSpec, MainnetEthSpec};
 
     /// The data from a deposit event, using the v0.8.3 version of the deposit contract.
     pub const EXAMPLE_LOG: &[u8] = &[
@@ -102,6 +101,7 @@ pub mod tests {
             block_number: 42,
             data: EXAMPLE_LOG.to_vec(),
         };
-        DepositLog::from_log(&log).expect("should decode log");
+        log.to_deposit_log(&MainnetEthSpec::default_spec())
+            .expect("should decode log");
     }
 }

beacon_node/eth1/src/http.rs

@@ -10,12 +10,14 @@
 //!
 //! There is no ABI parsing here, all function signatures and topics are hard-coded as constants.
 
-use futures::{Future, Stream};
-use libflate::gzip::Decoder;
-use reqwest::{header::CONTENT_TYPE, r#async::ClientBuilder, StatusCode};
+use futures::future::TryFutureExt;
+use reqwest::{header::CONTENT_TYPE, ClientBuilder, StatusCode};
+use sensitive_url::SensitiveUrl;
+use serde::{Deserialize, Serialize};
 use serde_json::{json, Value};
-use std::io::prelude::*;
+use std::fmt;
 use std::ops::Range;
+use std::str::FromStr;
 use std::time::Duration;
 use types::Hash256;
 
@@ -32,6 +34,107 @@ pub const DEPOSIT_COUNT_RESPONSE_BYTES: usize = 96;
 /// Number of bytes in deposit contract deposit root (value only).
 pub const DEPOSIT_ROOT_BYTES: usize = 32;
 
+/// This error is returned during a `chainId` call by Geth.
+pub const EIP155_ERROR_STR: &str = "chain not synced beyond EIP-155 replay-protection fork block";
+
+/// Represents an eth1 chain/network id.
+#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
+pub enum Eth1Id {
+    Goerli,
+    Mainnet,
+    Custom(u64),
+}
+
+/// Used to identify a block when querying the Eth1 node.
+#[derive(Clone, Copy)]
+pub enum BlockQuery {
+    Number(u64),
+    Latest,
+}
+
+/// Represents an error received from a remote procecdure call.
+#[derive(Debug, Serialize, Deserialize)]
+pub enum RpcError {
+    NoResultField,
+    Eip155Error,
+    InvalidJson(String),
+    Error(String),
+}
+
+impl fmt::Display for RpcError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            RpcError::NoResultField => write!(f, "No result field in response"),
+            RpcError::Eip155Error => write!(f, "Not synced past EIP-155"),
+            RpcError::InvalidJson(e) => write!(f, "Malformed JSON received: {}", e),
+            RpcError::Error(s) => write!(f, "{}", s),
+        }
+    }
+}
+
+impl From<RpcError> for String {
+    fn from(e: RpcError) -> String {
+        e.to_string()
+    }
+}
+
+impl Into<u64> for Eth1Id {
+    fn into(self) -> u64 {
+        match self {
+            Eth1Id::Mainnet => 1,
+            Eth1Id::Goerli => 5,
+            Eth1Id::Custom(id) => id,
+        }
+    }
+}
+
+impl From<u64> for Eth1Id {
+    fn from(id: u64) -> Self {
+        let into = |x: Eth1Id| -> u64 { x.into() };
+        match id {
+            id if id == into(Eth1Id::Mainnet) => Eth1Id::Mainnet,
+            id if id == into(Eth1Id::Goerli) => Eth1Id::Goerli,
+            id => Eth1Id::Custom(id),
+        }
+    }
+}
+
+impl FromStr for Eth1Id {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        s.parse::<u64>()
+            .map(Into::into)
+            .map_err(|e| format!("Failed to parse eth1 network id {}", e))
+    }
+}
+
+/// Get the eth1 network id of the given endpoint.
+pub async fn get_network_id(endpoint: &SensitiveUrl, timeout: Duration) -> Result<Eth1Id, String> {
+    let response_body = send_rpc_request(endpoint, "net_version", json!([]), timeout).await?;
+    Eth1Id::from_str(
+        response_result_or_error(&response_body)?
+            .as_str()
+            .ok_or("Data was not string")?,
+    )
+}
+
+/// Get the eth1 chain id of the given endpoint.
+pub async fn get_chain_id(endpoint: &SensitiveUrl, timeout: Duration) -> Result<Eth1Id, String> {
+    let response_body: String =
+        send_rpc_request(endpoint, "eth_chainId", json!([]), timeout).await?;
+
+    match response_result_or_error(&response_body) {
+        Ok(chain_id) => {
+            hex_to_u64_be(chain_id.as_str().ok_or("Data was not string")?).map(|id| id.into())
+        }
+        // Geth returns this error when it's syncing lower blocks. Simply map this into `0` since
+        // Lighthouse does not raise errors for `0`, it simply waits for it to change.
+        Err(RpcError::Eip155Error) => Ok(Eth1Id::Custom(0)),
+        Err(e) => Err(e.to_string()),
+    }
+}
+
 #[derive(Debug, PartialEq, Clone)]
 pub struct Block {
     pub hash: Hash256,
@@ -42,80 +145,77 @@ pub struct Block {
 /// Returns the current block number.
 ///
 /// Uses HTTP JSON RPC at `endpoint`. E.g., `http://localhost:8545`.
-pub fn get_block_number(
-    endpoint: &str,
-    timeout: Duration,
-) -> impl Future<Item = u64, Error = String> {
-    send_rpc_request(endpoint, "eth_blockNumber", json!([]), timeout)
-        .and_then(|response_body| {
-            hex_to_u64_be(
-                response_result(&response_body)?
-                    .ok_or_else(|| "No result field was returned for block number".to_string())?
-                    .as_str()
-                    .ok_or_else(|| "Data was not string")?,
-            )
-        })
-        .map_err(|e| format!("Failed to get block number: {}", e))
+pub async fn get_block_number(endpoint: &SensitiveUrl, timeout: Duration) -> Result<u64, String> {
+    let response_body = send_rpc_request(endpoint, "eth_blockNumber", json!([]), timeout).await?;
+    hex_to_u64_be(
+        response_result_or_error(&response_body)
+            .map_err(|e| format!("eth_blockNumber failed: {}", e))?
+            .as_str()
+            .ok_or("Data was not string")?,
+    )
+    .map_err(|e| format!("Failed to get block number: {}", e))
 }
 
 /// Gets a block hash by block number.
 ///
 /// Uses HTTP JSON RPC at `endpoint`. E.g., `http://localhost:8545`.
-pub fn get_block(
-    endpoint: &str,
-    block_number: u64,
+pub async fn get_block(
+    endpoint: &SensitiveUrl,
+    query: BlockQuery,
     timeout: Duration,
-) -> impl Future<Item = Block, Error = String> {
+) -> Result<Block, String> {
+    let query_param = match query {
+        BlockQuery::Number(block_number) => format!("0x{:x}", block_number),
+        BlockQuery::Latest => "latest".to_string(),
+    };
     let params = json!([
-        format!("0x{:x}", block_number),
+        query_param,
         false // do not return full tx objects.
     ]);
 
-    send_rpc_request(endpoint, "eth_getBlockByNumber", params, timeout)
-        .and_then(|response_body| {
-            let hash = hex_to_bytes(
-                response_result(&response_body)?
-                    .ok_or_else(|| "No result field was returned for block".to_string())?
-                    .get("hash")
-                    .ok_or_else(|| "No hash for block")?
-                    .as_str()
-                    .ok_or_else(|| "Block hash was not string")?,
-            )?;
-            let hash = if hash.len() == 32 {
-                Ok(Hash256::from_slice(&hash))
-            } else {
-                Err(format!("Block has was not 32 bytes: {:?}", hash))
-            }?;
+    let response_body = send_rpc_request(endpoint, "eth_getBlockByNumber", params, timeout).await?;
+    let response = response_result_or_error(&response_body)
+        .map_err(|e| format!("eth_getBlockByNumber failed: {}", e))?;
 
-            let timestamp = hex_to_u64_be(
-                response_result(&response_body)?
-                    .ok_or_else(|| "No result field was returned for timestamp".to_string())?
-                    .get("timestamp")
-                    .ok_or_else(|| "No timestamp for block")?
-                    .as_str()
-                    .ok_or_else(|| "Block timestamp was not string")?,
-            )?;
+    let hash: Vec<u8> = hex_to_bytes(
+        response
+            .get("hash")
+            .ok_or("No hash for block")?
+            .as_str()
+            .ok_or("Block hash was not string")?,
+    )?;
+    let hash: Hash256 = if hash.len() == 32 {
+        Hash256::from_slice(&hash)
+    } else {
+        return Err(format!("Block has was not 32 bytes: {:?}", hash));
+    };
 
-            let number = hex_to_u64_be(
-                response_result(&response_body)?
-                    .ok_or_else(|| "No result field was returned for number".to_string())?
-                    .get("number")
-                    .ok_or_else(|| "No number for block")?
-                    .as_str()
-                    .ok_or_else(|| "Block number was not string")?,
-            )?;
+    let timestamp = hex_to_u64_be(
+        response
+            .get("timestamp")
+            .ok_or("No timestamp for block")?
+            .as_str()
+            .ok_or("Block timestamp was not string")?,
+    )?;
 
-            if number <= usize::max_value() as u64 {
-                Ok(Block {
-                    hash,
-                    timestamp,
-                    number,
-                })
-            } else {
-                Err(format!("Block number {} is larger than a usize", number))
-            }
+    let number = hex_to_u64_be(
+        response
+            .get("number")
+            .ok_or("No number for block")?
+            .as_str()
+            .ok_or("Block number was not string")?,
+    )?;
+
+    if number <= usize::max_value() as u64 {
+        Ok(Block {
+            hash,
+            timestamp,
+            number,
         })
-        .map_err(|e| format!("Failed to get block number: {}", e))
+    } else {
+        Err(format!("Block number {} is larger than a usize", number))
+    }
+    .map_err(|e| format!("Failed to get block number: {}", e))
 }
 
 /// Returns the value of the `get_deposit_count()` call at the given `address` for the given
@@ -124,21 +224,22 @@ pub fn get_block(
 /// Assumes that the `address` has the same ABI as the eth2 deposit contract.
 ///
 /// Uses HTTP JSON RPC at `endpoint`. E.g., `http://localhost:8545`.
-pub fn get_deposit_count(
-    endpoint: &str,
+pub async fn get_deposit_count(
+    endpoint: &SensitiveUrl,
     address: &str,
     block_number: u64,
     timeout: Duration,
-) -> impl Future<Item = Option<u64>, Error = String> {
-    call(
+) -> Result<Option<u64>, String> {
+    let result = call(
         endpoint,
         address,
         DEPOSIT_COUNT_FN_SIGNATURE,
         block_number,
         timeout,
     )
-    .and_then(|result| match result {
-        None => Err(format!("Deposit root response was none")),
+    .await?;
+    match result {
+        None => Err("Deposit root response was none".to_string()),
         Some(bytes) => {
             if bytes.is_empty() {
                 Ok(None)
@@ -153,7 +254,7 @@ pub fn get_deposit_count(
                 ))
             }
         }
-    })
+    }
 }
 
 /// Returns the value of the `get_hash_tree_root()` call at the given `block_number`.
@@ -161,21 +262,22 @@ pub fn get_deposit_count(
 /// Assumes that the `address` has the same ABI as the eth2 deposit contract.
 ///
 /// Uses HTTP JSON RPC at `endpoint`. E.g., `http://localhost:8545`.
-pub fn get_deposit_root(
-    endpoint: &str,
+pub async fn get_deposit_root(
+    endpoint: &SensitiveUrl,
     address: &str,
     block_number: u64,
     timeout: Duration,
-) -> impl Future<Item = Option<Hash256>, Error = String> {
-    call(
+) -> Result<Option<Hash256>, String> {
+    let result = call(
         endpoint,
         address,
         DEPOSIT_ROOT_FN_SIGNATURE,
         block_number,
         timeout,
     )
-    .and_then(|result| match result {
-        None => Err(format!("Deposit root response was none")),
+    .await?;
+    match result {
+        None => Err("Deposit root response was none".to_string()),
         Some(bytes) => {
             if bytes.is_empty() {
                 Ok(None)
@@ -188,7 +290,7 @@ pub fn get_deposit_root(
                 ))
             }
         }
-    })
+    }
 }
 
 /// Performs a instant, no-transaction call to the contract `address` with the given `0x`-prefixed
@@ -197,13 +299,13 @@ pub fn get_deposit_root(
 /// Returns bytes, if any.
 ///
 /// Uses HTTP JSON RPC at `endpoint`. E.g., `http://localhost:8545`.
-fn call(
-    endpoint: &str,
+async fn call(
+    endpoint: &SensitiveUrl,
     address: &str,
     hex_data: &str,
     block_number: u64,
     timeout: Duration,
-) -> impl Future<Item = Option<Vec<u8>>, Error = String> {
+) -> Result<Option<Vec<u8>>, String> {
     let params = json! ([
         {
             "to": address,
@@ -212,19 +314,21 @@ fn call(
         format!("0x{:x}", block_number)
     ]);
 
-    send_rpc_request(endpoint, "eth_call", params, timeout).and_then(|response_body| {
-        match response_result(&response_body)? {
-            None => Ok(None),
-            Some(result) => {
-                let hex = result
-                    .as_str()
-                    .map(|s| s.to_string())
-                    .ok_or_else(|| "'result' value was not a string".to_string())?;
+    let response_body = send_rpc_request(endpoint, "eth_call", params, timeout).await?;
 
-                Ok(Some(hex_to_bytes(&hex)?))
-            }
+    match response_result_or_error(&response_body) {
+        Ok(result) => {
+            let hex = result
+                .as_str()
+                .map(|s| s.to_string())
+                .ok_or("'result' value was not a string")?;
+
+            Ok(Some(hex_to_bytes(&hex)?))
         }
-    })
+        // It's valid for `eth_call` to return without a result.
+        Err(RpcError::NoResultField) => Ok(None),
+        Err(e) => Err(format!("eth_call failed: {}", e)),
+    }
 }
 
 /// A reduced set of fields from an Eth1 contract log.
@@ -240,12 +344,12 @@ pub struct Log {
 /// It's not clear from the Ethereum JSON-RPC docs if this range is inclusive or not.
 ///
 /// Uses HTTP JSON RPC at `endpoint`. E.g., `http://localhost:8545`.
-pub fn get_deposit_logs_in_range(
-    endpoint: &str,
+pub async fn get_deposit_logs_in_range(
+    endpoint: &SensitiveUrl,
     address: &str,
     block_height_range: Range<u64>,
     timeout: Duration,
-) -> impl Future<Item = Vec<Log>, Error = String> {
+) -> Result<Vec<Log>, String> {
     let params = json! ([{
         "address": address,
         "topics": [DEPOSIT_EVENT_TOPIC],
@@ -253,46 +357,44 @@ pub fn get_deposit_logs_in_range(
         "toBlock": format!("0x{:x}", block_height_range.end),
     }]);
 
-    send_rpc_request(endpoint, "eth_getLogs", params, timeout)
-        .and_then(|response_body| {
-            response_result(&response_body)?
-                .ok_or_else(|| "No result field was returned for deposit logs".to_string())?
-                .as_array()
-                .cloned()
-                .ok_or_else(|| "'result' value was not an array".to_string())?
-                .into_iter()
-                .map(|value| {
-                    let block_number = value
-                        .get("blockNumber")
-                        .ok_or_else(|| "No block number field in log")?
-                        .as_str()
-                        .ok_or_else(|| "Block number was not string")?;
+    let response_body = send_rpc_request(endpoint, "eth_getLogs", params, timeout).await?;
+    Ok(response_result_or_error(&response_body)
+        .map_err(|e| format!("eth_getLogs failed: {}", e))?
+        .as_array()
+        .cloned()
+        .ok_or("'result' value was not an array")?
+        .into_iter()
+        .map(|value| {
+            let block_number = value
+                .get("blockNumber")
+                .ok_or("No block number field in log")?
+                .as_str()
+                .ok_or("Block number was not string")?;
 
-                    let data = value
-                        .get("data")
-                        .ok_or_else(|| "No block number field in log")?
-                        .as_str()
-                        .ok_or_else(|| "Data was not string")?;
+            let data = value
+                .get("data")
+                .ok_or("No block number field in log")?
+                .as_str()
+                .ok_or("Data was not string")?;
 
-                    Ok(Log {
-                        block_number: hex_to_u64_be(&block_number)?,
-                        data: hex_to_bytes(data)?,
-                    })
-                })
-                .collect::<Result<Vec<Log>, String>>()
+            Ok(Log {
+                block_number: hex_to_u64_be(block_number)?,
+                data: hex_to_bytes(data)?,
+            })
         })
-        .map_err(|e| format!("Failed to get logs in range: {}", e))
+        .collect::<Result<Vec<Log>, String>>()
+        .map_err(|e| format!("Failed to get logs in range: {}", e))?)
 }
 
 /// Sends an RPC request to `endpoint`, using a POST with the given `body`.
 ///
 /// Tries to receive the response and parse the body as a `String`.
-pub fn send_rpc_request(
-    endpoint: &str,
+pub async fn send_rpc_request(
+    endpoint: &SensitiveUrl,
     method: &str,
     params: Value,
     timeout: Duration,
-) -> impl Future<Item = String, Error = String> {
+) -> Result<String, String> {
     let body = json! ({
         "jsonrpc": "2.0",
         "method": method,
@@ -305,85 +407,57 @@ pub fn send_rpc_request(
     //
     // A better solution would be to create some struct that contains a built client and pass it
     // around (similar to the `web3` crate's `Transport` structs).
-    ClientBuilder::new()
+    let response = ClientBuilder::new()
         .timeout(timeout)
         .build()
         .expect("The builder should always build a client")
-        .post(endpoint)
+        .post(endpoint.full.clone())
         .header(CONTENT_TYPE, "application/json")
         .body(body)
         .send()
         .map_err(|e| format!("Request failed: {:?}", e))
-        .and_then(|response| {
-            if response.status() != StatusCode::OK {
-                Err(format!(
-                    "Response HTTP status was not 200 OK:  {}.",
-                    response.status()
-                ))
-            } else {
-                Ok(response)
-            }
-        })
-        .and_then(|response| {
-            response
-                .headers()
-                .get(CONTENT_TYPE)
-                .ok_or_else(|| "No content-type header in response".to_string())
-                .and_then(|encoding| {
-                    encoding
-                        .to_str()
-                        .map(|s| s.to_string())
-                        .map_err(|e| format!("Failed to parse content-type header: {}", e))
-                })
-                .map(|encoding| (response, encoding))
-        })
-        .and_then(|(response, encoding)| {
-            response
-                .into_body()
-                .concat2()
-                .map(|chunk| chunk.iter().cloned().collect::<Vec<u8>>())
-                .map_err(|e| format!("Failed to receive body: {:?}", e))
-                .and_then(move |bytes| match encoding.as_str() {
-                    "application/json" => Ok(bytes),
-                    "application/json; charset=utf-8" => Ok(bytes),
-                    // Note: gzip is not presently working because we always seem to get an empty
-                    // response from the server.
-                    //
-                    // I expect this is some simple-to-solve issue for someone who is familiar with
-                    // the eth1 JSON RPC.
-                    //
-                    // Some public-facing web3 servers use gzip to compress their traffic, it would
-                    // be good to support this.
-                    "application/x-gzip" => {
-                        let mut decoder = Decoder::new(&bytes[..])
-                            .map_err(|e| format!("Failed to create gzip decoder: {}", e))?;
-                        let mut decompressed = vec![];
-                        decoder
-                            .read_to_end(&mut decompressed)
-                            .map_err(|e| format!("Failed to decompress gzip data: {}", e))?;
+        .await?;
+    if response.status() != StatusCode::OK {
+        return Err(format!(
+            "Response HTTP status was not 200 OK:  {}.",
+            response.status()
+        ));
+    };
+    let encoding = response
+        .headers()
+        .get(CONTENT_TYPE)
+        .ok_or("No content-type header in response")?
+        .to_str()
+        .map(|s| s.to_string())
+        .map_err(|e| format!("Failed to parse content-type header: {}", e))?;
 
-                        Ok(decompressed)
-                    }
-                    other => Err(format!("Unsupported encoding: {}", other)),
-                })
-                .map(|bytes| String::from_utf8_lossy(&bytes).into_owned())
-                .map_err(|e| format!("Failed to receive body: {:?}", e))
+    response
+        .bytes()
+        .map_err(|e| format!("Failed to receive body: {:?}", e))
+        .await
+        .and_then(move |bytes| match encoding.as_str() {
+            "application/json" => Ok(bytes),
+            "application/json; charset=utf-8" => Ok(bytes),
+            other => Err(format!("Unsupported encoding: {}", other)),
         })
+        .map(|bytes| String::from_utf8_lossy(&bytes).into_owned())
+        .map_err(|e| format!("Failed to receive body: {:?}", e))
 }
 
-/// Accepts an entire HTTP body (as a string) and returns the `result` field, as a serde `Value`.
-fn response_result(response: &str) -> Result<Option<Value>, String> {
-    let json = serde_json::from_str::<Value>(&response)
-        .map_err(|e| format!("Failed to parse response: {:?}", e))?;
+/// Accepts an entire HTTP body (as a string) and returns either the `result` field or the `error['message']` field, as a serde `Value`.
+fn response_result_or_error(response: &str) -> Result<Value, RpcError> {
+    let json = serde_json::from_str::<Value>(response)
+        .map_err(|e| RpcError::InvalidJson(e.to_string()))?;
 
-    if let Some(error) = json.get("error") {
-        Err(format!("Eth1 node returned error: {}", error))
+    if let Some(error) = json.get("error").map(|e| e.get("message")).flatten() {
+        let error = error.to_string();
+        if error.contains(EIP155_ERROR_STR) {
+            Err(RpcError::Eip155Error)
+        } else {
+            Err(RpcError::Error(error))
+        }
     } else {
-        Ok(json
-            .get("result")
-            .cloned()
-            .map(Some)
-            .unwrap_or_else(|| None))
+        json.get("result").cloned().ok_or(RpcError::NoResultField)
     }
 }
 
@@ -407,8 +481,8 @@ fn hex_to_bytes(hex: &str) -> Result<Vec<u8>, String> {
 
 /// Removes the `0x` prefix from some bytes. Returns an error if the prefix is not present.
 fn strip_prefix(hex: &str) -> Result<&str, String> {
-    if hex.starts_with("0x") {
-        Ok(&hex[2..])
+    if let Some(stripped) = hex.strip_prefix("0x") {
+        Ok(stripped)
     } else {
         Err("Hex string did not start with `0x`".to_string())
     }

beacon_node/eth1/src/inner.rs

@@ -1,6 +1,14 @@
 use crate::Config;
-use crate::{block_cache::BlockCache, deposit_cache::DepositCache};
+use crate::{
+    block_cache::{BlockCache, Eth1Block},
+    deposit_cache::{DepositCache, SszDepositCache},
+    service::EndpointsCache,
+};
 use parking_lot::RwLock;
+use ssz::{Decode, Encode};
+use ssz_derive::{Decode, Encode};
+use std::sync::Arc;
+use types::ChainSpec;
 
 #[derive(Default)]
 pub struct DepositUpdater {
@@ -8,11 +16,24 @@ pub struct DepositUpdater {
     pub last_processed_block: Option<u64>,
 }
 
+impl DepositUpdater {
+    pub fn new(deposit_contract_deploy_block: u64) -> Self {
+        let cache = DepositCache::new(deposit_contract_deploy_block);
+        DepositUpdater {
+            cache,
+            last_processed_block: None,
+        }
+    }
+}
+
 #[derive(Default)]
 pub struct Inner {
     pub block_cache: RwLock<BlockCache>,
     pub deposit_cache: RwLock<DepositUpdater>,
+    pub endpoints_cache: RwLock<Option<Arc<EndpointsCache>>>,
     pub config: RwLock<Config>,
+    pub remote_head_block: RwLock<Option<Eth1Block>>,
+    pub spec: ChainSpec,
 }
 
 impl Inner {
@@ -24,4 +45,57 @@ impl Inner {
             self.block_cache.write().truncate(block_cache_truncation);
         }
     }
+
+    /// Encode the eth1 block and deposit cache as bytes.
+    pub fn as_bytes(&self) -> Vec<u8> {
+        let ssz_eth1_cache = SszEth1Cache::from_inner(self);
+        ssz_eth1_cache.as_ssz_bytes()
+    }
+
+    /// Recover `Inner` given byte representation of eth1 deposit and block caches.
+    pub fn from_bytes(bytes: &[u8], config: Config, spec: ChainSpec) -> Result<Self, String> {
+        let ssz_cache = SszEth1Cache::from_ssz_bytes(bytes)
+            .map_err(|e| format!("Ssz decoding error: {:?}", e))?;
+        ssz_cache.to_inner(config, spec)
+    }
+
+    /// Returns a reference to the specification.
+    pub fn spec(&self) -> &ChainSpec {
+        &self.spec
+    }
+}
+
+#[derive(Encode, Decode, Clone)]
+pub struct SszEth1Cache {
+    block_cache: BlockCache,
+    deposit_cache: SszDepositCache,
+    last_processed_block: Option<u64>,
+}
+
+impl SszEth1Cache {
+    pub fn from_inner(inner: &Inner) -> Self {
+        let deposit_updater = inner.deposit_cache.read();
+        let block_cache = inner.block_cache.read();
+        Self {
+            block_cache: (*block_cache).clone(),
+            deposit_cache: SszDepositCache::from_deposit_cache(&deposit_updater.cache),
+            last_processed_block: deposit_updater.last_processed_block,
+        }
+    }
+
+    pub fn to_inner(&self, config: Config, spec: ChainSpec) -> Result<Inner, String> {
+        Ok(Inner {
+            block_cache: RwLock::new(self.block_cache.clone()),
+            deposit_cache: RwLock::new(DepositUpdater {
+                cache: self.deposit_cache.to_deposit_cache()?,
+                last_processed_block: self.last_processed_block,
+            }),
+            endpoints_cache: RwLock::new(None),
+            // Set the remote head_block zero when creating a new instance. We only care about
+            // present and future eth1 nodes.
+            remote_head_block: RwLock::new(None),
+            config: RwLock::new(config),
+            spec,
+        })
+    }
 }

beacon_node/eth1/src/lib.rs

@@ -1,11 +1,19 @@
+#[macro_use]
+extern crate lazy_static;
+
 mod block_cache;
 mod deposit_cache;
 mod deposit_log;
 pub mod http;
 mod inner;
+mod metrics;
 mod service;
 
 pub use block_cache::{BlockCache, Eth1Block};
 pub use deposit_cache::DepositCache;
 pub use deposit_log::DepositLog;
-pub use service::{BlockCacheUpdateOutcome, Config, DepositCacheUpdateOutcome, Error, Service};
+pub use inner::SszEth1Cache;
+pub use service::{
+    BlockCacheUpdateOutcome, Config, DepositCacheUpdateOutcome, Error, Service, DEFAULT_CHAIN_ID,
+    DEFAULT_NETWORK_ID,
+};

beacon_node/eth1/src/metrics.rs

@@ -0,0 +1,48 @@
+pub use lighthouse_metrics::*;
+
+lazy_static! {
+    /*
+     * Eth1 blocks
+     */
+    pub static ref BLOCK_CACHE_LEN: Result<IntGauge> =
+        try_create_int_gauge("eth1_block_cache_len", "Count of eth1 blocks in cache");
+    pub static ref LATEST_CACHED_BLOCK_TIMESTAMP: Result<IntGauge> =
+        try_create_int_gauge("eth1_latest_cached_block_timestamp", "Timestamp of latest block in eth1 cache");
+
+    /*
+     * Eth1 deposits
+     */
+    pub static ref DEPOSIT_CACHE_LEN: Result<IntGauge> =
+        try_create_int_gauge("eth1_deposit_cache_len", "Number of deposits in the eth1 cache");
+    pub static ref HIGHEST_PROCESSED_DEPOSIT_BLOCK: Result<IntGauge> =
+        try_create_int_gauge("eth1_highest_processed_deposit_block", "Number of the last block checked for deposits");
+
+    /*
+     * Eth1 endpoint errors
+     */
+    pub static ref ENDPOINT_ERRORS: Result<IntCounterVec> = try_create_int_counter_vec(
+        "eth1_endpoint_errors", "The number of eth1 request errors for each endpoint", &["endpoint"]
+    );
+    pub static ref ENDPOINT_REQUESTS: Result<IntCounterVec> = try_create_int_counter_vec(
+        "eth1_endpoint_requests", "The number of eth1 requests for each endpoint", &["endpoint"]
+    );
+
+    /*
+     * Eth1 rpc connection
+     */
+
+    pub static ref ETH1_CONNECTED: Result<IntGauge> = try_create_int_gauge(
+        "sync_eth1_connected", "Set to 1 if connected to an eth1 node, otherwise set to 0"
+    );
+
+    pub static ref ETH1_FALLBACK_CONFIGURED: Result<IntGauge> = try_create_int_gauge(
+        "sync_eth1_fallback_configured", "Number of configured eth1 fallbacks"
+    );
+
+    // Note: This metric only checks if an eth1 fallback is configured, not if it is connected and synced.
+    // Checking for liveness of the fallback would require moving away from lazy checking of fallbacks.
+    pub static ref ETH1_FALLBACK_CONNECTED: Result<IntGauge> = try_create_int_gauge(
+        "eth1_sync_fallback_connected", "Set to 1 if an eth1 fallback is connected, otherwise set to 0"
+    );
+
+}

beacon_node/eth2-libp2p/Cargo.toml

@@ -1,34 +0,0 @@
-[package]
-name = "eth2-libp2p"
-version = "0.1.0"
-authors = ["Age Manning <Age@AgeManning.com>"]
-edition = "2018"
-
-[dependencies]
-hex = "0.3"
-# rust-libp2p is presently being sourced from a Sigma Prime fork of the
-# `libp2p/rust-libp2p` repository.
-libp2p =  { git = "https://github.com/SigP/rust-libp2p", rev = "3f9b030e29c9b31f9fe6f2ed27be4a813e2b3701" }
-enr =  { git = "https://github.com/SigP/rust-libp2p/", rev = "3f9b030e29c9b31f9fe6f2ed27be4a813e2b3701", features = ["serde"] }
-types = { path =  "../../eth2/types" }
-serde = "1.0.102"
-serde_derive = "1.0.102"
-eth2_ssz = "0.1.2"
-eth2_ssz_derive = "0.1.0"
-slog = { version = "2.5.2", features = ["max_level_trace"] }
-version = { path = "../version" }
-tokio = "0.1.22"
-futures = "0.1.29"
-error-chain = "0.12.1"
-dirs = "2.0.2"
-fnv = "1.0.6"
-unsigned-varint = "0.2.3"
-lazy_static = "1.4.0"
-lighthouse_metrics = { path = "../../eth2/utils/lighthouse_metrics" }
-tokio-io-timeout = "0.3.1"
-smallvec = "1.0.0"
-
-[dev-dependencies]
-slog-stdlog = "4.0.0"
-slog-term = "2.4.2"
-slog-async = "2.3.0"

beacon_node/eth2-libp2p/src/behaviour.rs

@@ -1,334 +0,0 @@
-use crate::config::*;
-use crate::discovery::Discovery;
-use crate::rpc::{RPCEvent, RPCMessage, RPC};
-use crate::{error, NetworkConfig};
-use crate::{Topic, TopicHash};
-use crate::{BEACON_ATTESTATION_TOPIC, BEACON_BLOCK_TOPIC};
-use futures::prelude::*;
-use libp2p::{
-    core::identity::Keypair,
-    discv5::Discv5Event,
-    gossipsub::{Gossipsub, GossipsubEvent},
-    identify::{Identify, IdentifyEvent},
-    ping::{Ping, PingConfig, PingEvent},
-    swarm::{NetworkBehaviourAction, NetworkBehaviourEventProcess},
-    tokio_io::{AsyncRead, AsyncWrite},
-    NetworkBehaviour, PeerId,
-};
-use slog::{debug, o};
-use std::num::NonZeroU32;
-use std::time::Duration;
-
-const MAX_IDENTIFY_ADDRESSES: usize = 20;
-
-/// Builds the network behaviour that manages the core protocols of eth2.
-/// This core behaviour is managed by `Behaviour` which adds peer management to all core
-/// behaviours.
-#[derive(NetworkBehaviour)]
-#[behaviour(out_event = "BehaviourEvent", poll_method = "poll")]
-pub struct Behaviour<TSubstream: AsyncRead + AsyncWrite> {
-    /// The routing pub-sub mechanism for eth2.
-    gossipsub: Gossipsub<TSubstream>,
-    /// The Eth2 RPC specified in the wire-0 protocol.
-    eth2_rpc: RPC<TSubstream>,
-    /// Keep regular connection to peers and disconnect if absent.
-    // TODO: Remove Libp2p ping in favour of discv5 ping.
-    ping: Ping<TSubstream>,
-    // TODO: Using id for initial interop. This will be removed by mainnet.
-    /// Provides IP addresses and peer information.
-    identify: Identify<TSubstream>,
-    /// Discovery behaviour.
-    discovery: Discovery<TSubstream>,
-    #[behaviour(ignore)]
-    /// The events generated by this behaviour to be consumed in the swarm poll.
-    events: Vec<BehaviourEvent>,
-    /// Logger for behaviour actions.
-    #[behaviour(ignore)]
-    log: slog::Logger,
-}
-
-impl<TSubstream: AsyncRead + AsyncWrite> Behaviour<TSubstream> {
-    pub fn new(
-        local_key: &Keypair,
-        net_conf: &NetworkConfig,
-        log: &slog::Logger,
-    ) -> error::Result<Self> {
-        let local_peer_id = local_key.public().clone().into_peer_id();
-        let behaviour_log = log.new(o!());
-
-        let ping_config = PingConfig::new()
-            .with_timeout(Duration::from_secs(30))
-            .with_interval(Duration::from_secs(20))
-            .with_max_failures(NonZeroU32::new(2).expect("2 != 0"))
-            .with_keep_alive(false);
-
-        let identify = Identify::new(
-            "lighthouse/libp2p".into(),
-            version::version(),
-            local_key.public(),
-        );
-
-        Ok(Behaviour {
-            eth2_rpc: RPC::new(log.clone()),
-            gossipsub: Gossipsub::new(local_peer_id.clone(), net_conf.gs_config.clone()),
-            discovery: Discovery::new(local_key, net_conf, log)?,
-            ping: Ping::new(ping_config),
-            identify,
-            events: Vec::new(),
-            log: behaviour_log,
-        })
-    }
-
-    pub fn discovery(&self) -> &Discovery<TSubstream> {
-        &self.discovery
-    }
-
-    pub fn gs(&self) -> &Gossipsub<TSubstream> {
-        &self.gossipsub
-    }
-}
-
-// Implement the NetworkBehaviourEventProcess trait so that we can derive NetworkBehaviour for Behaviour
-impl<TSubstream: AsyncRead + AsyncWrite> NetworkBehaviourEventProcess<GossipsubEvent>
-    for Behaviour<TSubstream>
-{
-    fn inject_event(&mut self, event: GossipsubEvent) {
-        match event {
-            GossipsubEvent::Message(propagation_source, gs_msg) => {
-                let id = gs_msg.id();
-                let msg = PubsubMessage::from_topics(&gs_msg.topics, gs_msg.data);
-
-                // Note: We are keeping track here of the peer that sent us the message, not the
-                // peer that originally published the message.
-                self.events.push(BehaviourEvent::GossipMessage {
-                    id,
-                    source: propagation_source,
-                    topics: gs_msg.topics,
-                    message: msg,
-                });
-            }
-            GossipsubEvent::Subscribed { peer_id, topic } => {
-                self.events
-                    .push(BehaviourEvent::PeerSubscribed(peer_id, topic));
-            }
-            GossipsubEvent::Unsubscribed { .. } => {}
-        }
-    }
-}
-
-impl<TSubstream: AsyncRead + AsyncWrite> NetworkBehaviourEventProcess<RPCMessage>
-    for Behaviour<TSubstream>
-{
-    fn inject_event(&mut self, event: RPCMessage) {
-        match event {
-            RPCMessage::PeerDialed(peer_id) => {
-                self.events.push(BehaviourEvent::PeerDialed(peer_id))
-            }
-            RPCMessage::PeerDisconnected(peer_id) => {
-                self.events.push(BehaviourEvent::PeerDisconnected(peer_id))
-            }
-            RPCMessage::RPC(peer_id, rpc_event) => {
-                self.events.push(BehaviourEvent::RPC(peer_id, rpc_event))
-            }
-        }
-    }
-}
-
-impl<TSubstream: AsyncRead + AsyncWrite> NetworkBehaviourEventProcess<PingEvent>
-    for Behaviour<TSubstream>
-{
-    fn inject_event(&mut self, _event: PingEvent) {
-        // not interested in ping responses at the moment.
-    }
-}
-
-impl<TSubstream: AsyncRead + AsyncWrite> Behaviour<TSubstream> {
-    /// Consumes the events list when polled.
-    fn poll<TBehaviourIn>(
-        &mut self,
-    ) -> Async<NetworkBehaviourAction<TBehaviourIn, BehaviourEvent>> {
-        if !self.events.is_empty() {
-            return Async::Ready(NetworkBehaviourAction::GenerateEvent(self.events.remove(0)));
-        }
-
-        Async::NotReady
-    }
-}
-
-impl<TSubstream: AsyncRead + AsyncWrite> NetworkBehaviourEventProcess<IdentifyEvent>
-    for Behaviour<TSubstream>
-{
-    fn inject_event(&mut self, event: IdentifyEvent) {
-        match event {
-            IdentifyEvent::Received {
-                peer_id,
-                mut info,
-                observed_addr,
-            } => {
-                if info.listen_addrs.len() > MAX_IDENTIFY_ADDRESSES {
-                    debug!(
-                        self.log,
-                        "More than 20 addresses have been identified, truncating"
-                    );
-                    info.listen_addrs.truncate(MAX_IDENTIFY_ADDRESSES);
-                }
-                debug!(self.log, "Identified Peer"; "peer" => format!("{}", peer_id),
-                "protocol_version" => info.protocol_version,
-                "agent_version" => info.agent_version,
-                "listening_ addresses" => format!("{:?}", info.listen_addrs),
-                "observed_address" => format!("{:?}", observed_addr),
-                "protocols" => format!("{:?}", info.protocols)
-                );
-            }
-            IdentifyEvent::Sent { .. } => {}
-            IdentifyEvent::Error { .. } => {}
-        }
-    }
-}
-
-impl<TSubstream: AsyncRead + AsyncWrite> NetworkBehaviourEventProcess<Discv5Event>
-    for Behaviour<TSubstream>
-{
-    fn inject_event(&mut self, _event: Discv5Event) {
-        // discv5 has no events to inject
-    }
-}
-
-/// Implements the combined behaviour for the libp2p service.
-impl<TSubstream: AsyncRead + AsyncWrite> Behaviour<TSubstream> {
-    /* Pubsub behaviour functions */
-
-    /// Subscribes to a gossipsub topic.
-    pub fn subscribe(&mut self, topic: Topic) -> bool {
-        self.gossipsub.subscribe(topic)
-    }
-
-    /// Unsubscribe from a gossipsub topic.
-    pub fn unsubscribe(&mut self, topic: Topic) -> bool {
-        self.gossipsub.unsubscribe(topic)
-    }
-
-    /// Publishes a message on the pubsub (gossipsub) behaviour.
-    pub fn publish(&mut self, topics: &[Topic], message: PubsubMessage) {
-        let message_data = message.into_data();
-        for topic in topics {
-            self.gossipsub.publish(topic, message_data.clone());
-        }
-    }
-
-    /// Forwards a message that is waiting in gossipsub's mcache. Messages are only propagated
-    /// once validated by the beacon chain.
-    pub fn propagate_message(&mut self, propagation_source: &PeerId, message_id: String) {
-        self.gossipsub
-            .propagate_message(&message_id, propagation_source);
-    }
-
-    /* Eth2 RPC behaviour functions */
-
-    /// Sends an RPC Request/Response via the RPC protocol.
-    pub fn send_rpc(&mut self, peer_id: PeerId, rpc_event: RPCEvent) {
-        self.eth2_rpc.send_rpc(peer_id, rpc_event);
-    }
-
-    /* Discovery / Peer management functions */
-    /// Return the list of currently connected peers.
-    pub fn connected_peers(&self) -> usize {
-        self.discovery.connected_peers()
-    }
-
-    /// Notify discovery that the peer has been banned.
-    pub fn peer_banned(&mut self, peer_id: PeerId) {
-        self.discovery.peer_banned(peer_id);
-    }
-
-    /// Notify discovery that the peer has been unbanned.
-    pub fn peer_unbanned(&mut self, peer_id: &PeerId) {
-        self.discovery.peer_unbanned(peer_id);
-    }
-
-    /// Informs the discovery behaviour if a new IP/Port is set at the application layer
-    pub fn update_local_enr_socket(&mut self, socket: std::net::SocketAddr, is_tcp: bool) {
-        self.discovery.update_local_enr(socket, is_tcp);
-    }
-}
-
-/// The types of events than can be obtained from polling the behaviour.
-pub enum BehaviourEvent {
-    /// A received RPC event and the peer that it was received from.
-    RPC(PeerId, RPCEvent),
-    /// We have completed an initial connection to a new peer.
-    PeerDialed(PeerId),
-    /// A peer has disconnected.
-    PeerDisconnected(PeerId),
-    /// A gossipsub message has been received.
-    GossipMessage {
-        /// The gossipsub message id. Used when propagating blocks after validation.
-        id: String,
-        /// The peer from which we received this message, not the peer that published it.
-        source: PeerId,
-        /// The topics that this message was sent on.
-        topics: Vec<TopicHash>,
-        /// The message itself.
-        message: PubsubMessage,
-    },
-    /// Subscribed to peer for given topic
-    PeerSubscribed(PeerId, TopicHash),
-}
-
-/// Messages that are passed to and from the pubsub (Gossipsub) behaviour. These are encoded and
-/// decoded upstream.
-#[derive(Debug, Clone, PartialEq)]
-pub enum PubsubMessage {
-    /// Gossipsub message providing notification of a new block.
-    Block(Vec<u8>),
-    /// Gossipsub message providing notification of a new attestation.
-    Attestation(Vec<u8>),
-    /// Gossipsub message providing notification of a voluntary exit.
-    VoluntaryExit(Vec<u8>),
-    /// Gossipsub message providing notification of a new proposer slashing.
-    ProposerSlashing(Vec<u8>),
-    /// Gossipsub message providing notification of a new attester slashing.
-    AttesterSlashing(Vec<u8>),
-    /// Gossipsub message from an unknown topic.
-    Unknown(Vec<u8>),
-}
-
-impl PubsubMessage {
-    /* Note: This is assuming we are not hashing topics. If we choose to hash topics, these will
-     * need to be modified.
-     *
-     * Also note that a message can be associated with many topics. As soon as one of the topics is
-     * known we match. If none of the topics are known we return an unknown state.
-     */
-    fn from_topics(topics: &[TopicHash], data: Vec<u8>) -> Self {
-        for topic in topics {
-            // compare the prefix and postfix, then match on the topic
-            let topic_parts: Vec<&str> = topic.as_str().split('/').collect();
-            if topic_parts.len() == 4
-                && topic_parts[1] == TOPIC_PREFIX
-                && topic_parts[3] == TOPIC_ENCODING_POSTFIX
-            {
-                match topic_parts[2] {
-                    BEACON_BLOCK_TOPIC => return PubsubMessage::Block(data),
-                    BEACON_ATTESTATION_TOPIC => return PubsubMessage::Attestation(data),
-                    VOLUNTARY_EXIT_TOPIC => return PubsubMessage::VoluntaryExit(data),
-                    PROPOSER_SLASHING_TOPIC => return PubsubMessage::ProposerSlashing(data),
-                    ATTESTER_SLASHING_TOPIC => return PubsubMessage::AttesterSlashing(data),
-                    _ => {}
-                }
-            }
-        }
-        PubsubMessage::Unknown(data)
-    }
-
-    fn into_data(self) -> Vec<u8> {
-        match self {
-            PubsubMessage::Block(data)
-            | PubsubMessage::Attestation(data)
-            | PubsubMessage::VoluntaryExit(data)
-            | PubsubMessage::ProposerSlashing(data)
-            | PubsubMessage::AttesterSlashing(data)
-            | PubsubMessage::Unknown(data) => data,
-        }
-    }
-}

beacon_node/eth2-libp2p/src/config.rs

@@ -1,98 +0,0 @@
-use enr::Enr;
-use libp2p::gossipsub::{GossipsubConfig, GossipsubConfigBuilder};
-use libp2p::Multiaddr;
-use serde_derive::{Deserialize, Serialize};
-use std::path::PathBuf;
-use std::time::Duration;
-
-/// The gossipsub topic names.
-// These constants form a topic name of the form /TOPIC_PREFIX/TOPIC/ENCODING_POSTFIX
-// For example /eth2/beacon_block/ssz
-pub const TOPIC_PREFIX: &str = "eth2";
-pub const TOPIC_ENCODING_POSTFIX: &str = "ssz";
-pub const BEACON_BLOCK_TOPIC: &str = "beacon_block";
-pub const BEACON_ATTESTATION_TOPIC: &str = "beacon_attestation";
-pub const VOLUNTARY_EXIT_TOPIC: &str = "voluntary_exit";
-pub const PROPOSER_SLASHING_TOPIC: &str = "proposer_slashing";
-pub const ATTESTER_SLASHING_TOPIC: &str = "attester_slashing";
-pub const SHARD_TOPIC_PREFIX: &str = "shard";
-
-#[derive(Clone, Debug, Serialize, Deserialize)]
-#[serde(default)]
-/// Network configuration for lighthouse.
-pub struct Config {
-    /// Data directory where node's keyfile is stored
-    pub network_dir: PathBuf,
-
-    /// IP address to listen on.
-    pub listen_address: std::net::IpAddr,
-
-    /// The TCP port that libp2p listens on.
-    pub libp2p_port: u16,
-
-    /// The address to broadcast to peers about which address we are listening on.
-    pub discovery_address: std::net::IpAddr,
-
-    /// UDP port that discovery listens on.
-    pub discovery_port: u16,
-
-    /// Target number of connected peers.
-    pub max_peers: usize,
-
-    /// A secp256k1 secret key, as bytes in ASCII-encoded hex.
-    ///
-    /// With or without `0x` prefix.
-    #[serde(skip)]
-    pub secret_key_hex: Option<String>,
-
-    /// Gossipsub configuration parameters.
-    #[serde(skip)]
-    pub gs_config: GossipsubConfig,
-
-    /// List of nodes to initially connect to.
-    pub boot_nodes: Vec<Enr>,
-
-    /// List of libp2p nodes to initially connect to.
-    pub libp2p_nodes: Vec<Multiaddr>,
-
-    /// Client version
-    pub client_version: String,
-
-    /// List of extra topics to initially subscribe to as strings.
-    pub topics: Vec<String>,
-
-    /// Introduces randomization in network propagation of messages. This should only be set for
-    /// testing purposes and will likely be removed in future versions.
-    // TODO: Remove this functionality for mainnet
-    pub propagation_percentage: Option<u8>,
-}
-
-impl Default for Config {
-    /// Generate a default network configuration.
-    fn default() -> Self {
-        let mut network_dir = dirs::home_dir().unwrap_or_else(|| PathBuf::from("."));
-        network_dir.push(".lighthouse");
-        network_dir.push("network");
-        Config {
-            network_dir,
-            listen_address: "127.0.0.1".parse().expect("valid ip address"),
-            libp2p_port: 9000,
-            discovery_address: "127.0.0.1".parse().expect("valid ip address"),
-            discovery_port: 9000,
-            max_peers: 10,
-            secret_key_hex: None,
-            // Note: The topics by default are sent as plain strings. Hashes are an optional
-            // parameter.
-            gs_config: GossipsubConfigBuilder::new()
-                .max_transmit_size(1_048_576)
-                .heartbeat_interval(Duration::from_secs(20)) // TODO: Reduce for mainnet
-                .manual_propagation(true) // require validation before propagation
-                .build(),
-            boot_nodes: vec![],
-            libp2p_nodes: vec![],
-            client_version: version::version(),
-            topics: Vec::new(),
-            propagation_percentage: None,
-        }
-    }
-}

beacon_node/eth2-libp2p/src/discovery.rs

@@ -1,391 +0,0 @@
-use crate::metrics;
-use crate::{error, NetworkConfig};
-/// This manages the discovery and management of peers.
-///
-/// Currently using discv5 for peer discovery.
-///
-use futures::prelude::*;
-use libp2p::core::{identity::Keypair, ConnectedPoint, Multiaddr, PeerId};
-use libp2p::discv5::{Discv5, Discv5Event};
-use libp2p::enr::{Enr, EnrBuilder, NodeId};
-use libp2p::multiaddr::Protocol;
-use libp2p::swarm::{NetworkBehaviour, NetworkBehaviourAction, PollParameters, ProtocolsHandler};
-use slog::{debug, info, warn};
-use std::collections::HashSet;
-use std::fs::File;
-use std::io::prelude::*;
-use std::path::Path;
-use std::str::FromStr;
-use std::time::{Duration, Instant};
-use tokio::io::{AsyncRead, AsyncWrite};
-use tokio::timer::Delay;
-
-/// Maximum seconds before searching for extra peers.
-const MAX_TIME_BETWEEN_PEER_SEARCHES: u64 = 120;
-/// Initial delay between peer searches.
-const INITIAL_SEARCH_DELAY: u64 = 5;
-/// Local ENR storage filename.
-const ENR_FILENAME: &str = "enr.dat";
-
-/// Lighthouse discovery behaviour. This provides peer management and discovery using the Discv5
-/// libp2p protocol.
-pub struct Discovery<TSubstream> {
-    /// The peers currently connected to libp2p streams.
-    connected_peers: HashSet<PeerId>,
-
-    /// The currently banned peers.
-    banned_peers: HashSet<PeerId>,
-
-    /// The target number of connected peers on the libp2p interface.
-    max_peers: usize,
-
-    /// The directory where the ENR is stored.
-    enr_dir: String,
-
-    /// The delay between peer discovery searches.
-    peer_discovery_delay: Delay,
-
-    /// Tracks the last discovery delay. The delay is doubled each round until the max
-    /// time is reached.
-    past_discovery_delay: u64,
-
-    /// The TCP port for libp2p. Used to convert an updated IP address to a multiaddr. Note: This
-    /// assumes that the external TCP port is the same as the internal TCP port if behind a NAT.
-    //TODO: Improve NAT handling limit the above restriction
-    tcp_port: u16,
-
-    /// The discovery behaviour used to discover new peers.
-    discovery: Discv5<TSubstream>,
-
-    /// Logger for the discovery behaviour.
-    log: slog::Logger,
-}
-
-impl<TSubstream> Discovery<TSubstream> {
-    pub fn new(
-        local_key: &Keypair,
-        config: &NetworkConfig,
-        log: &slog::Logger,
-    ) -> error::Result<Self> {
-        let log = log.clone();
-
-        // checks if current ENR matches that found on disk
-        let local_enr = load_enr(local_key, config, &log)?;
-
-        let enr_dir = match config.network_dir.to_str() {
-            Some(path) => String::from(path),
-            None => String::from(""),
-        };
-
-        info!(log, "ENR Initialised"; "enr" => local_enr.to_base64(), "seq" => local_enr.seq());
-        debug!(log, "Discv5 Node ID Initialised"; "node_id" => format!("{}",local_enr.node_id()));
-
-        // the last parameter enables IP limiting. 2 Nodes on the same /24 subnet per bucket and 10
-        // nodes on the same /24 subnet per table.
-        // TODO: IP filtering is currently disabled for the DHT. Enable for production
-        let mut discovery = Discv5::new(local_enr, local_key.clone(), config.listen_address, false)
-            .map_err(|e| format!("Discv5 service failed. Error: {:?}", e))?;
-
-        // Add bootnodes to routing table
-        for bootnode_enr in config.boot_nodes.clone() {
-            debug!(
-                log,
-                "Adding node to routing table";
-                "node_id" => format!("{}",
-                bootnode_enr.node_id())
-            );
-            discovery.add_enr(bootnode_enr);
-        }
-
-        Ok(Self {
-            connected_peers: HashSet::new(),
-            banned_peers: HashSet::new(),
-            max_peers: config.max_peers,
-            peer_discovery_delay: Delay::new(Instant::now()),
-            past_discovery_delay: INITIAL_SEARCH_DELAY,
-            tcp_port: config.libp2p_port,
-            discovery,
-            log,
-            enr_dir,
-        })
-    }
-
-    /// Allows the application layer to update the `IP` and `port` of the local ENR. The second
-    /// parameter defines whether the port is a TCP port. If false, this is interpreted as a UDP
-    /// port.
-    pub fn update_local_enr(&mut self, socket: std::net::SocketAddr, is_tcp: bool) {
-        // discv5 checks to see if an update is necessary before performing it, so we do not
-        // need to check here
-        if self.discovery.update_local_enr_socket(socket, is_tcp) {
-            let enr = self.discovery.local_enr();
-            info!(
-            self.log,
-            "ENR Updated";
-            "enr" => enr.to_base64(), 
-            "seq" => enr.seq(),
-            "address" => format!("{:?}", socket));
-            save_enr_to_disc(Path::new(&self.enr_dir), enr, &self.log)
-        }
-    }
-
-    /// Return the nodes local ENR.
-    pub fn local_enr(&self) -> &Enr {
-        self.discovery.local_enr()
-    }
-
-    /// Manually search for peers. This restarts the discovery round, sparking multiple rapid
-    /// queries.
-    pub fn discover_peers(&mut self) {
-        self.past_discovery_delay = INITIAL_SEARCH_DELAY;
-        self.find_peers();
-    }
-
-    /// Add an ENR to the routing table of the discovery mechanism.
-    pub fn add_enr(&mut self, enr: Enr) {
-        self.discovery.add_enr(enr);
-    }
-
-    /// The current number of connected libp2p peers.
-    pub fn connected_peers(&self) -> usize {
-        self.connected_peers.len()
-    }
-
-    /// The current number of connected libp2p peers.
-    pub fn connected_peer_set(&self) -> &HashSet<PeerId> {
-        &self.connected_peers
-    }
-
-    /// The peer has been banned. Add this peer to the banned list to prevent any future
-    /// re-connections.
-    // TODO: Remove the peer from the DHT if present
-    pub fn peer_banned(&mut self, peer_id: PeerId) {
-        self.banned_peers.insert(peer_id);
-    }
-
-    pub fn peer_unbanned(&mut self, peer_id: &PeerId) {
-        self.banned_peers.remove(peer_id);
-    }
-
-    /// Search for new peers using the underlying discovery mechanism.
-    fn find_peers(&mut self) {
-        // pick a random NodeId
-        let random_node = NodeId::random();
-        debug!(self.log, "Searching for peers");
-        self.discovery.find_node(random_node);
-    }
-}
-
-// Redirect all behaviour events to underlying discovery behaviour.
-impl<TSubstream> NetworkBehaviour for Discovery<TSubstream>
-where
-    TSubstream: AsyncRead + AsyncWrite,
-{
-    type ProtocolsHandler = <Discv5<TSubstream> as NetworkBehaviour>::ProtocolsHandler;
-    type OutEvent = <Discv5<TSubstream> as NetworkBehaviour>::OutEvent;
-
-    fn new_handler(&mut self) -> Self::ProtocolsHandler {
-        NetworkBehaviour::new_handler(&mut self.discovery)
-    }
-
-    fn addresses_of_peer(&mut self, peer_id: &PeerId) -> Vec<Multiaddr> {
-        // Let discovery track possible known peers.
-        self.discovery.addresses_of_peer(peer_id)
-    }
-
-    fn inject_connected(&mut self, peer_id: PeerId, _endpoint: ConnectedPoint) {
-        self.connected_peers.insert(peer_id);
-        // TODO: Drop peers if over max_peer limit
-
-        metrics::inc_counter(&metrics::PEER_CONNECT_EVENT_COUNT);
-        metrics::set_gauge(&metrics::PEERS_CONNECTED, self.connected_peers() as i64);
-    }
-
-    fn inject_disconnected(&mut self, peer_id: &PeerId, _endpoint: ConnectedPoint) {
-        self.connected_peers.remove(peer_id);
-
-        metrics::inc_counter(&metrics::PEER_DISCONNECT_EVENT_COUNT);
-        metrics::set_gauge(&metrics::PEERS_CONNECTED, self.connected_peers() as i64);
-    }
-
-    fn inject_replaced(
-        &mut self,
-        _peer_id: PeerId,
-        _closed: ConnectedPoint,
-        _opened: ConnectedPoint,
-    ) {
-        // discv5 doesn't implement
-    }
-
-    fn inject_node_event(
-        &mut self,
-        _peer_id: PeerId,
-        _event: <Self::ProtocolsHandler as ProtocolsHandler>::OutEvent,
-    ) {
-        // discv5 doesn't implement
-    }
-
-    fn poll(
-        &mut self,
-        params: &mut impl PollParameters,
-    ) -> Async<
-        NetworkBehaviourAction<
-            <Self::ProtocolsHandler as ProtocolsHandler>::InEvent,
-            Self::OutEvent,
-        >,
-    > {
-        // search for peers if it is time
-        loop {
-            match self.peer_discovery_delay.poll() {
-                Ok(Async::Ready(_)) => {
-                    if self.connected_peers.len() < self.max_peers {
-                        self.find_peers();
-                    }
-                    // Set to maximum, and update to earlier, once we get our results back.
-                    self.peer_discovery_delay.reset(
-                        Instant::now() + Duration::from_secs(MAX_TIME_BETWEEN_PEER_SEARCHES),
-                    );
-                }
-                Ok(Async::NotReady) => break,
-                Err(e) => {
-                    warn!(self.log, "Discovery peer search failed"; "error" => format!("{:?}", e));
-                }
-            }
-        }
-
-        // Poll discovery
-        loop {
-            match self.discovery.poll(params) {
-                Async::Ready(NetworkBehaviourAction::GenerateEvent(event)) => {
-                    match event {
-                        Discv5Event::Discovered(_enr) => {
-                            // not concerned about FINDNODE results, rather the result of an entire
-                            // query.
-                        }
-                        Discv5Event::SocketUpdated(socket) => {
-                            info!(self.log, "Address updated"; "ip" => format!("{}",socket.ip()), "udp_port" => format!("{}", socket.port()));
-                            metrics::inc_counter(&metrics::ADDRESS_UPDATE_COUNT);
-                            let mut address = Multiaddr::from(socket.ip());
-                            address.push(Protocol::Tcp(self.tcp_port));
-                            let enr = self.discovery.local_enr();
-                            save_enr_to_disc(Path::new(&self.enr_dir), enr, &self.log);
-
-                            return Async::Ready(NetworkBehaviourAction::ReportObservedAddr {
-                                address,
-                            });
-                        }
-                        Discv5Event::FindNodeResult { closer_peers, .. } => {
-                            debug!(self.log, "Discovery query completed"; "peers_found" => closer_peers.len());
-                            // update the time to the next query
-                            if self.past_discovery_delay < MAX_TIME_BETWEEN_PEER_SEARCHES {
-                                self.past_discovery_delay *= 2;
-                            }
-                            let delay = std::cmp::max(
-                                self.past_discovery_delay,
-                                MAX_TIME_BETWEEN_PEER_SEARCHES,
-                            );
-                            self.peer_discovery_delay
-                                .reset(Instant::now() + Duration::from_secs(delay));
-
-                            if closer_peers.is_empty() {
-                                debug!(self.log, "Discovery random query found no peers");
-                            }
-                            for peer_id in closer_peers {
-                                // if we need more peers, attempt a connection
-                                if self.connected_peers.len() < self.max_peers
-                                    && self.connected_peers.get(&peer_id).is_none()
-                                    && !self.banned_peers.contains(&peer_id)
-                                {
-                                    debug!(self.log, "Peer discovered"; "peer_id"=> format!("{:?}", peer_id));
-                                    return Async::Ready(NetworkBehaviourAction::DialPeer {
-                                        peer_id,
-                                    });
-                                }
-                            }
-                        }
-                        _ => {}
-                    }
-                }
-                // discv5 does not output any other NetworkBehaviourAction
-                Async::Ready(_) => {}
-                Async::NotReady => break,
-            }
-        }
-        Async::NotReady
-    }
-}
-
-/// Loads an ENR from file if it exists and matches the current NodeId and sequence number. If none
-/// exists, generates a new one.
-///
-/// If an ENR exists, with the same NodeId and IP address, we use the disk-generated one as its
-/// ENR sequence will be equal or higher than a newly generated one.
-fn load_enr(
-    local_key: &Keypair,
-    config: &NetworkConfig,
-    log: &slog::Logger,
-) -> Result<Enr, String> {
-    // Build the local ENR.
-    // Note: Discovery should update the ENR record's IP to the external IP as seen by the
-    // majority of our peers.
-    let mut local_enr = EnrBuilder::new("v4")
-        .ip(config.discovery_address)
-        .tcp(config.libp2p_port)
-        .udp(config.discovery_port)
-        .build(&local_key)
-        .map_err(|e| format!("Could not build Local ENR: {:?}", e))?;
-
-    let enr_f = config.network_dir.join(ENR_FILENAME);
-    if let Ok(mut enr_file) = File::open(enr_f.clone()) {
-        let mut enr_string = String::new();
-        match enr_file.read_to_string(&mut enr_string) {
-            Err(_) => debug!(log, "Could not read ENR from file"),
-            Ok(_) => {
-                match Enr::from_str(&enr_string) {
-                    Ok(enr) => {
-                        if enr.node_id() == local_enr.node_id() {
-                            if enr.ip().map(Into::into) == Some(config.discovery_address)
-                                && enr.tcp() == Some(config.libp2p_port)
-                                && enr.udp() == Some(config.discovery_port)
-                            {
-                                debug!(log, "ENR loaded from file"; "file" => format!("{:?}", enr_f));
-                                // the stored ENR has the same configuration, use it
-                                return Ok(enr);
-                            }
-
-                            // same node id, different configuration - update the sequence number
-                            let new_seq_no = enr.seq().checked_add(1).ok_or_else(|| "ENR sequence number on file is too large. Remove it to generate a new NodeId")?;
-                            local_enr.set_seq(new_seq_no, local_key).map_err(|e| {
-                                format!("Could not update ENR sequence number: {:?}", e)
-                            })?;
-                            debug!(log, "ENR sequence number increased"; "seq" =>  new_seq_no);
-                        }
-                    }
-                    Err(e) => {
-                        warn!(log, "ENR from file could not be decoded"; "error" => format!("{:?}", e));
-                    }
-                }
-            }
-        }
-    }
-
-    save_enr_to_disc(&config.network_dir, &local_enr, log);
-
-    Ok(local_enr)
-}
-
-fn save_enr_to_disc(dir: &Path, enr: &Enr, log: &slog::Logger) {
-    let _ = std::fs::create_dir_all(dir);
-    match File::create(dir.join(Path::new(ENR_FILENAME)))
-        .and_then(|mut f| f.write_all(&enr.to_base64().as_bytes()))
-    {
-        Ok(_) => {
-            debug!(log, "ENR written to disk");
-        }
-        Err(e) => {
-            warn!(
-                log,
-                "Could not write ENR to file"; "file" => format!("{:?}{:?}",dir, ENR_FILENAME),  "error" => format!("{}", e)
-            );
-        }
-    }
-}

beacon_node/eth2-libp2p/src/lib.rs

@@ -1,31 +0,0 @@
-/// This crate contains the main link for lighthouse to rust-libp2p. It therefore re-exports
-/// all required libp2p functionality.
-///
-/// This crate builds and manages the libp2p services required by the beacon node.
-#[macro_use]
-extern crate lazy_static;
-
-pub mod behaviour;
-mod config;
-mod discovery;
-pub mod error;
-mod metrics;
-pub mod rpc;
-mod service;
-
-pub use behaviour::PubsubMessage;
-pub use config::{
-    Config as NetworkConfig, BEACON_ATTESTATION_TOPIC, BEACON_BLOCK_TOPIC, SHARD_TOPIC_PREFIX,
-    TOPIC_ENCODING_POSTFIX, TOPIC_PREFIX,
-};
-pub use libp2p::enr::Enr;
-pub use libp2p::gossipsub::{Topic, TopicHash};
-pub use libp2p::multiaddr;
-pub use libp2p::Multiaddr;
-pub use libp2p::{
-    gossipsub::{GossipsubConfig, GossipsubConfigBuilder},
-    PeerId, Swarm,
-};
-pub use rpc::RPCEvent;
-pub use service::Libp2pEvent;
-pub use service::Service;