v1.1.0 (#2168)

## Issue Addressed

NA

## Proposed Changes

- Bump version
- ~~Run `cargo update`~~

## Additional Info

NA

.dockerignore

@@ -1,4 +1,4 @@
-tests/ef_tests/eth2.0-spec-tests
+testing/ef_tests/eth2.0-spec-tests
 target/
 *.data
 *.tar.gz

.github/workflows/book.yml

@@ -3,7 +3,7 @@ name: mdbook
 on:
   push:
     branches:
-      - master
+      - unstable
 
 jobs:
   build-and-upload-to-s3:

.github/workflows/docker.yml

@@ -0,0 +1,94 @@
+name: docker
+
+on:
+    push:
+        branches:
+            - unstable
+            - stable
+
+env:
+    DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+    DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+    IMAGE_NAME: sigp/lighthouse
+
+jobs:
+    extract-branch-name:
+        runs-on: ubuntu-18.04
+        steps:
+            - name: Extract branch name
+              run: echo "::set-output name=BRANCH_NAME::$(echo ${GITHUB_REF#refs/heads/})"
+              id: extract_branch
+        outputs:
+            BRANCH_NAME: ${{ steps.extract_branch.outputs.BRANCH_NAME }}
+    build-docker-arm64:
+        runs-on: ubuntu-18.04
+        needs: [extract-branch-name]
+        # We need to enable experimental docker features in order to use `docker buildx`
+        env:
+            DOCKER_CLI_EXPERIMENTAL: enabled
+        steps:
+            - uses: actions/checkout@v2
+            - name: Dockerhub login
+              run: |
+                  echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin
+            - name: Cross build lighthouse binary
+              run:  |
+                  cargo install cross
+                  make build-aarch64-portable
+            - name: Move cross-built ARM binary into Docker scope
+              run: |
+                  mkdir ./bin;
+                  mv ./target/aarch64-unknown-linux-gnu/release/lighthouse ./bin;
+            - name: Set Env
+              if: needs.extract-branch-name.outputs.BRANCH_NAME == 'unstable'
+              run: |
+                  echo "TAG_SUFFIX=-unstable" >> $GITHUB_ENV;
+            # Install dependencies for emulation. Have to create a new builder to pick up emulation support.
+            - name: Build ARM64 dockerfile (with push)
+              run: |
+                  docker run --privileged --rm tonistiigi/binfmt --install arm64
+                  docker buildx create --use --name cross-builder
+                  docker buildx build \
+                      --platform=linux/arm64 \
+                      --file ./Dockerfile.cross . \
+                      --tag ${IMAGE_NAME}:latest-arm64${TAG_SUFFIX} \
+                      --push
+    build-docker-amd64:
+        runs-on: ubuntu-18.04
+        needs: [extract-branch-name]
+        steps:
+            - uses: actions/checkout@v2
+            - name: Dockerhub login
+              run: |
+                  echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin
+            - name: Set Env
+              if: needs.extract-branch-name.outputs.BRANCH_NAME == 'unstable'
+              run: |
+                  echo "TAG_SUFFIX=-unstable" >> $GITHUB_ENV;
+            - name: Build AMD64 dockerfile (with push)
+              run: |
+                  docker build \
+                      --build-arg PORTABLE=true \
+                      --tag ${IMAGE_NAME}:latest-amd64${TAG_SUFFIX} \
+                      --file ./Dockerfile .
+                  docker push ${IMAGE_NAME}:latest-amd64${TAG_SUFFIX}
+    build-docker-multiarch:
+        runs-on: ubuntu-18.04
+        needs: [build-docker-arm64, build-docker-amd64, extract-branch-name]
+        # We need to enable experimental docker features in order to use `docker manifest`
+        env:
+            DOCKER_CLI_EXPERIMENTAL: enabled
+        steps:
+            - name: Dockerhub login
+              run: |
+                  echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin
+            - name: Set Env
+              if: needs.extract-branch-name.outputs.BRANCH_NAME == 'unstable'
+              run: |
+                  echo "TAG_SUFFIX=-unstable" >> $GITHUB_ENV;
+            - name: Create and push multiarch manifest
+              run: |
+                  docker manifest create ${IMAGE_NAME}:latest${TAG_SUFFIX} \
+                      --amend ${IMAGE_NAME}:latest-arm64${TAG_SUFFIX} \
+                      --amend ${IMAGE_NAME}:latest-amd64${TAG_SUFFIX};
+                  docker manifest push ${IMAGE_NAME}:latest${TAG_SUFFIX}

.github/workflows/release.yml

@@ -0,0 +1,269 @@
+name:   Release Suite
+
+on:
+    push:
+        tags:
+            - v*
+
+env:
+    DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+    DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+    REPO_NAME: sigp/lighthouse
+    IMAGE_NAME: sigp/lighthouse
+
+jobs:
+    extract-version:
+        runs-on: ubuntu-latest
+        steps:
+            - name: Extract version
+              run: echo "::set-output name=VERSION::$(echo ${GITHUB_REF#refs/tags/})"
+              id: extract_version
+        outputs:
+            VERSION: ${{ steps.extract_version.outputs.VERSION }}
+
+    build-docker-arm64:
+        runs-on: ubuntu-18.04
+        needs: [extract-version]
+        # We need to enable experimental docker features in order to use `docker buildx`
+        env:
+            DOCKER_CLI_EXPERIMENTAL: enabled
+            VERSION: ${{ needs.extract-version.outputs.VERSION }}
+        steps:
+            - uses: actions/checkout@v2
+            - name: Dockerhub login
+              run: |
+                  echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin
+            - name: Cross build lighthouse binary
+              run:  |
+                  cargo install cross
+                  make build-aarch64-portable
+            - name: Move cross-built ARM binary into Docker scope
+              run: |
+                  mkdir ./bin;
+                  mv ./target/aarch64-unknown-linux-gnu/release/lighthouse ./bin;
+            # Install dependencies for emulation. Have to create a new builder to pick up emulation support.
+            - name: Build ARM64 dockerfile (with push)
+              run: |
+                  docker run --privileged --rm tonistiigi/binfmt --install arm64
+                  docker buildx create --use --name cross-builder
+                  docker buildx build \
+                      --platform=linux/arm64 \
+                      --file ./Dockerfile.cross . \
+                      --tag ${IMAGE_NAME}:${{ env.VERSION }}-arm64 \
+                      --push
+    build-docker-amd64:
+        runs-on: ubuntu-18.04
+        needs: [extract-version]
+        env:
+            DOCKER_CLI_EXPERIMENTAL: enabled
+            VERSION: ${{ needs.extract-version.outputs.VERSION }}
+        steps:
+            - uses: actions/checkout@v2
+            - name: Dockerhub login
+              run: |
+                  echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin
+            - name: Build AMD64 dockerfile (with push)
+              run: |
+                  docker build \
+                      --build-arg PORTABLE=true \
+                      --tag ${IMAGE_NAME}:${{ env.VERSION }}-amd64 \
+                      --file ./Dockerfile .
+                  docker push ${IMAGE_NAME}:${{ env.VERSION }}-amd64
+    build-docker-multiarch:
+        runs-on: ubuntu-18.04
+        needs: [build-docker-arm64, build-docker-amd64, extract-version]
+        # We need to enable experimental docker features in order to use `docker manifest`
+        env:
+            DOCKER_CLI_EXPERIMENTAL: enabled
+            VERSION: ${{ needs.extract-version.outputs.VERSION }}
+        steps:
+            - name: Dockerhub login
+              run: |
+                  echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin
+            - name: Create and push multiarch manifest
+              run: |
+                  docker manifest create ${IMAGE_NAME}:${{ env.VERSION }} \
+                      --amend ${IMAGE_NAME}:${{ env.VERSION }}-arm64 \
+                      --amend ${IMAGE_NAME}:${{ env.VERSION }}-amd64;
+                  docker manifest push ${IMAGE_NAME}:${{ env.VERSION }}
+    build:
+        name:   Build Release
+        strategy:
+            matrix:
+                arch: [aarch64-unknown-linux-gnu,
+                       aarch64-unknown-linux-gnu-portable,
+                       x86_64-unknown-linux-gnu,
+                       x86_64-unknown-linux-gnu-portable,
+                       x86_64-apple-darwin,
+                       x86_64-apple-darwin-portable]
+                include:
+                    -   arch: aarch64-unknown-linux-gnu
+                        platform: ubuntu-latest
+                    -   arch: aarch64-unknown-linux-gnu-portable
+                        platform: ubuntu-latest
+                    -   arch: x86_64-unknown-linux-gnu
+                        platform: ubuntu-latest
+                    -   arch: x86_64-unknown-linux-gnu-portable
+                        platform: ubuntu-latest
+                    -   arch: x86_64-apple-darwin
+                        platform: macos-latest
+                    -   arch: x86_64-apple-darwin-portable
+                        platform: macos-latest
+
+        runs-on:    ${{ matrix.platform }}
+        needs: extract-version
+        steps:
+            - name: Checkout sources
+              uses: actions/checkout@v2
+            - name: Build toolchain
+              uses: actions-rs/toolchain@v1
+              with:
+                  toolchain: stable
+                  profile:  minimal
+                  override: true
+
+            # ==============================
+            #       Builds
+            # ==============================
+
+            - name: Build Lighthouse for aarch64-unknown-linux-gnu-portable
+              if:   matrix.arch == 'aarch64-unknown-linux-gnu-portable'
+              run:  |
+                cargo install cross
+                make build-aarch64-portable
+
+            - name: Build Lighthouse for aarch64-unknown-linux-gnu
+              if:   matrix.arch == 'aarch64-unknown-linux-gnu'
+              run:  |
+                cargo install cross
+                make build-aarch64
+
+            - name: Build Lighthouse for x86_64-unknown-linux-gnu-portable
+              if:   matrix.arch == 'x86_64-unknown-linux-gnu-portable'
+              run:  |
+                cargo install cross
+                make build-x86_64-portable
+
+            - name: Build Lighthouse for x86_64-unknown-linux-gnu
+              if:   matrix.arch == 'x86_64-unknown-linux-gnu'
+              run:  |
+                cargo install cross
+                make build-x86_64
+
+            - name: Move cross-compiled binary
+              if:  startsWith(matrix.arch, 'aarch64')
+              run: mv target/aarch64-unknown-linux-gnu/release/lighthouse ~/.cargo/bin/lighthouse
+
+            - name: Move cross-compiled binary
+              if:  startsWith(matrix.arch, 'x86_64-unknown-linux-gnu')
+              run: mv target/x86_64-unknown-linux-gnu/release/lighthouse ~/.cargo/bin/lighthouse
+
+            - name: Build Lighthouse for x86_64-apple-darwin portable
+              if:   matrix.arch == 'x86_64-apple-darwin-portable'
+              run:  cargo install --path lighthouse --force --locked --features portable
+
+            - name: Build Lighthouse for x86_64-apple-darwin modern
+              if:   matrix.arch == 'x86_64-apple-darwin'
+              run:  cargo install --path lighthouse --force --locked --features modern
+
+            - name: Configure GPG and create artifacts
+              env:
+                  GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
+                  GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
+              run: |
+                  export GPG_TTY=$(tty)
+                  echo "$GPG_SIGNING_KEY" | gpg --batch --import
+                  mkdir artifacts
+                  mv ~/.cargo/bin/lighthouse ./artifacts
+                  cd artifacts
+                  tar -czf lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz lighthouse
+                  echo "$GPG_PASSPHRASE" | gpg --passphrase-fd 0 --pinentry-mode loopback --batch -ab lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz
+                  mv *tar.gz* ..
+
+            # =======================================================================
+            # Upload artifacts
+            # This is required to share artifacts between different jobs
+            # =======================================================================
+
+            - name:  Upload artifact
+              uses:  actions/upload-artifact@v2
+              with:
+                  name: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz
+                  path: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz
+
+            - name: Upload signature
+              uses: actions/upload-artifact@v2
+              with:
+                  name: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc
+                  path: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc
+
+    draft-release:
+        name:   Draft Release
+        needs:  [build, extract-version]
+        runs-on: ubuntu-latest
+        env:
+            VERSION:  ${{ needs.extract-version.outputs.VERSION }}
+        steps:
+            # This is necessary for generating the changelog. It has to come before "Download Artifacts" or else it deletes the artifacts.
+            - name: Checkout sources
+              uses: actions/checkout@v2
+              with:
+                  fetch-depth: 0
+
+            # ==============================
+            #       Download artifacts
+            # ==============================
+
+            - name: Download artifacts
+              uses: actions/download-artifact@v2
+
+            # ==============================
+            #       Create release draft
+            # ==============================
+
+            - name: Generate Full Changelog
+              id: changelog
+              run: echo "::set-output name=CHANGELOG::$(git log --pretty=format:"- %s" $(git describe --tags --abbrev=0 ${{ env.VERSION }}^)..${{ env.VERSION }})"
+
+            - name: Create Release Draft
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+              # The formatting here is borrowed from OpenEthereum: https://github.com/openethereum/openethereum/blob/main/.github/workflows/build.yml
+              run: |
+                  body=$(cat <<- "ENDBODY"
+                  <Rick and Morty character>
+
+                  ## Summary
+
+                  Add a summary.
+
+                  ## All Changes
+
+                  ${{ steps.changelog.outputs.CHANGELOG }}
+
+                  ## Binaries
+
+                  [See pre-built binaries documentation.](https://lighthouse-book.sigmaprime.io/installation-binaries.html)
+
+                  The binaries are signed with Sigma Prime's PGP key: `15E66D941F697E28F49381F426416DC3F30674B0`
+
+                  | System | Architecture | Binary | PGP Signature |
+                  |:---:|:---:|:---:|:---|
+                  | <img src="https://simpleicons.org/icons/apple.svg" style="width: 32px;"/> | x86_64 | [lighthouse-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz.asc) |
+                  | <img src="https://simpleicons.org/icons/apple.svg" style="width: 32px;"/> | x86_64 | [lighthouse-${{ env.VERSION }}-x86_64-apple-darwin-portable.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-apple-darwin-portable.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-apple-darwin-portable.tar.gz.asc) |
+                  | <img src="https://simpleicons.org/icons/linux.svg" style="width: 32px;"/> | x86_64 | [lighthouse-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz.asc) |
+                  | <img src="https://simpleicons.org/icons/linux.svg" style="width: 32px;"/> | x86_64 | [lighthouse-${{ env.VERSION }}-x86_64-unknown-linux-gnu-portable.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-unknown-linux-gnu-portable.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-unknown-linux-gnu-portable.tar.gz.asc) |
+                  | <img src="https://simpleicons.org/icons/raspberrypi.svg" style="width: 32px;"/> | aarch64 | [lighthouse-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz.asc) |
+                  | <img src="https://simpleicons.org/icons/raspberrypi.svg" style="width: 32px;"/> | aarch64 | [lighthouse-${{ env.VERSION }}-aarch64-unknown-linux-gnu-portable.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64-unknown-linux-gnu-portable.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64-unknown-linux-gnu-portable.tar.gz.asc) |
+                  | | | | |
+                  | **System** | **Option** | - | **Resource** |
+                  | <img src="https://simpleicons.org/icons/docker.svg" style="width: 32px;"/> | Docker | [${{ env.VERSION }}](https://hub.docker.com/r/${{ env.IMAGE_NAME }}/tags?page=1&ordering=last_updated&name=${{ env.VERSION }}) | [${{ env.IMAGE_NAME }}](https://hub.docker.com/r/${{ env.IMAGE_NAME }}) |
+                  ENDBODY
+                  )
+                  assets=()
+                  for asset in ./lighthouse-*.tar.gz*; do
+                      assets+=("-a" "$asset/$asset")
+                  done
+                  tag_name="${{ env.VERSION }}"
+                  echo "$body" | hub release create --draft "${assets[@]}" -F "-" "$tag_name"

.github/workflows/test-suite.yml

@@ -33,20 +33,6 @@ jobs:
       run: sudo npm install -g ganache-cli
     - name: Run tests in release
       run: make test-release
-  release-tests-and-install-macos:
-    name: release-tests-and-install-macos
-    runs-on: macos-latest
-    needs: cargo-fmt
-    steps:
-    - uses: actions/checkout@v1
-    - name: Get latest version of stable Rust
-      run: rustup update stable
-    - name: Install ganache-cli
-      run: sudo npm install -g ganache-cli
-    - name: Run tests in release
-      run: make test-release
-    - name: Install Lighthouse
-      run: make
   debug-tests-ubuntu:
     name: debug-tests-ubuntu
     runs-on: ubuntu-latest
@@ -107,6 +93,16 @@ jobs:
       run: sudo npm install -g ganache-cli
     - name: Run the beacon chain sim without an eth1 connection
       run: cargo run --release --bin simulator no-eth1-sim
+  syncing-simulator-ubuntu:
+    name: syncing-simulator-ubuntu
+    runs-on: ubuntu-latest
+    needs: cargo-fmt
+    steps:
+      - uses: actions/checkout@v1
+      - name: Install ganache-cli
+        run: sudo npm install -g ganache-cli
+      - name: Run the syncing simulator
+        run: cargo run --release --bin simulator syncing-sim
   check-benchmarks:
     name: check-benchmarks
     runs-on: ubuntu-latest
@@ -115,6 +111,14 @@ jobs:
     - uses: actions/checkout@v1
     - name: Typecheck benchmark code without running it
       run: make check-benches
+  check-consensus:
+    name: check-consensus
+    runs-on: ubuntu-latest
+    needs: cargo-fmt
+    steps:
+    - uses: actions/checkout@v1
+    - name: Typecheck consensus code in strict mode
+      run: make check-consensus
   clippy:
     name: clippy
     runs-on: ubuntu-latest
@@ -123,6 +127,8 @@ jobs:
     - uses: actions/checkout@v1
     - name: Lint code for quality and style with Clippy
       run: make lint
+    - name: Certify Cargo.lock freshness
+      run: git diff --exit-code Cargo.lock
   arbitrary-check:
     name: arbitrary-check
     runs-on: ubuntu-latest

.gitignore

@@ -6,3 +6,5 @@ target/
 flamegraph.svg
 perf.data*
 *.tar.gz
+/bin
+genesis.ssz

Cargo.toml

@@ -7,11 +7,11 @@ members = [
     "beacon_node/client",
     "beacon_node/eth1",
     "beacon_node/eth2_libp2p",
+    "beacon_node/http_api",
+    "beacon_node/http_metrics",
     "beacon_node/network",
-    "beacon_node/rest_api",
     "beacon_node/store",
     "beacon_node/timer",
-    "beacon_node/websocket_server",
 
 	"boot_node",
 
@@ -20,19 +20,25 @@ members = [
     "common/compare_fields",
     "common/compare_fields_derive",
     "common/deposit_contract",
+    "common/directory",
+    "common/eth2",
     "common/eth2_config",
     "common/eth2_interop_keypairs",
-    "common/eth2_testnet_config",
+    "common/eth2_network_config",
     "common/eth2_wallet_manager",
     "common/hashset_delay",
     "common/lighthouse_metrics",
     "common/lighthouse_version",
+    "common/lockfile",
     "common/logging",
-    "common/remote_beacon_node",
-    "common/rest_types",
+    "common/lru_cache",
+    "common/remote_signer_consumer",
     "common/slot_clock",
+    "common/task_executor",
     "common/test_random_derive",
     "common/validator_dir",
+    "common/warp_utils",
+    "common/fallback",
 
     "consensus/cached_tree_hash",
     "consensus/int_to_bytes",
@@ -42,7 +48,7 @@ members = [
     "consensus/ssz",
     "consensus/ssz_derive",
     "consensus/ssz_types",
-    "consensus/serde_hex",
+    "consensus/serde_utils",
     "consensus/state_processing",
     "consensus/swap_or_not_shuffle",
     "consensus/tree_hash",
@@ -59,10 +65,18 @@ members = [
     "lighthouse",
     "lighthouse/environment",
 
-    "testing/simulator",
+    "remote_signer",
+    "remote_signer/backend",
+    "remote_signer/client",
+
+    "slasher",
+    "slasher/service",
+
     "testing/ef_tests",
     "testing/eth1_test_rig",
     "testing/node_test_rig",
+    "testing/remote_signer_test",
+    "testing/simulator",
     "testing/state_transition_vectors",
 
     "validator_client",
@@ -77,4 +91,3 @@ eth2_ssz = { path = "consensus/ssz" }
 eth2_ssz_derive = { path = "consensus/ssz_derive" }
 eth2_ssz_types = { path = "consensus/ssz_types" }
 eth2_hashing = { path = "crypto/eth2_hashing" }
-leveldb-sys = { git = "https://github.com/michaelsproul/leveldb-sys", branch = "v2.0.6-cmake" }

Cross.toml

@@ -0,0 +1,4 @@
+[build.env]
+passthrough = [
+    "RUSTFLAGS",
+]

Dockerfile

@@ -1,10 +1,9 @@
-FROM rust:1.45.1 AS builder
+FROM rust:1.47.0 AS builder
 RUN apt-get update && apt-get install -y cmake
 COPY . lighthouse
 ARG PORTABLE
 ENV PORTABLE $PORTABLE
 RUN cd lighthouse && make
-RUN cd lighthouse && make install-lcli
 
 FROM debian:buster-slim
 RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -13,4 +12,3 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/*
 COPY --from=builder /usr/local/cargo/bin/lighthouse /usr/local/bin/lighthouse
-COPY --from=builder /usr/local/cargo/bin/lcli /usr/local/bin/lcli

Dockerfile.cross

@@ -0,0 +1,10 @@
+# This image is meant to enable cross-architecture builds.
+# It assumes the lighthouse binary has already been
+# compiled for `$TARGETPLATFORM` and moved to `./bin`.
+FROM --platform=$TARGETPLATFORM debian:buster-slim
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  libssl-dev \
+  ca-certificates \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+COPY ./bin/lighthouse /usr/local/bin/lighthouse

Makefile

@@ -2,6 +2,13 @@
 
 EF_TESTS = "testing/ef_tests"
 STATE_TRANSITION_VECTORS = "testing/state_transition_vectors"
+GIT_TAG := $(shell git describe --tags --candidates 1)
+BIN_DIR = "bin"
+
+X86_64_TAG = "x86_64-unknown-linux-gnu"
+BUILD_PATH_X86_64 = "target/$(X86_64_TAG)/release"
+AARCH64_TAG = "aarch64-unknown-linux-gnu"
+BUILD_PATH_AARCH64 = "target/$(AARCH64_TAG)/release"
 
 # Builds the Lighthouse binary in release (optimized).
 #
@@ -21,6 +28,52 @@ else
 	cargo install --path lcli --force --locked
 endif
 
+# The following commands use `cross` to build a cross-compile.
+#
+# These commands require that:
+#
+# - `cross` is installed (`cargo install cross`).
+# - Docker is running.
+# - The current user is in the `docker` group.
+#
+# The resulting binaries will be created in the `target/` directory.
+#
+# The *-portable options compile the blst library *without* the use of some
+# optimized CPU functions that may not be available on some systems. This
+# results in a more portable binary with ~20% slower BLS verification.
+build-x86_64:
+	cross build --release --manifest-path lighthouse/Cargo.toml --target x86_64-unknown-linux-gnu --features modern
+build-x86_64-portable:
+	cross build --release --manifest-path lighthouse/Cargo.toml --target x86_64-unknown-linux-gnu --features portable
+build-aarch64:
+	cross build --release --manifest-path lighthouse/Cargo.toml --target aarch64-unknown-linux-gnu
+build-aarch64-portable:
+	cross build --release --manifest-path lighthouse/Cargo.toml --target aarch64-unknown-linux-gnu --features portable
+
+# Create a `.tar.gz` containing a binary for a specific target.
+define tarball_release_binary
+	cp $(1)/lighthouse $(BIN_DIR)/lighthouse
+	cd $(BIN_DIR) && \
+		tar -czf lighthouse-$(GIT_TAG)-$(2)$(3).tar.gz lighthouse && \
+		rm lighthouse
+endef
+
+# Create a series of `.tar.gz` files in the BIN_DIR directory, each containing
+# a `lighthouse` binary for a different target.
+#
+# The current git tag will be used as the version in the output file names. You
+# will likely need to use `git tag` and create a semver tag (e.g., `v0.2.3`).
+build-release-tarballs:
+	[ -d $(BIN_DIR) ] || mkdir -p $(BIN_DIR)
+	$(MAKE) build-x86_64
+	$(call tarball_release_binary,$(BUILD_PATH_X86_64),$(X86_64_TAG),"")
+	$(MAKE) build-x86_64-portable
+	$(call tarball_release_binary,$(BUILD_PATH_X86_64),$(X86_64_TAG),"-portable")
+	$(MAKE) build-aarch64
+	$(call tarball_release_binary,$(BUILD_PATH_AARCH64),$(AARCH64_TAG),"")
+	$(MAKE) build-aarch64-portable
+	$(call tarball_release_binary,$(BUILD_PATH_AARCH64),$(AARCH64_TAG),"-portable")
+
 # Runs the full workspace tests in **release**, without downloading any additional
 # test vectors.
 test-release:
@@ -39,6 +92,10 @@ cargo-fmt:
 check-benches:
 	cargo check --all --benches
 
+# Typechecks consensus code *without* allowing deprecated legacy arithmetic
+check-consensus:
+	cargo check --manifest-path=consensus/state_processing/Cargo.toml --no-default-features
+
 # Runs only the ef-test vectors.
 run-ef-tests:
 	cargo test --release --manifest-path=$(EF_TESTS)/Cargo.toml --features "ef_tests"
@@ -79,7 +136,8 @@ arbitrary-fuzz:
 # Runs cargo audit (Audit Cargo.lock files for crates with security vulnerabilities reported to the RustSec Advisory Database)
 audit:
 	cargo install --force cargo-audit
-	cargo audit
+	# TODO: we should address this --ignore.
+	cargo audit --ignore RUSTSEC-2016-0002 --ignore RUSTSEC-2020-0008 --ignore RUSTSEC-2017-0002
 
 # Runs `cargo udeps` to check for unused dependencies
 udeps:

README.md

@@ -2,76 +2,65 @@
 
 An open-source Ethereum 2.0 client, written in Rust and maintained by Sigma Prime.
 
-[![Build Status]][Build Link] [![Book Status]][Book Link] [![RustDoc Status]][RustDoc Link] [![Chat Badge]][Chat Link]
+[![Build Status]][Build Link] [![Book Status]][Book Link] [![Chat Badge]][Chat Link]
 
 [Build Status]: https://github.com/sigp/lighthouse/workflows/test-suite/badge.svg?branch=master
 [Build Link]: https://github.com/sigp/lighthouse/actions
 [Chat Badge]: https://img.shields.io/badge/chat-discord-%237289da
 [Chat Link]: https://discord.gg/cyAszAh
 [Book Status]:https://img.shields.io/badge/user--docs-master-informational
-[Book Link]: http://lighthouse-book.sigmaprime.io/
-[RustDoc Status]:https://img.shields.io/badge/code--docs-master-orange
-[RustDoc Link]: http://lighthouse-docs.sigmaprime.io/
+[Book Link]: https://lighthouse-book.sigmaprime.io
+[stable]: https://github.com/sigp/lighthouse/tree/stable
+[unstable]: https://github.com/sigp/lighthouse/tree/unstable
+[blog]: https://lighthouse.sigmaprime.io
 
-[Documentation](http://lighthouse-book.sigmaprime.io/)
+[Documentation](https://lighthouse-book.sigmaprime.io)
 
-![terminalize](https://i.postimg.cc/kG11dpCW/lighthouse-cli-png.gif)
+![Banner](https://i.postimg.cc/hjdTGKPd/photo-2020-10-23-09-52-16.jpg)
 
 ## Overview
 
 Lighthouse is:
 
+- Ready for use on Eth2 mainnet.
 - Fully open-source, licensed under Apache 2.0.
-- Security-focused. Fuzzing has begun and security reviews are underway.
-- Built in [Rust](https://www.rust-lang.org/), a modern language providing unique safety guarantees and
+- Security-focused. Fuzzing techniques have been continuously applied and several external security reviews have been performed.
+- Built in [Rust](https://www.rust-lang.org), a modern language providing unique safety guarantees and
 	excellent performance (comparable to C++).
 - Funded by various organisations, including Sigma Prime, the
-	Ethereum Foundation, ConsenSys and private individuals.
-- Actively involved in the specification and security analysis of the emerging
-    Ethereum 2.0 specification.
+	Ethereum Foundation, ConsenSys, the Decentralization Foundation and private individuals.
+- Actively involved in the specification and security analysis of the
+	Ethereum 2.0 specification.
 
-Like all Ethereum 2.0 clients, Lighthouse is a work-in-progress.
-
-## Development Status
-
-Current development overview:
-
-- Specification `v0.12.1` implemented, optimized and passing test vectors.
-- Rust-native libp2p with Gossipsub and Discv5.
-- RESTful JSON API via HTTP server.
-- Events via WebSocket.
-- Metrics via Prometheus.
-
-### Roadmap
-
-- ~~**April 2019**: Inital single-client testnets.~~
-- ~~**September 2019**: Inter-operability with other Ethereum 2.0 clients.~~
-- ~~**Q1 2020**: `lighthouse-0.1.0` release: All major phase 0 features implemented.~~
-- ~~**Q2 2020**: Public, multi-client testnet with user-facing functionality.~~
-- ~~**Q2 2020**: Third-party security review.~~
-- **Q3 2020**: Additional third-party security reviews.
-- **Q3 2020**: Long-lived, multi-client Beacon Chain testnet
-- **Q4 2020**: Production Beacon Chain (tentative).
+## Eth2 Deposit Contract
 
+The Lighthouse team acknowledges
+[`0x00000000219ab540356cBB839Cbe05303d7705Fa`](https://etherscan.io/address/0x00000000219ab540356cbb839cbe05303d7705fa)
+as the canonical Eth2 deposit contract address.
 
 ## Documentation
 
-The [Lighthouse Book](http://lighthouse-book.sigmaprime.io/) contains information
-for testnet users and developers.
+The [Lighthouse Book](https://lighthouse-book.sigmaprime.io) contains information for users and
+developers.
 
-Code documentation is generated via `cargo doc` and hosted at
-[lighthouse-docs.sigmaprime.io](http://lighthouse-docs.sigmaprime.io/).
+The Lighthouse team maintains a blog at [lighthouse.sigmaprime.io][blog] which contains periodical
+progress updates, roadmap insights and interesting findings.
 
-If you'd like some background on Sigma Prime, please see the [Lighthouse Update
-\#00](https://lighthouse.sigmaprime.io/update-00.html) blog post or
-[sigmaprime.io](https://sigmaprime.io).
+## Branches
+
+Lighthouse maintains two permanent branches:
+
+- [`stable`][stable]: Always points to the latest stable release.
+  - This is ideal for most users.
+- [`unstable`][unstable]: Used for development, contains the latest PRs.
+  - Developers should base thier PRs on this branch.
 
 ## Contributing
 
 Lighthouse welcomes contributors.
 
 If you are looking to contribute, please head to the
-[Contributing](http://lighthouse-book.sigmaprime.io/contributing.html) section
+[Contributing](https://lighthouse-book.sigmaprime.io/contributing.html) section
 of the Lighthouse book.
 
 ## Contact
@@ -80,8 +69,11 @@ The best place for discussion is the [Lighthouse Discord
 server](https://discord.gg/cyAszAh). Alternatively, you may use the
 [sigp/lighthouse gitter](https://gitter.im/sigp/lighthouse).
 
+Sign up to the [Lighthouse Development Updates](https://mailchi.mp/3d9df0417779/lighthouse-dev-updates)
+mailing list for email notifications about releases, network status and other important information.
+
 Encrypt sensitive messages using our [PGP
-key](https://keybase.io/sigp/pgp_keys.asc?fingerprint=dcf37e025d6c9d42ea795b119e7c6cf9988604be).
+key](https://keybase.io/sigp/pgp_keys.asc?fingerprint=15e66d941f697e28f49381f426416dc3f30674b0).
 
 ## Donations
 

account_manager/Cargo.toml

@@ -1,33 +1,40 @@
 [package]
 name = "account_manager"
-version = "0.2.0"
+version = "0.3.5"
 authors = ["Paul Hauner <paul@paulhauner.com>", "Luke Anderson <luke@sigmaprime.io>"]
 edition = "2018"
 
 [dependencies]
 bls = { path = "../crypto/bls" }
-clap = "2.33.0"
+clap = "2.33.3"
 slog = "2.5.2"
-slog-term = "2.5.0"
+slog-term = "2.6.0"
 slog-async = "2.5.0"
 types = { path = "../consensus/types" }
-state_processing = { path = "../consensus/state_processing" }
-dirs = "2.0.2"
+dirs = "3.0.1"
 environment = { path = "../lighthouse/environment" }
 deposit_contract = { path = "../common/deposit_contract" }
-libc = "0.2.65"
+libc = "0.2.79"
 eth2_ssz = "0.1.2"
 eth2_ssz_derive = "0.1.0"
 hex = "0.4.2"
-rayon = "1.3.0"
-eth2_testnet_config = { path = "../common/eth2_testnet_config" }
-web3 = "0.11.0"
-futures = { version = "0.3.5", features = ["compat"] }
+rayon = "1.4.1"
+eth2_network_config = { path = "../common/eth2_network_config" }
+futures = "0.3.7"
 clap_utils = { path = "../common/clap_utils" }
+directory = { path = "../common/directory" }
 eth2_wallet = { path = "../crypto/eth2_wallet" }
 eth2_wallet_manager = { path = "../common/eth2_wallet_manager" }
-rand = "0.7.2"
+rand = "0.7.3"
 validator_dir = { path = "../common/validator_dir" }
-tokio = { version = "0.2.21", features = ["full"] }
+tokio = { version = "0.3.5", features = ["full"] }
 eth2_keystore = { path = "../crypto/eth2_keystore" }
 account_utils = { path = "../common/account_utils" }
+slashing_protection = { path = "../validator_client/slashing_protection" }
+eth2 = {path = "../common/eth2"}
+safe_arith = {path = "../consensus/safe_arith"}
+slot_clock = { path = "../common/slot_clock" }
+tokio-compat-02 = "0.1"
+
+[dev-dependencies]
+tempfile = "3.1.0"

account_manager/src/common.rs

@@ -1,21 +1,67 @@
-use clap::ArgMatches;
-use std::fs::create_dir_all;
-use std::path::{Path, PathBuf};
+use account_utils::PlainText;
+use account_utils::{read_input_from_user, strip_off_newlines};
+use eth2_wallet::bip39::{Language, Mnemonic};
+use std::fs;
+use std::path::PathBuf;
+use std::str::from_utf8;
+use std::thread::sleep;
+use std::time::Duration;
 
-pub fn ensure_dir_exists<P: AsRef<Path>>(path: P) -> Result<(), String> {
-    let path = path.as_ref();
+pub const MNEMONIC_PROMPT: &str = "Enter the mnemonic phrase:";
+pub const WALLET_NAME_PROMPT: &str = "Enter wallet name:";
 
-    if !path.exists() {
-        create_dir_all(path).map_err(|e| format!("Unable to create {:?}: {:?}", path, e))?;
+pub fn read_mnemonic_from_cli(
+    mnemonic_path: Option<PathBuf>,
+    stdin_inputs: bool,
+) -> Result<Mnemonic, String> {
+    let mnemonic = match mnemonic_path {
+        Some(path) => fs::read(&path)
+            .map_err(|e| format!("Unable to read {:?}: {:?}", path, e))
+            .and_then(|bytes| {
+                let bytes_no_newlines: PlainText = strip_off_newlines(bytes).into();
+                let phrase = from_utf8(&bytes_no_newlines.as_ref())
+                    .map_err(|e| format!("Unable to derive mnemonic: {:?}", e))?;
+                Mnemonic::from_phrase(phrase, Language::English).map_err(|e| {
+                    format!(
+                        "Unable to derive mnemonic from string {:?}: {:?}",
+                        phrase, e
+                    )
+                })
+            })?,
+        None => loop {
+            eprintln!("");
+            eprintln!("{}", MNEMONIC_PROMPT);
+
+            let mnemonic = read_input_from_user(stdin_inputs)?;
+
+            match Mnemonic::from_phrase(mnemonic.as_str(), Language::English) {
+                Ok(mnemonic_m) => {
+                    eprintln!("Valid mnemonic provided.");
+                    eprintln!("");
+                    sleep(Duration::from_secs(1));
+                    break mnemonic_m;
+                }
+                Err(_) => {
+                    eprintln!("Invalid mnemonic");
+                }
+            }
+        },
+    };
+    Ok(mnemonic)
+}
+
+/// Reads in a wallet name from the user. If the `--wallet-name` flag is provided, use it. Otherwise
+/// read from an interactive prompt using tty unless the `--stdin-inputs` flag is provided.
+pub fn read_wallet_name_from_cli(
+    wallet_name: Option<String>,
+    stdin_inputs: bool,
+) -> Result<String, String> {
+    match wallet_name {
+        Some(name) => Ok(name),
+        None => {
+            eprintln!("{}", WALLET_NAME_PROMPT);
+
+            read_input_from_user(stdin_inputs)
+        }
     }
-
-    Ok(())
-}
-
-pub fn base_wallet_dir(matches: &ArgMatches, arg: &'static str) -> Result<PathBuf, String> {
-    clap_utils::parse_path_with_default_in_home_dir(
-        matches,
-        arg,
-        PathBuf::new().join(".lighthouse").join("wallets"),
-    )
 }

account_manager/src/lib.rs

@@ -10,7 +10,7 @@ use types::EthSpec;
 pub const CMD: &str = "account_manager";
 pub const SECRETS_DIR_FLAG: &str = "secrets-dir";
 pub const VALIDATOR_DIR_FLAG: &str = "validator-dir";
-pub const BASE_DIR_FLAG: &str = "base-dir";
+pub const WALLETS_DIR_FLAG: &str = "wallets-dir";
 
 pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
     App::new(CMD)

account_manager/src/validator/create.rs

@@ -1,9 +1,16 @@
-use crate::{common::ensure_dir_exists, SECRETS_DIR_FLAG, VALIDATOR_DIR_FLAG};
-use account_utils::{random_password, strip_off_newlines, validator_definitions};
+use crate::common::read_wallet_name_from_cli;
+use crate::wallet::create::STDIN_INPUTS_FLAG;
+use crate::{SECRETS_DIR_FLAG, WALLETS_DIR_FLAG};
+use account_utils::{
+    random_password, read_password_from_user, strip_off_newlines, validator_definitions, PlainText,
+};
 use clap::{App, Arg, ArgMatches};
+use directory::{
+    ensure_dir_exists, parse_path_or_default_with_flag, DEFAULT_SECRET_DIR, DEFAULT_WALLET_DIR,
+};
 use environment::Environment;
-use eth2_wallet::PlainText;
 use eth2_wallet_manager::WalletManager;
+use slashing_protection::{SlashingDatabase, SLASHING_PROTECTION_FILENAME};
 use std::ffi::OsStr;
 use std::fs;
 use std::path::{Path, PathBuf};
@@ -11,13 +18,13 @@ use types::EthSpec;
 use validator_dir::Builder as ValidatorDirBuilder;
 
 pub const CMD: &str = "create";
-pub const BASE_DIR_FLAG: &str = "base-dir";
 pub const WALLET_NAME_FLAG: &str = "wallet-name";
-pub const WALLET_PASSPHRASE_FLAG: &str = "wallet-passphrase";
+pub const WALLET_PASSWORD_FLAG: &str = "wallet-password";
 pub const DEPOSIT_GWEI_FLAG: &str = "deposit-gwei";
 pub const STORE_WITHDRAW_FLAG: &str = "store-withdrawal-keystore";
 pub const COUNT_FLAG: &str = "count";
 pub const AT_MOST_FLAG: &str = "at-most";
+pub const WALLET_PASSWORD_PROMPT: &str = "Enter your wallet's password:";
 
 pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
     App::new(CMD)
@@ -30,26 +37,22 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .long(WALLET_NAME_FLAG)
                 .value_name("WALLET_NAME")
                 .help("Use the wallet identified by this name")
-                .takes_value(true)
-                .required(true),
+                .takes_value(true),
         )
         .arg(
-            Arg::with_name(WALLET_PASSPHRASE_FLAG)
-                .long(WALLET_PASSPHRASE_FLAG)
+            Arg::with_name(WALLET_PASSWORD_FLAG)
+                .long(WALLET_PASSWORD_FLAG)
                 .value_name("WALLET_PASSWORD_PATH")
                 .help("A path to a file containing the password which will unlock the wallet.")
-                .takes_value(true)
-                .required(true),
+                .takes_value(true),
         )
         .arg(
-            Arg::with_name(VALIDATOR_DIR_FLAG)
-                .long(VALIDATOR_DIR_FLAG)
-                .value_name("VALIDATOR_DIRECTORY")
-                .help(
-                    "The path where the validator directories will be created. \
-                    Defaults to ~/.lighthouse/validators",
-                )
-                .takes_value(true),
+            Arg::with_name(WALLETS_DIR_FLAG)
+                .long(WALLETS_DIR_FLAG)
+                .value_name(WALLETS_DIR_FLAG)
+                .help("A path containing Eth2 EIP-2386 wallets. Defaults to ~/.lighthouse/{network}/wallets")
+                .takes_value(true)
+                .conflicts_with("datadir"),
         )
         .arg(
             Arg::with_name(SECRETS_DIR_FLAG)
@@ -57,8 +60,9 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .value_name("SECRETS_DIR")
                 .help(
                     "The path where the validator keystore passwords will be stored. \
-                    Defaults to ~/.lighthouse/secrets",
+                    Defaults to ~/.lighthouse/{network}/secrets",
                 )
+                .conflicts_with("datadir")
                 .takes_value(true),
         )
         .arg(
@@ -99,36 +103,46 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .conflicts_with("count")
                 .takes_value(true),
         )
+        .arg(
+            Arg::with_name(STDIN_INPUTS_FLAG)
+                .long(STDIN_INPUTS_FLAG)
+                .help("If present, read all user inputs from stdin instead of tty."),
+        )
 }
 
 pub fn cli_run<T: EthSpec>(
     matches: &ArgMatches,
     mut env: Environment<T>,
-    wallet_base_dir: PathBuf,
+    validator_dir: PathBuf,
 ) -> Result<(), String> {
     let spec = env.core_context().eth2_config.spec;
 
-    let name: String = clap_utils::parse_required(matches, WALLET_NAME_FLAG)?;
-    let wallet_password_path: PathBuf =
-        clap_utils::parse_required(matches, WALLET_PASSPHRASE_FLAG)?;
-    let validator_dir = clap_utils::parse_path_with_default_in_home_dir(
-        matches,
-        VALIDATOR_DIR_FLAG,
-        PathBuf::new().join(".lighthouse").join("validators"),
-    )?;
-    let secrets_dir = clap_utils::parse_path_with_default_in_home_dir(
-        matches,
-        SECRETS_DIR_FLAG,
-        PathBuf::new().join(".lighthouse").join("secrets"),
-    )?;
+    let name: Option<String> = clap_utils::parse_optional(matches, WALLET_NAME_FLAG)?;
+    let stdin_inputs = matches.is_present(STDIN_INPUTS_FLAG);
+    let wallet_base_dir = if matches.value_of("datadir").is_some() {
+        let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
+        path.join(DEFAULT_WALLET_DIR)
+    } else {
+        parse_path_or_default_with_flag(matches, WALLETS_DIR_FLAG, DEFAULT_WALLET_DIR)?
+    };
+    let secrets_dir = if matches.value_of("datadir").is_some() {
+        let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
+        path.join(DEFAULT_SECRET_DIR)
+    } else {
+        parse_path_or_default_with_flag(matches, SECRETS_DIR_FLAG, DEFAULT_SECRET_DIR)?
+    };
+
     let deposit_gwei = clap_utils::parse_optional(matches, DEPOSIT_GWEI_FLAG)?
-        .unwrap_or_else(|| spec.max_effective_balance);
+        .unwrap_or(spec.max_effective_balance);
     let count: Option<usize> = clap_utils::parse_optional(matches, COUNT_FLAG)?;
     let at_most: Option<usize> = clap_utils::parse_optional(matches, AT_MOST_FLAG)?;
 
     ensure_dir_exists(&validator_dir)?;
     ensure_dir_exists(&secrets_dir)?;
 
+    eprintln!("secrets-dir path {:?}", secrets_dir);
+    eprintln!("wallets-dir path {:?}", wallet_base_dir);
+
     let starting_validator_count = existing_validator_count(&validator_dir)?;
 
     let n = match (count, at_most) {
@@ -152,17 +166,37 @@ pub fn cli_run<T: EthSpec>(
         return Ok(());
     }
 
-    let wallet_password = fs::read(&wallet_password_path)
-        .map_err(|e| format!("Unable to read {:?}: {:?}", wallet_password_path, e))
-        .map(|bytes| PlainText::from(strip_off_newlines(bytes)))?;
+    let wallet_password_path: Option<PathBuf> =
+        clap_utils::parse_optional(matches, WALLET_PASSWORD_FLAG)?;
+
+    let wallet_name = read_wallet_name_from_cli(name, stdin_inputs)?;
+    let wallet_password = read_wallet_password_from_cli(wallet_password_path, stdin_inputs)?;
 
     let mgr = WalletManager::open(&wallet_base_dir)
-        .map_err(|e| format!("Unable to open --{}: {:?}", BASE_DIR_FLAG, e))?;
+        .map_err(|e| format!("Unable to open --{}: {:?}", WALLETS_DIR_FLAG, e))?;
 
     let mut wallet = mgr
-        .wallet_by_name(&name)
+        .wallet_by_name(&wallet_name)
         .map_err(|e| format!("Unable to open wallet: {:?}", e))?;
 
+    let slashing_protection_path = validator_dir.join(SLASHING_PROTECTION_FILENAME);
+    let slashing_protection =
+        SlashingDatabase::open_or_create(&slashing_protection_path).map_err(|e| {
+            format!(
+                "Unable to open or create slashing protection database at {}: {:?}",
+                slashing_protection_path.display(),
+                e
+            )
+        })?;
+
+    // Create an empty transaction and drops it. Used to test if the database is locked.
+    slashing_protection.test_transaction().map_err(|e| {
+        format!(
+            "Cannot create keys while the validator client is running: {:?}",
+            e
+        )
+    })?;
+
     for i in 0..n {
         let voting_password = random_password();
         let withdrawal_password = random_password();
@@ -175,9 +209,25 @@ pub fn cli_run<T: EthSpec>(
             )
             .map_err(|e| format!("Unable to create validator keys: {:?}", e))?;
 
-        let voting_pubkey = keystores.voting.pubkey().to_string();
+        let voting_pubkey = keystores.voting.public_key().ok_or_else(|| {
+            format!(
+                "Keystore public key is invalid: {}",
+                keystores.voting.pubkey()
+            )
+        })?;
 
-        ValidatorDirBuilder::new(validator_dir.clone(), secrets_dir.clone())
+        slashing_protection
+            .register_validator(&voting_pubkey)
+            .map_err(|e| {
+                format!(
+                    "Error registering validator {}: {:?}",
+                    voting_pubkey.to_hex_string(),
+                    e
+                )
+            })?;
+
+        ValidatorDirBuilder::new(validator_dir.clone())
+            .password_dir(secrets_dir.clone())
             .voting_keystore(keystores.voting, voting_password.as_bytes())
             .withdrawal_keystore(keystores.withdrawal, withdrawal_password.as_bytes())
             .create_eth1_tx_data(deposit_gwei, &spec)
@@ -185,7 +235,7 @@ pub fn cli_run<T: EthSpec>(
             .build()
             .map_err(|e| format!("Unable to build validator directory: {:?}", e))?;
 
-        println!("{}/{}\t0x{}", i + 1, n, voting_pubkey);
+        println!("{}/{}\t{}", i + 1, n, voting_pubkey.to_hex_string());
     }
 
     Ok(())
@@ -193,15 +243,40 @@ pub fn cli_run<T: EthSpec>(
 
 /// Returns the number of validators that exist in the given `validator_dir`.
 ///
-/// This function just assumes all files and directories, excluding the validator definitions YAML,
-/// are validator directories, making it likely to return a higher number than accurate
-/// but never a lower one.
+/// This function just assumes all files and directories, excluding the validator definitions YAML
+/// and slashing protection database are validator directories, making it likely to return a higher
+/// number than accurate but never a lower one.
 fn existing_validator_count<P: AsRef<Path>>(validator_dir: P) -> Result<usize, String> {
     fs::read_dir(validator_dir.as_ref())
         .map(|iter| {
             iter.filter_map(|e| e.ok())
-                .filter(|e| e.file_name() != OsStr::new(validator_definitions::CONFIG_FILENAME))
+                .filter(|e| {
+                    e.file_name() != OsStr::new(validator_definitions::CONFIG_FILENAME)
+                        && e.file_name()
+                            != OsStr::new(slashing_protection::SLASHING_PROTECTION_FILENAME)
+                })
                 .count()
         })
         .map_err(|e| format!("Unable to read {:?}: {}", validator_dir.as_ref(), e))
 }
+
+/// Used when a user is accessing an existing wallet. Read in a wallet password from a file if the password file
+/// path is provided. Otherwise, read from an interactive prompt using tty unless the `--stdin-inputs`
+/// flag is provided.
+pub fn read_wallet_password_from_cli(
+    password_file_path: Option<PathBuf>,
+    stdin_inputs: bool,
+) -> Result<PlainText, String> {
+    match password_file_path {
+        Some(path) => fs::read(&path)
+            .map_err(|e| format!("Unable to read {:?}: {:?}", path, e))
+            .map(|bytes| strip_off_newlines(bytes).into()),
+        None => {
+            eprintln!("");
+            eprintln!("{}", WALLET_PASSWORD_PROMPT);
+            let password =
+                PlainText::from(read_password_from_user(stdin_inputs)?.as_ref().to_vec());
+            Ok(password)
+        }
+    }
+}

account_manager/src/validator/deposit.rs

@@ -1,405 +0,0 @@
-use crate::VALIDATOR_DIR_FLAG;
-use clap::{App, Arg, ArgMatches};
-use deposit_contract::DEPOSIT_GAS;
-use environment::Environment;
-use futures::{
-    compat::Future01CompatExt,
-    stream::{FuturesUnordered, StreamExt},
-};
-use slog::{info, Logger};
-use state_processing::per_block_processing::verify_deposit_signature;
-use std::path::PathBuf;
-use tokio::time::{delay_until, Duration, Instant};
-use types::EthSpec;
-use validator_dir::{Eth1DepositData, Manager as ValidatorManager, ValidatorDir};
-use web3::{
-    transports::Http,
-    transports::Ipc,
-    types::{Address, SyncInfo, SyncState, TransactionRequest, U256},
-    Transport, Web3,
-};
-
-pub const CMD: &str = "deposit";
-pub const VALIDATOR_FLAG: &str = "validator";
-pub const ETH1_IPC_FLAG: &str = "eth1-ipc";
-pub const ETH1_HTTP_FLAG: &str = "eth1-http";
-pub const FROM_ADDRESS_FLAG: &str = "from-address";
-pub const CONFIRMATION_COUNT_FLAG: &str = "confirmation-count";
-pub const CONFIRMATION_BATCH_SIZE_FLAG: &str = "confirmation-batch-size";
-
-const GWEI: u64 = 1_000_000_000;
-
-const SYNCING_STATE_RETRY_DELAY: Duration = Duration::from_secs(2);
-
-const CONFIRMATIONS_POLL_TIME: Duration = Duration::from_secs(2);
-
-pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
-    App::new("deposit")
-        .about(
-            "Submits a deposit to an Eth1 validator registration contract via an IPC endpoint \
-            of an Eth1 client (e.g., Geth, OpenEthereum, etc.). The validators must already \
-            have been created and exist on the file-system. The process will exit immediately \
-            with an error if any error occurs. After each deposit is submitted to the Eth1 \
-            node, a file will be saved in the validator directory with the transaction hash. \
-            If confirmations are set to non-zero then the application will wait for confirmations \
-            before saving the transaction hash and moving onto the next batch of deposits. \
-            The deposit contract address will be determined by the --testnet-dir flag on the \
-            primary Lighthouse binary.",
-        )
-        .arg(
-            Arg::with_name(VALIDATOR_DIR_FLAG)
-                .long(VALIDATOR_DIR_FLAG)
-                .value_name("VALIDATOR_DIRECTORY")
-                .help(
-                    "The path to the validator client data directory. \
-                    Defaults to ~/.lighthouse/validators",
-                )
-                .takes_value(true),
-        )
-        .arg(
-            Arg::with_name(VALIDATOR_FLAG)
-                .long(VALIDATOR_FLAG)
-                .value_name("VALIDATOR_NAME")
-                .help(
-                    "The name of the directory in --data-dir for which to deposit. \
-                    Set to 'all' to deposit all validators in the --data-dir.",
-                )
-                .takes_value(true)
-                .required(true),
-        )
-        .arg(
-            Arg::with_name(ETH1_IPC_FLAG)
-                .long(ETH1_IPC_FLAG)
-                .value_name("ETH1_IPC_PATH")
-                .help("Path to an Eth1 JSON-RPC IPC endpoint")
-                .takes_value(true)
-                .required(false),
-        )
-        .arg(
-            Arg::with_name(ETH1_HTTP_FLAG)
-                .long(ETH1_HTTP_FLAG)
-                .value_name("ETH1_HTTP_URL")
-                .help("URL to an Eth1 JSON-RPC endpoint")
-                .takes_value(true)
-                .required(false),
-        )
-        .arg(
-            Arg::with_name(FROM_ADDRESS_FLAG)
-                .long(FROM_ADDRESS_FLAG)
-                .value_name("FROM_ETH1_ADDRESS")
-                .help(
-                    "The address that will submit the eth1 deposit. \
-                    Must be unlocked on the node at --eth1-ipc.",
-                )
-                .takes_value(true)
-                .required(true),
-        )
-        .arg(
-            Arg::with_name(CONFIRMATION_COUNT_FLAG)
-                .long(CONFIRMATION_COUNT_FLAG)
-                .value_name("CONFIRMATION_COUNT")
-                .help(
-                    "The number of Eth1 block confirmations required \
-                    before a transaction is considered complete. Set to \
-                    0 for no confirmations.",
-                )
-                .takes_value(true)
-                .default_value("1"),
-        )
-        .arg(
-            Arg::with_name(CONFIRMATION_BATCH_SIZE_FLAG)
-                .long(CONFIRMATION_BATCH_SIZE_FLAG)
-                .value_name("BATCH_SIZE")
-                .help(
-                    "Perform BATCH_SIZE deposits and wait for confirmations \
-                    in parallel. Useful for achieving faster bulk deposits.",
-                )
-                .takes_value(true)
-                .default_value("10"),
-        )
-}
-
-#[allow(clippy::too_many_arguments)]
-fn send_deposit_transactions<T1, T2: 'static>(
-    mut env: Environment<T1>,
-    log: Logger,
-    mut eth1_deposit_datas: Vec<(ValidatorDir, Eth1DepositData)>,
-    from_address: Address,
-    deposit_contract: Address,
-    transport: T2,
-    confirmation_count: usize,
-    confirmation_batch_size: usize,
-) -> Result<(), String>
-where
-    T1: EthSpec,
-    T2: Transport + std::marker::Send,
-    <T2 as web3::Transport>::Out: std::marker::Send,
-{
-    let web3 = Web3::new(transport);
-    let spec = env.eth2_config.spec.clone();
-
-    let deposits_fut = async {
-        poll_until_synced(web3.clone(), log.clone()).await?;
-
-        for chunk in eth1_deposit_datas.chunks_mut(confirmation_batch_size) {
-            let futures = FuturesUnordered::default();
-
-            for (ref mut validator_dir, eth1_deposit_data) in chunk.iter_mut() {
-                verify_deposit_signature(&eth1_deposit_data.deposit_data, &spec).map_err(|e| {
-                    format!(
-                        "Deposit for {:?} fails verification, \
-                         are you using the correct testnet configuration?\nError: {:?}",
-                        eth1_deposit_data.deposit_data.pubkey, e
-                    )
-                })?;
-
-                let web3 = web3.clone();
-                let log = log.clone();
-                futures.push(async move {
-                    let tx_hash = web3
-                        .send_transaction_with_confirmation(
-                            TransactionRequest {
-                                from: from_address,
-                                to: Some(deposit_contract),
-                                gas: Some(DEPOSIT_GAS.into()),
-                                gas_price: None,
-                                value: Some(from_gwei(eth1_deposit_data.deposit_data.amount)),
-                                data: Some(eth1_deposit_data.rlp.clone().into()),
-                                nonce: None,
-                                condition: None,
-                            },
-                            CONFIRMATIONS_POLL_TIME,
-                            confirmation_count,
-                        )
-                        .compat()
-                        .await
-                        .map_err(|e| format!("Failed to send transaction: {:?}", e))?;
-
-                    info!(
-                        log,
-                        "Submitted deposit";
-                        "tx_hash" => format!("{:?}", tx_hash),
-                    );
-
-                    validator_dir
-                        .save_eth1_deposit_tx_hash(&format!("{:?}", tx_hash))
-                        .map_err(|e| {
-                            format!("Failed to save tx hash {:?} to disk: {:?}", tx_hash, e)
-                        })?;
-
-                    Ok::<(), String>(())
-                });
-            }
-
-            futures
-                .collect::<Vec<_>>()
-                .await
-                .into_iter()
-                .collect::<Result<_, _>>()?;
-        }
-
-        Ok::<(), String>(())
-    };
-
-    env.runtime().block_on(deposits_fut)?;
-
-    Ok(())
-}
-
-pub fn cli_run<T: EthSpec>(
-    matches: &ArgMatches<'_>,
-    mut env: Environment<T>,
-) -> Result<(), String> {
-    let log = env.core_context().log().clone();
-
-    let data_dir = clap_utils::parse_path_with_default_in_home_dir(
-        matches,
-        VALIDATOR_DIR_FLAG,
-        PathBuf::new().join(".lighthouse").join("validators"),
-    )?;
-    let validator: String = clap_utils::parse_required(matches, VALIDATOR_FLAG)?;
-    let eth1_ipc_path: Option<PathBuf> = clap_utils::parse_optional(matches, ETH1_IPC_FLAG)?;
-    let eth1_http_url: Option<String> = clap_utils::parse_optional(matches, ETH1_HTTP_FLAG)?;
-    let from_address: Address = clap_utils::parse_required(matches, FROM_ADDRESS_FLAG)?;
-    let confirmation_count: usize = clap_utils::parse_required(matches, CONFIRMATION_COUNT_FLAG)?;
-    let confirmation_batch_size: usize =
-        clap_utils::parse_required(matches, CONFIRMATION_BATCH_SIZE_FLAG)?;
-
-    let manager = ValidatorManager::open(&data_dir)
-        .map_err(|e| format!("Unable to read --{}: {:?}", VALIDATOR_DIR_FLAG, e))?;
-
-    let validators = match validator.as_ref() {
-        "all" => manager
-            .open_all_validators()
-            .map_err(|e| format!("Unable to read all validators: {:?}", e)),
-        name => {
-            let path = manager
-                .directory_names()
-                .map_err(|e| {
-                    format!(
-                        "Unable to read --{} directory names: {:?}",
-                        VALIDATOR_DIR_FLAG, e
-                    )
-                })?
-                .get(name)
-                .ok_or_else(|| format!("Unknown validator:  {}", name))?
-                .clone();
-
-            manager
-                .open_validator(&path)
-                .map_err(|e| format!("Unable to open {}: {:?}", name, e))
-                .map(|v| vec![v])
-        }
-    }?;
-
-    let eth1_deposit_datas = validators
-        .into_iter()
-        .filter(|v| !v.eth1_deposit_tx_hash_exists())
-        .map(|v| match v.eth1_deposit_data() {
-            Ok(Some(data)) => Ok((v, data)),
-            Ok(None) => Err(format!(
-                "Validator is missing deposit data file: {:?}",
-                v.dir()
-            )),
-            Err(e) => Err(format!(
-                "Unable to read deposit data for {:?}: {:?}",
-                v.dir(),
-                e
-            )),
-        })
-        .collect::<Result<Vec<_>, _>>()?;
-
-    let total_gwei: u64 = eth1_deposit_datas
-        .iter()
-        .map(|(_, d)| d.deposit_data.amount)
-        .sum();
-
-    if eth1_deposit_datas.is_empty() {
-        info!(log, "No validators to deposit");
-
-        return Ok(());
-    }
-
-    info!(
-        log,
-        "Starting deposits";
-        "deposit_count" => eth1_deposit_datas.len(),
-        "total_eth" => total_gwei / GWEI,
-    );
-
-    let deposit_contract = env
-        .testnet
-        .as_ref()
-        .ok_or_else(|| "Unable to run account manager without a testnet dir".to_string())?
-        .deposit_contract_address()
-        .map_err(|e| format!("Unable to parse deposit contract address: {}", e))?;
-
-    if deposit_contract == Address::zero() {
-        return Err("Refusing to deposit to the zero address. Check testnet configuration.".into());
-    }
-
-    match (eth1_ipc_path, eth1_http_url) {
-        (Some(_), Some(_)) => Err(format!(
-            "error: Cannot supply both --{} and --{}",
-            ETH1_IPC_FLAG, ETH1_HTTP_FLAG
-        )),
-        (None, None) => Err(format!(
-            "error: Must supply one of --{} or --{}",
-            ETH1_IPC_FLAG, ETH1_HTTP_FLAG
-        )),
-        (Some(ipc_path), None) => {
-            let (_event_loop_handle, ipc_transport) = Ipc::new(ipc_path)
-                .map_err(|e| format!("Unable to connect to eth1 IPC: {:?}", e))?;
-            send_deposit_transactions(
-                env,
-                log,
-                eth1_deposit_datas,
-                from_address,
-                deposit_contract,
-                ipc_transport,
-                confirmation_count,
-                confirmation_batch_size,
-            )
-        }
-        (None, Some(http_url)) => {
-            let (_event_loop_handle, http_transport) = Http::new(http_url.as_str())
-                .map_err(|e| format!("Unable to connect to eth1 http RPC: {:?}", e))?;
-            send_deposit_transactions(
-                env,
-                log,
-                eth1_deposit_datas,
-                from_address,
-                deposit_contract,
-                http_transport,
-                confirmation_count,
-                confirmation_batch_size,
-            )
-        }
-    }
-}
-
-/// Converts gwei to wei.
-fn from_gwei(gwei: u64) -> U256 {
-    U256::from(gwei) * U256::exp10(9)
-}
-
-/// Run a poll on the `eth_syncing` endpoint, blocking until the node is synced.
-async fn poll_until_synced<T>(web3: Web3<T>, log: Logger) -> Result<(), String>
-where
-    T: Transport + Send + 'static,
-    <T as Transport>::Out: Send,
-{
-    loop {
-        let sync_state = web3
-            .clone()
-            .eth()
-            .syncing()
-            .compat()
-            .await
-            .map_err(|e| format!("Unable to read syncing state from eth1 node: {:?}", e))?;
-
-        match sync_state {
-            SyncState::Syncing(SyncInfo {
-                current_block,
-                highest_block,
-                ..
-            }) => {
-                info!(
-                    log,
-                    "Waiting for eth1 node to sync";
-                    "est_highest_block" => format!("{}", highest_block),
-                    "current_block" => format!("{}", current_block),
-                );
-
-                delay_until(Instant::now() + SYNCING_STATE_RETRY_DELAY).await;
-            }
-            SyncState::NotSyncing => {
-                let block_number = web3
-                    .clone()
-                    .eth()
-                    .block_number()
-                    .compat()
-                    .await
-                    .map_err(|e| format!("Unable to read block number from eth1 node: {:?}", e))?;
-
-                if block_number > 0.into() {
-                    info!(
-                        log,
-                        "Eth1 node is synced";
-                        "head_block" => format!("{}", block_number),
-                    );
-                    break;
-                } else {
-                    delay_until(Instant::now() + SYNCING_STATE_RETRY_DELAY).await;
-                    info!(
-                        log,
-                        "Waiting for eth1 node to sync";
-                        "current_block" => 0,
-                    );
-                }
-            }
-        }
-    }
-
-    Ok(())
-}

account_manager/src/validator/exit.rs

@@ -0,0 +1,352 @@
+use crate::wallet::create::STDIN_INPUTS_FLAG;
+use bls::{Keypair, PublicKey};
+use clap::{App, Arg, ArgMatches};
+use environment::Environment;
+use eth2::{
+    types::{GenesisData, StateId, ValidatorId, ValidatorStatus},
+    BeaconNodeHttpClient, Url,
+};
+use eth2_keystore::Keystore;
+use eth2_network_config::Eth2NetworkConfig;
+use safe_arith::SafeArith;
+use slot_clock::{SlotClock, SystemTimeSlotClock};
+use std::path::PathBuf;
+use std::time::Duration;
+use tokio_compat_02::FutureExt;
+use types::{ChainSpec, Epoch, EthSpec, Fork, VoluntaryExit};
+
+pub const CMD: &str = "exit";
+pub const KEYSTORE_FLAG: &str = "keystore";
+pub const PASSWORD_FILE_FLAG: &str = "password-file";
+pub const BEACON_SERVER_FLAG: &str = "beacon-node";
+pub const PASSWORD_PROMPT: &str = "Enter the keystore password";
+
+pub const DEFAULT_BEACON_NODE: &str = "http://localhost:5052/";
+pub const CONFIRMATION_PHRASE: &str = "Exit my validator";
+pub const WEBSITE_URL: &str = "https://lighthouse-book.sigmaprime.io/voluntary-exit.html";
+pub const PROMPT: &str = "WARNING: WITHDRAWING STAKED ETH IS NOT CURRENTLY POSSIBLE";
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new("exit")
+        .about("Submits a VoluntaryExit to the beacon chain for a given validator keystore.")
+        .arg(
+            Arg::with_name(KEYSTORE_FLAG)
+                .long(KEYSTORE_FLAG)
+                .value_name("KEYSTORE_PATH")
+                .help("The path to the EIP-2335 voting keystore for the validator")
+                .takes_value(true)
+                .required(true),
+        )
+        .arg(
+            Arg::with_name(PASSWORD_FILE_FLAG)
+                .long(PASSWORD_FILE_FLAG)
+                .value_name("PASSWORD_FILE_PATH")
+                .help("The path to the password file which unlocks the validator voting keystore")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(BEACON_SERVER_FLAG)
+                .long(BEACON_SERVER_FLAG)
+                .value_name("NETWORK_ADDRESS")
+                .help("Address to a beacon node HTTP API")
+                .default_value(&DEFAULT_BEACON_NODE)
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(STDIN_INPUTS_FLAG)
+                .long(STDIN_INPUTS_FLAG)
+                .help("If present, read all user inputs from stdin instead of tty."),
+        )
+}
+
+pub fn cli_run<E: EthSpec>(matches: &ArgMatches, env: Environment<E>) -> Result<(), String> {
+    let keystore_path: PathBuf = clap_utils::parse_required(matches, KEYSTORE_FLAG)?;
+    let password_file_path: Option<PathBuf> =
+        clap_utils::parse_optional(matches, PASSWORD_FILE_FLAG)?;
+    let stdin_inputs = matches.is_present(STDIN_INPUTS_FLAG);
+
+    let spec = env.eth2_config().spec.clone();
+    let server_url: String = clap_utils::parse_required(matches, BEACON_SERVER_FLAG)?;
+    let client = BeaconNodeHttpClient::new(
+        Url::parse(&server_url)
+            .map_err(|e| format!("Failed to parse beacon http server: {:?}", e))?,
+    );
+
+    let testnet_config = env
+        .testnet
+        .clone()
+        .expect("network should have a valid config");
+
+    env.runtime().block_on(
+        publish_voluntary_exit::<E>(
+            &keystore_path,
+            password_file_path.as_ref(),
+            &client,
+            &spec,
+            stdin_inputs,
+            &testnet_config,
+        )
+        .compat(),
+    )?;
+
+    Ok(())
+}
+
+/// Gets the keypair and validator_index for every validator and calls `publish_voluntary_exit` on it.
+async fn publish_voluntary_exit<E: EthSpec>(
+    keystore_path: &PathBuf,
+    password_file_path: Option<&PathBuf>,
+    client: &BeaconNodeHttpClient,
+    spec: &ChainSpec,
+    stdin_inputs: bool,
+    testnet_config: &Eth2NetworkConfig,
+) -> Result<(), String> {
+    let genesis_data = get_geneisis_data(client).await?;
+    let testnet_genesis_root = testnet_config
+        .beacon_state::<E>()
+        .as_ref()
+        .expect("network should have valid genesis state")
+        .genesis_validators_root;
+
+    // Verify that the beacon node and validator being exited are on the same network.
+    if genesis_data.genesis_validators_root != testnet_genesis_root {
+        return Err(
+            "Invalid genesis state. Please ensure that your beacon node is on the same network \
+                 as the validator you are publishing an exit for"
+                .to_string(),
+        );
+    }
+
+    // Return immediately if beacon node is not synced
+    if is_syncing(client).await? {
+        return Err("Beacon node is still syncing".to_string());
+    }
+
+    let keypair = load_voting_keypair(keystore_path, password_file_path, stdin_inputs)?;
+
+    let epoch = get_current_epoch::<E>(genesis_data.genesis_time, spec)
+        .ok_or("Failed to get current epoch. Please check your system time")?;
+    let validator_index = get_validator_index_for_exit(client, &keypair.pk, epoch, spec).await?;
+
+    let fork = get_beacon_state_fork(client).await?;
+    let voluntary_exit = VoluntaryExit {
+        epoch,
+        validator_index,
+    };
+
+    eprintln!(
+        "Publishing a voluntary exit for validator: {} \n",
+        keypair.pk
+    );
+    eprintln!("WARNING: THIS IS AN IRREVERSIBLE OPERATION\n");
+    eprintln!("{}\n", PROMPT);
+    eprintln!(
+        "PLEASE VISIT {} TO MAKE SURE YOU UNDERSTAND THE IMPLICATIONS OF A VOLUNTARY EXIT.",
+        WEBSITE_URL
+    );
+    eprintln!("Enter the exit phrase from the above URL to confirm the voluntary exit: ");
+
+    let confirmation = account_utils::read_input_from_user(stdin_inputs)?;
+    if confirmation == CONFIRMATION_PHRASE {
+        // Sign and publish the voluntary exit to network
+        let signed_voluntary_exit = voluntary_exit.sign(
+            &keypair.sk,
+            &fork,
+            genesis_data.genesis_validators_root,
+            spec,
+        );
+        client
+            .post_beacon_pool_voluntary_exits(&signed_voluntary_exit)
+            .await
+            .map_err(|e| format!("Failed to publish voluntary exit: {}", e))?;
+        tokio::time::sleep(std::time::Duration::from_secs(1)).await; // Provides nicer UX.
+        eprintln!(
+            "Successfully validated and published voluntary exit for validator {}",
+            keypair.pk
+        );
+    } else {
+        eprintln!(
+            "Did not publish voluntary exit for validator {}. Please check that you entered the correct exit phrase.",
+            keypair.pk
+        );
+    }
+
+    Ok(())
+}
+
+/// Get the validator index of a given the validator public key by querying the beacon node endpoint.
+///
+/// Returns an error if the beacon endpoint returns an error or given validator is not eligible for an exit.
+async fn get_validator_index_for_exit(
+    client: &BeaconNodeHttpClient,
+    validator_pubkey: &PublicKey,
+    epoch: Epoch,
+    spec: &ChainSpec,
+) -> Result<u64, String> {
+    let validator_data = client
+        .get_beacon_states_validator_id(
+            StateId::Head,
+            &ValidatorId::PublicKey(validator_pubkey.into()),
+        )
+        .await
+        .map_err(|e| format!("Failed to get validator details: {:?}", e))?
+        .ok_or_else(|| {
+            format!(
+                "Validator {} is not present in the beacon state. \
+                Please ensure that your beacon node is synced and the validator has been deposited.",
+                validator_pubkey
+            )
+        })?
+        .data;
+
+    match validator_data.status {
+        ValidatorStatus::Active => {
+            let eligible_epoch = validator_data
+                .validator
+                .activation_epoch
+                .safe_add(spec.shard_committee_period)
+                .map_err(|e| format!("Failed to calculate eligible epoch, validator activation epoch too high: {:?}", e))?;
+
+            if epoch >= eligible_epoch {
+                Ok(validator_data.index)
+            } else {
+                Err(format!(
+                    "Validator {:?} is not eligible for exit. It will become eligible on epoch {}",
+                    validator_pubkey, eligible_epoch
+                ))
+            }
+        }
+        status => Err(format!(
+            "Validator {:?} is not eligible for voluntary exit. Validator status: {:?}",
+            validator_pubkey, status
+        )),
+    }
+}
+
+/// Get genesis data by querying the beacon node client.
+async fn get_geneisis_data(client: &BeaconNodeHttpClient) -> Result<GenesisData, String> {
+    Ok(client
+        .get_beacon_genesis()
+        .await
+        .map_err(|e| format!("Failed to get beacon genesis: {}", e))?
+        .data)
+}
+
+/// Gets syncing status from beacon node client and returns true if syncing and false otherwise.
+async fn is_syncing(client: &BeaconNodeHttpClient) -> Result<bool, String> {
+    Ok(client
+        .get_node_syncing()
+        .await
+        .map_err(|e| format!("Failed to get sync status: {:?}", e))?
+        .data
+        .is_syncing)
+}
+
+/// Get fork object for the current state by querying the beacon node client.
+async fn get_beacon_state_fork(client: &BeaconNodeHttpClient) -> Result<Fork, String> {
+    Ok(client
+        .get_beacon_states_fork(StateId::Head)
+        .await
+        .map_err(|e| format!("Failed to get get fork: {:?}", e))?
+        .ok_or("Failed to get fork, state not found")?
+        .data)
+}
+
+/// Calculates the current epoch from the genesis time and current time.
+fn get_current_epoch<E: EthSpec>(genesis_time: u64, spec: &ChainSpec) -> Option<Epoch> {
+    let slot_clock = SystemTimeSlotClock::new(
+        spec.genesis_slot,
+        Duration::from_secs(genesis_time),
+        Duration::from_secs(spec.seconds_per_slot),
+    );
+    slot_clock.now().map(|s| s.epoch(E::slots_per_epoch()))
+}
+
+/// Load the voting keypair by loading and decrypting the keystore.
+///
+/// If the `password_file_path` is Some, unlock keystore using password in given file
+/// otherwise, prompts user for a password to unlock the keystore.
+fn load_voting_keypair(
+    voting_keystore_path: &PathBuf,
+    password_file_path: Option<&PathBuf>,
+    stdin_inputs: bool,
+) -> Result<Keypair, String> {
+    let keystore = Keystore::from_json_file(&voting_keystore_path).map_err(|e| {
+        format!(
+            "Unable to read keystore JSON {:?}: {:?}",
+            voting_keystore_path, e
+        )
+    })?;
+
+    // Get password from password file.
+    if let Some(password_file) = password_file_path {
+        validator_dir::unlock_keypair_from_password_path(voting_keystore_path, password_file)
+            .map_err(|e| format!("Error while decrypting keypair: {:?}", e))
+    } else {
+        // Prompt password from user.
+        eprintln!("");
+        eprintln!(
+            "{} for validator in {:?}: ",
+            PASSWORD_PROMPT, voting_keystore_path
+        );
+        let password = account_utils::read_password_from_user(stdin_inputs)?;
+        match keystore.decrypt_keypair(password.as_ref()) {
+            Ok(keypair) => {
+                eprintln!("Password is correct.");
+                eprintln!("");
+                std::thread::sleep(std::time::Duration::from_secs(1)); // Provides nicer UX.
+                Ok(keypair)
+            }
+            Err(eth2_keystore::Error::InvalidPassword) => Err("Invalid password".to_string()),
+            Err(e) => Err(format!("Error while decrypting keypair: {:?}", e)),
+        }
+    }
+}
+
+#[cfg(test)]
+#[cfg(not(debug_assertions))]
+mod tests {
+    use super::*;
+    use eth2_keystore::KeystoreBuilder;
+    use std::fs::File;
+    use std::io::Write;
+    use tempfile::{tempdir, TempDir};
+
+    const PASSWORD: &str = "cats";
+    const KEYSTORE_NAME: &str = "keystore-m_12381_3600_0_0_0-1595406747.json";
+    const PASSWORD_FILE: &str = "password.pass";
+
+    fn create_and_save_keystore(dir: &TempDir, save_password: bool) -> PublicKey {
+        let keypair = Keypair::random();
+        let keystore = KeystoreBuilder::new(&keypair, PASSWORD.as_bytes(), "".into())
+            .unwrap()
+            .build()
+            .unwrap();
+
+        // Create a keystore.
+        File::create(dir.path().join(KEYSTORE_NAME))
+            .map(|mut file| keystore.to_json_writer(&mut file).unwrap())
+            .unwrap();
+        if save_password {
+            File::create(dir.path().join(PASSWORD_FILE))
+                .map(|mut file| file.write_all(PASSWORD.as_bytes()).unwrap())
+                .unwrap();
+        }
+        keystore.public_key().unwrap()
+    }
+
+    #[test]
+    fn test_load_keypair_password_file() {
+        let dir = tempdir().unwrap();
+        let expected_pk = create_and_save_keystore(&dir, true);
+
+        let kp = load_voting_keypair(
+            &dir.path().join(KEYSTORE_NAME),
+            Some(&dir.path().join(PASSWORD_FILE)),
+            false,
+        )
+        .unwrap();
+
+        assert_eq!(expected_pk, kp.pk.into());
+    }
+}

account_manager/src/validator/import.rs

@@ -1,4 +1,4 @@
-use crate::{common::ensure_dir_exists, VALIDATOR_DIR_FLAG};
+use crate::wallet::create::STDIN_INPUTS_FLAG;
 use account_utils::{
     eth2_keystore::Keystore,
     read_password_from_user,
@@ -6,8 +6,10 @@ use account_utils::{
         recursively_find_voting_keystores, ValidatorDefinition, ValidatorDefinitions,
         CONFIG_FILENAME,
     },
+    ZeroizeString,
 };
 use clap::{App, Arg, ArgMatches};
+use slashing_protection::{SlashingDatabase, SLASHING_PROTECTION_FILENAME};
 use std::fs;
 use std::path::PathBuf;
 use std::thread::sleep;
@@ -16,7 +18,7 @@ use std::time::Duration;
 pub const CMD: &str = "import";
 pub const KEYSTORE_FLAG: &str = "keystore";
 pub const DIR_FLAG: &str = "directory";
-pub const STDIN_PASSWORD_FLAG: &str = "stdin-passwords";
+pub const REUSE_PASSWORD_FLAG: &str = "reuse-password";
 
 pub const PASSWORD_PROMPT: &str = "Enter the keystore password, or press enter to omit it:";
 pub const KEYSTORE_REUSE_WARNING: &str = "DO NOT USE THE ORIGINAL KEYSTORES TO VALIDATE WITH \
@@ -54,37 +56,44 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .takes_value(true),
         )
         .arg(
-            Arg::with_name(VALIDATOR_DIR_FLAG)
-                .long(VALIDATOR_DIR_FLAG)
-                .value_name("VALIDATOR_DIRECTORY")
-                .help(
-                    "The path where the validator directories will be created. \
-                    Defaults to ~/.lighthouse/validators",
-                )
-                .takes_value(true),
+            Arg::with_name(STDIN_INPUTS_FLAG)
+                .long(STDIN_INPUTS_FLAG)
+                .help("If present, read all user inputs from stdin instead of tty."),
         )
         .arg(
-            Arg::with_name(STDIN_PASSWORD_FLAG)
-                .long(STDIN_PASSWORD_FLAG)
-                .help("If present, read passwords from stdin instead of tty."),
+            Arg::with_name(REUSE_PASSWORD_FLAG)
+                .long(REUSE_PASSWORD_FLAG)
+                .help("If present, the same password will be used for all imported keystores."),
         )
 }
 
-pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
+pub fn cli_run(matches: &ArgMatches, validator_dir: PathBuf) -> Result<(), String> {
     let keystore: Option<PathBuf> = clap_utils::parse_optional(matches, KEYSTORE_FLAG)?;
     let keystores_dir: Option<PathBuf> = clap_utils::parse_optional(matches, DIR_FLAG)?;
-    let validator_dir = clap_utils::parse_path_with_default_in_home_dir(
-        matches,
-        VALIDATOR_DIR_FLAG,
-        PathBuf::new().join(".lighthouse").join("validators"),
-    )?;
-    let stdin_password = matches.is_present(STDIN_PASSWORD_FLAG);
-
-    ensure_dir_exists(&validator_dir)?;
+    let stdin_inputs = matches.is_present(STDIN_INPUTS_FLAG);
+    let reuse_password = matches.is_present(REUSE_PASSWORD_FLAG);
 
     let mut defs = ValidatorDefinitions::open_or_create(&validator_dir)
         .map_err(|e| format!("Unable to open {}: {:?}", CONFIG_FILENAME, e))?;
 
+    let slashing_protection_path = validator_dir.join(SLASHING_PROTECTION_FILENAME);
+    let slashing_protection =
+        SlashingDatabase::open_or_create(&slashing_protection_path).map_err(|e| {
+            format!(
+                "Unable to open or create slashing protection database at {}: {:?}",
+                slashing_protection_path.display(),
+                e
+            )
+        })?;
+
+    // Create an empty transaction and drop it. Used to test if the database is locked.
+    slashing_protection.test_transaction().map_err(|e| {
+        format!(
+            "Cannot import keys while the validator client is running: {:?}",
+            e
+        )
+    })?;
+
     // Collect the paths for the keystores that should be imported.
     let keystore_paths = match (keystore, keystores_dir) {
         (Some(keystore), None) => vec![keystore],
@@ -115,10 +124,13 @@ pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
     //
     // - Obtain the keystore password, if the user desires.
     // - Copy the keystore into the `validator_dir`.
+    // - Register the voting key with the slashing protection database.
     // - Add the keystore to the validator definitions file.
     //
     // Skip keystores that already exist, but exit early if any operation fails.
+    // Reuses the same password for all keystores if the `REUSE_PASSWORD_FLAG` flag is set.
     let mut num_imported_keystores = 0;
+    let mut previous_password: Option<ZeroizeString> = None;
     for src_keystore in &keystore_paths {
         let keystore = Keystore::from_json_file(src_keystore)
             .map_err(|e| format!("Unable to read keystore JSON {:?}: {:?}", src_keystore, e))?;
@@ -136,10 +148,14 @@ pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
         );
 
         let password_opt = loop {
+            if let Some(password) = previous_password.clone() {
+                eprintln!("Reuse previous password.");
+                break Some(password);
+            }
             eprintln!("");
             eprintln!("{}", PASSWORD_PROMPT);
 
-            let password = read_password_from_user(stdin_password)?;
+            let password = read_password_from_user(stdin_inputs)?;
 
             if password.as_ref().is_empty() {
                 eprintln!("Continuing without password.");
@@ -152,6 +168,9 @@ pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
                     eprintln!("Password is correct.");
                     eprintln!("");
                     sleep(Duration::from_secs(1)); // Provides nicer UX.
+                    if reuse_password {
+                        previous_password = Some(password.clone());
+                    }
                     break Some(password);
                 }
                 Err(eth2_keystore::Error::InvalidPassword) => {
@@ -186,6 +205,20 @@ pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
         fs::copy(&src_keystore, &dest_keystore)
             .map_err(|e| format!("Unable to copy keystore: {:?}", e))?;
 
+        // Register with slashing protection.
+        let voting_pubkey = keystore
+            .public_key()
+            .ok_or_else(|| format!("Keystore public key is invalid: {}", keystore.pubkey()))?;
+        slashing_protection
+            .register_validator(&voting_pubkey)
+            .map_err(|e| {
+                format!(
+                    "Error registering validator {}: {:?}",
+                    voting_pubkey.to_hex_string(),
+                    e
+                )
+            })?;
+
         eprintln!("Successfully imported keystore.");
         num_imported_keystores += 1;
 

account_manager/src/validator/list.rs

@@ -1,40 +1,27 @@
-use crate::VALIDATOR_DIR_FLAG;
-use clap::{App, Arg, ArgMatches};
+use account_utils::validator_definitions::ValidatorDefinitions;
+use clap::App;
 use std::path::PathBuf;
-use validator_dir::Manager as ValidatorManager;
 
 pub const CMD: &str = "list";
 
 pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
-    App::new(CMD)
-        .arg(
-            Arg::with_name(VALIDATOR_DIR_FLAG)
-                .long(VALIDATOR_DIR_FLAG)
-                .value_name("VALIDATOR_DIRECTORY")
-                .help(
-                    "The path to search for validator directories. \
-                    Defaults to ~/.lighthouse/validators",
-                )
-                .takes_value(true),
-        )
-        .about("Lists the names of all validators.")
+    App::new(CMD).about("Lists the public keys of all validators.")
 }
 
-pub fn cli_run(matches: &ArgMatches<'_>) -> Result<(), String> {
-    let data_dir = clap_utils::parse_path_with_default_in_home_dir(
-        matches,
-        VALIDATOR_DIR_FLAG,
-        PathBuf::new().join(".lighthouse").join("validators"),
-    )?;
+pub fn cli_run(validator_dir: PathBuf) -> Result<(), String> {
+    let validator_definitions = ValidatorDefinitions::open(&validator_dir).map_err(|e| {
+        format!(
+            "No validator definitions found in {:?}: {:?}",
+            validator_dir, e
+        )
+    })?;
 
-    let mgr = ValidatorManager::open(&data_dir)
-        .map_err(|e| format!("Unable to read --{}: {:?}", VALIDATOR_DIR_FLAG, e))?;
-
-    for (name, _path) in mgr
-        .directory_names()
-        .map_err(|e| format!("Unable to list wallets: {:?}", e))?
-    {
-        println!("{}", name)
+    for def in validator_definitions.as_slice() {
+        println!(
+            "{} ({})",
+            def.voting_public_key,
+            if def.enabled { "enabled" } else { "disabled" }
+        );
     }
 
     Ok(())

account_manager/src/validator/mod.rs

@@ -1,11 +1,15 @@
 pub mod create;
-pub mod deposit;
+pub mod exit;
 pub mod import;
 pub mod list;
+pub mod recover;
+pub mod slashing_protection;
 
-use crate::common::base_wallet_dir;
+use crate::VALIDATOR_DIR_FLAG;
 use clap::{App, Arg, ArgMatches};
+use directory::{parse_path_or_default_with_flag, DEFAULT_VALIDATOR_DIR};
 use environment::Environment;
+use std::path::PathBuf;
 use types::EthSpec;
 
 pub const CMD: &str = "validator";
@@ -14,26 +18,42 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
     App::new(CMD)
         .about("Provides commands for managing Eth2 validators.")
         .arg(
-            Arg::with_name("base-dir")
-                .long("base-dir")
-                .value_name("BASE_DIRECTORY")
-                .help("A path containing Eth2 EIP-2386 wallets. Defaults to ~/.lighthouse/wallets")
-                .takes_value(true),
+            Arg::with_name(VALIDATOR_DIR_FLAG)
+                .long(VALIDATOR_DIR_FLAG)
+                .value_name("VALIDATOR_DIRECTORY")
+                .help(
+                    "The path to search for validator directories. \
+                    Defaults to ~/.lighthouse/{network}/validators",
+                )
+                .takes_value(true)
+                .conflicts_with("datadir"),
         )
         .subcommand(create::cli_app())
-        .subcommand(deposit::cli_app())
         .subcommand(import::cli_app())
         .subcommand(list::cli_app())
+        .subcommand(recover::cli_app())
+        .subcommand(slashing_protection::cli_app())
+        .subcommand(exit::cli_app())
 }
 
 pub fn cli_run<T: EthSpec>(matches: &ArgMatches, env: Environment<T>) -> Result<(), String> {
-    let base_wallet_dir = base_wallet_dir(matches, "base-dir")?;
+    let validator_base_dir = if matches.value_of("datadir").is_some() {
+        let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
+        path.join(DEFAULT_VALIDATOR_DIR)
+    } else {
+        parse_path_or_default_with_flag(matches, VALIDATOR_DIR_FLAG, DEFAULT_VALIDATOR_DIR)?
+    };
+    eprintln!("validator-dir path: {:?}", validator_base_dir);
 
     match matches.subcommand() {
-        (create::CMD, Some(matches)) => create::cli_run::<T>(matches, env, base_wallet_dir),
-        (deposit::CMD, Some(matches)) => deposit::cli_run::<T>(matches, env),
-        (import::CMD, Some(matches)) => import::cli_run(matches),
-        (list::CMD, Some(matches)) => list::cli_run(matches),
+        (create::CMD, Some(matches)) => create::cli_run::<T>(matches, env, validator_base_dir),
+        (import::CMD, Some(matches)) => import::cli_run(matches, validator_base_dir),
+        (list::CMD, Some(_)) => list::cli_run(validator_base_dir),
+        (recover::CMD, Some(matches)) => recover::cli_run(matches, validator_base_dir),
+        (slashing_protection::CMD, Some(matches)) => {
+            slashing_protection::cli_run(matches, env, validator_base_dir)
+        }
+        (exit::CMD, Some(matches)) => exit::cli_run(matches, env),
         (unknown, _) => Err(format!(
             "{} does not have a {} command. See --help",
             CMD, unknown

account_manager/src/validator/recover.rs

@@ -0,0 +1,147 @@
+use super::create::STORE_WITHDRAW_FLAG;
+use crate::common::read_mnemonic_from_cli;
+use crate::validator::create::COUNT_FLAG;
+use crate::wallet::create::STDIN_INPUTS_FLAG;
+use crate::SECRETS_DIR_FLAG;
+use account_utils::eth2_keystore::{keypair_from_secret, Keystore, KeystoreBuilder};
+use account_utils::random_password;
+use clap::{App, Arg, ArgMatches};
+use directory::ensure_dir_exists;
+use directory::{parse_path_or_default_with_flag, DEFAULT_SECRET_DIR};
+use eth2_wallet::bip39::Seed;
+use eth2_wallet::{recover_validator_secret_from_mnemonic, KeyType, ValidatorKeystores};
+use std::path::PathBuf;
+use validator_dir::Builder as ValidatorDirBuilder;
+pub const CMD: &str = "recover";
+pub const FIRST_INDEX_FLAG: &str = "first-index";
+pub const MNEMONIC_FLAG: &str = "mnemonic-path";
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD)
+        .about(
+            "Recovers validator private keys given a BIP-39 mnemonic phrase. \
+            If you did not specify a `--first-index` or count `--count`, by default this will \
+            only recover the keys associated with the validator at index 0 for an HD wallet \
+            in accordance with the EIP-2333 spec.")
+        .arg(
+            Arg::with_name(FIRST_INDEX_FLAG)
+                .long(FIRST_INDEX_FLAG)
+                .value_name("FIRST_INDEX")
+                .help("The first of consecutive key indexes you wish to recover.")
+                .takes_value(true)
+                .required(false)
+                .default_value("0"),
+        )
+        .arg(
+            Arg::with_name(COUNT_FLAG)
+                .long(COUNT_FLAG)
+                .value_name("COUNT")
+                .help("The number of validator keys you wish to recover. Counted consecutively from the provided `--first_index`.")
+                .takes_value(true)
+                .required(false)
+                .default_value("1"),
+        )
+        .arg(
+            Arg::with_name(MNEMONIC_FLAG)
+                .long(MNEMONIC_FLAG)
+                .value_name("MNEMONIC_PATH")
+                .help(
+                    "If present, the mnemonic will be read in from this file.",
+                )
+                .takes_value(true)
+        )
+        .arg(
+            Arg::with_name(SECRETS_DIR_FLAG)
+                .long(SECRETS_DIR_FLAG)
+                .value_name("SECRETS_DIR")
+                .help(
+                    "The path where the validator keystore passwords will be stored. \
+                    Defaults to ~/.lighthouse/{network}/secrets",
+                )
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(STORE_WITHDRAW_FLAG)
+                .long(STORE_WITHDRAW_FLAG)
+                .help(
+                    "If present, the withdrawal keystore will be stored alongside the voting \
+                    keypair. It is generally recommended to *not* store the withdrawal key and \
+                    instead generate them from the wallet seed when required.",
+                ),
+        )
+        .arg(
+            Arg::with_name(STDIN_INPUTS_FLAG)
+                .long(STDIN_INPUTS_FLAG)
+                .help("If present, read all user inputs from stdin instead of tty."),
+        )
+}
+
+pub fn cli_run(matches: &ArgMatches, validator_dir: PathBuf) -> Result<(), String> {
+    let secrets_dir = if matches.value_of("datadir").is_some() {
+        let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
+        path.join(DEFAULT_SECRET_DIR)
+    } else {
+        parse_path_or_default_with_flag(matches, SECRETS_DIR_FLAG, DEFAULT_SECRET_DIR)?
+    };
+    let first_index: u32 = clap_utils::parse_required(matches, FIRST_INDEX_FLAG)?;
+    let count: u32 = clap_utils::parse_required(matches, COUNT_FLAG)?;
+    let mnemonic_path: Option<PathBuf> = clap_utils::parse_optional(matches, MNEMONIC_FLAG)?;
+    let stdin_inputs = matches.is_present(STDIN_INPUTS_FLAG);
+
+    eprintln!("secrets-dir path: {:?}", secrets_dir);
+
+    ensure_dir_exists(&validator_dir)?;
+    ensure_dir_exists(&secrets_dir)?;
+
+    eprintln!("");
+    eprintln!("WARNING: KEY RECOVERY CAN LEAD TO DUPLICATING VALIDATORS KEYS, WHICH CAN LEAD TO SLASHING.");
+    eprintln!("");
+
+    let mnemonic = read_mnemonic_from_cli(mnemonic_path, stdin_inputs)?;
+
+    let seed = Seed::new(&mnemonic, "");
+
+    for index in first_index..first_index + count {
+        let voting_password = random_password();
+        let withdrawal_password = random_password();
+
+        let derive = |key_type: KeyType, password: &[u8]| -> Result<Keystore, String> {
+            let (secret, path) =
+                recover_validator_secret_from_mnemonic(seed.as_bytes(), index, key_type)
+                    .map_err(|e| format!("Unable to recover validator keys: {:?}", e))?;
+
+            let keypair = keypair_from_secret(secret.as_bytes())
+                .map_err(|e| format!("Unable build keystore: {:?}", e))?;
+
+            KeystoreBuilder::new(&keypair, password, format!("{}", path))
+                .map_err(|e| format!("Unable build keystore: {:?}", e))?
+                .build()
+                .map_err(|e| format!("Unable build keystore: {:?}", e))
+        };
+
+        let keystores = ValidatorKeystores {
+            voting: derive(KeyType::Voting, voting_password.as_bytes())?,
+            withdrawal: derive(KeyType::Withdrawal, withdrawal_password.as_bytes())?,
+        };
+
+        let voting_pubkey = keystores.voting.pubkey().to_string();
+
+        ValidatorDirBuilder::new(validator_dir.clone())
+            .password_dir(secrets_dir.clone())
+            .voting_keystore(keystores.voting, voting_password.as_bytes())
+            .withdrawal_keystore(keystores.withdrawal, withdrawal_password.as_bytes())
+            .store_withdrawal_keystore(matches.is_present(STORE_WITHDRAW_FLAG))
+            .build()
+            .map_err(|e| format!("Unable to build validator directory: {:?}", e))?;
+
+        println!(
+            "{}/{}\tIndex: {}\t0x{}",
+            index - first_index,
+            count - first_index,
+            index,
+            voting_pubkey
+        );
+    }
+
+    Ok(())
+}

account_manager/src/validator/slashing_protection.rs

@@ -0,0 +1,185 @@
+use clap::{App, Arg, ArgMatches};
+use environment::Environment;
+use slashing_protection::{
+    interchange::Interchange, InterchangeImportOutcome, SlashingDatabase,
+    SLASHING_PROTECTION_FILENAME,
+};
+use std::fs::File;
+use std::path::PathBuf;
+use types::{BeaconState, Epoch, EthSpec, Slot};
+
+pub const CMD: &str = "slashing-protection";
+pub const IMPORT_CMD: &str = "import";
+pub const EXPORT_CMD: &str = "export";
+
+pub const IMPORT_FILE_ARG: &str = "IMPORT-FILE";
+pub const EXPORT_FILE_ARG: &str = "EXPORT-FILE";
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD)
+        .about("Import or export slashing protection data to or from another client")
+        .subcommand(
+            App::new(IMPORT_CMD)
+                .about("Import an interchange file")
+                .arg(
+                    Arg::with_name(IMPORT_FILE_ARG)
+                        .takes_value(true)
+                        .value_name("FILE")
+                        .help("The slashing protection interchange file to import (.json)"),
+                ),
+        )
+        .subcommand(
+            App::new(EXPORT_CMD)
+                .about("Export an interchange file")
+                .arg(
+                    Arg::with_name(EXPORT_FILE_ARG)
+                        .takes_value(true)
+                        .value_name("FILE")
+                        .help("The filename to export the interchange file to"),
+                ),
+        )
+}
+
+pub fn cli_run<T: EthSpec>(
+    matches: &ArgMatches<'_>,
+    env: Environment<T>,
+    validator_base_dir: PathBuf,
+) -> Result<(), String> {
+    let slashing_protection_db_path = validator_base_dir.join(SLASHING_PROTECTION_FILENAME);
+
+    let testnet_config = env
+        .testnet
+        .ok_or("Unable to get testnet configuration from the environment")?;
+
+    let genesis_validators_root = testnet_config
+        .beacon_state::<T>()
+        .map(|state: BeaconState<T>| state.genesis_validators_root)
+        .map_err(|e| {
+            format!(
+                "Unable to get genesis state, has genesis occurred? Detail: {:?}",
+                e
+            )
+        })?;
+
+    match matches.subcommand() {
+        (IMPORT_CMD, Some(matches)) => {
+            let import_filename: PathBuf = clap_utils::parse_required(&matches, IMPORT_FILE_ARG)?;
+            let import_file = File::open(&import_filename).map_err(|e| {
+                format!(
+                    "Unable to open import file at {}: {:?}",
+                    import_filename.display(),
+                    e
+                )
+            })?;
+
+            let interchange = Interchange::from_json_reader(&import_file)
+                .map_err(|e| format!("Error parsing file for import: {:?}", e))?;
+
+            let slashing_protection_database =
+                SlashingDatabase::open_or_create(&slashing_protection_db_path).map_err(|e| {
+                    format!(
+                        "Unable to open database at {}: {:?}",
+                        slashing_protection_db_path.display(),
+                        e
+                    )
+                })?;
+
+            let outcomes = slashing_protection_database
+                .import_interchange_info(interchange, genesis_validators_root)
+                .map_err(|e| {
+                    format!(
+                        "Error during import: {:?}\n\
+                         IT IS NOT SAFE TO START VALIDATING",
+                        e
+                    )
+                })?;
+
+            let display_slot = |slot: Option<Slot>| {
+                slot.map_or("none".to_string(), |slot| format!("{}", slot.as_u64()))
+            };
+            let display_epoch = |epoch: Option<Epoch>| {
+                epoch.map_or("?".to_string(), |epoch| format!("{}", epoch.as_u64()))
+            };
+            let display_attestation = |source, target| match (source, target) {
+                (None, None) => "none".to_string(),
+                (source, target) => format!("{}=>{}", display_epoch(source), display_epoch(target)),
+            };
+
+            let mut num_failed = 0;
+
+            for outcome in &outcomes {
+                match outcome {
+                    InterchangeImportOutcome::Success { pubkey, summary } => {
+                        eprintln!("- {:?} SUCCESS min block: {}, max block: {}, min attestation: {}, max attestation: {}",
+                            pubkey,
+                            display_slot(summary.min_block_slot),
+                            display_slot(summary.max_block_slot),
+                            display_attestation(summary.min_attestation_source, summary.min_attestation_target),
+                            display_attestation(summary.max_attestation_source,
+                            summary.max_attestation_target),
+                        );
+                    }
+                    InterchangeImportOutcome::Failure { pubkey, error } => {
+                        eprintln!("- {:?} ERROR: {:?}", pubkey, error);
+                        num_failed += 1;
+                    }
+                }
+            }
+
+            if num_failed == 0 {
+                eprintln!("Import completed successfully.");
+                eprintln!(
+                    "Please double-check that the minimum and maximum blocks and slots above \
+                     match your expectations."
+                );
+            } else {
+                eprintln!(
+                    "WARNING: history was NOT imported for {} of {} records",
+                    num_failed,
+                    outcomes.len()
+                );
+                eprintln!("IT IS NOT SAFE TO START VALIDATING");
+                eprintln!("Please see https://lighthouse-book.sigmaprime.io/slashing-protection.html#slashable-data-in-import");
+                return Err("Partial import".to_string());
+            }
+
+            Ok(())
+        }
+        (EXPORT_CMD, Some(matches)) => {
+            let export_filename: PathBuf = clap_utils::parse_required(&matches, EXPORT_FILE_ARG)?;
+
+            if !slashing_protection_db_path.exists() {
+                return Err(format!(
+                    "No slashing protection database exists at: {}",
+                    slashing_protection_db_path.display()
+                ));
+            }
+
+            let slashing_protection_database = SlashingDatabase::open(&slashing_protection_db_path)
+                .map_err(|e| {
+                    format!(
+                        "Unable to open database at {}: {:?}",
+                        slashing_protection_db_path.display(),
+                        e
+                    )
+                })?;
+
+            let interchange = slashing_protection_database
+                .export_interchange_info(genesis_validators_root)
+                .map_err(|e| format!("Error during export: {:?}", e))?;
+
+            let output_file = File::create(export_filename)
+                .map_err(|e| format!("Error creating output file: {:?}", e))?;
+
+            interchange
+                .write_to(&output_file)
+                .map_err(|e| format!("Error writing output file: {:?}", e))?;
+
+            eprintln!("Export completed successfully");
+
+            Ok(())
+        }
+        ("", _) => Err("No subcommand provided, see --help for options".to_string()),
+        (command, _) => Err(format!("No such subcommand `{}`", command)),
+    }
+}

account_manager/src/wallet/create.rs

@@ -1,13 +1,17 @@
-use crate::BASE_DIR_FLAG;
-use account_utils::{random_password, strip_off_newlines};
+use crate::common::read_wallet_name_from_cli;
+use crate::WALLETS_DIR_FLAG;
+use account_utils::{
+    is_password_sufficiently_complex, random_password, read_password_from_user, strip_off_newlines,
+};
 use clap::{App, Arg, ArgMatches};
 use eth2_wallet::{
     bip39::{Language, Mnemonic, MnemonicType},
     PlainText,
 };
-use eth2_wallet_manager::{WalletManager, WalletType};
+use eth2_wallet_manager::{LockedWallet, WalletManager, WalletType};
 use std::ffi::OsStr;
-use std::fs::{self, File};
+use std::fs;
+use std::fs::File;
 use std::io::prelude::*;
 use std::os::unix::fs::PermissionsExt;
 use std::path::{Path, PathBuf};
@@ -15,9 +19,21 @@ use std::path::{Path, PathBuf};
 pub const CMD: &str = "create";
 pub const HD_TYPE: &str = "hd";
 pub const NAME_FLAG: &str = "name";
-pub const PASSPHRASE_FLAG: &str = "passphrase-file";
+pub const PASSWORD_FLAG: &str = "password-file";
 pub const TYPE_FLAG: &str = "type";
 pub const MNEMONIC_FLAG: &str = "mnemonic-output-path";
+pub const STDIN_INPUTS_FLAG: &str = "stdin-inputs";
+pub const MNEMONIC_LENGTH_FLAG: &str = "mnemonic-length";
+pub const MNEMONIC_TYPES: &[MnemonicType] = &[
+    MnemonicType::Words12,
+    MnemonicType::Words15,
+    MnemonicType::Words18,
+    MnemonicType::Words21,
+    MnemonicType::Words24,
+];
+pub const NEW_WALLET_PASSWORD_PROMPT: &str =
+    "Enter a password for your new wallet that is at least 12 characters long:";
+pub const RETYPE_PASSWORD_PROMPT: &str = "Please re-enter your wallet's new password:";
 
 pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
     App::new(CMD)
@@ -30,12 +46,11 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                     "The wallet will be created with this name. It is not allowed to \
                             create two wallets with the same name for the same --base-dir.",
                 )
-                .takes_value(true)
-                .required(true),
+                .takes_value(true),
         )
         .arg(
-            Arg::with_name(PASSPHRASE_FLAG)
-                .long(PASSPHRASE_FLAG)
+            Arg::with_name(PASSWORD_FLAG)
+                .long(PASSWORD_FLAG)
                 .value_name("WALLET_PASSWORD_PATH")
                 .help(
                     "A path to a file containing the password which will unlock the wallet. \
@@ -43,8 +58,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                     saved at that path. To avoid confusion, if the file does not already \
                     exist it must include a '.pass' suffix.",
                 )
-                .takes_value(true)
-                .required(true),
+                .takes_value(true),
         )
         .arg(
             Arg::with_name(TYPE_FLAG)
@@ -67,56 +81,47 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 )
                 .takes_value(true)
         )
+        .arg(
+            Arg::with_name(STDIN_INPUTS_FLAG)
+                .long(STDIN_INPUTS_FLAG)
+                .help("If present, read all user inputs from stdin instead of tty."),
+        )
+        .arg(
+            Arg::with_name(MNEMONIC_LENGTH_FLAG)
+                .long(MNEMONIC_LENGTH_FLAG)
+                .value_name("MNEMONIC_LENGTH")
+                .help("The number of words to use for the mnemonic phrase.")
+                .takes_value(true)
+                .validator(|len| {
+                    match len.parse::<usize>().ok().and_then(|words| MnemonicType::for_word_count(words).ok()) {
+                        Some(_) => Ok(()),
+                        None => Err(format!("Mnemonic length must be one of {}", MNEMONIC_TYPES.iter().map(|t| t.word_count().to_string()).collect::<Vec<_>>().join(", "))),
+                    }
+                })
+                .default_value("24"),
+        )
 }
 
-pub fn cli_run(matches: &ArgMatches, base_dir: PathBuf) -> Result<(), String> {
-    let name: String = clap_utils::parse_required(matches, NAME_FLAG)?;
-    let wallet_password_path: PathBuf = clap_utils::parse_required(matches, PASSPHRASE_FLAG)?;
+pub fn cli_run(matches: &ArgMatches, wallet_base_dir: PathBuf) -> Result<(), String> {
     let mnemonic_output_path: Option<PathBuf> = clap_utils::parse_optional(matches, MNEMONIC_FLAG)?;
-    let type_field: String = clap_utils::parse_required(matches, TYPE_FLAG)?;
-
-    let wallet_type = match type_field.as_ref() {
-        HD_TYPE => WalletType::Hd,
-        unknown => return Err(format!("--{} {} is not supported", TYPE_FLAG, unknown)),
-    };
-
-    let mgr = WalletManager::open(&base_dir)
-        .map_err(|e| format!("Unable to open --{}: {:?}", BASE_DIR_FLAG, e))?;
 
     // Create a new random mnemonic.
     //
     // The `tiny-bip39` crate uses `thread_rng()` for this entropy.
-    let mnemonic = Mnemonic::new(MnemonicType::Words12, Language::English);
+    let mnemonic_length = clap_utils::parse_required(matches, MNEMONIC_LENGTH_FLAG)?;
+    let mnemonic = Mnemonic::new(
+        MnemonicType::for_word_count(mnemonic_length).expect("Mnemonic length already validated"),
+        Language::English,
+    );
 
-    // Create a random password if the file does not exist.
-    if !wallet_password_path.exists() {
-        // To prevent users from accidentally supplying their password to the PASSPHRASE_FLAG and
-        // create a file with that name, we require that the password has a .pass suffix.
-        if wallet_password_path.extension() != Some(&OsStr::new("pass")) {
-            return Err(format!(
-                "Only creates a password file if that file ends in .pass: {:?}",
-                wallet_password_path
-            ));
-        }
-
-        create_with_600_perms(&wallet_password_path, random_password().as_bytes())
-            .map_err(|e| format!("Unable to write to {:?}: {:?}", wallet_password_path, e))?;
-    }
-
-    let wallet_password = fs::read(&wallet_password_path)
-        .map_err(|e| format!("Unable to read {:?}: {:?}", wallet_password_path, e))
-        .map(|bytes| PlainText::from(strip_off_newlines(bytes)))?;
-
-    let wallet = mgr
-        .create_wallet(name, wallet_type, &mnemonic, wallet_password.as_bytes())
-        .map_err(|e| format!("Unable to create wallet: {:?}", e))?;
+    let wallet = create_wallet_from_mnemonic(matches, &wallet_base_dir.as_path(), &mnemonic)?;
 
     if let Some(path) = mnemonic_output_path {
         create_with_600_perms(&path, mnemonic.phrase().as_bytes())
             .map_err(|e| format!("Unable to write mnemonic to {:?}: {:?}", path, e))?;
     }
 
-    println!("Your wallet's 12-word BIP-39 mnemonic is:");
+    println!("Your wallet's {}-word BIP-39 mnemonic is:", mnemonic_length);
     println!();
     println!("\t{}", mnemonic.phrase());
     println!();
@@ -140,6 +145,99 @@ pub fn cli_run(matches: &ArgMatches, base_dir: PathBuf) -> Result<(), String> {
     Ok(())
 }
 
+pub fn create_wallet_from_mnemonic(
+    matches: &ArgMatches,
+    wallet_base_dir: &Path,
+    mnemonic: &Mnemonic,
+) -> Result<LockedWallet, String> {
+    let name: Option<String> = clap_utils::parse_optional(matches, NAME_FLAG)?;
+    let wallet_password_path: Option<PathBuf> = clap_utils::parse_optional(matches, PASSWORD_FLAG)?;
+    let type_field: String = clap_utils::parse_required(matches, TYPE_FLAG)?;
+    let stdin_inputs = matches.is_present(STDIN_INPUTS_FLAG);
+
+    let wallet_type = match type_field.as_ref() {
+        HD_TYPE => WalletType::Hd,
+        unknown => return Err(format!("--{} {} is not supported", TYPE_FLAG, unknown)),
+    };
+
+    let mgr = WalletManager::open(&wallet_base_dir)
+        .map_err(|e| format!("Unable to open --{}: {:?}", WALLETS_DIR_FLAG, e))?;
+
+    let wallet_password: PlainText = match wallet_password_path {
+        Some(path) => {
+            // Create a random password if the file does not exist.
+            if !path.exists() {
+                // To prevent users from accidentally supplying their password to the PASSWORD_FLAG and
+                // create a file with that name, we require that the password has a .pass suffix.
+                if path.extension() != Some(&OsStr::new("pass")) {
+                    return Err(format!(
+                        "Only creates a password file if that file ends in .pass: {:?}",
+                        path
+                    ));
+                }
+
+                create_with_600_perms(&path, random_password().as_bytes())
+                    .map_err(|e| format!("Unable to write to {:?}: {:?}", path, e))?;
+            }
+            read_new_wallet_password_from_cli(Some(path), stdin_inputs)?
+        }
+        None => read_new_wallet_password_from_cli(None, stdin_inputs)?,
+    };
+
+    let wallet_name = read_wallet_name_from_cli(name, stdin_inputs)?;
+
+    let wallet = mgr
+        .create_wallet(
+            wallet_name,
+            wallet_type,
+            &mnemonic,
+            wallet_password.as_bytes(),
+        )
+        .map_err(|e| format!("Unable to create wallet: {:?}", e))?;
+    Ok(wallet)
+}
+
+/// Used when a user is creating a new wallet. Read in a wallet password from a file if the password file
+/// path is provided. Otherwise, read from an interactive prompt using tty unless the `--stdin-inputs`
+/// flag is provided. This verifies the password complexity and verifies the password is correctly re-entered.
+pub fn read_new_wallet_password_from_cli(
+    password_file_path: Option<PathBuf>,
+    stdin_inputs: bool,
+) -> Result<PlainText, String> {
+    match password_file_path {
+        Some(path) => {
+            let password: PlainText = fs::read(&path)
+                .map_err(|e| format!("Unable to read {:?}: {:?}", path, e))
+                .map(|bytes| strip_off_newlines(bytes).into())?;
+
+            // Ensure the password meets the minimum requirements.
+            is_password_sufficiently_complex(password.as_bytes())?;
+            Ok(password)
+        }
+        None => loop {
+            eprintln!("");
+            eprintln!("{}", NEW_WALLET_PASSWORD_PROMPT);
+            let password =
+                PlainText::from(read_password_from_user(stdin_inputs)?.as_ref().to_vec());
+
+            // Ensure the password meets the minimum requirements.
+            match is_password_sufficiently_complex(password.as_bytes()) {
+                Ok(_) => {
+                    eprintln!("{}", RETYPE_PASSWORD_PROMPT);
+                    let retyped_password =
+                        PlainText::from(read_password_from_user(stdin_inputs)?.as_ref().to_vec());
+                    if retyped_password == password {
+                        break Ok(password);
+                    } else {
+                        eprintln!("Passwords do not match.");
+                    }
+                }
+                Err(message) => eprintln!("{}", message),
+            }
+        },
+    }
+}
+
 /// Creates a file with `600 (-rw-------)` permissions.
 pub fn create_with_600_perms<P: AsRef<Path>>(path: P, bytes: &[u8]) -> Result<(), String> {
     let path = path.as_ref();

account_manager/src/wallet/list.rs

@@ -1,4 +1,4 @@
-use crate::BASE_DIR_FLAG;
+use crate::WALLETS_DIR_FLAG;
 use clap::App;
 use eth2_wallet_manager::WalletManager;
 use std::path::PathBuf;
@@ -9,9 +9,9 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
     App::new(CMD).about("Lists the names of all wallets.")
 }
 
-pub fn cli_run(base_dir: PathBuf) -> Result<(), String> {
-    let mgr = WalletManager::open(&base_dir)
-        .map_err(|e| format!("Unable to open --{}: {:?}", BASE_DIR_FLAG, e))?;
+pub fn cli_run(wallet_base_dir: PathBuf) -> Result<(), String> {
+    let mgr = WalletManager::open(&wallet_base_dir)
+        .map_err(|e| format!("Unable to open --{}: {:?}", WALLETS_DIR_FLAG, e))?;
 
     for (name, _uuid) in mgr
         .wallets()

account_manager/src/wallet/mod.rs

@@ -1,11 +1,11 @@
 pub mod create;
 pub mod list;
+pub mod recover;
 
-use crate::{
-    common::{base_wallet_dir, ensure_dir_exists},
-    BASE_DIR_FLAG,
-};
+use crate::WALLETS_DIR_FLAG;
 use clap::{App, Arg, ArgMatches};
+use directory::{ensure_dir_exists, parse_path_or_default_with_flag, DEFAULT_WALLET_DIR};
+use std::path::PathBuf;
 
 pub const CMD: &str = "wallet";
 
@@ -13,23 +13,33 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
     App::new(CMD)
         .about("Manage wallets, from which validator keys can be derived.")
         .arg(
-            Arg::with_name(BASE_DIR_FLAG)
-                .long(BASE_DIR_FLAG)
-                .value_name("BASE_DIRECTORY")
-                .help("A path containing Eth2 EIP-2386 wallets. Defaults to ~/.lighthouse/wallets")
-                .takes_value(true),
+            Arg::with_name(WALLETS_DIR_FLAG)
+                .long(WALLETS_DIR_FLAG)
+                .value_name("WALLETS_DIRECTORY")
+                .help("A path containing Eth2 EIP-2386 wallets. Defaults to ~/.lighthouse/{network}/wallets")
+                .takes_value(true)
+                .conflicts_with("datadir"),
         )
         .subcommand(create::cli_app())
         .subcommand(list::cli_app())
+        .subcommand(recover::cli_app())
 }
 
 pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
-    let base_dir = base_wallet_dir(matches, BASE_DIR_FLAG)?;
-    ensure_dir_exists(&base_dir)?;
+    let wallet_base_dir = if matches.value_of("datadir").is_some() {
+        let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
+        path.join(DEFAULT_WALLET_DIR)
+    } else {
+        parse_path_or_default_with_flag(matches, WALLETS_DIR_FLAG, DEFAULT_WALLET_DIR)?
+    };
+    ensure_dir_exists(&wallet_base_dir)?;
+
+    eprintln!("wallet-dir path: {:?}", wallet_base_dir);
 
     match matches.subcommand() {
-        (create::CMD, Some(matches)) => create::cli_run(matches, base_dir),
-        (list::CMD, Some(_)) => list::cli_run(base_dir),
+        (create::CMD, Some(matches)) => create::cli_run(matches, wallet_base_dir),
+        (list::CMD, Some(_)) => list::cli_run(wallet_base_dir),
+        (recover::CMD, Some(matches)) => recover::cli_run(matches, wallet_base_dir),
         (unknown, _) => Err(format!(
             "{} does not have a {} command. See --help",
             CMD, unknown

account_manager/src/wallet/recover.rs

@@ -0,0 +1,84 @@
+use crate::common::read_mnemonic_from_cli;
+use crate::wallet::create::{create_wallet_from_mnemonic, STDIN_INPUTS_FLAG};
+use crate::wallet::create::{HD_TYPE, NAME_FLAG, PASSWORD_FLAG, TYPE_FLAG};
+use clap::{App, Arg, ArgMatches};
+use std::path::PathBuf;
+
+pub const CMD: &str = "recover";
+pub const MNEMONIC_FLAG: &str = "mnemonic-path";
+
+pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
+    App::new(CMD)
+        .about("Recovers an EIP-2386 wallet from a given a BIP-39 mnemonic phrase.")
+        .arg(
+            Arg::with_name(NAME_FLAG)
+                .long(NAME_FLAG)
+                .value_name("WALLET_NAME")
+                .help(
+                    "The wallet will be created with this name. It is not allowed to \
+                            create two wallets with the same name for the same --base-dir.",
+                )
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(PASSWORD_FLAG)
+                .long(PASSWORD_FLAG)
+                .value_name("PASSWORD_FILE_PATH")
+                .help(
+                    "This will be the new password for your recovered wallet. \
+                    A path to a file containing the password which will unlock the wallet. \
+                    If the file does not exist, a random password will be generated and \
+                    saved at that path. To avoid confusion, if the file does not already \
+                    exist it must include a '.pass' suffix.",
+                )
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(MNEMONIC_FLAG)
+                .long(MNEMONIC_FLAG)
+                .value_name("MNEMONIC_PATH")
+                .help("If present, the mnemonic will be read in from this file.")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name(TYPE_FLAG)
+                .long(TYPE_FLAG)
+                .value_name("WALLET_TYPE")
+                .help(
+                    "The type of wallet to create. Only HD (hierarchical-deterministic) \
+                            wallets are supported presently..",
+                )
+                .takes_value(true)
+                .possible_values(&[HD_TYPE])
+                .default_value(HD_TYPE),
+        )
+        .arg(
+            Arg::with_name(STDIN_INPUTS_FLAG)
+                .long(STDIN_INPUTS_FLAG)
+                .help("If present, read all user inputs from stdin instead of tty."),
+        )
+}
+
+pub fn cli_run(matches: &ArgMatches, wallet_base_dir: PathBuf) -> Result<(), String> {
+    let mnemonic_path: Option<PathBuf> = clap_utils::parse_optional(matches, MNEMONIC_FLAG)?;
+    let stdin_inputs = matches.is_present(STDIN_INPUTS_FLAG);
+
+    eprintln!("");
+    eprintln!("WARNING: KEY RECOVERY CAN LEAD TO DUPLICATING VALIDATORS KEYS, WHICH CAN LEAD TO SLASHING.");
+    eprintln!("");
+
+    let mnemonic = read_mnemonic_from_cli(mnemonic_path, stdin_inputs)?;
+
+    let wallet = create_wallet_from_mnemonic(matches, &wallet_base_dir.as_path(), &mnemonic)
+        .map_err(|e| format!("Unable to create wallet: {:?}", e))?;
+
+    println!("Your wallet has been successfully recovered.");
+    println!();
+    println!("Your wallet's UUID is:");
+    println!();
+    println!("\t{}", wallet.wallet().uuid());
+    println!();
+    println!("You do not need to backup your UUID or keep it secret.");
+
+    Ok(())
+}

beacon_node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "beacon_node"
-version = "0.2.2"
+version = "1.1.0"
 authors = ["Paul Hauner <paul@paulhauner.com>", "Age Manning <Age@AgeManning.com"]
 edition = "2018"
 
@@ -10,6 +10,7 @@ path = "src/lib.rs"
 
 [dev-dependencies]
 node_test_rig = { path = "../testing/node_test_rig" }
+tokio-compat-02 = "0.1"
 
 [features]
 write_ssz_files = ["beacon_chain/write_ssz_files"]  # Writes debugging .ssz files to /tmp during block processing.
@@ -20,23 +21,27 @@ beacon_chain = { path = "beacon_chain" }
 types = { path = "../consensus/types" }
 store = { path = "./store" }
 client = { path = "client" }
-clap = "2.33.0"
+clap = "2.33.3"
 rand = "0.7.3"
 slog = { version = "2.5.2", features = ["max_level_trace", "release_max_level_trace"] }
-slog-term = "2.5.0"
+slog-term = "2.6.0"
 slog-async = "2.5.0"
-ctrlc = { version = "3.1.4", features = ["termination"] }
-tokio = { version = "0.2.21", features = ["time"] }
+ctrlc = { version = "3.1.6", features = ["termination"] }
+tokio = { version = "0.3.2", features = ["time"] }
 exit-future = "0.2.0"
-dirs = "2.0.2"
+dirs = "3.0.1"
 logging = { path = "../common/logging" }
-futures = "0.3.5"
+directory = {path = "../common/directory"}
+futures = "0.3.7"
 environment = { path = "../lighthouse/environment" }
+task_executor = { path = "../common/task_executor" }
 genesis = { path = "genesis" }
-eth2_testnet_config = { path = "../common/eth2_testnet_config" }
+eth2_network_config = { path = "../common/eth2_network_config" }
 eth2_libp2p = { path = "./eth2_libp2p" }
 eth2_ssz = "0.1.2"
-serde = "1.0.110"
+serde = "1.0.116"
 clap_utils = { path = "../common/clap_utils" }
-hyper = "0.13.5"
+hyper = "0.13.8"
 lighthouse_version = { path = "../common/lighthouse_version" }
+hex = "0.4.2"
+slasher = { path = "../slasher" }

beacon_node/beacon_chain/Cargo.toml

@@ -8,9 +8,12 @@ edition = "2018"
 default = ["participation_metrics"]
 write_ssz_files = []  # Writes debugging .ssz files to /tmp during block processing.
 participation_metrics = []  # Exposes validator participation metrics to Prometheus.
+test_logger = [] # Print log output to stderr when running tests instead of dropping it
 
 [dev-dependencies]
 int_to_bytes = { path = "../../consensus/int_to_bytes" }
+maplit = "1.0.2"
+environment = { path = "../../lighthouse/environment" }
 
 [dependencies]
 eth2_config = { path = "../../common/eth2_config" }
@@ -18,41 +21,44 @@ merkle_proof = { path = "../../consensus/merkle_proof" }
 store = { path = "../store" }
 parking_lot = "0.11.0"
 lazy_static = "1.4.0"
-smallvec = "1.4.1"
+smallvec = "1.6.1"
 lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
-log = "0.4.8"
+log = "0.4.11"
 operation_pool = { path = "../operation_pool" }
-rayon = "1.3.0"
-serde = "1.0.110"
-serde_derive = "1.0.110"
-serde_yaml = "0.8.11"
-serde_json = "1.0.52"
+rayon = "1.4.1"
+serde = "1.0.116"
+serde_derive = "1.0.116"
+serde_yaml = "0.8.13"
+serde_json = "1.0.58"
 slog = { version = "2.5.2", features = ["max_level_trace"] }
-slog-term = "2.6.0"
-sloggers = "1.0.0"
+sloggers = "1.0.1"
 slot_clock = { path = "../../common/slot_clock" }
 eth2_hashing = "0.1.0"
 eth2_ssz = "0.1.2"
 eth2_ssz_types = { path = "../../consensus/ssz_types" }
 eth2_ssz_derive = "0.1.0"
 state_processing = { path = "../../consensus/state_processing" }
-tree_hash = "0.1.0"
+tree_hash = "0.1.1"
 types = { path = "../../consensus/types" }
-tokio = "0.2.21"
+tokio = "0.3.2"
 eth1 = { path = "../eth1" }
-websocket_server = { path = "../websocket_server" }
-futures = "0.3.5"
+futures = "0.3.7"
 genesis = { path = "../genesis" }
-integer-sqrt = "0.1.3"
+integer-sqrt = "0.1.5"
 rand = "0.7.3"
+rand_core = "0.5.1"
 proto_array = { path = "../../consensus/proto_array" }
-lru = "0.5.1"
+lru = "0.6.0"
 tempfile = "3.1.0"
-bitvec = "0.17.4"
+bitvec = "0.19.3"
 bls = { path = "../../crypto/bls" }
 safe_arith = { path = "../../consensus/safe_arith" }
 fork_choice = { path = "../../consensus/fork_choice" }
-environment = { path = "../../lighthouse/environment" }
-bus = "2.2.3"
+task_executor = { path = "../../common/task_executor" }
 derivative = "2.1.1"
 itertools = "0.9.0"
+regex = "1.3.9"
+exit-future = "0.2.0"
+slasher = { path = "../../slasher" }
+eth2 = { path = "../../common/eth2" }
+strum = { version = "0.20", features = ["derive"] }

beacon_node/beacon_chain/src/attestation_verification.rs

@@ -28,8 +28,7 @@
 
 use crate::{
     beacon_chain::{
-        ATTESTATION_CACHE_LOCK_TIMEOUT, HEAD_LOCK_TIMEOUT, MAXIMUM_GOSSIP_CLOCK_DISPARITY,
-        VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT,
+        HEAD_LOCK_TIMEOUT, MAXIMUM_GOSSIP_CLOCK_DISPARITY, VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT,
     },
     metrics,
     observed_attestations::ObserveOutcome,
@@ -37,22 +36,23 @@ use crate::{
     BeaconChain, BeaconChainError, BeaconChainTypes,
 };
 use bls::verify_signature_sets;
+use proto_array::Block as ProtoBlock;
 use slog::debug;
 use slot_clock::SlotClock;
 use state_processing::{
     common::get_indexed_attestation,
     per_block_processing::errors::AttestationValidationError,
-    per_slot_processing,
     signature_sets::{
         indexed_attestation_signature_set_from_pubkeys,
         signed_aggregate_selection_proof_signature_set, signed_aggregate_signature_set,
     },
 };
 use std::borrow::Cow;
+use strum::AsRefStr;
 use tree_hash::TreeHash;
 use types::{
     Attestation, BeaconCommittee, CommitteeIndex, Epoch, EthSpec, Hash256, IndexedAttestation,
-    RelativeEpoch, SelectionProof, SignedAggregateAndProof, Slot, SubnetId,
+    SelectionProof, SignedAggregateAndProof, Slot, SubnetId,
 };
 
 /// Returned when an attestation was not successfully verified. It might not have been verified for
@@ -62,7 +62,7 @@ use types::{
 ///   other than `BeaconChainError`).
 /// - The application encountered an internal error whilst attempting to determine validity
 ///   (the `BeaconChainError` variant)
-#[derive(Debug)]
+#[derive(Debug, AsRefStr)]
 pub enum Error {
     /// The attestation is from a slot that is later than the current slot (with respect to the
     /// gossip clock disparity).
@@ -220,6 +220,27 @@ pub enum Error {
     ///
     /// The peer has sent an invalid message.
     Invalid(AttestationValidationError),
+    /// The attestation head block is too far behind the attestation slot, causing many skip slots.
+    /// This is deemed a DoS risk.
+    TooManySkippedSlots {
+        head_block_slot: Slot,
+        attestation_slot: Slot,
+    },
+    /// The attestation has an invalid target epoch.
+    ///
+    /// ## Peer scoring
+    ///
+    /// The peer has sent an invalid message.
+    InvalidTargetEpoch { slot: Slot, epoch: Epoch },
+    /// The attestation references an invalid target block.
+    ///
+    /// ## Peer scoring
+    ///
+    /// The peer has sent an invalid message.
+    InvalidTargetRoot {
+        attestation: Hash256,
+        expected: Option<Hash256>,
+    },
     /// There was an error whilst processing the attestation. It is not known if it is valid or invalid.
     ///
     /// ## Peer scoring
@@ -245,6 +266,7 @@ pub struct VerifiedAggregatedAttestation<T: BeaconChainTypes> {
 pub struct VerifiedUnaggregatedAttestation<T: BeaconChainTypes> {
     attestation: Attestation<T::EthSpec>,
     indexed_attestation: IndexedAttestation<T::EthSpec>,
+    subnet_id: SubnetId,
 }
 
 /// Custom `Clone` implementation is to avoid the restrictive trait bounds applied by the usual derive
@@ -254,6 +276,7 @@ impl<T: BeaconChainTypes> Clone for VerifiedUnaggregatedAttestation<T> {
         Self {
             attestation: self.attestation.clone(),
             indexed_attestation: self.indexed_attestation.clone(),
+            subnet_id: self.subnet_id,
         }
     }
 }
@@ -276,6 +299,76 @@ impl<T: BeaconChainTypes> SignatureVerifiedAttestation<T> for VerifiedUnaggregat
     }
 }
 
+/// Information about invalid attestations which might still be slashable despite being invalid.
+pub enum AttestationSlashInfo<T: BeaconChainTypes, TErr> {
+    /// The attestation is invalid, but its signature wasn't checked.
+    SignatureNotChecked(Attestation<T::EthSpec>, TErr),
+    /// As for `SignatureNotChecked`, but we know the `IndexedAttestation`.
+    SignatureNotCheckedIndexed(IndexedAttestation<T::EthSpec>, TErr),
+    /// The attestation's signature is invalid, so it will never be slashable.
+    SignatureInvalid(TErr),
+    /// The signature is valid but the attestation is invalid in some other way.
+    SignatureValid(IndexedAttestation<T::EthSpec>, TErr),
+}
+
+/// After processing an attestation normally, optionally process it further for the slasher.
+///
+/// This maps an `AttestationSlashInfo` error back into a regular `Error`, performing signature
+/// checks on attestations that failed verification for other reasons.
+///
+/// No substantial extra work will be done if there is no slasher configured.
+fn process_slash_info<T: BeaconChainTypes>(
+    slash_info: AttestationSlashInfo<T, Error>,
+    chain: &BeaconChain<T>,
+) -> Error {
+    use AttestationSlashInfo::*;
+
+    if let Some(slasher) = chain.slasher.as_ref() {
+        let (indexed_attestation, check_signature, err) = match slash_info {
+            SignatureNotChecked(attestation, err) => {
+                match obtain_indexed_attestation_and_committees_per_slot(chain, &attestation) {
+                    Ok((indexed, _)) => (indexed, true, err),
+                    Err(e) => {
+                        debug!(
+                            chain.log,
+                            "Unable to obtain indexed form of attestation for slasher";
+                            "attestation_root" => format!("{:?}", attestation.tree_hash_root()),
+                            "error" => format!("{:?}", e)
+                        );
+                        return err;
+                    }
+                }
+            }
+            SignatureNotCheckedIndexed(indexed, err) => (indexed, true, err),
+            SignatureInvalid(e) => return e,
+            SignatureValid(indexed, err) => (indexed, false, err),
+        };
+
+        if check_signature {
+            if let Err(e) = verify_attestation_signature(chain, &indexed_attestation) {
+                debug!(
+                    chain.log,
+                    "Signature verification for slasher failed";
+                    "error" => format!("{:?}", e),
+                );
+                return err;
+            }
+        }
+
+        // Supply to slasher.
+        slasher.accept_attestation(indexed_attestation);
+
+        err
+    } else {
+        match slash_info {
+            SignatureNotChecked(_, e)
+            | SignatureNotCheckedIndexed(_, e)
+            | SignatureInvalid(e)
+            | SignatureValid(_, e) => e,
+        }
+    }
+}
+
 impl<T: BeaconChainTypes> VerifiedAggregatedAttestation<T> {
     /// Returns `Ok(Self)` if the `signed_aggregate` is valid to be (re)published on the gossip
     /// network.
@@ -283,6 +376,21 @@ impl<T: BeaconChainTypes> VerifiedAggregatedAttestation<T> {
         signed_aggregate: SignedAggregateAndProof<T::EthSpec>,
         chain: &BeaconChain<T>,
     ) -> Result<Self, Error> {
+        Self::verify_slashable(signed_aggregate, chain)
+            .map(|verified_aggregate| {
+                if let Some(slasher) = chain.slasher.as_ref() {
+                    slasher.accept_attestation(verified_aggregate.indexed_attestation.clone());
+                }
+                verified_aggregate
+            })
+            .map_err(|slash_info| process_slash_info(slash_info, chain))
+    }
+
+    /// Run the checks that happen before an indexed attestation is constructed.
+    fn verify_early_checks(
+        signed_aggregate: &SignedAggregateAndProof<T::EthSpec>,
+        chain: &BeaconChain<T>,
+    ) -> Result<Hash256, Error> {
         let attestation = &signed_aggregate.message.aggregate;
 
         // Ensure attestation is within the last ATTESTATION_PROPAGATION_SLOT_RANGE slots (within a
@@ -295,6 +403,7 @@ impl<T: BeaconChainTypes> VerifiedAggregatedAttestation<T> {
         let attestation_root = attestation.tree_hash_root();
         if chain
             .observed_attestations
+            .write()
             .is_known(attestation, attestation_root)
             .map_err(|e| Error::BeaconChainError(e.into()))?
         {
@@ -305,9 +414,10 @@ impl<T: BeaconChainTypes> VerifiedAggregatedAttestation<T> {
 
         // Ensure there has been no other observed aggregate for the given `aggregator_index`.
         //
-        // Note: do not observe yet, only observe once the attestation has been verfied.
+        // Note: do not observe yet, only observe once the attestation has been verified.
         match chain
             .observed_aggregators
+            .read()
             .validator_has_been_observed(attestation, aggregator_index as usize)
         {
             Ok(true) => Err(Error::AggregatorAlreadyKnown(aggregator_index)),
@@ -319,6 +429,7 @@ impl<T: BeaconChainTypes> VerifiedAggregatedAttestation<T> {
         }?;
 
         // Ensure the block being voted for (attestation.data.beacon_block_root) passes validation.
+        // Don't enforce the skip slot restriction for aggregates.
         //
         // This indirectly checks to see if the `attestation.data.beacon_block_root` is in our fork
         // choice. Any known, non-finalized, processed block should be in fork choice, so this
@@ -327,15 +438,82 @@ impl<T: BeaconChainTypes> VerifiedAggregatedAttestation<T> {
         //
         // Attestations must be for a known block. If the block is unknown, we simply drop the
         // attestation and do not delay consideration for later.
-        verify_head_block_is_known(chain, &attestation)?;
+        let head_block = verify_head_block_is_known(chain, &attestation, None)?;
+
+        // Check the attestation target root is consistent with the head root.
+        //
+        // This check is not in the specification, however we guard against it since it opens us up
+        // to weird edge cases during verification.
+        //
+        // Whilst this attestation *technically* could be used to add value to a block, it is
+        // invalid in the spirit of the protocol. Here we choose safety over profit.
+        verify_attestation_target_root::<T::EthSpec>(&head_block, &attestation)?;
 
         // Ensure that the attestation has participants.
         if attestation.aggregation_bits.is_zero() {
-            return Err(Error::EmptyAggregationBitfield);
+            Err(Error::EmptyAggregationBitfield)
+        } else {
+            Ok(attestation_root)
+        }
+    }
+
+    /// Run the checks that happen after the indexed attestation and signature have been checked.
+    fn verify_late_checks(
+        signed_aggregate: &SignedAggregateAndProof<T::EthSpec>,
+        attestation_root: Hash256,
+        chain: &BeaconChain<T>,
+    ) -> Result<(), Error> {
+        let attestation = &signed_aggregate.message.aggregate;
+        let aggregator_index = signed_aggregate.message.aggregator_index;
+
+        // Observe the valid attestation so we do not re-process it.
+        //
+        // It's important to double check that the attestation is not already known, otherwise two
+        // attestations processed at the same time could be published.
+        if let ObserveOutcome::AlreadyKnown = chain
+            .observed_attestations
+            .write()
+            .observe_attestation(attestation, Some(attestation_root))
+            .map_err(|e| Error::BeaconChainError(e.into()))?
+        {
+            return Err(Error::AttestationAlreadyKnown(attestation_root));
         }
 
+        // Observe the aggregator so we don't process another aggregate from them.
+        //
+        // It's important to double check that the attestation is not already known, otherwise two
+        // attestations processed at the same time could be published.
+        if chain
+            .observed_aggregators
+            .write()
+            .observe_validator(&attestation, aggregator_index as usize)
+            .map_err(BeaconChainError::from)?
+        {
+            return Err(Error::PriorAttestationKnown {
+                validator_index: aggregator_index,
+                epoch: attestation.data.target.epoch,
+            });
+        }
+
+        Ok(())
+    }
+
+    /// Verify the attestation, producing extra information about whether it might be slashable.
+    pub fn verify_slashable(
+        signed_aggregate: SignedAggregateAndProof<T::EthSpec>,
+        chain: &BeaconChain<T>,
+    ) -> Result<Self, AttestationSlashInfo<T, Error>> {
+        use AttestationSlashInfo::*;
+
+        let attestation = &signed_aggregate.message.aggregate;
+        let aggregator_index = signed_aggregate.message.aggregator_index;
+        let attestation_root = match Self::verify_early_checks(&signed_aggregate, chain) {
+            Ok(root) => root,
+            Err(e) => return Err(SignatureNotChecked(signed_aggregate.message.aggregate, e)),
+        };
+
         let indexed_attestation =
-            map_attestation_committee(chain, attestation, |(committee, _)| {
+            match map_attestation_committee(chain, attestation, |(committee, _)| {
                 // Note: this clones the signature which is known to be a relatively slow operation.
                 //
                 // Future optimizations should remove this clone.
@@ -354,40 +532,29 @@ impl<T: BeaconChainTypes> VerifiedAggregatedAttestation<T> {
                     return Err(Error::AggregatorNotInCommittee { aggregator_index });
                 }
 
-                get_indexed_attestation(committee.committee, &attestation)
+                get_indexed_attestation(committee.committee, attestation)
                     .map_err(|e| BeaconChainError::from(e).into())
-            })?;
+            }) {
+                Ok(indexed_attestation) => indexed_attestation,
+                Err(e) => return Err(SignatureNotChecked(signed_aggregate.message.aggregate, e)),
+            };
 
         // Ensure that all signatures are valid.
-        if !verify_signed_aggregate_signatures(chain, &signed_aggregate, &indexed_attestation)? {
-            return Err(Error::InvalidSignature);
+        if let Err(e) =
+            verify_signed_aggregate_signatures(chain, &signed_aggregate, &indexed_attestation)
+                .and_then(|is_valid| {
+                    if !is_valid {
+                        Err(Error::InvalidSignature)
+                    } else {
+                        Ok(())
+                    }
+                })
+        {
+            return Err(SignatureInvalid(e));
         }
 
-        // Observe the valid attestation so we do not re-process it.
-        //
-        // It's important to double check that the attestation is not already known, otherwise two
-        // attestations processed at the same time could be published.
-        if let ObserveOutcome::AlreadyKnown = chain
-            .observed_attestations
-            .observe_attestation(attestation, Some(attestation_root))
-            .map_err(|e| Error::BeaconChainError(e.into()))?
-        {
-            return Err(Error::AttestationAlreadyKnown(attestation_root));
-        }
-
-        // Observe the aggregator so we don't process another aggregate from them.
-        //
-        // It's important to double check that the attestation is not already known, otherwise two
-        // attestations processed at the same time could be published.
-        if chain
-            .observed_aggregators
-            .observe_validator(&attestation, aggregator_index as usize)
-            .map_err(BeaconChainError::from)?
-        {
-            return Err(Error::PriorAttestationKnown {
-                validator_index: aggregator_index,
-                epoch: attestation.data.target.epoch,
-            });
+        if let Err(e) = Self::verify_late_checks(&signed_aggregate, attestation_root, chain) {
+            return Err(SignatureValid(indexed_attestation, e));
         }
 
         Ok(VerifiedAggregatedAttestation {
@@ -405,19 +572,29 @@ impl<T: BeaconChainTypes> VerifiedAggregatedAttestation<T> {
     pub fn attestation(&self) -> &Attestation<T::EthSpec> {
         &self.signed_aggregate.message.aggregate
     }
+
+    /// Returns the underlying `signed_aggregate`.
+    pub fn aggregate(&self) -> &SignedAggregateAndProof<T::EthSpec> {
+        &self.signed_aggregate
+    }
 }
 
 impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
-    /// Returns `Ok(Self)` if the `attestation` is valid to be (re)published on the gossip
-    /// network.
-    ///
-    /// `subnet_id` is the subnet from which we received this attestation. This function will
-    /// verify that it was received on the correct subnet.
-    pub fn verify(
-        attestation: Attestation<T::EthSpec>,
-        subnet_id: SubnetId,
+    /// Run the checks that happen before an indexed attestation is constructed.
+    pub fn verify_early_checks(
+        attestation: &Attestation<T::EthSpec>,
         chain: &BeaconChain<T>,
-    ) -> Result<Self, Error> {
+    ) -> Result<(), Error> {
+        let attestation_epoch = attestation.data.slot.epoch(T::EthSpec::slots_per_epoch());
+
+        // Check the attestation's epoch matches its target.
+        if attestation_epoch != attestation.data.target.epoch {
+            return Err(Error::InvalidTargetEpoch {
+                slot: attestation.data.slot,
+                epoch: attestation.data.target.epoch,
+            });
+        }
+
         // Ensure attestation is within the last ATTESTATION_PROPAGATION_SLOT_RANGE slots (within a
         // MAXIMUM_GOSSIP_CLOCK_DISPARITY allowance).
         //
@@ -426,18 +603,32 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
 
         // Check to ensure that the attestation is "unaggregated". I.e., it has exactly one
         // aggregation bit set.
-        let num_aggreagtion_bits = attestation.aggregation_bits.num_set_bits();
-        if num_aggreagtion_bits != 1 {
-            return Err(Error::NotExactlyOneAggregationBitSet(num_aggreagtion_bits));
+        let num_aggregation_bits = attestation.aggregation_bits.num_set_bits();
+        if num_aggregation_bits != 1 {
+            return Err(Error::NotExactlyOneAggregationBitSet(num_aggregation_bits));
         }
 
         // Attestations must be for a known block. If the block is unknown, we simply drop the
         // attestation and do not delay consideration for later.
-        verify_head_block_is_known(chain, &attestation)?;
+        //
+        // Enforce a maximum skip distance for unaggregated attestations.
+        let head_block =
+            verify_head_block_is_known(chain, &attestation, chain.config.import_max_skip_slots)?;
 
-        let (indexed_attestation, committees_per_slot) =
-            obtain_indexed_attestation_and_committees_per_slot(chain, &attestation)?;
+        // Check the attestation target root is consistent with the head root.
+        verify_attestation_target_root::<T::EthSpec>(&head_block, &attestation)?;
 
+        Ok(())
+    }
+
+    /// Run the checks that apply to the indexed attestation before the signature is checked.
+    pub fn verify_middle_checks(
+        attestation: &Attestation<T::EthSpec>,
+        indexed_attestation: &IndexedAttestation<T::EthSpec>,
+        committees_per_slot: u64,
+        subnet_id: Option<SubnetId>,
+        chain: &BeaconChain<T>,
+    ) -> Result<(u64, SubnetId), Error> {
         let expected_subnet_id = SubnetId::compute_subnet_for_attestation_data::<T::EthSpec>(
             &indexed_attestation.data,
             committees_per_slot,
@@ -445,18 +636,20 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
         )
         .map_err(BeaconChainError::from)?;
 
-        // Ensure the attestation is from the correct subnet.
-        if subnet_id != expected_subnet_id {
-            return Err(Error::InvalidSubnetId {
-                received: subnet_id,
-                expected: expected_subnet_id,
-            });
-        }
+        // If a subnet was specified, ensure that subnet is correct.
+        if let Some(subnet_id) = subnet_id {
+            if subnet_id != expected_subnet_id {
+                return Err(Error::InvalidSubnetId {
+                    received: subnet_id,
+                    expected: expected_subnet_id,
+                });
+            }
+        };
 
         let validator_index = *indexed_attestation
             .attesting_indices
             .first()
-            .ok_or_else(|| Error::NotExactlyOneAggregationBitSet(0))?;
+            .ok_or(Error::NotExactlyOneAggregationBitSet(0))?;
 
         /*
          * The attestation is the first valid attestation received for the participating validator
@@ -464,6 +657,7 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
          */
         if chain
             .observed_attesters
+            .read()
             .validator_has_been_observed(&attestation, validator_index as usize)
             .map_err(BeaconChainError::from)?
         {
@@ -473,9 +667,15 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
             });
         }
 
-        // The aggregate signature of the attestation is valid.
-        verify_attestation_signature(chain, &indexed_attestation)?;
+        Ok((validator_index, expected_subnet_id))
+    }
 
+    /// Run the checks that apply after the signature has been checked.
+    fn verify_late_checks(
+        attestation: &Attestation<T::EthSpec>,
+        validator_index: u64,
+        chain: &BeaconChain<T>,
+    ) -> Result<(), Error> {
         // Now that the attestation has been fully verified, store that we have received a valid
         // attestation from this validator.
         //
@@ -484,6 +684,7 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
         // process them in different threads.
         if chain
             .observed_attesters
+            .write()
             .observe_validator(&attestation, validator_index as usize)
             .map_err(BeaconChainError::from)?
         {
@@ -492,10 +693,73 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
                 epoch: attestation.data.target.epoch,
             });
         }
+        Ok(())
+    }
+
+    /// Returns `Ok(Self)` if the `attestation` is valid to be (re)published on the gossip
+    /// network.
+    ///
+    /// `subnet_id` is the subnet from which we received this attestation. This function will
+    /// verify that it was received on the correct subnet.
+    pub fn verify(
+        attestation: Attestation<T::EthSpec>,
+        subnet_id: Option<SubnetId>,
+        chain: &BeaconChain<T>,
+    ) -> Result<Self, Error> {
+        Self::verify_slashable(attestation, subnet_id, chain)
+            .map(|verified_unaggregated| {
+                if let Some(slasher) = chain.slasher.as_ref() {
+                    slasher.accept_attestation(verified_unaggregated.indexed_attestation.clone());
+                }
+                verified_unaggregated
+            })
+            .map_err(|slash_info| process_slash_info(slash_info, chain))
+    }
+
+    /// Verify the attestation, producing extra information about whether it might be slashable.
+    pub fn verify_slashable(
+        attestation: Attestation<T::EthSpec>,
+        subnet_id: Option<SubnetId>,
+        chain: &BeaconChain<T>,
+    ) -> Result<Self, AttestationSlashInfo<T, Error>> {
+        use AttestationSlashInfo::*;
+
+        if let Err(e) = Self::verify_early_checks(&attestation, chain) {
+            return Err(SignatureNotChecked(attestation, e));
+        }
+
+        let (indexed_attestation, committees_per_slot) =
+            match obtain_indexed_attestation_and_committees_per_slot(chain, &attestation) {
+                Ok(x) => x,
+                Err(e) => {
+                    return Err(SignatureNotChecked(attestation, e));
+                }
+            };
+
+        let (validator_index, expected_subnet_id) = match Self::verify_middle_checks(
+            &attestation,
+            &indexed_attestation,
+            committees_per_slot,
+            subnet_id,
+            chain,
+        ) {
+            Ok(t) => t,
+            Err(e) => return Err(SignatureNotCheckedIndexed(indexed_attestation, e)),
+        };
+
+        // The aggregate signature of the attestation is valid.
+        if let Err(e) = verify_attestation_signature(chain, &indexed_attestation) {
+            return Err(SignatureInvalid(e));
+        }
+
+        if let Err(e) = Self::verify_late_checks(&attestation, validator_index, chain) {
+            return Err(SignatureValid(indexed_attestation, e));
+        }
 
         Ok(Self {
             attestation,
             indexed_attestation,
+            subnet_id: expected_subnet_id,
         })
     }
 
@@ -504,11 +768,21 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
         chain.add_to_naive_aggregation_pool(self)
     }
 
+    /// Returns the correct subnet for the attestation.
+    pub fn subnet_id(&self) -> SubnetId {
+        self.subnet_id
+    }
+
     /// Returns the wrapped `attestation`.
     pub fn attestation(&self) -> &Attestation<T::EthSpec> {
         &self.attestation
     }
 
+    /// Returns the wrapped `indexed_attestation`.
+    pub fn indexed_attestation(&self) -> &IndexedAttestation<T::EthSpec> {
+        &self.indexed_attestation
+    }
+
     /// Returns a mutable reference to the underlying attestation.
     ///
     /// Only use during testing since modifying the `IndexedAttestation` can cause the attestation
@@ -519,6 +793,7 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
 }
 
 /// Returns `Ok(())` if the `attestation.data.beacon_block_root` is known to this chain.
+/// You can use this `shuffling_id` to read from the shuffling cache.
 ///
 /// The block root may not be known for two reasons:
 ///
@@ -531,13 +806,24 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
 fn verify_head_block_is_known<T: BeaconChainTypes>(
     chain: &BeaconChain<T>,
     attestation: &Attestation<T::EthSpec>,
-) -> Result<(), Error> {
-    if chain
+    max_skip_slots: Option<u64>,
+) -> Result<ProtoBlock, Error> {
+    if let Some(block) = chain
         .fork_choice
         .read()
-        .contains_block(&attestation.data.beacon_block_root)
+        .get_block(&attestation.data.beacon_block_root)
     {
-        Ok(())
+        // Reject any block that exceeds our limit on skipped slots.
+        if let Some(max_skip_slots) = max_skip_slots {
+            if attestation.data.slot > block.slot + max_skip_slots {
+                return Err(Error::TooManySkippedSlots {
+                    head_block_slot: block.slot,
+                    attestation_slot: attestation.data.slot,
+                });
+            }
+        }
+
+        Ok(block)
     } else {
         Err(Error::UnknownHeadBlock {
             beacon_block_root: attestation.data.beacon_block_root,
@@ -558,7 +844,7 @@ pub fn verify_propagation_slot_range<T: BeaconChainTypes>(
     let latest_permissible_slot = chain
         .slot_clock
         .now_with_future_tolerance(MAXIMUM_GOSSIP_CLOCK_DISPARITY)
-        .ok_or_else(|| BeaconChainError::UnableToReadSlot)?;
+        .ok_or(BeaconChainError::UnableToReadSlot)?;
     if attestation_slot > latest_permissible_slot {
         return Err(Error::FutureSlot {
             attestation_slot,
@@ -570,7 +856,7 @@ pub fn verify_propagation_slot_range<T: BeaconChainTypes>(
     let earliest_permissible_slot = chain
         .slot_clock
         .now_with_past_tolerance(MAXIMUM_GOSSIP_CLOCK_DISPARITY)
-        .ok_or_else(|| BeaconChainError::UnableToReadSlot)?
+        .ok_or(BeaconChainError::UnableToReadSlot)?
         - T::EthSpec::slots_per_epoch();
     if attestation_slot < earliest_permissible_slot {
         return Err(Error::PastSlot {
@@ -593,12 +879,12 @@ pub fn verify_attestation_signature<T: BeaconChainTypes>(
     let pubkey_cache = chain
         .validator_pubkey_cache
         .try_read_for(VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT)
-        .ok_or_else(|| BeaconChainError::ValidatorPubkeyCacheLockTimeout)?;
+        .ok_or(BeaconChainError::ValidatorPubkeyCacheLockTimeout)?;
 
     let fork = chain
         .canonical_head
         .try_read_for(HEAD_LOCK_TIMEOUT)
-        .ok_or_else(|| BeaconChainError::CanonicalHeadLockTimeout)
+        .ok_or(BeaconChainError::CanonicalHeadLockTimeout)
         .map(|head| head.beacon_state.fork)?;
 
     let signature_set = indexed_attestation_signature_set_from_pubkeys(
@@ -623,6 +909,57 @@ pub fn verify_attestation_signature<T: BeaconChainTypes>(
     }
 }
 
+/// Verifies that the `attestation.data.target.root` is indeed the target root of the block at
+/// `attestation.data.beacon_block_root`.
+pub fn verify_attestation_target_root<T: EthSpec>(
+    head_block: &ProtoBlock,
+    attestation: &Attestation<T>,
+) -> Result<(), Error> {
+    // Check the attestation target root.
+    let head_block_epoch = head_block.slot.epoch(T::slots_per_epoch());
+    let attestation_epoch = attestation.data.slot.epoch(T::slots_per_epoch());
+    if head_block_epoch > attestation_epoch {
+        // The epoch references an invalid head block from a future epoch.
+        //
+        // This check is not in the specification, however we guard against it since it opens us up
+        // to weird edge cases during verification.
+        //
+        // Whilst this attestation *technically* could be used to add value to a block, it is
+        // invalid in the spirit of the protocol. Here we choose safety over profit.
+        //
+        // Reference:
+        // https://github.com/ethereum/eth2.0-specs/pull/2001#issuecomment-699246659
+        return Err(Error::InvalidTargetRoot {
+            attestation: attestation.data.target.root,
+            // It is not clear what root we should expect in this case, since the attestation is
+            // fundamentally invalid.
+            expected: None,
+        });
+    } else {
+        let target_root = if head_block_epoch == attestation_epoch {
+            // If the block is in the same epoch as the attestation, then use the target root
+            // from the block.
+            head_block.target_root
+        } else {
+            // If the head block is from a previous epoch then skip slots will cause the head block
+            // root to become the target block root.
+            //
+            // We know the head block is from a previous epoch due to a previous check.
+            head_block.root
+        };
+
+        // Reject any attestation with an invalid target root.
+        if target_root != attestation.data.target.root {
+            return Err(Error::InvalidTargetRoot {
+                attestation: attestation.data.target.root,
+                expected: Some(target_root),
+            });
+        }
+    }
+
+    Ok(())
+}
+
 /// Verifies all the signatures in a `SignedAggregateAndProof` using BLS batch verification. This
 /// includes three signatures:
 ///
@@ -643,7 +980,7 @@ pub fn verify_signed_aggregate_signatures<T: BeaconChainTypes>(
     let pubkey_cache = chain
         .validator_pubkey_cache
         .try_read_for(VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT)
-        .ok_or_else(|| BeaconChainError::ValidatorPubkeyCacheLockTimeout)?;
+        .ok_or(BeaconChainError::ValidatorPubkeyCacheLockTimeout)?;
 
     let aggregator_index = signed_aggregate.message.aggregator_index;
     if aggregator_index >= pubkey_cache.len() as u64 {
@@ -653,7 +990,7 @@ pub fn verify_signed_aggregate_signatures<T: BeaconChainTypes>(
     let fork = chain
         .canonical_head
         .try_read_for(HEAD_LOCK_TIMEOUT)
-        .ok_or_else(|| BeaconChainError::CanonicalHeadLockTimeout)
+        .ok_or(BeaconChainError::CanonicalHeadLockTimeout)
         .map(|head| head.beacon_state.fork)?;
 
     let signature_sets = vec![
@@ -692,14 +1029,14 @@ type CommitteesPerSlot = u64;
 
 /// Returns the `indexed_attestation` and committee count per slot for the `attestation` using the
 /// public keys cached in the `chain`.
-pub fn obtain_indexed_attestation_and_committees_per_slot<T: BeaconChainTypes>(
+fn obtain_indexed_attestation_and_committees_per_slot<T: BeaconChainTypes>(
     chain: &BeaconChain<T>,
     attestation: &Attestation<T::EthSpec>,
 ) -> Result<(IndexedAttestation<T::EthSpec>, CommitteesPerSlot), Error> {
     map_attestation_committee(chain, attestation, |(committee, committees_per_slot)| {
         get_indexed_attestation(committee.committee, &attestation)
             .map(|attestation| (attestation, committees_per_slot))
-            .map_err(|e| BeaconChainError::from(e).into())
+            .map_err(Error::Invalid)
     })
 }
 
@@ -712,8 +1049,8 @@ pub fn obtain_indexed_attestation_and_committees_per_slot<T: BeaconChainTypes>(
 ///
 /// If the committee for `attestation` isn't found in the `shuffling_cache`, we will read a state
 /// from disk and then update the `shuffling_cache`.
-pub fn map_attestation_committee<'a, T, F, R>(
-    chain: &'a BeaconChain<T>,
+fn map_attestation_committee<T, F, R>(
+    chain: &BeaconChain<T>,
     attestation: &Attestation<T::EthSpec>,
     map_fn: F,
 ) -> Result<R, Error>
@@ -731,99 +1068,23 @@ where
     // processing an attestation that does not include our latest finalized block in its chain.
     //
     // We do not delay consideration for later, we simply drop the attestation.
-    let target_block = chain
-        .fork_choice
-        .read()
-        .get_block(&target.root)
-        .ok_or_else(|| Error::UnknownTargetRoot(target.root))?;
-
-    // Obtain the shuffling cache, timing how long we wait.
-    let cache_wait_timer =
-        metrics::start_timer(&metrics::ATTESTATION_PROCESSING_SHUFFLING_CACHE_WAIT_TIMES);
-
-    let mut shuffling_cache = chain
-        .shuffling_cache
-        .try_write_for(ATTESTATION_CACHE_LOCK_TIMEOUT)
-        .ok_or_else(|| BeaconChainError::AttestationCacheLockTimeout)?;
-
-    metrics::stop_timer(cache_wait_timer);
-
-    if let Some(committee_cache) = shuffling_cache.get(attestation_epoch, target.root) {
-        let committees_per_slot = committee_cache.committees_per_slot();
-        committee_cache
-            .get_beacon_committee(attestation.data.slot, attestation.data.index)
-            .map(|committee| map_fn((committee, committees_per_slot)))
-            .unwrap_or_else(|| {
-                Err(Error::NoCommitteeForSlotAndIndex {
-                    slot: attestation.data.slot,
-                    index: attestation.data.index,
-                })
-            })
-    } else {
-        // Drop the shuffling cache to avoid holding the lock for any longer than
-        // required.
-        drop(shuffling_cache);
-
-        debug!(
-            chain.log,
-            "Attestation processing cache miss";
-            "attn_epoch" => attestation_epoch.as_u64(),
-            "target_block_epoch" => target_block.slot.epoch(T::EthSpec::slots_per_epoch()).as_u64(),
-        );
-
-        let state_read_timer =
-            metrics::start_timer(&metrics::ATTESTATION_PROCESSING_STATE_READ_TIMES);
-
-        let mut state = chain
-            .get_state(&target_block.state_root, Some(target_block.slot))?
-            .ok_or_else(|| BeaconChainError::MissingBeaconState(target_block.state_root))?;
-
-        metrics::stop_timer(state_read_timer);
-        let state_skip_timer =
-            metrics::start_timer(&metrics::ATTESTATION_PROCESSING_STATE_SKIP_TIMES);
-
-        while state.current_epoch() + 1 < attestation_epoch {
-            // Here we tell `per_slot_processing` to skip hashing the state and just
-            // use the zero hash instead.
-            //
-            // The state roots are not useful for the shuffling, so there's no need to
-            // compute them.
-            per_slot_processing(&mut state, Some(Hash256::zero()), &chain.spec)
-                .map_err(BeaconChainError::from)?;
-        }
-
-        metrics::stop_timer(state_skip_timer);
-        let committee_building_timer =
-            metrics::start_timer(&metrics::ATTESTATION_PROCESSING_COMMITTEE_BUILDING_TIMES);
-
-        let relative_epoch = RelativeEpoch::from_epoch(state.current_epoch(), attestation_epoch)
-            .map_err(BeaconChainError::IncorrectStateForAttestation)?;
-
-        state
-            .build_committee_cache(relative_epoch, &chain.spec)
-            .map_err(BeaconChainError::from)?;
-
-        let committee_cache = state
-            .committee_cache(relative_epoch)
-            .map_err(BeaconChainError::from)?;
-
-        chain
-            .shuffling_cache
-            .try_write_for(ATTESTATION_CACHE_LOCK_TIMEOUT)
-            .ok_or_else(|| BeaconChainError::AttestationCacheLockTimeout)?
-            .insert(attestation_epoch, target.root, committee_cache);
-
-        metrics::stop_timer(committee_building_timer);
-
-        let committees_per_slot = committee_cache.committees_per_slot();
-        committee_cache
-            .get_beacon_committee(attestation.data.slot, attestation.data.index)
-            .map(|committee| map_fn((committee, committees_per_slot)))
-            .unwrap_or_else(|| {
-                Err(Error::NoCommitteeForSlotAndIndex {
-                    slot: attestation.data.slot,
-                    index: attestation.data.index,
-                })
-            })
+    if !chain.fork_choice.read().contains_block(&target.root) {
+        return Err(Error::UnknownTargetRoot(target.root));
     }
+
+    chain
+        .with_committee_cache(target.root, attestation_epoch, |committee_cache| {
+            let committees_per_slot = committee_cache.committees_per_slot();
+
+            Ok(committee_cache
+                .get_beacon_committee(attestation.data.slot, attestation.data.index)
+                .map(|committee| map_fn((committee, committees_per_slot)))
+                .unwrap_or_else(|| {
+                    Err(Error::NoCommitteeForSlotAndIndex {
+                        slot: attestation.data.slot,
+                        index: attestation.data.index,
+                    })
+                }))
+        })
+        .map_err(BeaconChainError::from)?
 }

beacon_node/beacon_chain/src/beacon_fork_choice_store.rs

@@ -320,14 +320,14 @@ where
                 .store
                 .get_item::<SignedBeaconBlock<E>>(&self.justified_checkpoint.root)
                 .map_err(Error::FailedToReadBlock)?
-                .ok_or_else(|| Error::MissingBlock(self.justified_checkpoint.root))?
+                .ok_or(Error::MissingBlock(self.justified_checkpoint.root))?
                 .message;
 
             self.justified_balances = self
                 .store
                 .get_state(&justified_block.state_root, Some(justified_block.slot))
                 .map_err(Error::FailedToReadState)?
-                .ok_or_else(|| Error::MissingState(justified_block.state_root))?
+                .ok_or(Error::MissingState(justified_block.state_root))?
                 .balances
                 .into();
         }

beacon_node/beacon_chain/src/beacon_snapshot.rs

@@ -1,6 +1,6 @@
 use serde_derive::Serialize;
 use ssz_derive::{Decode, Encode};
-use types::{BeaconState, EthSpec, Hash256, SignedBeaconBlock};
+use types::{beacon_state::CloneConfig, BeaconState, EthSpec, Hash256, SignedBeaconBlock};
 
 /// Represents some block and its associated state. Generally, this will be used for tracking the
 /// head, justified head and finalized head.
@@ -42,11 +42,11 @@ impl<E: EthSpec> BeaconSnapshot<E> {
         self.beacon_state_root = beacon_state_root;
     }
 
-    pub fn clone_with_only_committee_caches(&self) -> Self {
+    pub fn clone_with(&self, clone_config: CloneConfig) -> Self {
         Self {
             beacon_block: self.beacon_block.clone(),
             beacon_block_root: self.beacon_block_root,
-            beacon_state: self.beacon_state.clone_with_only_committee_caches(),
+            beacon_state: self.beacon_state.clone_with(clone_config),
             beacon_state_root: self.beacon_state_root,
         }
     }

beacon_node/beacon_chain/src/block_verification.rs

@@ -48,6 +48,7 @@ use crate::{
     },
     metrics, BeaconChain, BeaconChainError, BeaconChainTypes, BeaconSnapshot,
 };
+use fork_choice::{ForkChoice, ForkChoiceStore};
 use parking_lot::RwLockReadGuard;
 use slog::{error, Logger};
 use slot_clock::SlotClock;
@@ -62,11 +63,11 @@ use std::borrow::Cow;
 use std::convert::TryFrom;
 use std::fs;
 use std::io::Write;
-use store::{Error as DBError, HotStateSummary, StoreOp};
+use store::{Error as DBError, HotColdDB, HotStateSummary, KeyValueStore, StoreOp};
 use tree_hash::TreeHash;
 use types::{
     BeaconBlock, BeaconState, BeaconStateError, ChainSpec, CloneConfig, EthSpec, Hash256,
-    PublicKey, RelativeEpoch, SignedBeaconBlock, Slot,
+    PublicKey, RelativeEpoch, SignedBeaconBlock, SignedBeaconBlockHeader, Slot,
 };
 
 /// Maximum block slot number. Block with slots bigger than this constant will NOT be processed.
@@ -91,6 +92,8 @@ pub enum BlockError<T: EthSpec> {
     /// It's unclear if this block is valid, but it cannot be processed without already knowing
     /// its parent.
     ParentUnknown(Box<SignedBeaconBlock<T>>),
+    /// The block skips too many slots and is a DoS risk.
+    TooManySkippedSlots { parent_slot: Slot, block_slot: Slot },
     /// The block slot is greater than the present slot.
     ///
     /// ## Peer scoring
@@ -118,6 +121,13 @@ pub enum BlockError<T: EthSpec> {
         block_slot: Slot,
         finalized_slot: Slot,
     },
+    /// The block conflicts with finalization, no need to propagate.
+    ///
+    /// ## Peer scoring
+    ///
+    /// It's unclear if this block is valid, but it conflicts with finality and shouldn't be
+    /// imported.
+    NotFinalizedDescendant { block_parent_root: Hash256 },
     /// Block is already known, no need to re-import.
     ///
     /// ## Peer scoring
@@ -197,6 +207,24 @@ pub enum BlockError<T: EthSpec> {
     /// We were unable to process this block due to an internal error. It's unclear if the block is
     /// valid.
     BeaconChainError(BeaconChainError),
+    /// There was an error whilst verifying weak subjectivity. This block conflicts with the
+    /// configured weak subjectivity checkpoint and was not imported.
+    ///
+    /// ## Peer scoring
+    ///
+    /// The block is invalid and the peer is faulty.
+    WeakSubjectivityConflict,
+}
+
+impl<T: EthSpec> std::fmt::Display for BlockError<T> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            BlockError::ParentUnknown(block) => {
+                write!(f, "ParentUnknown(parent_root:{})", block.parent_root())
+            }
+            other => write!(f, "{:?}", other),
+        }
+    }
 }
 
 impl<T: EthSpec> From<BlockSignatureVerifierError> for BlockError<T> {
@@ -240,6 +268,58 @@ impl<T: EthSpec> From<DBError> for BlockError<T> {
     }
 }
 
+/// Information about invalid blocks which might still be slashable despite being invalid.
+pub enum BlockSlashInfo<TErr> {
+    /// The block is invalid, but its proposer signature wasn't checked.
+    SignatureNotChecked(SignedBeaconBlockHeader, TErr),
+    /// The block's proposer signature is invalid, so it will never be slashable.
+    SignatureInvalid(TErr),
+    /// The signature is valid but the attestation is invalid in some other way.
+    SignatureValid(SignedBeaconBlockHeader, TErr),
+}
+
+impl<E: EthSpec> BlockSlashInfo<BlockError<E>> {
+    pub fn from_early_error(header: SignedBeaconBlockHeader, e: BlockError<E>) -> Self {
+        match e {
+            BlockError::ProposalSignatureInvalid => BlockSlashInfo::SignatureInvalid(e),
+            // `InvalidSignature` could indicate any signature in the block, so we want
+            // to recheck the proposer signature alone.
+            _ => BlockSlashInfo::SignatureNotChecked(header, e),
+        }
+    }
+}
+
+/// Process invalid blocks to see if they are suitable for the slasher.
+///
+/// If no slasher is configured, this is a no-op.
+fn process_block_slash_info<T: BeaconChainTypes>(
+    chain: &BeaconChain<T>,
+    slash_info: BlockSlashInfo<BlockError<T::EthSpec>>,
+) -> BlockError<T::EthSpec> {
+    if let Some(slasher) = chain.slasher.as_ref() {
+        let (verified_header, error) = match slash_info {
+            BlockSlashInfo::SignatureNotChecked(header, e) => {
+                if verify_header_signature(chain, &header).is_ok() {
+                    (header, e)
+                } else {
+                    return e;
+                }
+            }
+            BlockSlashInfo::SignatureInvalid(e) => return e,
+            BlockSlashInfo::SignatureValid(header, e) => (header, e),
+        };
+
+        slasher.accept_block_header(verified_header);
+        error
+    } else {
+        match slash_info {
+            BlockSlashInfo::SignatureNotChecked(_, e)
+            | BlockSlashInfo::SignatureInvalid(e)
+            | BlockSlashInfo::SignatureValid(_, e) => e,
+        }
+    }
+}
+
 /// Verify all signatures (except deposit signatures) on all blocks in the `chain_segment`. If all
 /// signatures are valid, the `chain_segment` is mapped to a `Vec<SignatureVerifiedBlock>` that can
 /// later be transformed into a `FullyVerifiedBlock` without re-checking the signatures. If any
@@ -335,17 +415,33 @@ pub struct FullyVerifiedBlock<'a, T: BeaconChainTypes> {
     pub block_root: Hash256,
     pub state: BeaconState<T::EthSpec>,
     pub parent_block: SignedBeaconBlock<T::EthSpec>,
-    pub intermediate_states: Vec<StoreOp<'a, T::EthSpec>>,
+    pub confirmation_db_batch: Vec<StoreOp<'a, T::EthSpec>>,
 }
 
 /// Implemented on types that can be converted into a `FullyVerifiedBlock`.
 ///
 /// Used to allow functions to accept blocks at various stages of verification.
-pub trait IntoFullyVerifiedBlock<T: BeaconChainTypes> {
+pub trait IntoFullyVerifiedBlock<T: BeaconChainTypes>: Sized {
     fn into_fully_verified_block(
         self,
         chain: &BeaconChain<T>,
-    ) -> Result<FullyVerifiedBlock<T>, BlockError<T::EthSpec>>;
+    ) -> Result<FullyVerifiedBlock<T>, BlockError<T::EthSpec>> {
+        self.into_fully_verified_block_slashable(chain)
+            .map(|fully_verified| {
+                // Supply valid block to slasher.
+                if let Some(slasher) = chain.slasher.as_ref() {
+                    slasher.accept_block_header(fully_verified.block.signed_block_header());
+                }
+                fully_verified
+            })
+            .map_err(|slash_info| process_block_slash_info(chain, slash_info))
+    }
+
+    /// Convert the block to fully-verified form while producing data to aid checking slashability.
+    fn into_fully_verified_block_slashable(
+        self,
+        chain: &BeaconChain<T>,
+    ) -> Result<FullyVerifiedBlock<T>, BlockSlashInfo<BlockError<T::EthSpec>>>;
 
     fn block(&self) -> &SignedBeaconBlock<T::EthSpec>;
 }
@@ -358,12 +454,27 @@ impl<T: BeaconChainTypes> GossipVerifiedBlock<T> {
     pub fn new(
         block: SignedBeaconBlock<T::EthSpec>,
         chain: &BeaconChain<T>,
+    ) -> Result<Self, BlockError<T::EthSpec>> {
+        // If the block is valid for gossip we don't supply it to the slasher here because
+        // we assume it will be transformed into a fully verified block. We *do* need to supply
+        // it to the slasher if an error occurs, because that's the end of this block's journey,
+        // and it could be a repeat proposal (a likely cause for slashing!).
+        let header = block.signed_block_header();
+        Self::new_without_slasher_checks(block, chain).map_err(|e| {
+            process_block_slash_info(chain, BlockSlashInfo::from_early_error(header, e))
+        })
+    }
+
+    /// As for new, but doesn't pass the block to the slasher.
+    fn new_without_slasher_checks(
+        block: SignedBeaconBlock<T::EthSpec>,
+        chain: &BeaconChain<T>,
     ) -> Result<Self, BlockError<T::EthSpec>> {
         // Do not gossip or process blocks from future slots.
         let present_slot_with_tolerance = chain
             .slot_clock
             .now_with_future_tolerance(MAXIMUM_GOSSIP_CLOCK_DISPARITY)
-            .ok_or_else(|| BeaconChainError::UnableToReadSlot)?;
+            .ok_or(BeaconChainError::UnableToReadSlot)?;
         if block.slot() > present_slot_with_tolerance {
             return Err(BlockError::FutureSlot {
                 present_slot: present_slot_with_tolerance,
@@ -371,12 +482,26 @@ impl<T: BeaconChainTypes> GossipVerifiedBlock<T> {
             });
         }
 
+        let block_root = get_block_root(&block);
+
         // Do not gossip a block from a finalized slot.
         check_block_against_finalized_slot(&block.message, chain)?;
 
+        // Check if the block is already known. We know it is post-finalization, so it is
+        // sufficient to check the fork choice.
+        //
+        // In normal operation this isn't necessary, however it is useful immediately after a
+        // reboot if the `observed_block_producers` cache is empty. In that case, without this
+        // check, we will load the parent and state from disk only to find out later that we
+        // already know this block.
+        if chain.fork_choice.read().contains_block(&block_root) {
+            return Err(BlockError::BlockIsAlreadyKnown);
+        }
+
         // Check that we have not already received a block with a valid signature for this slot.
         if chain
             .observed_block_producers
+            .read()
             .proposer_has_been_observed(&block.message)
             .map_err(|e| BlockError::BeaconChainError(e.into()))?
         {
@@ -386,8 +511,19 @@ impl<T: BeaconChainTypes> GossipVerifiedBlock<T> {
             });
         }
 
+        // Do not process a block that doesn't descend from the finalized root.
+        //
+        // We check this *before* we load the parent so that we can return a more detailed error.
+        let block = check_block_is_finalized_descendant::<T, _>(
+            block,
+            &chain.fork_choice.read(),
+            &chain.store,
+        )?;
+
         let (mut parent, block) = load_parent(block, chain)?;
-        let block_root = get_block_root(&block);
+
+        // Reject any block that exceeds our limit on skipped slots.
+        check_block_skip_slots(chain, &parent.beacon_block.message, &block.message)?;
 
         let state = cheap_state_advance_to_obtain_committees(
             &mut parent.beacon_state,
@@ -399,7 +535,7 @@ impl<T: BeaconChainTypes> GossipVerifiedBlock<T> {
             let pubkey_cache = get_validator_pubkey_cache(chain)?;
             let pubkey = pubkey_cache
                 .get(block.message.proposer_index as usize)
-                .ok_or_else(|| BlockError::UnknownValidator(block.message.proposer_index))?;
+                .ok_or(BlockError::UnknownValidator(block.message.proposer_index))?;
             block.verify_signature(
                 Some(block_root),
                 pubkey,
@@ -420,6 +556,7 @@ impl<T: BeaconChainTypes> GossipVerifiedBlock<T> {
         // have a race-condition when verifying two blocks simultaneously.
         if chain
             .observed_block_producers
+            .write()
             .observe_proposer(&block.message)
             .map_err(|e| BlockError::BeaconChainError(e.into()))?
         {
@@ -452,12 +589,13 @@ impl<T: BeaconChainTypes> GossipVerifiedBlock<T> {
 
 impl<T: BeaconChainTypes> IntoFullyVerifiedBlock<T> for GossipVerifiedBlock<T> {
     /// Completes verification of the wrapped `block`.
-    fn into_fully_verified_block(
+    fn into_fully_verified_block_slashable(
         self,
         chain: &BeaconChain<T>,
-    ) -> Result<FullyVerifiedBlock<T>, BlockError<T::EthSpec>> {
-        let fully_verified = SignatureVerifiedBlock::from_gossip_verified_block(self, chain)?;
-        fully_verified.into_fully_verified_block(chain)
+    ) -> Result<FullyVerifiedBlock<T>, BlockSlashInfo<BlockError<T::EthSpec>>> {
+        let fully_verified =
+            SignatureVerifiedBlock::from_gossip_verified_block_check_slashable(self, chain)?;
+        fully_verified.into_fully_verified_block_slashable(chain)
     }
 
     fn block(&self) -> &SignedBeaconBlock<T::EthSpec> {
@@ -475,6 +613,10 @@ impl<T: BeaconChainTypes> SignatureVerifiedBlock<T> {
         chain: &BeaconChain<T>,
     ) -> Result<Self, BlockError<T::EthSpec>> {
         let (mut parent, block) = load_parent(block, chain)?;
+
+        // Reject any block that exceeds our limit on skipped slots.
+        check_block_skip_slots(chain, &parent.beacon_block.message, &block.message)?;
+
         let block_root = get_block_root(&block);
 
         let state = cheap_state_advance_to_obtain_committees(
@@ -500,6 +642,15 @@ impl<T: BeaconChainTypes> SignatureVerifiedBlock<T> {
         }
     }
 
+    /// As for `new` above but producing `BlockSlashInfo`.
+    pub fn check_slashable(
+        block: SignedBeaconBlock<T::EthSpec>,
+        chain: &BeaconChain<T>,
+    ) -> Result<Self, BlockSlashInfo<BlockError<T::EthSpec>>> {
+        let header = block.signed_block_header();
+        Self::new(block, chain).map_err(|e| BlockSlashInfo::from_early_error(header, e))
+    }
+
     /// Finishes signature verification on the provided `GossipVerifedBlock`. Does not re-verify
     /// the proposer signature.
     pub fn from_gossip_verified_block(
@@ -531,18 +682,30 @@ impl<T: BeaconChainTypes> SignatureVerifiedBlock<T> {
             Err(BlockError::InvalidSignature)
         }
     }
+
+    /// Same as `from_gossip_verified_block` but producing slashing-relevant data as well.
+    pub fn from_gossip_verified_block_check_slashable(
+        from: GossipVerifiedBlock<T>,
+        chain: &BeaconChain<T>,
+    ) -> Result<Self, BlockSlashInfo<BlockError<T::EthSpec>>> {
+        let header = from.block.signed_block_header();
+        Self::from_gossip_verified_block(from, chain)
+            .map_err(|e| BlockSlashInfo::from_early_error(header, e))
+    }
 }
 
 impl<T: BeaconChainTypes> IntoFullyVerifiedBlock<T> for SignatureVerifiedBlock<T> {
     /// Completes verification of the wrapped `block`.
-    fn into_fully_verified_block(
+    fn into_fully_verified_block_slashable(
         self,
         chain: &BeaconChain<T>,
-    ) -> Result<FullyVerifiedBlock<T>, BlockError<T::EthSpec>> {
+    ) -> Result<FullyVerifiedBlock<T>, BlockSlashInfo<BlockError<T::EthSpec>>> {
+        let header = self.block.signed_block_header();
         let (parent, block) = if let Some(parent) = self.parent {
             (parent, self.block)
         } else {
-            load_parent(self.block, chain)?
+            load_parent(self.block, chain)
+                .map_err(|e| BlockSlashInfo::SignatureValid(header.clone(), e))?
         };
 
         FullyVerifiedBlock::from_signature_verified_components(
@@ -551,6 +714,7 @@ impl<T: BeaconChainTypes> IntoFullyVerifiedBlock<T> for SignatureVerifiedBlock<T
             parent,
             chain,
         )
+        .map_err(|e| BlockSlashInfo::SignatureValid(header, e))
     }
 
     fn block(&self) -> &SignedBeaconBlock<T::EthSpec> {
@@ -561,11 +725,12 @@ impl<T: BeaconChainTypes> IntoFullyVerifiedBlock<T> for SignatureVerifiedBlock<T
 impl<T: BeaconChainTypes> IntoFullyVerifiedBlock<T> for SignedBeaconBlock<T::EthSpec> {
     /// Verifies the `SignedBeaconBlock` by first transforming it into a `SignatureVerifiedBlock`
     /// and then using that implementation of `IntoFullyVerifiedBlock` to complete verification.
-    fn into_fully_verified_block(
+    fn into_fully_verified_block_slashable(
         self,
         chain: &BeaconChain<T>,
-    ) -> Result<FullyVerifiedBlock<T>, BlockError<T::EthSpec>> {
-        SignatureVerifiedBlock::new(self, chain)?.into_fully_verified_block(chain)
+    ) -> Result<FullyVerifiedBlock<T>, BlockSlashInfo<BlockError<T::EthSpec>>> {
+        SignatureVerifiedBlock::check_slashable(self, chain)?
+            .into_fully_verified_block_slashable(chain)
     }
 
     fn block(&self) -> &SignedBeaconBlock<T::EthSpec> {
@@ -605,6 +770,9 @@ impl<'a, T: BeaconChainTypes> FullyVerifiedBlock<'a, T> {
             return Err(BlockError::ParentUnknown(Box::new(block)));
         }
 
+        // Reject any block that exceeds our limit on skipped slots.
+        check_block_skip_slots(chain, &parent.beacon_block.message, &block.message)?;
+
         /*
          *  Perform cursory checks to see if the block is even worth processing.
          */
@@ -617,9 +785,9 @@ impl<'a, T: BeaconChainTypes> FullyVerifiedBlock<'a, T> {
 
         let catchup_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_CATCHUP_STATE);
 
-        // Keep a batch of any states that were "skipped" (block-less) in between the parent state
-        // slot and the block slot. These will be stored in the database.
-        let mut intermediate_states: Vec<StoreOp<T::EthSpec>> = Vec::new();
+        // Stage a batch of operations to be completed atomically if this block is imported
+        // successfully.
+        let mut confirmation_db_batch = vec![];
 
         // The block must have a higher slot than its parent.
         if block.slot() <= parent.beacon_state.slot {
@@ -643,15 +811,36 @@ impl<'a, T: BeaconChainTypes> FullyVerifiedBlock<'a, T> {
                 // processing, but we get early access to it.
                 let state_root = state.update_tree_hash_cache()?;
 
-                let op = if state.slot % T::EthSpec::slots_per_epoch() == 0 {
-                    StoreOp::PutState(state_root.into(), Cow::Owned(state.clone()))
+                // Store the state immediately, marking it as temporary, and staging the deletion
+                // of its temporary status as part of the larger atomic operation.
+                let txn_lock = chain.store.hot_db.begin_rw_transaction();
+                let state_already_exists =
+                    chain.store.load_hot_state_summary(&state_root)?.is_some();
+
+                let state_batch = if state_already_exists {
+                    // If the state exists, it could be temporary or permanent, but in neither case
+                    // should we rewrite it or store a new temporary flag for it. We *will* stage
+                    // the temporary flag for deletion because it's OK to double-delete the flag,
+                    // and we don't mind if another thread gets there first.
+                    vec![]
                 } else {
-                    StoreOp::PutStateSummary(
-                        state_root.into(),
-                        HotStateSummary::new(&state_root, &state)?,
-                    )
+                    vec![
+                        if state.slot % T::EthSpec::slots_per_epoch() == 0 {
+                            StoreOp::PutState(state_root, &state)
+                        } else {
+                            StoreOp::PutStateSummary(
+                                state_root,
+                                HotStateSummary::new(&state_root, &state)?,
+                            )
+                        },
+                        StoreOp::PutStateTemporaryFlag(state_root),
+                    ]
                 };
-                intermediate_states.push(op);
+                chain.store.do_atomically(state_batch)?;
+                drop(txn_lock);
+
+                confirmation_db_batch.push(StoreOp::DeleteStateTemporaryFlag(state_root));
+
                 state_root
             };
 
@@ -739,11 +928,35 @@ impl<'a, T: BeaconChainTypes> FullyVerifiedBlock<'a, T> {
             block_root,
             state,
             parent_block: parent.beacon_block,
-            intermediate_states,
+            confirmation_db_batch,
         })
     }
 }
 
+/// Check that the count of skip slots between the block and its parent does not exceed our maximum
+/// value.
+///
+/// Whilst this is not part of the specification, we include this to help prevent us from DoS
+/// attacks. In times of dire network circumstance, the user can configure the
+/// `import_max_skip_slots` value.
+fn check_block_skip_slots<T: BeaconChainTypes>(
+    chain: &BeaconChain<T>,
+    parent: &BeaconBlock<T::EthSpec>,
+    block: &BeaconBlock<T::EthSpec>,
+) -> Result<(), BlockError<T::EthSpec>> {
+    // Reject any block that exceeds our limit on skipped slots.
+    if let Some(max_skip_slots) = chain.config.import_max_skip_slots {
+        if block.slot > parent.slot + max_skip_slots {
+            return Err(BlockError::TooManySkippedSlots {
+                parent_slot: parent.slot,
+                block_slot: block.slot,
+            });
+        }
+    }
+
+    Ok(())
+}
+
 /// Returns `Ok(())` if the block is later than the finalized slot on `chain`.
 ///
 /// Returns an error if the block is earlier or equal to the finalized slot, or there was an error
@@ -768,6 +981,36 @@ fn check_block_against_finalized_slot<T: BeaconChainTypes>(
     }
 }
 
+/// Returns `Ok(block)` if the block descends from the finalized root.
+pub fn check_block_is_finalized_descendant<T: BeaconChainTypes, F: ForkChoiceStore<T::EthSpec>>(
+    block: SignedBeaconBlock<T::EthSpec>,
+    fork_choice: &ForkChoice<F, T::EthSpec>,
+    store: &HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>,
+) -> Result<SignedBeaconBlock<T::EthSpec>, BlockError<T::EthSpec>> {
+    if fork_choice.is_descendant_of_finalized(block.parent_root()) {
+        Ok(block)
+    } else {
+        // If fork choice does *not* consider the parent to be a descendant of the finalized block,
+        // then there are two more cases:
+        //
+        // 1. We have the parent stored in our database. Because fork-choice has confirmed the
+        //    parent is *not* in our post-finalization DAG, all other blocks must be either
+        //    pre-finalization or conflicting with finalization.
+        // 2. The parent is unknown to us, we probably want to download it since it might actually
+        //    descend from the finalized root.
+        if store
+            .item_exists::<SignedBeaconBlock<T::EthSpec>>(&block.parent_root())
+            .map_err(|e| BlockError::BeaconChainError(e.into()))?
+        {
+            Err(BlockError::NotFinalizedDescendant {
+                block_parent_root: block.parent_root(),
+            })
+        } else {
+            Err(BlockError::ParentUnknown(Box::new(block)))
+        }
+    }
+}
+
 /// Performs simple, cheap checks to ensure that the block is relevant to be imported.
 ///
 /// `Ok(block_root)` is returned if the block passes these checks and should progress with
@@ -871,14 +1114,17 @@ fn load_parent<T: BeaconChainTypes>(
         // exist in fork choice but not in the database yet. In such a case we simply
         // indicate that we don't yet know the parent.
         let root = block.parent_root();
-        let parent_block = if let Some(block) = chain
+        let parent_block = chain
             .get_block(&block.parent_root())
             .map_err(BlockError::BeaconChainError)?
-        {
-            block
-        } else {
-            return Err(BlockError::ParentUnknown(Box::new(block)));
-        };
+            .ok_or_else(|| {
+                // Return a `MissingBeaconBlock` error instead of a `ParentUnknown` error since
+                // we've already checked fork choice for this block.
+                //
+                // It's an internal error if the block exists in fork choice but not in the
+                // database.
+                BlockError::from(BeaconChainError::MissingBeaconBlock(block.parent_root()))
+            })?;
 
         // Load the parent blocks state from the database, returning an error if it is not found.
         // It is an error because if we know the parent block we should also know the parent state.
@@ -956,7 +1202,7 @@ fn get_validator_pubkey_cache<T: BeaconChainTypes>(
     chain
         .validator_pubkey_cache
         .try_read_for(VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT)
-        .ok_or_else(|| BeaconChainError::ValidatorPubkeyCacheLockTimeout)
+        .ok_or(BeaconChainError::ValidatorPubkeyCacheLockTimeout)
         .map_err(BlockError::BeaconChainError)
 }
 
@@ -986,6 +1232,38 @@ fn get_signature_verifier<'a, E: EthSpec>(
     )
 }
 
+/// Verify that `header` was signed with a valid signature from its proposer.
+///
+/// Return `Ok(())` if the signature is valid, and an `Err` otherwise.
+fn verify_header_signature<T: BeaconChainTypes>(
+    chain: &BeaconChain<T>,
+    header: &SignedBeaconBlockHeader,
+) -> Result<(), BlockError<T::EthSpec>> {
+    let proposer_pubkey = get_validator_pubkey_cache(chain)?
+        .get(header.message.proposer_index as usize)
+        .cloned()
+        .ok_or(BlockError::UnknownValidator(header.message.proposer_index))?;
+    let (fork, genesis_validators_root) = chain
+        .with_head(|head| {
+            Ok((
+                head.beacon_state.fork,
+                head.beacon_state.genesis_validators_root,
+            ))
+        })
+        .map_err(|e: BlockError<T::EthSpec>| e)?;
+
+    if header.verify_signature::<T::EthSpec>(
+        &proposer_pubkey,
+        &fork,
+        genesis_validators_root,
+        &chain.spec,
+    ) {
+        Ok(())
+    } else {
+        Err(BlockError::ProposalSignatureInvalid)
+    }
+}
+
 fn expose_participation_metrics(summaries: &[EpochProcessingSummary]) {
     if !cfg!(feature = "participation_metrics") {
         return;

beacon_node/beacon_chain/src/builder.rs

@@ -1,25 +1,25 @@
-use crate::beacon_chain::{
-    BEACON_CHAIN_DB_KEY, ETH1_CACHE_DB_KEY, FORK_CHOICE_DB_KEY, OP_POOL_DB_KEY,
-};
+use crate::beacon_chain::{BEACON_CHAIN_DB_KEY, ETH1_CACHE_DB_KEY, OP_POOL_DB_KEY};
 use crate::eth1_chain::{CachingEth1Backend, SszEth1};
-use crate::events::NullEventHandler;
 use crate::head_tracker::HeadTracker;
-use crate::migrate::Migrate;
+use crate::migrate::{BackgroundMigrator, MigratorConfig};
 use crate::persisted_beacon_chain::PersistedBeaconChain;
-use crate::persisted_fork_choice::PersistedForkChoice;
 use crate::shuffling_cache::ShufflingCache;
 use crate::snapshot_cache::{SnapshotCache, DEFAULT_SNAPSHOT_CACHE_SIZE};
 use crate::timeout_rw_lock::TimeoutRwLock;
+use crate::validator_monitor::ValidatorMonitor;
 use crate::validator_pubkey_cache::ValidatorPubkeyCache;
+use crate::ChainConfig;
 use crate::{
     BeaconChain, BeaconChainTypes, BeaconForkChoiceStore, BeaconSnapshot, Eth1Chain,
-    Eth1ChainBackend, EventHandler,
+    Eth1ChainBackend, ServerSentEventHandler,
 };
 use eth1::Config as Eth1Config;
 use fork_choice::ForkChoice;
+use futures::channel::mpsc::Sender;
 use operation_pool::{OperationPool, PersistedOperationPool};
 use parking_lot::RwLock;
-use slog::{info, Logger};
+use slasher::Slasher;
+use slog::{crit, info, Logger};
 use slot_clock::{SlotClock, TestingSlotClock};
 use std::marker::PhantomData;
 use std::path::PathBuf;
@@ -27,61 +27,32 @@ use std::sync::Arc;
 use std::time::Duration;
 use store::{HotColdDB, ItemStore};
 use types::{
-    BeaconBlock, BeaconState, ChainSpec, EthSpec, Graffiti, Hash256, Signature, SignedBeaconBlock,
-    Slot,
+    BeaconBlock, BeaconState, ChainSpec, EthSpec, Graffiti, Hash256, PublicKeyBytes, Signature,
+    SignedBeaconBlock, Slot,
 };
 
 pub const PUBKEY_CACHE_FILENAME: &str = "pubkey_cache.ssz";
 
 /// An empty struct used to "witness" all the `BeaconChainTypes` traits. It has no user-facing
 /// functionality and only exists to satisfy the type system.
-pub struct Witness<
-    TStoreMigrator,
-    TSlotClock,
-    TEth1Backend,
-    TEthSpec,
-    TEventHandler,
-    THotStore,
-    TColdStore,
->(
-    PhantomData<(
-        TStoreMigrator,
-        TSlotClock,
-        TEth1Backend,
-        TEthSpec,
-        TEventHandler,
-        THotStore,
-        TColdStore,
-    )>,
+pub struct Witness<TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>(
+    PhantomData<(TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore)>,
 );
 
-impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
-    BeaconChainTypes
-    for Witness<
-        TStoreMigrator,
-        TSlotClock,
-        TEth1Backend,
-        TEthSpec,
-        TEventHandler,
-        THotStore,
-        TColdStore,
-    >
+impl<TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore> BeaconChainTypes
+    for Witness<TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>
 where
     THotStore: ItemStore<TEthSpec> + 'static,
     TColdStore: ItemStore<TEthSpec> + 'static,
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore> + 'static,
     TSlotClock: SlotClock + 'static,
     TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
     TEthSpec: EthSpec + 'static,
-    TEventHandler: EventHandler<TEthSpec> + 'static,
 {
     type HotStore = THotStore;
     type ColdStore = TColdStore;
-    type StoreMigrator = TStoreMigrator;
     type SlotClock = TSlotClock;
     type Eth1Chain = TEth1Backend;
     type EthSpec = TEthSpec;
-    type EventHandler = TEventHandler;
 }
 
 /// Builds a `BeaconChain` by either creating anew from genesis, or, resuming from an existing chain
@@ -95,46 +66,40 @@ where
 pub struct BeaconChainBuilder<T: BeaconChainTypes> {
     #[allow(clippy::type_complexity)]
     store: Option<Arc<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>>,
-    store_migrator: Option<T::StoreMigrator>,
-    canonical_head: Option<BeaconSnapshot<T::EthSpec>>,
-    /// The finalized checkpoint to anchor the chain. May be genesis or a higher
-    /// checkpoint.
-    pub finalized_snapshot: Option<BeaconSnapshot<T::EthSpec>>,
+    store_migrator_config: Option<MigratorConfig>,
+    pub genesis_time: Option<u64>,
     genesis_block_root: Option<Hash256>,
+    genesis_state_root: Option<Hash256>,
+    #[allow(clippy::type_complexity)]
+    fork_choice: Option<
+        ForkChoice<BeaconForkChoiceStore<T::EthSpec, T::HotStore, T::ColdStore>, T::EthSpec>,
+    >,
     op_pool: Option<OperationPool<T::EthSpec>>,
     eth1_chain: Option<Eth1Chain<T::Eth1Chain, T::EthSpec>>,
-    event_handler: Option<T::EventHandler>,
+    event_handler: Option<ServerSentEventHandler<T::EthSpec>>,
     slot_clock: Option<T::SlotClock>,
+    shutdown_sender: Option<Sender<&'static str>>,
     head_tracker: Option<HeadTracker>,
     data_dir: Option<PathBuf>,
     pubkey_cache_path: Option<PathBuf>,
     validator_pubkey_cache: Option<ValidatorPubkeyCache>,
     spec: ChainSpec,
+    chain_config: ChainConfig,
     disabled_forks: Vec<String>,
     log: Option<Logger>,
     graffiti: Graffiti,
+    slasher: Option<Arc<Slasher<T::EthSpec>>>,
+    validator_monitor: Option<ValidatorMonitor<T::EthSpec>>,
 }
 
-impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
-    BeaconChainBuilder<
-        Witness<
-            TStoreMigrator,
-            TSlotClock,
-            TEth1Backend,
-            TEthSpec,
-            TEventHandler,
-            THotStore,
-            TColdStore,
-        >,
-    >
+impl<TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>
+    BeaconChainBuilder<Witness<TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>>
 where
     THotStore: ItemStore<TEthSpec> + 'static,
     TColdStore: ItemStore<TEthSpec> + 'static,
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore> + 'static,
     TSlotClock: SlotClock + 'static,
     TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
     TEthSpec: EthSpec + 'static,
-    TEventHandler: EventHandler<TEthSpec> + 'static,
 {
     /// Returns a new builder.
     ///
@@ -143,22 +108,27 @@ where
     pub fn new(_eth_spec_instance: TEthSpec) -> Self {
         Self {
             store: None,
-            store_migrator: None,
-            canonical_head: None,
-            finalized_snapshot: None,
+            store_migrator_config: None,
+            genesis_time: None,
             genesis_block_root: None,
+            genesis_state_root: None,
+            fork_choice: None,
             op_pool: None,
             eth1_chain: None,
             event_handler: None,
             slot_clock: None,
+            shutdown_sender: None,
             head_tracker: None,
             pubkey_cache_path: None,
             data_dir: None,
             disabled_forks: Vec::new(),
             validator_pubkey_cache: None,
             spec: TEthSpec::default_spec(),
+            chain_config: ChainConfig::default(),
             log: None,
             graffiti: Graffiti::default(),
+            slasher: None,
+            validator_monitor: None,
         }
     }
 
@@ -171,6 +141,15 @@ where
         self
     }
 
+    /// Sets the maximum number of blocks that will be skipped when processing
+    /// some consensus messages.
+    ///
+    /// Set to `None` for no limit.
+    pub fn import_max_skip_slots(mut self, n: Option<u64>) -> Self {
+        self.chain_config.import_max_skip_slots = n;
+        self
+    }
+
     /// Sets the store (database).
     ///
     /// Should generally be called early in the build chain.
@@ -179,17 +158,23 @@ where
         self
     }
 
-    /// Sets the store migrator.
-    pub fn store_migrator(mut self, store_migrator: TStoreMigrator) -> Self {
-        self.store_migrator = Some(store_migrator);
+    /// Sets the store migrator config (optional).
+    pub fn store_migrator_config(mut self, config: MigratorConfig) -> Self {
+        self.store_migrator_config = Some(config);
+        self
+    }
+
+    /// Sets the slasher.
+    pub fn slasher(mut self, slasher: Arc<Slasher<TEthSpec>>) -> Self {
+        self.slasher = Some(slasher);
         self
     }
 
     /// Sets the logger.
     ///
     /// Should generally be called early in the build chain.
-    pub fn logger(mut self, logger: Logger) -> Self {
-        self.log = Some(logger);
+    pub fn logger(mut self, log: Logger) -> Self {
+        self.log = Some(log);
         self
     }
 
@@ -213,10 +198,10 @@ where
         let store = self
             .store
             .clone()
-            .ok_or_else(|| "get_persisted_eth1_backend requires a store.".to_string())?;
+            .ok_or("get_persisted_eth1_backend requires a store.")?;
 
         store
-            .get_item::<SszEth1>(&Hash256::from_slice(&ETH1_CACHE_DB_KEY))
+            .get_item::<SszEth1>(&ETH1_CACHE_DB_KEY)
             .map_err(|e| format!("DB error whilst reading eth1 cache: {:?}", e))
     }
 
@@ -225,10 +210,10 @@ where
         let store = self
             .store
             .clone()
-            .ok_or_else(|| "store_contains_beacon_chain requires a store.".to_string())?;
+            .ok_or("store_contains_beacon_chain requires a store.")?;
 
         Ok(store
-            .get_item::<PersistedBeaconChain>(&Hash256::from_slice(&BEACON_CHAIN_DB_KEY))
+            .get_item::<PersistedBeaconChain>(&BEACON_CHAIN_DB_KEY)
             .map_err(|e| format!("DB error when reading persisted beacon chain: {:?}", e))?
             .is_some())
     }
@@ -237,15 +222,12 @@ where
     ///
     /// May initialize several components; including the op_pool and finalized checkpoints.
     pub fn resume_from_db(mut self) -> Result<Self, String> {
-        let log = self
-            .log
-            .as_ref()
-            .ok_or_else(|| "resume_from_db requires a log".to_string())?;
+        let log = self.log.as_ref().ok_or("resume_from_db requires a log")?;
 
         let pubkey_cache_path = self
             .pubkey_cache_path
             .as_ref()
-            .ok_or_else(|| "resume_from_db requires a data_dir".to_string())?;
+            .ok_or("resume_from_db requires a data_dir")?;
 
         info!(
             log,
@@ -256,70 +238,53 @@ where
         let store = self
             .store
             .clone()
-            .ok_or_else(|| "resume_from_db requires a store.".to_string())?;
+            .ok_or("resume_from_db requires a store.")?;
 
         let chain = store
-            .get_item::<PersistedBeaconChain>(&Hash256::from_slice(&BEACON_CHAIN_DB_KEY))
+            .get_item::<PersistedBeaconChain>(&BEACON_CHAIN_DB_KEY)
             .map_err(|e| format!("DB error when reading persisted beacon chain: {:?}", e))?
             .ok_or_else(|| {
                 "No persisted beacon chain found in store. Try purging the beacon chain database."
                     .to_string()
             })?;
 
-        self.genesis_block_root = Some(chain.genesis_block_root);
-        self.head_tracker = Some(
-            HeadTracker::from_ssz_container(&chain.ssz_head_tracker)
-                .map_err(|e| format!("Failed to decode head tracker for database: {:?}", e))?,
-        );
+        let fork_choice =
+            BeaconChain::<Witness<TSlotClock, TEth1Backend, _, _, _>>::load_fork_choice(
+                store.clone(),
+            )
+            .map_err(|e| format!("Unable to load fork choice from disk: {:?}", e))?
+            .ok_or("Fork choice not found in store")?;
 
-        let head_block_root = chain.canonical_head_block_root;
-        let head_block = store
-            .get_item::<SignedBeaconBlock<TEthSpec>>(&head_block_root)
-            .map_err(|e| format!("DB error when reading head block: {:?}", e))?
-            .ok_or_else(|| "Head block not found in store".to_string())?;
-        let head_state_root = head_block.state_root();
-        let head_state = store
-            .get_state(&head_state_root, Some(head_block.slot()))
-            .map_err(|e| format!("DB error when reading head state: {:?}", e))?
-            .ok_or_else(|| "Head state not found in store".to_string())?;
+        let genesis_block = store
+            .get_item::<SignedBeaconBlock<TEthSpec>>(&chain.genesis_block_root)
+            .map_err(|e| format!("DB error when reading genesis block: {:?}", e))?
+            .ok_or("Genesis block not found in store")?;
+        let genesis_state = store
+            .get_state(&genesis_block.state_root(), Some(genesis_block.slot()))
+            .map_err(|e| format!("DB error when reading genesis state: {:?}", e))?
+            .ok_or("Genesis block not found in store")?;
+
+        self.genesis_time = Some(genesis_state.genesis_time);
 
         self.op_pool = Some(
             store
-                .get_item::<PersistedOperationPool<TEthSpec>>(&Hash256::from_slice(&OP_POOL_DB_KEY))
+                .get_item::<PersistedOperationPool<TEthSpec>>(&OP_POOL_DB_KEY)
                 .map_err(|e| format!("DB error whilst reading persisted op pool: {:?}", e))?
                 .map(PersistedOperationPool::into_operation_pool)
                 .unwrap_or_else(OperationPool::new),
         );
 
-        let finalized_block_root = head_state.finalized_checkpoint.root;
-        let finalized_block = store
-            .get_item::<SignedBeaconBlock<TEthSpec>>(&finalized_block_root)
-            .map_err(|e| format!("DB error when reading finalized block: {:?}", e))?
-            .ok_or_else(|| "Finalized block not found in store".to_string())?;
-        let finalized_state_root = finalized_block.state_root();
-        let finalized_state = store
-            .get_state(&finalized_state_root, Some(finalized_block.slot()))
-            .map_err(|e| format!("DB error when reading finalized state: {:?}", e))?
-            .ok_or_else(|| "Finalized state not found in store".to_string())?;
-
-        self.finalized_snapshot = Some(BeaconSnapshot {
-            beacon_block_root: finalized_block_root,
-            beacon_block: finalized_block,
-            beacon_state_root: finalized_state_root,
-            beacon_state: finalized_state,
-        });
-
-        self.canonical_head = Some(BeaconSnapshot {
-            beacon_block_root: head_block_root,
-            beacon_block: head_block,
-            beacon_state_root: head_state_root,
-            beacon_state: head_state,
-        });
-
         let pubkey_cache = ValidatorPubkeyCache::load_from_file(pubkey_cache_path)
             .map_err(|e| format!("Unable to open persisted pubkey cache: {:?}", e))?;
 
+        self.genesis_block_root = Some(chain.genesis_block_root);
+        self.genesis_state_root = Some(genesis_block.state_root());
+        self.head_tracker = Some(
+            HeadTracker::from_ssz_container(&chain.ssz_head_tracker)
+                .map_err(|e| format!("Failed to decode head tracker for database: {:?}", e))?,
+        );
         self.validator_pubkey_cache = Some(pubkey_cache);
+        self.fork_choice = Some(fork_choice);
 
         Ok(self)
     }
@@ -329,10 +294,7 @@ where
         mut self,
         mut beacon_state: BeaconState<TEthSpec>,
     ) -> Result<Self, String> {
-        let store = self
-            .store
-            .clone()
-            .ok_or_else(|| "genesis_state requires a store")?;
+        let store = self.store.clone().ok_or("genesis_state requires a store")?;
 
         let beacon_block = genesis_block(&mut beacon_state, &self.spec)?;
 
@@ -343,6 +305,7 @@ where
         let beacon_state_root = beacon_block.message.state_root;
         let beacon_block_root = beacon_block.canonical_root();
 
+        self.genesis_state_root = Some(beacon_state_root);
         self.genesis_block_root = Some(beacon_block_root);
 
         store
@@ -362,12 +325,25 @@ where
                 )
             })?;
 
-        self.finalized_snapshot = Some(BeaconSnapshot {
+        let genesis = BeaconSnapshot {
             beacon_block_root,
             beacon_block,
             beacon_state_root,
             beacon_state,
-        });
+        };
+
+        let fc_store = BeaconForkChoiceStore::get_forkchoice_store(store, &genesis);
+
+        let fork_choice = ForkChoice::from_genesis(
+            fc_store,
+            genesis.beacon_block_root,
+            &genesis.beacon_block.message,
+            &genesis.beacon_state,
+        )
+        .map_err(|e| format!("Unable to build initialize ForkChoice: {:?}", e))?;
+
+        self.fork_choice = Some(fork_choice);
+        self.genesis_time = Some(genesis.beacon_state.genesis_time);
 
         Ok(self.empty_op_pool())
     }
@@ -380,9 +356,9 @@ where
 
     /// Sets the `BeaconChain` event handler backend.
     ///
-    /// For example, provide `WebSocketSender` as a `handler`.
-    pub fn event_handler(mut self, handler: TEventHandler) -> Self {
-        self.event_handler = Some(handler);
+    /// For example, provide `ServerSentEventHandler` as a `handler`.
+    pub fn event_handler(mut self, handler: Option<ServerSentEventHandler<TEthSpec>>) -> Self {
+        self.event_handler = handler;
         self
     }
 
@@ -394,6 +370,12 @@ where
         self
     }
 
+    /// Sets a `Sender` to allow the beacon chain to send shutdown signals.
+    pub fn shutdown_sender(mut self, sender: Sender<&'static str>) -> Self {
+        self.shutdown_sender = Some(sender);
+        self
+    }
+
     /// Creates a new, empty operation pool.
     fn empty_op_pool(mut self) -> Self {
         self.op_pool = Some(OperationPool::new());
@@ -406,6 +388,29 @@ where
         self
     }
 
+    /// Sets the `ChainConfig` that determines `BeaconChain` runtime behaviour.
+    pub fn chain_config(mut self, config: ChainConfig) -> Self {
+        self.chain_config = config;
+        self
+    }
+
+    /// Register some validators for additional monitoring.
+    ///
+    /// `validators` is a comma-separated string of 0x-formatted BLS pubkeys.
+    pub fn monitor_validators(
+        mut self,
+        auto_register: bool,
+        validators: Vec<PublicKeyBytes>,
+        log: Logger,
+    ) -> Self {
+        self.validator_monitor = Some(ValidatorMonitor::new(
+            validators,
+            auto_register,
+            log.clone(),
+        ));
+        self
+    }
+
     /// Consumes `self`, returning a `BeaconChain` if all required parameters have been supplied.
     ///
     /// An error will be returned at runtime if all required parameters have not been configured.
@@ -416,87 +421,118 @@ where
     pub fn build(
         self,
     ) -> Result<
-        BeaconChain<
-            Witness<
-                TStoreMigrator,
-                TSlotClock,
-                TEth1Backend,
-                TEthSpec,
-                TEventHandler,
-                THotStore,
-                TColdStore,
-            >,
-        >,
+        BeaconChain<Witness<TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>>,
         String,
     > {
-        let log = self
-            .log
-            .ok_or_else(|| "Cannot build without a logger".to_string())?;
+        let log = self.log.ok_or("Cannot build without a logger")?;
         let slot_clock = self
             .slot_clock
-            .ok_or_else(|| "Cannot build without a slot_clock.".to_string())?;
-        let store = self
-            .store
-            .clone()
-            .ok_or_else(|| "Cannot build without a store.".to_string())?;
+            .ok_or("Cannot build without a slot_clock.")?;
+        let store = self.store.clone().ok_or("Cannot build without a store.")?;
+        let mut fork_choice = self
+            .fork_choice
+            .ok_or("Cannot build without fork choice.")?;
+        let genesis_block_root = self
+            .genesis_block_root
+            .ok_or("Cannot build without a genesis block root")?;
+        let genesis_state_root = self
+            .genesis_state_root
+            .ok_or("Cannot build without a genesis state root")?;
+        let mut validator_monitor = self
+            .validator_monitor
+            .ok_or("Cannot build without a validator monitor")?;
 
-        // If this beacon chain is being loaded from disk, use the stored head. Otherwise, just use
-        // the finalized checkpoint (which is probably genesis).
-        let mut canonical_head = if let Some(head) = self.canonical_head {
-            head
+        let current_slot = if slot_clock
+            .is_prior_to_genesis()
+            .ok_or("Unable to read slot clock")?
+        {
+            self.spec.genesis_slot
         } else {
-            self.finalized_snapshot
-                .ok_or_else(|| "Cannot build without a state".to_string())?
+            slot_clock.now().ok_or("Unable to read slot")?
         };
 
+        let head_block_root = fork_choice
+            .get_head(current_slot)
+            .map_err(|e| format!("Unable to get fork choice head: {:?}", e))?;
+
+        let head_block = store
+            .get_item::<SignedBeaconBlock<TEthSpec>>(&head_block_root)
+            .map_err(|e| format!("DB error when reading head block: {:?}", e))?
+            .ok_or("Head block not found in store")?;
+        let head_state_root = head_block.state_root();
+        let head_state = store
+            .get_state(&head_state_root, Some(head_block.slot()))
+            .map_err(|e| format!("DB error when reading head state: {:?}", e))?
+            .ok_or("Head state not found in store")?;
+
+        let mut canonical_head = BeaconSnapshot {
+            beacon_block_root: head_block_root,
+            beacon_block: head_block,
+            beacon_state_root: head_state_root,
+            beacon_state: head_state,
+        };
+
+        if canonical_head.beacon_block.state_root() != canonical_head.beacon_state_root {
+            return Err("beacon_block.state_root != beacon_state".to_string());
+        }
+
         canonical_head
             .beacon_state
             .build_all_caches(&self.spec)
             .map_err(|e| format!("Failed to build state caches: {:?}", e))?;
 
-        if canonical_head.beacon_block.state_root() != canonical_head.beacon_state_root {
-            return Err("beacon_block.state_root != beacon_state".to_string());
+        // Perform a check to ensure that the finalization points of the head and fork choice are
+        // consistent.
+        //
+        // This is a sanity check to detect database corruption.
+        let fc_finalized = fork_choice.finalized_checkpoint();
+        let head_finalized = canonical_head.beacon_state.finalized_checkpoint;
+        if fc_finalized != head_finalized {
+            if head_finalized.root == Hash256::zero()
+                && head_finalized.epoch == fc_finalized.epoch
+                && fc_finalized.root == genesis_block_root
+            {
+                // This is a legal edge-case encountered during genesis.
+            } else {
+                return Err(format!(
+                    "Database corrupt: fork choice is finalized at {:?} whilst head is finalized at \
+                    {:?}",
+                    fc_finalized, head_finalized
+                ));
+            }
         }
 
         let pubkey_cache_path = self
             .pubkey_cache_path
-            .ok_or_else(|| "Cannot build without a pubkey cache path".to_string())?;
+            .ok_or("Cannot build without a pubkey cache path")?;
 
         let validator_pubkey_cache = self.validator_pubkey_cache.map(Ok).unwrap_or_else(|| {
             ValidatorPubkeyCache::new(&canonical_head.beacon_state, pubkey_cache_path)
                 .map_err(|e| format!("Unable to init validator pubkey cache: {:?}", e))
         })?;
 
-        let persisted_fork_choice = store
-            .get_item::<PersistedForkChoice>(&Hash256::from_slice(&FORK_CHOICE_DB_KEY))
-            .map_err(|e| format!("DB error when reading persisted fork choice: {:?}", e))?;
+        let migrator_config = self.store_migrator_config.unwrap_or_default();
+        let store_migrator = BackgroundMigrator::new(
+            store.clone(),
+            migrator_config,
+            genesis_block_root,
+            log.clone(),
+        );
 
-        let fork_choice = if let Some(persisted) = persisted_fork_choice {
-            let fc_store =
-                BeaconForkChoiceStore::from_persisted(persisted.fork_choice_store, store.clone())
-                    .map_err(|e| format!("Unable to load ForkChoiceStore: {:?}", e))?;
-
-            ForkChoice::from_persisted(persisted.fork_choice, fc_store)
-                .map_err(|e| format!("Unable to parse persisted fork choice from disk: {:?}", e))?
-        } else {
-            let genesis = &canonical_head;
-
-            let fc_store = BeaconForkChoiceStore::get_forkchoice_store(store.clone(), genesis);
-
-            ForkChoice::from_genesis(fc_store, &genesis.beacon_block.message)
-                .map_err(|e| format!("Unable to build initialize ForkChoice: {:?}", e))?
-        };
+        if let Some(slot) = slot_clock.now() {
+            validator_monitor.process_valid_state(
+                slot.epoch(TEthSpec::slots_per_epoch()),
+                &canonical_head.beacon_state,
+            );
+        }
 
         let beacon_chain = BeaconChain {
             spec: self.spec,
+            config: self.chain_config,
             store,
-            store_migrator: self
-                .store_migrator
-                .ok_or_else(|| "Cannot build without store migrator".to_string())?,
+            store_migrator,
             slot_clock,
-            op_pool: self
-                .op_pool
-                .ok_or_else(|| "Cannot build without op pool".to_string())?,
+            op_pool: self.op_pool.ok_or("Cannot build without op pool")?,
             // TODO: allow for persisting and loading the pool from disk.
             naive_aggregation_pool: <_>::default(),
             // TODO: allow for persisting and loading the pool from disk.
@@ -514,13 +550,10 @@ where
             eth1_chain: self.eth1_chain,
             genesis_validators_root: canonical_head.beacon_state.genesis_validators_root,
             canonical_head: TimeoutRwLock::new(canonical_head.clone()),
-            genesis_block_root: self
-                .genesis_block_root
-                .ok_or_else(|| "Cannot build without a genesis block root".to_string())?,
+            genesis_block_root,
+            genesis_state_root,
             fork_choice: RwLock::new(fork_choice),
-            event_handler: self
-                .event_handler
-                .ok_or_else(|| "Cannot build without an event handler".to_string())?,
+            event_handler: self.event_handler,
             head_tracker: Arc::new(self.head_tracker.unwrap_or_default()),
             snapshot_cache: TimeoutRwLock::new(SnapshotCache::new(
                 DEFAULT_SNAPSHOT_CACHE_SIZE,
@@ -529,14 +562,40 @@ where
             shuffling_cache: TimeoutRwLock::new(ShufflingCache::new()),
             validator_pubkey_cache: TimeoutRwLock::new(validator_pubkey_cache),
             disabled_forks: self.disabled_forks,
+            shutdown_sender: self
+                .shutdown_sender
+                .ok_or("Cannot build without a shutdown sender.")?,
             log: log.clone(),
             graffiti: self.graffiti,
+            slasher: self.slasher.clone(),
+            validator_monitor: RwLock::new(validator_monitor),
         };
 
         let head = beacon_chain
             .head()
             .map_err(|e| format!("Failed to get head: {:?}", e))?;
 
+        // Only perform the check if it was configured.
+        if let Some(wss_checkpoint) = beacon_chain.config.weak_subjectivity_checkpoint {
+            if let Err(e) = beacon_chain.verify_weak_subjectivity_checkpoint(
+                wss_checkpoint,
+                head.beacon_block_root,
+                &head.beacon_state,
+            ) {
+                crit!(
+                    log,
+                    "Weak subjectivity checkpoint verification failed on startup!";
+                    "head_block_root" => format!("{}", head.beacon_block_root),
+                    "head_slot" => format!("{}", head.beacon_block.slot()),
+                    "finalized_epoch" => format!("{}", head.beacon_state.finalized_checkpoint.epoch),
+                    "wss_checkpoint_epoch" => format!("{}", wss_checkpoint.epoch),
+                    "error" => format!("{:?}", e),
+                );
+                crit!(log, "You must use the `--purge-db` flag to clear the database and restart sync. You may be on a hostile network.");
+                return Err(format!("Weak subjectivity verification failed: {:?}", e));
+            }
+        }
+
         info!(
             log,
             "Beacon chain initialized";
@@ -549,25 +608,15 @@ where
     }
 }
 
-impl<TStoreMigrator, TSlotClock, TEthSpec, TEventHandler, THotStore, TColdStore>
+impl<TSlotClock, TEthSpec, THotStore, TColdStore>
     BeaconChainBuilder<
-        Witness<
-            TStoreMigrator,
-            TSlotClock,
-            CachingEth1Backend<TEthSpec>,
-            TEthSpec,
-            TEventHandler,
-            THotStore,
-            TColdStore,
-        >,
+        Witness<TSlotClock, CachingEth1Backend<TEthSpec>, TEthSpec, THotStore, TColdStore>,
     >
 where
     THotStore: ItemStore<TEthSpec> + 'static,
     TColdStore: ItemStore<TEthSpec> + 'static,
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore> + 'static,
     TSlotClock: SlotClock + 'static,
     TEthSpec: EthSpec + 'static,
-    TEventHandler: EventHandler<TEthSpec> + 'static,
 {
     /// Do not use any eth1 backend. The client will not be able to produce beacon blocks.
     pub fn no_eth1_backend(self) -> Self {
@@ -579,7 +628,7 @@ where
         let log = self
             .log
             .as_ref()
-            .ok_or_else(|| "dummy_eth1_backend requires a log".to_string())?;
+            .ok_or("dummy_eth1_backend requires a log")?;
 
         let backend =
             CachingEth1Backend::new(Eth1Config::default(), log.clone(), self.spec.clone());
@@ -590,36 +639,21 @@ where
     }
 }
 
-impl<TStoreMigrator, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
-    BeaconChainBuilder<
-        Witness<
-            TStoreMigrator,
-            TestingSlotClock,
-            TEth1Backend,
-            TEthSpec,
-            TEventHandler,
-            THotStore,
-            TColdStore,
-        >,
-    >
+impl<TEth1Backend, TEthSpec, THotStore, TColdStore>
+    BeaconChainBuilder<Witness<TestingSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>>
 where
     THotStore: ItemStore<TEthSpec> + 'static,
     TColdStore: ItemStore<TEthSpec> + 'static,
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore> + 'static,
     TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
     TEthSpec: EthSpec + 'static,
-    TEventHandler: EventHandler<TEthSpec> + 'static,
 {
     /// Sets the `BeaconChain` slot clock to `TestingSlotClock`.
     ///
     /// Requires the state to be initialized.
     pub fn testing_slot_clock(self, slot_duration: Duration) -> Result<Self, String> {
         let genesis_time = self
-            .finalized_snapshot
-            .as_ref()
-            .ok_or_else(|| "testing_slot_clock requires an initialized state")?
-            .beacon_state
-            .genesis_time;
+            .genesis_time
+            .ok_or("testing_slot_clock requires an initialized state")?;
 
         let slot_clock = TestingSlotClock::new(
             Slot::new(0),
@@ -631,33 +665,6 @@ where
     }
 }
 
-impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>
-    BeaconChainBuilder<
-        Witness<
-            TStoreMigrator,
-            TSlotClock,
-            TEth1Backend,
-            TEthSpec,
-            NullEventHandler<TEthSpec>,
-            THotStore,
-            TColdStore,
-        >,
-    >
-where
-    THotStore: ItemStore<TEthSpec> + 'static,
-    TColdStore: ItemStore<TEthSpec> + 'static,
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore> + 'static,
-    TSlotClock: SlotClock + 'static,
-    TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
-    TEthSpec: EthSpec + 'static,
-{
-    /// Sets the `BeaconChain` event handler to `NullEventHandler`.
-    pub fn null_event_handler(self) -> Self {
-        let handler = NullEventHandler::default();
-        self.event_handler(handler)
-    }
-}
-
 fn genesis_block<T: EthSpec>(
     genesis_state: &mut BeaconState<T>,
     spec: &ChainSpec,
@@ -678,7 +685,6 @@ fn genesis_block<T: EthSpec>(
 #[cfg(test)]
 mod test {
     use super::*;
-    use crate::migrate::NullMigrator;
     use eth2_hashing::hash;
     use genesis::{generate_deterministic_keypairs, interop_genesis_state};
     use sloggers::{null::NullLoggerBuilder, Build};
@@ -718,18 +724,20 @@ mod test {
         )
         .expect("should create interop genesis state");
 
+        let (shutdown_tx, _) = futures::channel::mpsc::channel(1);
+
         let chain = BeaconChainBuilder::new(MinimalEthSpec)
             .logger(log.clone())
             .store(Arc::new(store))
-            .store_migrator(NullMigrator)
             .data_dir(data_dir.path().to_path_buf())
             .genesis_state(genesis_state)
             .expect("should build state using recent genesis")
             .dummy_eth1_backend()
             .expect("should build the dummy eth1 backend")
-            .null_event_handler()
             .testing_slot_clock(Duration::from_secs(1))
             .expect("should configure testing slot clock")
+            .shutdown_sender(shutdown_tx)
+            .monitor_validators(true, vec![], log.clone())
             .build()
             .expect("should build");
 

beacon_node/beacon_chain/src/chain_config.rs

@@ -0,0 +1,24 @@
+use serde_derive::{Deserialize, Serialize};
+use types::Checkpoint;
+
+#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]
+pub struct ChainConfig {
+    /// Maximum number of slots to skip when importing a consensus message (e.g., block,
+    /// attestation, etc).
+    ///
+    /// If `None`, there is no limit.
+    pub import_max_skip_slots: Option<u64>,
+    /// A user-input `Checkpoint` that must exist in the beacon chain's sync path.
+    ///
+    /// If `None`, there is no weak subjectivity verification.
+    pub weak_subjectivity_checkpoint: Option<Checkpoint>,
+}
+
+impl Default for ChainConfig {
+    fn default() -> Self {
+        Self {
+            import_max_skip_slots: None,
+            weak_subjectivity_checkpoint: None,
+        }
+    }
+}

beacon_node/beacon_chain/src/errors.rs

@@ -1,9 +1,12 @@
 use crate::beacon_chain::ForkChoiceError;
+use crate::beacon_fork_choice_store::Error as ForkChoiceStoreError;
 use crate::eth1_chain::Error as Eth1ChainError;
+use crate::migrate::PruningError;
 use crate::naive_aggregation_pool::Error as NaiveAggregationError;
 use crate::observed_attestations::Error as ObservedAttestationsError;
 use crate::observed_attesters::Error as ObservedAttestersError;
 use crate::observed_block_producers::Error as ObservedBlockProducersError;
+use futures::channel::mpsc::TrySendError;
 use operation_pool::OpPoolError;
 use safe_arith::ArithError;
 use ssz_types::Error as SszTypesError;
@@ -44,6 +47,7 @@ pub enum BeaconChainError {
     DBInconsistent(String),
     DBError(store::Error),
     ForkChoiceError(ForkChoiceError),
+    ForkChoiceStoreError(ForkChoiceStoreError),
     MissingBeaconBlock(Hash256),
     MissingBeaconState(Hash256),
     SlotProcessingError(SlotProcessingError),
@@ -61,6 +65,7 @@ pub enum BeaconChainError {
         requested_slot: Slot,
         max_task_runtime: Duration,
     },
+    MissingFinalizedStateRoot(Slot),
     /// Returned when an internal check fails, indicating corrupt data.
     InvariantViolated(String),
     SszTypesError(SszTypesError),
@@ -79,7 +84,18 @@ pub enum BeaconChainError {
     ObservedAttestationsError(ObservedAttestationsError),
     ObservedAttestersError(ObservedAttestersError),
     ObservedBlockProducersError(ObservedBlockProducersError),
+    PruningError(PruningError),
     ArithError(ArithError),
+    InvalidShufflingId {
+        shuffling_epoch: Epoch,
+        head_block_epoch: Epoch,
+    },
+    WeakSubjectivtyVerificationFailure,
+    WeakSubjectivtyShutdownError(TrySendError<&'static str>),
+    AttestingPriorToHead {
+        head_slot: Slot,
+        request_slot: Slot,
+    },
 }
 
 easy_from_to!(SlotProcessingError, BeaconChainError);
@@ -94,10 +110,13 @@ easy_from_to!(ObservedAttestationsError, BeaconChainError);
 easy_from_to!(ObservedAttestersError, BeaconChainError);
 easy_from_to!(ObservedBlockProducersError, BeaconChainError);
 easy_from_to!(BlockSignatureVerifierError, BeaconChainError);
+easy_from_to!(PruningError, BeaconChainError);
 easy_from_to!(ArithError, BeaconChainError);
+easy_from_to!(ForkChoiceStoreError, BeaconChainError);
 
 #[derive(Debug)]
 pub enum BlockProductionError {
+    UnableToGetHeadInfo(BeaconChainError),
     UnableToGetBlockRootFromState,
     UnableToReadSlot,
     UnableToProduceAtSlot(Slot),

beacon_node/beacon_chain/src/eth1_chain.rs

@@ -1,6 +1,6 @@
 use crate::metrics;
-use environment::TaskExecutor;
 use eth1::{Config as Eth1Config, Eth1Block, Service as HttpService};
+use eth2::lighthouse::Eth1SyncStatusData;
 use eth2_hashing::hash;
 use slog::{debug, error, trace, Logger};
 use ssz::{Decode, Encode};
@@ -10,7 +10,9 @@ use std::cmp::Ordering;
 use std::collections::HashMap;
 use std::iter::DoubleEndedIterator;
 use std::marker::PhantomData;
+use std::time::{SystemTime, UNIX_EPOCH};
 use store::{DBColumn, Error as StoreError, StoreItem};
+use task_executor::TaskExecutor;
 use types::{
     BeaconState, BeaconStateError, ChainSpec, Deposit, Eth1Data, EthSpec, Hash256, Slot, Unsigned,
     DEPOSIT_TREE_DEPTH,
@@ -19,6 +21,11 @@ use types::{
 type BlockNumber = u64;
 type Eth1DataVoteCount = HashMap<(Eth1Data, BlockNumber), u64>;
 
+/// We will declare ourself synced with the Eth1 chain, even if we are this many blocks behind.
+///
+/// This number (8) was chosen somewhat arbitrarily.
+const ETH1_SYNC_TOLERANCE: u64 = 8;
+
 #[derive(Debug)]
 pub enum Error {
     /// Unable to return an Eth1Data for the given epoch.
@@ -53,6 +60,113 @@ impl From<safe_arith::ArithError> for Error {
     }
 }
 
+/// Returns an `Eth1SyncStatusData` given some parameters:
+///
+/// - `latest_cached_block`: The latest eth1 block in our cache, if any.
+/// - `head_block`: The block at the very head of our eth1 node (ignoring follow distance, etc).
+/// - `genesis_time`: beacon chain genesis time.
+/// - `current_slot`: current beacon chain slot.
+/// - `spec`: current beacon chain specification.
+fn get_sync_status<T: EthSpec>(
+    latest_cached_block: Option<&Eth1Block>,
+    head_block: Option<&Eth1Block>,
+    genesis_time: u64,
+    current_slot: Option<Slot>,
+    spec: &ChainSpec,
+) -> Option<Eth1SyncStatusData> {
+    let eth1_follow_distance_seconds = spec
+        .seconds_per_eth1_block
+        .saturating_mul(spec.eth1_follow_distance);
+
+    // The voting target timestamp needs to be special-cased when we're before
+    // genesis (as defined by `current_slot == None`).
+    //
+    // For the sake of this status, when prior to genesis we want to invent some voting periods
+    // that are *before* genesis, so that we can indicate to users that we're actually adequately
+    // cached for where they are in time.
+    let voting_target_timestamp = if let Some(current_slot) = current_slot {
+        let period = T::SlotsPerEth1VotingPeriod::to_u64();
+        let voting_period_start_slot = (current_slot / period) * period;
+
+        let period_start = slot_start_seconds::<T>(
+            genesis_time,
+            spec.seconds_per_slot,
+            voting_period_start_slot,
+        );
+
+        period_start.saturating_sub(eth1_follow_distance_seconds)
+    } else {
+        // The number of seconds in an eth1 voting period.
+        let voting_period_duration =
+            T::slots_per_eth1_voting_period() as u64 * spec.seconds_per_slot;
+
+        let now = SystemTime::now().duration_since(UNIX_EPOCH).ok()?.as_secs();
+
+        // The number of seconds between now and genesis.
+        let seconds_till_genesis = genesis_time.saturating_sub(now);
+
+        // Determine how many voting periods are contained in distance between
+        // now and genesis, rounding up.
+        let voting_periods_past =
+            (seconds_till_genesis + voting_period_duration - 1) / voting_period_duration;
+
+        // Return the start time of the current voting period*.
+        //
+        // *: This voting period doesn't *actually* exist, we're just using it to
+        // give useful logs prior to genesis.
+        genesis_time
+            .saturating_sub(voting_periods_past * voting_period_duration)
+            .saturating_sub(eth1_follow_distance_seconds)
+    };
+
+    let latest_cached_block_number = latest_cached_block.map(|b| b.number);
+    let latest_cached_block_timestamp = latest_cached_block.map(|b| b.timestamp);
+    let head_block_number = head_block.map(|b| b.number);
+    let head_block_timestamp = head_block.map(|b| b.timestamp);
+
+    let eth1_node_sync_status_percentage = if let Some(head_block) = head_block {
+        let now = SystemTime::now().duration_since(UNIX_EPOCH).ok()?.as_secs();
+        let head_age = now.saturating_sub(head_block.timestamp);
+
+        if head_age < ETH1_SYNC_TOLERANCE * spec.seconds_per_eth1_block {
+            // Always indicate we are fully synced if it's within the sync threshold.
+            100.0
+        } else {
+            let blocks_behind = head_age
+                .checked_div(spec.seconds_per_eth1_block)
+                .unwrap_or(0);
+
+            let part = f64::from(head_block.number as u32);
+            let whole = f64::from(head_block.number.saturating_add(blocks_behind) as u32);
+
+            if whole > 0.0 {
+                (part / whole) * 100.0
+            } else {
+                // Avoids a divide-by-zero.
+                0.0
+            }
+        }
+    } else {
+        // Always return 0% synced if the head block of the eth1 chain is unknown.
+        0.0
+    };
+
+    // Lighthouse is "cached and ready" when it has cached enough blocks to cover the start of the
+    // current voting period.
+    let lighthouse_is_cached_and_ready =
+        latest_cached_block_timestamp.map_or(false, |t| t >= voting_target_timestamp);
+
+    Some(Eth1SyncStatusData {
+        head_block_number,
+        head_block_timestamp,
+        latest_cached_block_number,
+        latest_cached_block_timestamp,
+        voting_target_timestamp,
+        eth1_node_sync_status_percentage,
+        lighthouse_is_cached_and_ready,
+    })
+}
+
 #[derive(Encode, Decode, Clone)]
 pub struct SszEth1 {
     use_dummy_backend: bool,
@@ -143,6 +257,22 @@ where
         }
     }
 
+    /// Returns a status indicating how synced our caches are with the eth1 chain.
+    pub fn sync_status(
+        &self,
+        genesis_time: u64,
+        current_slot: Option<Slot>,
+        spec: &ChainSpec,
+    ) -> Option<Eth1SyncStatusData> {
+        get_sync_status::<E>(
+            self.backend.latest_cached_block().as_ref(),
+            self.backend.head_block().as_ref(),
+            genesis_time,
+            current_slot,
+            spec,
+        )
+    }
+
     /// Instantiate `Eth1Chain` from a persisted `SszEth1`.
     ///
     /// The `Eth1Chain` will have the same caches as the persisted `SszEth1`.
@@ -195,6 +325,14 @@ pub trait Eth1ChainBackend<T: EthSpec>: Sized + Send + Sync {
         spec: &ChainSpec,
     ) -> Result<Vec<Deposit>, Error>;
 
+    /// Returns the latest block stored in the cache. Used to obtain an idea of how up-to-date the
+    /// beacon node eth1 cache is.
+    fn latest_cached_block(&self) -> Option<Eth1Block>;
+
+    /// Returns the block at the head of the chain (ignoring follow distance, etc). Used to obtain
+    /// an idea of how up-to-date the remote eth1 node is.
+    fn head_block(&self) -> Option<Eth1Block>;
+
     /// Encode the `Eth1ChainBackend` instance to bytes.
     fn as_bytes(&self) -> Vec<u8>;
 
@@ -241,6 +379,14 @@ impl<T: EthSpec> Eth1ChainBackend<T> for DummyEth1ChainBackend<T> {
         Ok(vec![])
     }
 
+    fn latest_cached_block(&self) -> Option<Eth1Block> {
+        None
+    }
+
+    fn head_block(&self) -> Option<Eth1Block> {
+        None
+    }
+
     /// Return empty Vec<u8> for dummy backend.
     fn as_bytes(&self) -> Vec<u8> {
         Vec::new()
@@ -308,7 +454,7 @@ impl<T: EthSpec> Eth1ChainBackend<T> for CachingEth1Backend<T> {
         let voting_period_start_slot = (state.slot / period) * period;
         let voting_period_start_seconds = slot_start_seconds::<T>(
             state.genesis_time,
-            spec.milliseconds_per_slot,
+            spec.seconds_per_slot,
             voting_period_start_slot,
         );
 
@@ -333,13 +479,14 @@ impl<T: EthSpec> Eth1ChainBackend<T> for CachingEth1Backend<T> {
             // If no votes exist, choose `state.eth1_data` as default vote.
             votes_to_consider
                 .iter()
-                .max_by(|(_, x), (_, y)| x.cmp(y))
+                .max_by_key(|(_, block_number)| *block_number)
                 .map(|vote| {
                     let vote = vote.0.clone();
                     debug!(
                         self.log,
                         "No valid eth1_data votes";
                         "outcome" => "Casting vote corresponding to last candidate eth1 block",
+                        "vote" => ?vote
                     );
                     vote
                 })
@@ -400,6 +547,14 @@ impl<T: EthSpec> Eth1ChainBackend<T> for CachingEth1Backend<T> {
         }
     }
 
+    fn latest_cached_block(&self) -> Option<Eth1Block> {
+        self.core.latest_cached_block()
+    }
+
+    fn head_block(&self) -> Option<Eth1Block> {
+        self.core.head_block()
+    }
+
     /// Return encoded byte representation of the block and deposit caches.
     fn as_bytes(&self) -> Vec<u8> {
         self.core.as_bytes()
@@ -490,10 +645,10 @@ fn int_to_bytes32(int: u64) -> Vec<u8> {
 /// Returns the unix-epoch seconds at the start of the given `slot`.
 fn slot_start_seconds<T: EthSpec>(
     genesis_unix_seconds: u64,
-    milliseconds_per_slot: u64,
+    seconds_per_slot: u64,
     slot: Slot,
 ) -> u64 {
-    genesis_unix_seconds + slot.as_u64() * milliseconds_per_slot / 1_000
+    genesis_unix_seconds + slot.as_u64() * seconds_per_slot
 }
 
 /// Returns a boolean denoting if a given `Eth1Block` is a candidate for `Eth1Data` calculation
@@ -531,7 +686,7 @@ mod test {
         let voting_period_start_slot = (state.slot / period) * period;
         slot_start_seconds::<E>(
             state.genesis_time,
-            spec.milliseconds_per_slot,
+            spec.seconds_per_slot,
             voting_period_start_slot,
         )
     }
@@ -541,21 +696,21 @@ mod test {
         let zero_sec = 0;
         assert_eq!(slot_start_seconds::<E>(100, zero_sec, Slot::new(2)), 100);
 
-        let half_sec = 500;
-        assert_eq!(slot_start_seconds::<E>(100, half_sec, Slot::new(0)), 100);
-        assert_eq!(slot_start_seconds::<E>(100, half_sec, Slot::new(1)), 100);
-        assert_eq!(slot_start_seconds::<E>(100, half_sec, Slot::new(2)), 101);
-        assert_eq!(slot_start_seconds::<E>(100, half_sec, Slot::new(3)), 101);
-
-        let one_sec = 1_000;
+        let one_sec = 1;
         assert_eq!(slot_start_seconds::<E>(100, one_sec, Slot::new(0)), 100);
         assert_eq!(slot_start_seconds::<E>(100, one_sec, Slot::new(1)), 101);
         assert_eq!(slot_start_seconds::<E>(100, one_sec, Slot::new(2)), 102);
 
-        let three_sec = 3_000;
+        let three_sec = 3;
         assert_eq!(slot_start_seconds::<E>(100, three_sec, Slot::new(0)), 100);
         assert_eq!(slot_start_seconds::<E>(100, three_sec, Slot::new(1)), 103);
         assert_eq!(slot_start_seconds::<E>(100, three_sec, Slot::new(2)), 106);
+
+        let five_sec = 5;
+        assert_eq!(slot_start_seconds::<E>(100, five_sec, Slot::new(0)), 100);
+        assert_eq!(slot_start_seconds::<E>(100, five_sec, Slot::new(1)), 105);
+        assert_eq!(slot_start_seconds::<E>(100, five_sec, Slot::new(2)), 110);
+        assert_eq!(slot_start_seconds::<E>(100, five_sec, Slot::new(3)), 115);
     }
 
     fn get_eth1_block(timestamp: u64, number: u64) -> Eth1Block {
@@ -658,7 +813,7 @@ mod test {
                         .write()
                         .cache
                         .insert_log(log.clone())
-                        .expect("should insert log")
+                        .expect("should insert log");
                 })
                 .collect();
 

beacon_node/beacon_chain/src/events.rs

@@ -1,147 +1,113 @@
-use bus::Bus;
-use parking_lot::Mutex;
-use serde_derive::{Deserialize, Serialize};
-use slog::{error, Logger};
-use std::marker::PhantomData;
-use std::sync::Arc;
-use types::{Attestation, Epoch, EthSpec, Hash256, SignedBeaconBlock, SignedBeaconBlockHash};
-pub use websocket_server::WebSocketSender;
+pub use eth2::types::{EventKind, SseBlock, SseFinalizedCheckpoint, SseHead};
+use slog::{trace, Logger};
+use tokio::sync::broadcast;
+use tokio::sync::broadcast::{error::SendError, Receiver, Sender};
+use types::EthSpec;
 
-pub trait EventHandler<T: EthSpec>: Sized + Send + Sync {
-    fn register(&self, kind: EventKind<T>) -> Result<(), String>;
-}
+const DEFAULT_CHANNEL_CAPACITY: usize = 16;
 
-pub struct NullEventHandler<T: EthSpec>(PhantomData<T>);
-
-impl<T: EthSpec> EventHandler<T> for WebSocketSender<T> {
-    fn register(&self, kind: EventKind<T>) -> Result<(), String> {
-        self.send_string(
-            serde_json::to_string(&kind)
-                .map_err(|e| format!("Unable to serialize event: {:?}", e))?,
-        )
-    }
-}
-
-pub struct ServerSentEvents<T: EthSpec> {
-    // Bus<> is itself Sync + Send.  We use Mutex<> here only because of the surrounding code does
-    // not enforce mutability statically (i.e. relies on interior mutability).
-    head_changed_queue: Arc<Mutex<Bus<SignedBeaconBlockHash>>>,
+pub struct ServerSentEventHandler<T: EthSpec> {
+    attestation_tx: Sender<EventKind<T>>,
+    block_tx: Sender<EventKind<T>>,
+    finalized_tx: Sender<EventKind<T>>,
+    head_tx: Sender<EventKind<T>>,
+    exit_tx: Sender<EventKind<T>>,
     log: Logger,
-    _phantom: PhantomData<T>,
 }
 
-impl<T: EthSpec> ServerSentEvents<T> {
-    pub fn new(log: Logger) -> (Self, Arc<Mutex<Bus<SignedBeaconBlockHash>>>) {
-        let bus = Bus::new(T::slots_per_epoch() as usize);
-        let mutex = Mutex::new(bus);
-        let arc = Arc::new(mutex);
-        let this = Self {
-            head_changed_queue: arc.clone(),
+impl<T: EthSpec> ServerSentEventHandler<T> {
+    pub fn new(log: Logger) -> Self {
+        let (attestation_tx, _) = broadcast::channel(DEFAULT_CHANNEL_CAPACITY);
+        let (block_tx, _) = broadcast::channel(DEFAULT_CHANNEL_CAPACITY);
+        let (finalized_tx, _) = broadcast::channel(DEFAULT_CHANNEL_CAPACITY);
+        let (head_tx, _) = broadcast::channel(DEFAULT_CHANNEL_CAPACITY);
+        let (exit_tx, _) = broadcast::channel(DEFAULT_CHANNEL_CAPACITY);
+
+        Self {
+            attestation_tx,
+            block_tx,
+            finalized_tx,
+            head_tx,
+            exit_tx,
             log,
-            _phantom: PhantomData,
-        };
-        (this, arc)
-    }
-}
-
-impl<T: EthSpec> EventHandler<T> for ServerSentEvents<T> {
-    fn register(&self, kind: EventKind<T>) -> Result<(), String> {
-        match kind {
-            EventKind::BeaconHeadChanged {
-                current_head_beacon_block_root,
-                ..
-            } => {
-                let mut guard = self.head_changed_queue.lock();
-                if guard
-                    .try_broadcast(current_head_beacon_block_root.into())
-                    .is_err()
-                {
-                    error!(
-                        self.log,
-                        "Head change streaming queue full";
-                        "dropped_change" => format!("{}", current_head_beacon_block_root),
-                    );
-                }
-                Ok(())
-            }
-            _ => Ok(()),
         }
     }
-}
 
-// An event handler that pushes events to both the websockets handler and the SSE handler.
-// Named after the unix `tee` command.  Meant as a temporary solution before ditching WebSockets
-// completely once SSE functions well enough.
-pub struct TeeEventHandler<E: EthSpec> {
-    websockets_handler: WebSocketSender<E>,
-    sse_handler: ServerSentEvents<E>,
-}
+    pub fn new_with_capacity(log: Logger, capacity: usize) -> Self {
+        let (attestation_tx, _) = broadcast::channel(capacity);
+        let (block_tx, _) = broadcast::channel(capacity);
+        let (finalized_tx, _) = broadcast::channel(capacity);
+        let (head_tx, _) = broadcast::channel(capacity);
+        let (exit_tx, _) = broadcast::channel(capacity);
 
-impl<E: EthSpec> TeeEventHandler<E> {
-    #[allow(clippy::type_complexity)]
-    pub fn new(
-        log: Logger,
-        websockets_handler: WebSocketSender<E>,
-    ) -> Result<(Self, Arc<Mutex<Bus<SignedBeaconBlockHash>>>), String> {
-        let (sse_handler, bus) = ServerSentEvents::new(log);
-        let result = Self {
-            websockets_handler,
-            sse_handler,
+        Self {
+            attestation_tx,
+            block_tx,
+            finalized_tx,
+            head_tx,
+            exit_tx,
+            log,
+        }
+    }
+
+    pub fn register(&self, kind: EventKind<T>) {
+        let result = match kind {
+            EventKind::Attestation(attestation) => self
+                .attestation_tx
+                .send(EventKind::Attestation(attestation))
+                .map(|count| trace!(self.log, "Registering server-sent attestation event"; "receiver_count" => count)),
+            EventKind::Block(block) => self.block_tx.send(EventKind::Block(block))
+                .map(|count| trace!(self.log, "Registering server-sent block event"; "receiver_count" => count)),
+            EventKind::FinalizedCheckpoint(checkpoint) => self.finalized_tx
+                .send(EventKind::FinalizedCheckpoint(checkpoint))
+                .map(|count| trace!(self.log, "Registering server-sent finalized checkpoint event"; "receiver_count" => count)),
+            EventKind::Head(head) => self.head_tx.send(EventKind::Head(head))
+                .map(|count| trace!(self.log, "Registering server-sent head event"; "receiver_count" => count)),
+            EventKind::VoluntaryExit(exit) => self.exit_tx.send(EventKind::VoluntaryExit(exit))
+                .map(|count| trace!(self.log, "Registering server-sent voluntary exit event"; "receiver_count" => count)),
         };
-        Ok((result, bus))
+        if let Err(SendError(event)) = result {
+            trace!(self.log, "No receivers registered to listen for event"; "event" => ?event);
+        }
+    }
+
+    pub fn subscribe_attestation(&self) -> Receiver<EventKind<T>> {
+        self.attestation_tx.subscribe()
+    }
+
+    pub fn subscribe_block(&self) -> Receiver<EventKind<T>> {
+        self.block_tx.subscribe()
+    }
+
+    pub fn subscribe_finalized(&self) -> Receiver<EventKind<T>> {
+        self.finalized_tx.subscribe()
+    }
+
+    pub fn subscribe_head(&self) -> Receiver<EventKind<T>> {
+        self.head_tx.subscribe()
+    }
+
+    pub fn subscribe_exit(&self) -> Receiver<EventKind<T>> {
+        self.exit_tx.subscribe()
+    }
+
+    pub fn has_attestation_subscribers(&self) -> bool {
+        self.attestation_tx.receiver_count() > 0
+    }
+
+    pub fn has_block_subscribers(&self) -> bool {
+        self.block_tx.receiver_count() > 0
+    }
+
+    pub fn has_finalized_subscribers(&self) -> bool {
+        self.finalized_tx.receiver_count() > 0
+    }
+
+    pub fn has_head_subscribers(&self) -> bool {
+        self.head_tx.receiver_count() > 0
+    }
+
+    pub fn has_exit_subscribers(&self) -> bool {
+        self.exit_tx.receiver_count() > 0
     }
 }
-
-impl<E: EthSpec> EventHandler<E> for TeeEventHandler<E> {
-    fn register(&self, kind: EventKind<E>) -> Result<(), String> {
-        self.websockets_handler.register(kind.clone())?;
-        self.sse_handler.register(kind)?;
-        Ok(())
-    }
-}
-
-impl<T: EthSpec> EventHandler<T> for NullEventHandler<T> {
-    fn register(&self, _kind: EventKind<T>) -> Result<(), String> {
-        Ok(())
-    }
-}
-
-impl<T: EthSpec> Default for NullEventHandler<T> {
-    fn default() -> Self {
-        NullEventHandler(PhantomData)
-    }
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-#[serde(
-    bound = "T: EthSpec",
-    rename_all = "snake_case",
-    tag = "event",
-    content = "data"
-)]
-pub enum EventKind<T: EthSpec> {
-    BeaconHeadChanged {
-        reorg: bool,
-        current_head_beacon_block_root: Hash256,
-        previous_head_beacon_block_root: Hash256,
-    },
-    BeaconFinalization {
-        epoch: Epoch,
-        root: Hash256,
-    },
-    BeaconBlockImported {
-        block_root: Hash256,
-        block: Box<SignedBeaconBlock<T>>,
-    },
-    BeaconBlockRejected {
-        reason: String,
-        block: Box<SignedBeaconBlock<T>>,
-    },
-    BeaconAttestationImported {
-        attestation: Box<Attestation<T>>,
-    },
-    BeaconAttestationRejected {
-        reason: String,
-        attestation: Box<Attestation<T>>,
-    },
-}

beacon_node/beacon_chain/src/head_tracker.rs

@@ -1,7 +1,6 @@
 use parking_lot::RwLock;
 use ssz_derive::{Decode, Encode};
 use std::collections::HashMap;
-use std::iter::FromIterator;
 use types::{Hash256, Slot};
 
 #[derive(Debug, PartialEq)]
@@ -15,7 +14,7 @@ pub enum Error {
 /// In order for this struct to be effective, every single block that is imported must be
 /// registered here.
 #[derive(Default, Debug)]
-pub struct HeadTracker(RwLock<HashMap<Hash256, Slot>>);
+pub struct HeadTracker(pub RwLock<HashMap<Hash256, Slot>>);
 
 impl HeadTracker {
     /// Register a block with `Self`, so it may or may not be included in a `Self::heads` call.
@@ -29,13 +28,6 @@ impl HeadTracker {
         map.insert(block_root, slot);
     }
 
-    /// Removes abandoned head.
-    pub fn remove_head(&self, block_root: Hash256) {
-        let mut map = self.0.write();
-        debug_assert!(map.contains_key(&block_root));
-        map.remove(&block_root);
-    }
-
     /// Returns true iff `block_root` is a recognized head.
     pub fn contains_head(&self, block_root: Hash256) -> bool {
         self.0.read().contains_key(&block_root)
@@ -53,14 +45,7 @@ impl HeadTracker {
     /// Returns a `SszHeadTracker`, which contains all necessary information to restore the state
     /// of `Self` at some later point.
     pub fn to_ssz_container(&self) -> SszHeadTracker {
-        let (roots, slots) = self
-            .0
-            .read()
-            .iter()
-            .map(|(hash, slot)| (*hash, *slot))
-            .unzip();
-
-        SszHeadTracker { roots, slots }
+        SszHeadTracker::from_map(&*self.0.read())
     }
 
     /// Creates a new `Self` from the given `SszHeadTracker`, restoring `Self` to the same state of
@@ -75,13 +60,12 @@ impl HeadTracker {
                 slots_len,
             })
         } else {
-            let map = HashMap::from_iter(
-                ssz_container
-                    .roots
-                    .iter()
-                    .zip(ssz_container.slots.iter())
-                    .map(|(root, slot)| (*root, *slot)),
-            );
+            let map = ssz_container
+                .roots
+                .iter()
+                .zip(ssz_container.slots.iter())
+                .map(|(root, slot)| (*root, *slot))
+                .collect::<HashMap<_, _>>();
 
             Ok(Self(RwLock::new(map)))
         }
@@ -103,6 +87,13 @@ pub struct SszHeadTracker {
     slots: Vec<Slot>,
 }
 
+impl SszHeadTracker {
+    pub fn from_map(map: &HashMap<Hash256, Slot>) -> Self {
+        let (roots, slots) = map.iter().map(|(hash, slot)| (*hash, *slot)).unzip();
+        SszHeadTracker { roots, slots }
+    }
+}
+
 #[cfg(test)]
 mod test {
     use super::*;

beacon_node/beacon_chain/src/lib.rs

@@ -1,13 +1,11 @@
 #![recursion_limit = "128"] // For lazy-static
-#[macro_use]
-extern crate lazy_static;
-
 pub mod attestation_verification;
 mod beacon_chain;
 mod beacon_fork_choice_store;
 mod beacon_snapshot;
 mod block_verification;
 pub mod builder;
+pub mod chain_config;
 mod errors;
 pub mod eth1_chain;
 pub mod events;
@@ -25,6 +23,7 @@ mod shuffling_cache;
 mod snapshot_cache;
 pub mod test_utils;
 mod timeout_rw_lock;
+pub mod validator_monitor;
 mod validator_pubkey_cache;
 
 pub use self::beacon_chain::{
@@ -32,12 +31,13 @@ pub use self::beacon_chain::{
     ForkChoiceError, StateSkipConfig,
 };
 pub use self::beacon_snapshot::BeaconSnapshot;
+pub use self::chain_config::ChainConfig;
 pub use self::errors::{BeaconChainError, BlockProductionError};
 pub use attestation_verification::Error as AttestationError;
 pub use beacon_fork_choice_store::{BeaconForkChoiceStore, Error as ForkChoiceStoreError};
 pub use block_verification::{BlockError, GossipVerifiedBlock};
 pub use eth1_chain::{Eth1Chain, Eth1ChainBackend};
-pub use events::EventHandler;
+pub use events::ServerSentEventHandler;
 pub use metrics::scrape_for_metrics;
 pub use parking_lot;
 pub use slot_clock;

beacon_node/beacon_chain/src/metrics.rs

@@ -1,4 +1,5 @@
 use crate::{BeaconChain, BeaconChainTypes};
+use lazy_static::lazy_static;
 pub use lighthouse_metrics::*;
 use slot_clock::SlotClock;
 use types::{BeaconState, Epoch, EthSpec, Hash256, Slot};
@@ -67,6 +68,30 @@ lazy_static! {
     );
     pub static ref BLOCK_PRODUCTION_TIMES: Result<Histogram> =
         try_create_histogram("beacon_block_production_seconds", "Full runtime of block production");
+    pub static ref BLOCK_PRODUCTION_STATE_LOAD_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_block_production_state_load_seconds",
+        "Time taken to load the base state for block production"
+    );
+    pub static ref BLOCK_PRODUCTION_SLOT_PROCESS_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_block_production_slot_process_seconds",
+        "Time taken to advance the state to the block production slot"
+    );
+    pub static ref BLOCK_PRODUCTION_UNAGGREGATED_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_block_production_unaggregated_seconds",
+        "Time taken to import the naive aggregation pool for block production"
+    );
+    pub static ref BLOCK_PRODUCTION_ATTESTATION_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_block_production_attestation_seconds",
+        "Time taken to pack attestations into a block"
+    );
+    pub static ref BLOCK_PRODUCTION_PROCESS_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_block_production_process_seconds",
+        "Time taken to process the block produced"
+    );
+    pub static ref BLOCK_PRODUCTION_STATE_ROOT_TIMES: Result<Histogram> = try_create_histogram(
+        "beacon_block_production_state_root_seconds",
+        "Time taken to calculate the block's state root"
+    );
 
     /*
      * Block Statistics
@@ -330,6 +355,223 @@ lazy_static! {
     );
 }
 
+// Third lazy-static block is used to account for macro recursion limit.
+lazy_static! {
+    /*
+     * Validator Monitor Metrics (balances, etc)
+     */
+    pub static ref VALIDATOR_MONITOR_BALANCE_GWEI: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_balance_gwei",
+            "The validator's balance in gwei.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_EFFECTIVE_BALANCE_GWEI: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_effective_balance_gwei",
+            "The validator's effective balance in gwei.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_SLASHED: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_slashed",
+            "Set to 1 if the validator is slashed.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_ACTIVE: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_active",
+            "Set to 1 if the validator is active.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_EXITED: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_exited",
+            "Set to 1 if the validator is exited.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_WITHDRAWABLE: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_withdrawable",
+            "Set to 1 if the validator is withdrawable.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_ACTIVATION_ELIGIBILITY_EPOCH: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_activation_eligibility_epoch",
+            "Set to the epoch where the validator will be eligible for activation.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_ACTIVATION_EPOCH: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_activation_epoch",
+            "Set to the epoch where the validator will activate.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_EXIT_EPOCH: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_exit_epoch",
+            "Set to the epoch where the validator will exit.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_WITHDRAWABLE_EPOCH: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_withdrawable_epoch",
+            "Set to the epoch where the validator will be withdrawable.",
+            &["validator"]
+        );
+
+    /*
+     * Validator Monitor Metrics (per-epoch summaries)
+     */
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATIONS_TOTAL: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_prev_epoch_attestations_total",
+            "The number of unagg. attestations seen in the previous epoch.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATIONS_MIN_DELAY_SECONDS: Result<HistogramVec> =
+        try_create_histogram_vec(
+            "validator_monitor_prev_epoch_attestations_min_delay_seconds",
+            "The min delay between when the validator should send the attestation and when it was received.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATION_AGGREGATE_INCLUSIONS: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_prev_epoch_attestation_aggregate_inclusions",
+            "The count of times an attestation was seen inside an aggregate.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATION_BLOCK_INCLUSIONS: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_prev_epoch_attestation_block_inclusions",
+            "The count of times an attestation was seen inside a block.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATION_BLOCK_MIN_INCLUSION_DISTANCE: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_prev_epoch_attestation_block_min_inclusion_distance",
+            "The minimum inclusion distance observed for the inclusion of an attestation in a block.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_BEACON_BLOCKS_TOTAL: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_prev_epoch_beacon_blocks_total",
+            "The number of beacon_blocks seen in the previous epoch.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_BEACON_BLOCKS_MIN_DELAY_SECONDS: Result<HistogramVec> =
+        try_create_histogram_vec(
+            "validator_monitor_prev_epoch_beacon_blocks_min_delay_seconds",
+            "The min delay between when the validator should send the block and when it was received.",
+            &["validator"]
+       );
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_AGGREGATES_TOTAL: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_prev_epoch_aggregates_total",
+            "The number of aggregates seen in the previous epoch.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_AGGREGATES_MIN_DELAY_SECONDS: Result<HistogramVec> =
+        try_create_histogram_vec(
+            "validator_monitor_prev_epoch_aggregates_min_delay_seconds",
+            "The min delay between when the validator should send the aggregate and when it was received.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_EXITS_TOTAL: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_prev_epoch_exits_total",
+            "The number of exits seen in the previous epoch.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_PROPOSER_SLASHINGS_TOTAL: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_prev_epoch_proposer_slashings_total",
+            "The number of proposer slashings seen in the previous epoch.",
+            &["validator"]
+        );
+    pub static ref VALIDATOR_MONITOR_PREV_EPOCH_ATTESTER_SLASHINGS_TOTAL: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "validator_monitor_prev_epoch_attester_slashings_total",
+            "The number of attester slashings seen in the previous epoch.",
+            &["validator"]
+        );
+
+    /*
+     * Validator Monitor Metrics (real-time)
+     */
+    pub static ref VALIDATOR_MONITOR_VALIDATORS_TOTAL: Result<IntGauge> = try_create_int_gauge(
+        "validator_monitor_validators_total",
+        "Count of validators that are specifically monitored by this beacon node"
+    );
+    pub static ref VALIDATOR_MONITOR_UNAGGREGATED_ATTESTATION_TOTAL: Result<IntCounterVec> = try_create_int_counter_vec(
+        "validator_monitor_unaggregated_attestation_total",
+        "Number of unaggregated attestations seen",
+        &["src", "validator"]
+    );
+    pub static ref VALIDATOR_MONITOR_UNAGGREGATED_ATTESTATION_DELAY_SECONDS: Result<HistogramVec> = try_create_histogram_vec(
+        "validator_monitor_unaggregated_attestation_delay_seconds",
+        "The delay between when the validator should send the attestation and when it was received.",
+        &["src", "validator"]
+    );
+    pub static ref VALIDATOR_MONITOR_AGGREGATED_ATTESTATION_TOTAL: Result<IntCounterVec> = try_create_int_counter_vec(
+        "validator_monitor_aggregated_attestation_total",
+        "Number of aggregated attestations seen",
+        &["src", "validator"]
+    );
+    pub static ref VALIDATOR_MONITOR_AGGREGATED_ATTESTATION_DELAY_SECONDS: Result<HistogramVec> = try_create_histogram_vec(
+        "validator_monitor_aggregated_attestation_delay_seconds",
+        "The delay between then the validator should send the aggregate and when it was received.",
+        &["src", "validator"]
+    );
+    pub static ref VALIDATOR_MONITOR_ATTESTATION_IN_AGGREGATE_TOTAL: Result<IntCounterVec> = try_create_int_counter_vec(
+        "validator_monitor_attestation_in_aggregate_total",
+        "Number of times an attestation has been seen in an aggregate",
+        &["src", "validator"]
+    );
+    pub static ref VALIDATOR_MONITOR_ATTESTATION_IN_AGGREGATE_DELAY_SECONDS: Result<HistogramVec> = try_create_histogram_vec(
+        "validator_monitor_attestation_in_aggregate_delay_seconds",
+        "The delay between when the validator should send the aggregate and when it was received.",
+        &["src", "validator"]
+    );
+    pub static ref VALIDATOR_MONITOR_ATTESTATION_IN_BLOCK_TOTAL: Result<IntCounterVec> = try_create_int_counter_vec(
+        "validator_monitor_attestation_in_block_total",
+        "Number of times an attestation has been seen in a block",
+        &["src", "validator"]
+    );
+    pub static ref VALIDATOR_MONITOR_ATTESTATION_IN_BLOCK_DELAY_SLOTS: Result<IntGaugeVec> = try_create_int_gauge_vec(
+        "validator_monitor_attestation_in_block_delay_slots",
+        "The excess slots (beyond the minimum delay) between the attestation slot and the block slot.",
+        &["src", "validator"]
+    );
+    pub static ref VALIDATOR_MONITOR_BEACON_BLOCK_TOTAL: Result<IntCounterVec> = try_create_int_counter_vec(
+        "validator_monitor_beacon_block_total",
+        "Number of beacon blocks seen",
+        &["src", "validator"]
+    );
+    pub static ref VALIDATOR_MONITOR_BEACON_BLOCK_DELAY_SECONDS: Result<HistogramVec> = try_create_histogram_vec(
+        "validator_monitor_beacon_block_delay_seconds",
+        "The delay between when the validator should send the block and when it was received.",
+        &["src", "validator"]
+    );
+    pub static ref VALIDATOR_MONITOR_EXIT_TOTAL: Result<IntCounterVec> = try_create_int_counter_vec(
+        "validator_monitor_exit_total",
+        "Number of beacon exits seen",
+        &["src", "validator"]
+    );
+    pub static ref VALIDATOR_MONITOR_PROPOSER_SLASHING_TOTAL: Result<IntCounterVec> = try_create_int_counter_vec(
+        "validator_monitor_proposer_slashing_total",
+        "Number of proposer slashings seen",
+        &["src", "validator"]
+    );
+    pub static ref VALIDATOR_MONITOR_ATTESTER_SLASHING_TOTAL: Result<IntCounterVec> = try_create_int_counter_vec(
+        "validator_monitor_attester_slashing_total",
+        "Number of attester slashings seen",
+        &["src", "validator"]
+    );
+
+}
+
 /// Scrape the `beacon_chain` for metrics that are not constantly updated (e.g., the present slot,
 /// head state info, etc) and update the Prometheus `DEFAULT_REGISTRY`.
 pub fn scrape_for_metrics<T: BeaconChainTypes>(beacon_chain: &BeaconChain<T>) {
@@ -357,6 +599,11 @@ pub fn scrape_for_metrics<T: BeaconChainTypes>(beacon_chain: &BeaconChain<T>) {
         &OP_POOL_NUM_VOLUNTARY_EXITS,
         beacon_chain.op_pool.num_voluntary_exits(),
     );
+
+    beacon_chain
+        .validator_monitor
+        .read()
+        .scrape_metrics(&beacon_chain.slot_clock, &beacon_chain.spec);
 }
 
 /// Scrape the given `state` assuming it's the head state, updating the `DEFAULT_REGISTRY`.
@@ -418,6 +665,7 @@ fn scrape_attestation_observation<T: BeaconChainTypes>(slot_now: Slot, chain: &B
 
     if let Some(count) = chain
         .observed_attesters
+        .read()
         .observed_validator_count(prev_epoch)
     {
         set_gauge_by_usize(&ATTN_OBSERVATION_PREV_EPOCH_ATTESTERS, count);
@@ -425,6 +673,7 @@ fn scrape_attestation_observation<T: BeaconChainTypes>(slot_now: Slot, chain: &B
 
     if let Some(count) = chain
         .observed_aggregators
+        .read()
         .observed_validator_count(prev_epoch)
     {
         set_gauge_by_usize(&ATTN_OBSERVATION_PREV_EPOCH_AGGREGATORS, count);

beacon_node/beacon_chain/src/migrate.rs

@@ -1,76 +1,322 @@
+use crate::beacon_chain::BEACON_CHAIN_DB_KEY;
 use crate::errors::BeaconChainError;
-use crate::head_tracker::HeadTracker;
+use crate::head_tracker::{HeadTracker, SszHeadTracker};
+use crate::persisted_beacon_chain::{PersistedBeaconChain, DUMMY_CANONICAL_HEAD_BLOCK_ROOT};
 use parking_lot::Mutex;
-use slog::{debug, warn, Logger};
+use slog::{debug, error, info, warn, Logger};
 use std::collections::{HashMap, HashSet};
 use std::mem;
-use std::sync::mpsc;
-use std::sync::Arc;
+use std::sync::{mpsc, Arc};
 use std::thread;
-use store::hot_cold_store::{process_finalization, HotColdDBError};
-use store::iter::{ParentRootBlockIterator, RootsIterator};
-use store::{Error, ItemStore, StoreOp};
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use store::hot_cold_store::{migrate_database, HotColdDBError};
+use store::iter::RootsIterator;
+use store::{Error, ItemStore, StoreItem, StoreOp};
 pub use store::{HotColdDB, MemoryStore};
-use types::*;
-use types::{BeaconState, EthSpec, Hash256, Slot};
+use types::{
+    BeaconState, BeaconStateError, BeaconStateHash, Checkpoint, Epoch, EthSpec, Hash256,
+    SignedBeaconBlockHash, Slot,
+};
 
-/// Trait for migration processes that update the database upon finalization.
-pub trait Migrate<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>:
-    Send + Sync + 'static
-{
-    fn new(db: Arc<HotColdDB<E, Hot, Cold>>, log: Logger) -> Self;
+/// Compact at least this frequently, finalization permitting (7 days).
+const MAX_COMPACTION_PERIOD_SECONDS: u64 = 604800;
+/// Compact at *most* this frequently, to prevent over-compaction during sync (2 hours).
+const MIN_COMPACTION_PERIOD_SECONDS: u64 = 7200;
+/// Compact after a large finality gap, if we respect `MIN_COMPACTION_PERIOD_SECONDS`.
+const COMPACTION_FINALITY_DISTANCE: u64 = 1024;
 
-    fn process_finalization(
+/// The background migrator runs a thread to perform pruning and migrate state from the hot
+/// to the cold database.
+pub struct BackgroundMigrator<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> {
+    db: Arc<HotColdDB<E, Hot, Cold>>,
+    #[allow(clippy::type_complexity)]
+    tx_thread: Option<Mutex<(mpsc::Sender<MigrationNotification>, thread::JoinHandle<()>)>>,
+    /// Genesis block root, for persisting the `PersistedBeaconChain`.
+    genesis_block_root: Hash256,
+    log: Logger,
+}
+
+#[derive(Debug, Default, Clone, PartialEq, Eq)]
+pub struct MigratorConfig {
+    pub blocking: bool,
+}
+
+impl MigratorConfig {
+    pub fn blocking(mut self) -> Self {
+        self.blocking = true;
+        self
+    }
+}
+
+/// Pruning can be successful, or in rare cases deferred to a later point.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum PruningOutcome {
+    /// The pruning succeeded and updated the pruning checkpoint from `old_finalized_checkpoint`.
+    Successful {
+        old_finalized_checkpoint: Checkpoint,
+    },
+    DeferredConcurrentMutation,
+}
+
+/// Logic errors that can occur during pruning, none of these should ever happen.
+#[derive(Debug)]
+pub enum PruningError {
+    IncorrectFinalizedState {
+        state_slot: Slot,
+        new_finalized_slot: Slot,
+    },
+    MissingInfoForCanonicalChain {
+        slot: Slot,
+    },
+    UnexpectedEqualStateRoots,
+    UnexpectedUnequalStateRoots,
+}
+
+/// Message sent to the migration thread containing the information it needs to run.
+pub struct MigrationNotification {
+    finalized_state_root: BeaconStateHash,
+    finalized_checkpoint: Checkpoint,
+    head_tracker: Arc<HeadTracker>,
+    genesis_block_root: Hash256,
+}
+
+impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Hot, Cold> {
+    /// Create a new `BackgroundMigrator` and spawn its thread if necessary.
+    pub fn new(
+        db: Arc<HotColdDB<E, Hot, Cold>>,
+        config: MigratorConfig,
+        genesis_block_root: Hash256,
+        log: Logger,
+    ) -> Self {
+        let tx_thread = if config.blocking {
+            None
+        } else {
+            Some(Mutex::new(Self::spawn_thread(db.clone(), log.clone())))
+        };
+        Self {
+            db,
+            tx_thread,
+            genesis_block_root,
+            log,
+        }
+    }
+
+    /// Process a finalized checkpoint from the `BeaconChain`.
+    ///
+    /// If successful, all forks descending from before the `finalized_checkpoint` will be
+    /// pruned, and the split point of the database will be advanced to the slot of the finalized
+    /// checkpoint.
+    pub fn process_finalization(
         &self,
-        _state_root: Hash256,
-        _new_finalized_state: BeaconState<E>,
-        _max_finality_distance: u64,
-        _head_tracker: Arc<HeadTracker>,
-        _old_finalized_block_hash: SignedBeaconBlockHash,
-        _new_finalized_block_hash: SignedBeaconBlockHash,
-    ) {
+        finalized_state_root: BeaconStateHash,
+        finalized_checkpoint: Checkpoint,
+        head_tracker: Arc<HeadTracker>,
+    ) -> Result<(), BeaconChainError> {
+        let notif = MigrationNotification {
+            finalized_state_root,
+            finalized_checkpoint,
+            head_tracker,
+            genesis_block_root: self.genesis_block_root,
+        };
+
+        // Async path, on the background thread.
+        if let Some(tx_thread) = &self.tx_thread {
+            let (ref mut tx, ref mut thread) = *tx_thread.lock();
+
+            // Restart the background thread if it has crashed.
+            if let Err(tx_err) = tx.send(notif) {
+                let (new_tx, new_thread) = Self::spawn_thread(self.db.clone(), self.log.clone());
+
+                *tx = new_tx;
+                let old_thread = mem::replace(thread, new_thread);
+
+                // Join the old thread, which will probably have panicked, or may have
+                // halted normally just now as a result of us dropping the old `mpsc::Sender`.
+                if let Err(thread_err) = old_thread.join() {
+                    warn!(
+                        self.log,
+                        "Migration thread died, so it was restarted";
+                        "reason" => format!("{:?}", thread_err)
+                    );
+                }
+
+                // Retry at most once, we could recurse but that would risk overflowing the stack.
+                let _ = tx.send(tx_err.0);
+            }
+        }
+        // Synchronous path, on the current thread.
+        else {
+            Self::run_migration(self.db.clone(), notif, &self.log)
+        }
+
+        Ok(())
+    }
+
+    /// Perform the actual work of `process_finalization`.
+    fn run_migration(db: Arc<HotColdDB<E, Hot, Cold>>, notif: MigrationNotification, log: &Logger) {
+        let finalized_state_root = notif.finalized_state_root;
+
+        let finalized_state = match db.get_state(&finalized_state_root.into(), None) {
+            Ok(Some(state)) => state,
+            other => {
+                error!(
+                    log,
+                    "Migrator failed to load state";
+                    "state_root" => ?finalized_state_root,
+                    "error" => ?other
+                );
+                return;
+            }
+        };
+
+        let old_finalized_checkpoint = match Self::prune_abandoned_forks(
+            db.clone(),
+            notif.head_tracker,
+            finalized_state_root,
+            &finalized_state,
+            notif.finalized_checkpoint,
+            notif.genesis_block_root,
+            log,
+        ) {
+            Ok(PruningOutcome::Successful {
+                old_finalized_checkpoint,
+            }) => old_finalized_checkpoint,
+            Ok(PruningOutcome::DeferredConcurrentMutation) => {
+                warn!(
+                    log,
+                    "Pruning deferred because of a concurrent mutation";
+                    "message" => "this is expected only very rarely!"
+                );
+                return;
+            }
+            Err(e) => {
+                warn!(log, "Block pruning failed"; "error" => format!("{:?}", e));
+                return;
+            }
+        };
+
+        match migrate_database(db.clone(), finalized_state_root.into(), &finalized_state) {
+            Ok(()) => {}
+            Err(Error::HotColdDBError(HotColdDBError::FreezeSlotUnaligned(slot))) => {
+                debug!(
+                    log,
+                    "Database migration postponed, unaligned finalized block";
+                    "slot" => slot.as_u64()
+                );
+            }
+            Err(e) => {
+                warn!(
+                    log,
+                    "Database migration failed";
+                    "error" => format!("{:?}", e)
+                );
+                return;
+            }
+        };
+
+        // Finally, compact the database so that new free space is properly reclaimed.
+        if let Err(e) = Self::run_compaction(
+            db,
+            old_finalized_checkpoint.epoch,
+            notif.finalized_checkpoint.epoch,
+            log,
+        ) {
+            warn!(log, "Database compaction failed"; "error" => format!("{:?}", e));
+        }
+    }
+
+    /// Spawn a new child thread to run the migration process.
+    ///
+    /// Return a channel handle for sending new finalized states to the thread.
+    fn spawn_thread(
+        db: Arc<HotColdDB<E, Hot, Cold>>,
+        log: Logger,
+    ) -> (mpsc::Sender<MigrationNotification>, thread::JoinHandle<()>) {
+        let (tx, rx) = mpsc::channel();
+        let thread = thread::spawn(move || {
+            while let Ok(notif) = rx.recv() {
+                // Read the rest of the messages in the channel, ultimately choosing the `notif`
+                // with the highest finalized epoch.
+                let notif = rx
+                    .try_iter()
+                    .fold(notif, |best, other: MigrationNotification| {
+                        if other.finalized_checkpoint.epoch > best.finalized_checkpoint.epoch {
+                            other
+                        } else {
+                            best
+                        }
+                    });
+
+                Self::run_migration(db.clone(), notif, &log);
+            }
+        });
+        (tx, thread)
     }
 
     /// Traverses live heads and prunes blocks and states of chains that we know can't be built
-    /// upon because finalization would prohibit it.  This is an optimisation intended to save disk
+    /// upon because finalization would prohibit it. This is an optimisation intended to save disk
     /// space.
-    ///
-    /// Assumptions:
-    ///  * It is called after every finalization.
+    #[allow(clippy::too_many_arguments)]
     fn prune_abandoned_forks(
         store: Arc<HotColdDB<E, Hot, Cold>>,
         head_tracker: Arc<HeadTracker>,
-        old_finalized_block_hash: SignedBeaconBlockHash,
-        new_finalized_block_hash: SignedBeaconBlockHash,
-        new_finalized_slot: Slot,
-    ) -> Result<(), BeaconChainError> {
-        // There will never be any blocks to prune if there is only a single head in the chain.
-        if head_tracker.heads().len() == 1 {
-            return Ok(());
+        new_finalized_state_hash: BeaconStateHash,
+        new_finalized_state: &BeaconState<E>,
+        new_finalized_checkpoint: Checkpoint,
+        genesis_block_root: Hash256,
+        log: &Logger,
+    ) -> Result<PruningOutcome, BeaconChainError> {
+        let old_finalized_checkpoint =
+            store
+                .load_pruning_checkpoint()?
+                .unwrap_or_else(|| Checkpoint {
+                    epoch: Epoch::new(0),
+                    root: Hash256::zero(),
+                });
+
+        let old_finalized_slot = old_finalized_checkpoint
+            .epoch
+            .start_slot(E::slots_per_epoch());
+        let new_finalized_slot = new_finalized_checkpoint
+            .epoch
+            .start_slot(E::slots_per_epoch());
+        let new_finalized_block_hash = new_finalized_checkpoint.root.into();
+
+        // The finalized state must be for the epoch boundary slot, not the slot of the finalized
+        // block.
+        if new_finalized_state.slot != new_finalized_slot {
+            return Err(PruningError::IncorrectFinalizedState {
+                state_slot: new_finalized_state.slot,
+                new_finalized_slot,
+            }
+            .into());
         }
 
-        let old_finalized_slot = store
-            .get_block(&old_finalized_block_hash.into())?
-            .ok_or_else(|| BeaconChainError::MissingBeaconBlock(old_finalized_block_hash.into()))?
-            .slot();
-
-        // Collect hashes from new_finalized_block back to old_finalized_block (inclusive)
-        let mut found_block = false; // hack for `take_until`
-        let newly_finalized_blocks: HashMap<SignedBeaconBlockHash, Slot> =
-            ParentRootBlockIterator::new(&*store, new_finalized_block_hash.into())
-                .take_while(|result| match result {
-                    Ok((block_hash, _)) => {
-                        if found_block {
-                            false
-                        } else {
-                            found_block |= *block_hash == old_finalized_block_hash.into();
-                            true
-                        }
-                    }
-                    Err(_) => true,
-                })
-                .map(|result| result.map(|(block_hash, block)| (block_hash.into(), block.slot())))
-                .collect::<Result<_, _>>()?;
+        debug!(
+            log,
+            "Starting database pruning";
+            "old_finalized_epoch" => old_finalized_checkpoint.epoch,
+            "new_finalized_epoch" => new_finalized_checkpoint.epoch,
+        );
+        // For each slot between the new finalized checkpoint and the old finalized checkpoint,
+        // collect the beacon block root and state root of the canonical chain.
+        let newly_finalized_chain: HashMap<Slot, (SignedBeaconBlockHash, BeaconStateHash)> =
+            std::iter::once(Ok((
+                new_finalized_slot,
+                (new_finalized_block_hash, new_finalized_state_hash),
+            )))
+            .chain(
+                RootsIterator::new(store.clone(), new_finalized_state).map(|res| {
+                    res.map(|(block_root, state_root, slot)| {
+                        (slot, (block_root.into(), state_root.into()))
+                    })
+                }),
+            )
+            .take_while(|res| {
+                res.as_ref()
+                    .map_or(true, |(slot, _)| *slot >= old_finalized_slot)
+            })
+            .collect::<Result<_, _>>()?;
 
         // We don't know which blocks are shared among abandoned chains, so we buffer and delete
         // everything in one fell swoop.
@@ -78,284 +324,220 @@ pub trait Migrate<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>:
         let mut abandoned_states: HashSet<(Slot, BeaconStateHash)> = HashSet::new();
         let mut abandoned_heads: HashSet<Hash256> = HashSet::new();
 
-        for (head_hash, head_slot) in head_tracker.heads() {
-            let mut potentially_abandoned_head: Option<Hash256> = Some(head_hash);
-            let mut potentially_abandoned_blocks: Vec<(
-                Slot,
-                Option<SignedBeaconBlockHash>,
-                Option<BeaconStateHash>,
-            )> = Vec::new();
+        let heads = head_tracker.heads();
+        debug!(
+            log,
+            "Extra pruning information";
+            "old_finalized_root" => format!("{:?}", old_finalized_checkpoint.root),
+            "new_finalized_root" => format!("{:?}", new_finalized_checkpoint.root),
+            "head_count" => heads.len(),
+        );
+
+        for (head_hash, head_slot) in heads {
+            let mut potentially_abandoned_head = Some(head_hash);
+            let mut potentially_abandoned_blocks = vec![];
 
             let head_state_hash = store
                 .get_block(&head_hash)?
                 .ok_or_else(|| BeaconStateError::MissingBeaconBlock(head_hash.into()))?
                 .state_root();
 
+            // Iterate backwards from this head, staging blocks and states for deletion.
             let iter = std::iter::once(Ok((head_hash, head_state_hash, head_slot)))
-                .chain(RootsIterator::from_block(Arc::clone(&store), head_hash)?);
+                .chain(RootsIterator::from_block(store.clone(), head_hash)?);
+
             for maybe_tuple in iter {
-                let (block_hash, state_hash, slot) = maybe_tuple?;
-                if slot < old_finalized_slot {
-                    // We must assume here any candidate chains include old_finalized_block_hash,
-                    // i.e. there aren't any forks starting at a block that is a strict ancestor of
-                    // old_finalized_block_hash.
-                    break;
-                }
-                match newly_finalized_blocks.get(&block_hash.into()).copied() {
-                    // Block is not finalized, mark it and its state for deletion
+                let (block_root, state_root, slot) = maybe_tuple?;
+                let block_root = SignedBeaconBlockHash::from(block_root);
+                let state_root = BeaconStateHash::from(state_root);
+
+                match newly_finalized_chain.get(&slot) {
+                    // If there's no information about a slot on the finalized chain, then
+                    // it should be because it's ahead of the new finalized slot. Stage
+                    // the fork's block and state for possible deletion.
                     None => {
-                        potentially_abandoned_blocks.push((
-                            slot,
-                            Some(block_hash.into()),
-                            Some(state_hash.into()),
-                        ));
+                        if slot > new_finalized_slot {
+                            potentially_abandoned_blocks.push((
+                                slot,
+                                Some(block_root),
+                                Some(state_root),
+                            ));
+                        } else if slot >= old_finalized_slot {
+                            return Err(PruningError::MissingInfoForCanonicalChain { slot }.into());
+                        } else {
+                            // We must assume here any candidate chains include the old finalized
+                            // checkpoint, i.e. there aren't any forks starting at a block that is a
+                            // strict ancestor of old_finalized_checkpoint.
+                            warn!(
+                                log,
+                                "Found a chain that should already have been pruned";
+                                "head_block_root" => format!("{:?}", head_hash),
+                                "head_slot" => head_slot,
+                            );
+                            potentially_abandoned_head.take();
+                            break;
+                        }
                     }
-                    Some(finalized_slot) => {
-                        // Block root is finalized, and we have reached the slot it was finalized
-                        // at: we've hit a shared part of the chain.
-                        if finalized_slot == slot {
-                            // The first finalized block of a candidate chain lies after (in terms
-                            // of slots order) the newly finalized block.  It's not a candidate for
-                            // prunning.
-                            if finalized_slot == new_finalized_slot {
+                    Some((finalized_block_root, finalized_state_root)) => {
+                        // This fork descends from a newly finalized block, we can stop.
+                        if block_root == *finalized_block_root {
+                            // Sanity check: if the slot and block root match, then the
+                            // state roots should match too.
+                            if state_root != *finalized_state_root {
+                                return Err(PruningError::UnexpectedUnequalStateRoots.into());
+                            }
+
+                            // If the fork descends from the whole finalized chain,
+                            // do not prune it. Otherwise continue to delete all
+                            // of the blocks and states that have been staged for
+                            // deletion so far.
+                            if slot == new_finalized_slot {
                                 potentially_abandoned_blocks.clear();
                                 potentially_abandoned_head.take();
                             }
-
+                            // If there are skipped slots on the fork to be pruned, then
+                            // we will have just staged the common block for deletion.
+                            // Unstage it.
+                            else {
+                                for (_, block_root, _) in
+                                    potentially_abandoned_blocks.iter_mut().rev()
+                                {
+                                    if block_root.as_ref() == Some(finalized_block_root) {
+                                        *block_root = None;
+                                    } else {
+                                        break;
+                                    }
+                                }
+                            }
                             break;
-                        }
-                        // Block root is finalized, but we're at a skip slot: delete the state only.
-                        else {
+                        } else {
+                            if state_root == *finalized_state_root {
+                                return Err(PruningError::UnexpectedEqualStateRoots.into());
+                            }
                             potentially_abandoned_blocks.push((
                                 slot,
-                                None,
-                                Some(state_hash.into()),
+                                Some(block_root),
+                                Some(state_root),
                             ));
                         }
                     }
                 }
             }
 
-            abandoned_heads.extend(potentially_abandoned_head.into_iter());
-            if !potentially_abandoned_blocks.is_empty() {
+            if let Some(abandoned_head) = potentially_abandoned_head {
+                debug!(
+                    log,
+                    "Pruning head";
+                    "head_block_root" => format!("{:?}", abandoned_head),
+                    "head_slot" => head_slot,
+                );
+                abandoned_heads.insert(abandoned_head);
                 abandoned_blocks.extend(
                     potentially_abandoned_blocks
                         .iter()
                         .filter_map(|(_, maybe_block_hash, _)| *maybe_block_hash),
                 );
                 abandoned_states.extend(potentially_abandoned_blocks.iter().filter_map(
-                    |(slot, _, maybe_state_hash)| match maybe_state_hash {
-                        None => None,
-                        Some(state_hash) => Some((*slot, *state_hash)),
-                    },
+                    |(slot, _, maybe_state_hash)| maybe_state_hash.map(|sr| (*slot, sr)),
                 ));
             }
         }
 
+        // Update the head tracker before the database, so that we maintain the invariant
+        // that a block present in the head tracker is present in the database.
+        // See https://github.com/sigp/lighthouse/issues/1557
+        let mut head_tracker_lock = head_tracker.0.write();
+
+        // Check that all the heads to be deleted are still present. The absence of any
+        // head indicates a race, that will likely resolve itself, so we defer pruning until
+        // later.
+        for head_hash in &abandoned_heads {
+            if !head_tracker_lock.contains_key(head_hash) {
+                return Ok(PruningOutcome::DeferredConcurrentMutation);
+            }
+        }
+
+        // Then remove them for real.
+        for head_hash in abandoned_heads {
+            head_tracker_lock.remove(&head_hash);
+        }
+
         let batch: Vec<StoreOp<E>> = abandoned_blocks
             .into_iter()
+            .map(Into::into)
             .map(StoreOp::DeleteBlock)
             .chain(
                 abandoned_states
                     .into_iter()
-                    .map(|(slot, state_hash)| StoreOp::DeleteState(state_hash, slot)),
+                    .map(|(slot, state_hash)| StoreOp::DeleteState(state_hash.into(), Some(slot))),
             )
             .collect();
-        store.do_atomically(batch)?;
-        for head_hash in abandoned_heads.into_iter() {
-            head_tracker.remove_head(head_hash);
+
+        let mut kv_batch = store.convert_to_kv_batch(&batch)?;
+
+        // Persist the head in case the process is killed or crashes here. This prevents
+        // the head tracker reverting after our mutation above.
+        let persisted_head = PersistedBeaconChain {
+            _canonical_head_block_root: DUMMY_CANONICAL_HEAD_BLOCK_ROOT,
+            genesis_block_root,
+            ssz_head_tracker: SszHeadTracker::from_map(&*head_tracker_lock),
+        };
+        drop(head_tracker_lock);
+        kv_batch.push(persisted_head.as_kv_store_op(BEACON_CHAIN_DB_KEY));
+
+        // Persist the new finalized checkpoint as the pruning checkpoint.
+        kv_batch.push(store.pruning_checkpoint_store_op(new_finalized_checkpoint));
+
+        store.hot_db.do_atomically(kv_batch)?;
+        debug!(log, "Database pruning complete");
+
+        Ok(PruningOutcome::Successful {
+            old_finalized_checkpoint,
+        })
+    }
+
+    /// Compact the database if it has been more than `COMPACTION_PERIOD_SECONDS` since it
+    /// was last compacted.
+    pub fn run_compaction(
+        db: Arc<HotColdDB<E, Hot, Cold>>,
+        old_finalized_epoch: Epoch,
+        new_finalized_epoch: Epoch,
+        log: &Logger,
+    ) -> Result<(), Error> {
+        if !db.compact_on_prune() {
+            return Ok(());
         }
 
+        let last_compaction_timestamp = db
+            .load_compaction_timestamp()?
+            .unwrap_or_else(|| Duration::from_secs(0));
+        let start_time = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .unwrap_or(last_compaction_timestamp);
+        let seconds_since_last_compaction = start_time
+            .checked_sub(last_compaction_timestamp)
+            .as_ref()
+            .map_or(0, Duration::as_secs);
+
+        if seconds_since_last_compaction > MAX_COMPACTION_PERIOD_SECONDS
+            || (new_finalized_epoch - old_finalized_epoch > COMPACTION_FINALITY_DISTANCE
+                && seconds_since_last_compaction > MIN_COMPACTION_PERIOD_SECONDS)
+        {
+            info!(
+                log,
+                "Starting database compaction";
+                "old_finalized_epoch" => old_finalized_epoch,
+                "new_finalized_epoch" => new_finalized_epoch,
+            );
+            db.compact()?;
+
+            let finish_time = SystemTime::now()
+                .duration_since(UNIX_EPOCH)
+                .unwrap_or(start_time);
+            db.store_compaction_timestamp(finish_time)?;
+
+            info!(log, "Database compaction complete");
+        }
         Ok(())
     }
 }
-
-/// Migrator that does nothing, for stores that don't need migration.
-pub struct NullMigrator;
-
-impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Migrate<E, Hot, Cold> for NullMigrator {
-    fn new(_: Arc<HotColdDB<E, Hot, Cold>>, _: Logger) -> Self {
-        NullMigrator
-    }
-}
-
-/// Migrator that immediately calls the store's migration function, blocking the current execution.
-///
-/// Mostly useful for tests.
-pub struct BlockingMigrator<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> {
-    db: Arc<HotColdDB<E, Hot, Cold>>,
-}
-
-impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Migrate<E, Hot, Cold>
-    for BlockingMigrator<E, Hot, Cold>
-{
-    fn new(db: Arc<HotColdDB<E, Hot, Cold>>, _: Logger) -> Self {
-        BlockingMigrator { db }
-    }
-
-    fn process_finalization(
-        &self,
-        state_root: Hash256,
-        new_finalized_state: BeaconState<E>,
-        _max_finality_distance: u64,
-        head_tracker: Arc<HeadTracker>,
-        old_finalized_block_hash: SignedBeaconBlockHash,
-        new_finalized_block_hash: SignedBeaconBlockHash,
-    ) {
-        if let Err(e) = process_finalization(self.db.clone(), state_root, &new_finalized_state) {
-            // This migrator is only used for testing, so we just log to stderr without a logger.
-            eprintln!("Migration error: {:?}", e);
-        }
-
-        if let Err(e) = Self::prune_abandoned_forks(
-            self.db.clone(),
-            head_tracker,
-            old_finalized_block_hash,
-            new_finalized_block_hash,
-            new_finalized_state.slot,
-        ) {
-            eprintln!("Pruning error: {:?}", e);
-        }
-    }
-}
-
-type MpscSender<E> = mpsc::Sender<(
-    Hash256,
-    BeaconState<E>,
-    Arc<HeadTracker>,
-    SignedBeaconBlockHash,
-    SignedBeaconBlockHash,
-    Slot,
-)>;
-
-/// Migrator that runs a background thread to migrate state from the hot to the cold database.
-pub struct BackgroundMigrator<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> {
-    db: Arc<HotColdDB<E, Hot, Cold>>,
-    tx_thread: Mutex<(MpscSender<E>, thread::JoinHandle<()>)>,
-    log: Logger,
-}
-
-impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Migrate<E, Hot, Cold>
-    for BackgroundMigrator<E, Hot, Cold>
-{
-    fn new(db: Arc<HotColdDB<E, Hot, Cold>>, log: Logger) -> Self {
-        let tx_thread = Mutex::new(Self::spawn_thread(db.clone(), log.clone()));
-        Self { db, tx_thread, log }
-    }
-
-    /// Perform the freezing operation on the database,
-    fn process_finalization(
-        &self,
-        finalized_state_root: Hash256,
-        new_finalized_state: BeaconState<E>,
-        max_finality_distance: u64,
-        head_tracker: Arc<HeadTracker>,
-        old_finalized_block_hash: SignedBeaconBlockHash,
-        new_finalized_block_hash: SignedBeaconBlockHash,
-    ) {
-        if !self.needs_migration(new_finalized_state.slot, max_finality_distance) {
-            return;
-        }
-
-        let (ref mut tx, ref mut thread) = *self.tx_thread.lock();
-
-        let new_finalized_slot = new_finalized_state.slot;
-        if let Err(tx_err) = tx.send((
-            finalized_state_root,
-            new_finalized_state,
-            head_tracker,
-            old_finalized_block_hash,
-            new_finalized_block_hash,
-            new_finalized_slot,
-        )) {
-            let (new_tx, new_thread) = Self::spawn_thread(self.db.clone(), self.log.clone());
-
-            drop(mem::replace(tx, new_tx));
-            let old_thread = mem::replace(thread, new_thread);
-
-            // Join the old thread, which will probably have panicked, or may have
-            // halted normally just now as a result of us dropping the old `mpsc::Sender`.
-            if let Err(thread_err) = old_thread.join() {
-                warn!(
-                    self.log,
-                    "Migration thread died, so it was restarted";
-                    "reason" => format!("{:?}", thread_err)
-                );
-            }
-
-            // Retry at most once, we could recurse but that would risk overflowing the stack.
-            let _ = tx.send(tx_err.0);
-        }
-    }
-}
-
-impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Hot, Cold> {
-    /// Return true if a migration needs to be performed, given a new `finalized_slot`.
-    fn needs_migration(&self, finalized_slot: Slot, max_finality_distance: u64) -> bool {
-        let finality_distance = finalized_slot - self.db.get_split_slot();
-        finality_distance > max_finality_distance
-    }
-
-    #[allow(clippy::type_complexity)]
-    /// Spawn a new child thread to run the migration process.
-    ///
-    /// Return a channel handle for sending new finalized states to the thread.
-    fn spawn_thread(
-        db: Arc<HotColdDB<E, Hot, Cold>>,
-        log: Logger,
-    ) -> (
-        mpsc::Sender<(
-            Hash256,
-            BeaconState<E>,
-            Arc<HeadTracker>,
-            SignedBeaconBlockHash,
-            SignedBeaconBlockHash,
-            Slot,
-        )>,
-        thread::JoinHandle<()>,
-    ) {
-        let (tx, rx) = mpsc::channel();
-        let thread = thread::spawn(move || {
-            while let Ok((
-                state_root,
-                state,
-                head_tracker,
-                old_finalized_block_hash,
-                new_finalized_block_hash,
-                new_finalized_slot,
-            )) = rx.recv()
-            {
-                match process_finalization(db.clone(), state_root, &state) {
-                    Ok(()) => {}
-                    Err(Error::HotColdDBError(HotColdDBError::FreezeSlotUnaligned(slot))) => {
-                        debug!(
-                            log,
-                            "Database migration postponed, unaligned finalized block";
-                            "slot" => slot.as_u64()
-                        );
-                    }
-                    Err(e) => {
-                        warn!(
-                            log,
-                            "Database migration failed";
-                            "error" => format!("{:?}", e)
-                        );
-                    }
-                };
-
-                match Self::prune_abandoned_forks(
-                    db.clone(),
-                    head_tracker,
-                    old_finalized_block_hash,
-                    new_finalized_block_hash,
-                    new_finalized_slot,
-                ) {
-                    Ok(()) => {}
-                    Err(e) => warn!(log, "Block pruning failed: {:?}", e),
-                }
-            }
-        });
-
-        (tx, thread)
-    }
-}

beacon_node/beacon_chain/src/naive_aggregation_pool.rs

@@ -1,7 +1,9 @@
 use crate::metrics;
 use std::collections::HashMap;
-use types::{Attestation, AttestationData, EthSpec, Slot};
+use tree_hash::TreeHash;
+use types::{Attestation, AttestationData, EthSpec, Hash256, Slot};
 
+type AttestationDataRoot = Hash256;
 /// The number of slots that will be stored in the pool.
 ///
 /// For example, if `SLOTS_RETAINED == 3` and the pool is pruned at slot `6`, then all attestations
@@ -53,7 +55,7 @@ pub enum Error {
 /// A collection of `Attestation` objects, keyed by their `attestation.data`. Enforces that all
 /// `attestation` are from the same slot.
 struct AggregatedAttestationMap<E: EthSpec> {
-    map: HashMap<AttestationData, Attestation<E>>,
+    map: HashMap<AttestationDataRoot, Attestation<E>>,
 }
 
 impl<E: EthSpec> AggregatedAttestationMap<E> {
@@ -81,13 +83,15 @@ impl<E: EthSpec> AggregatedAttestationMap<E> {
         let committee_index = set_bits
             .first()
             .copied()
-            .ok_or_else(|| Error::NoAggregationBitsSet)?;
+            .ok_or(Error::NoAggregationBitsSet)?;
 
         if set_bits.len() > 1 {
             return Err(Error::MoreThanOneAggregationBitSet(set_bits.len()));
         }
 
-        if let Some(existing_attestation) = self.map.get_mut(&a.data) {
+        let attestation_data_root = a.data.tree_hash_root();
+
+        if let Some(existing_attestation) = self.map.get_mut(&attestation_data_root) {
             if existing_attestation
                 .aggregation_bits
                 .get(committee_index)
@@ -107,7 +111,7 @@ impl<E: EthSpec> AggregatedAttestationMap<E> {
                 ));
             }
 
-            self.map.insert(a.data.clone(), a.clone());
+            self.map.insert(attestation_data_root, a.clone());
             Ok(InsertOutcome::NewAttestationData { committee_index })
         }
     }
@@ -115,8 +119,13 @@ impl<E: EthSpec> AggregatedAttestationMap<E> {
     /// Returns an aggregated `Attestation` with the given `data`, if any.
     ///
     /// The given `a.data.slot` must match the slot that `self` was initialized with.
-    pub fn get(&self, data: &AttestationData) -> Result<Option<Attestation<E>>, Error> {
-        Ok(self.map.get(data).cloned())
+    pub fn get(&self, data: &AttestationData) -> Option<Attestation<E>> {
+        self.map.get(&data.tree_hash_root()).cloned()
+    }
+
+    /// Returns an aggregated `Attestation` with the given `root`, if any.
+    pub fn get_by_root(&self, root: &AttestationDataRoot) -> Option<&Attestation<E>> {
+        self.map.get(root)
     }
 
     /// Iterate all attestations in `self`.
@@ -220,12 +229,19 @@ impl<E: EthSpec> NaiveAggregationPool<E> {
     }
 
     /// Returns an aggregated `Attestation` with the given `data`, if any.
-    pub fn get(&self, data: &AttestationData) -> Result<Option<Attestation<E>>, Error> {
+    pub fn get(&self, data: &AttestationData) -> Option<Attestation<E>> {
+        self.maps.get(&data.slot).and_then(|map| map.get(data))
+    }
+
+    /// Returns an aggregated `Attestation` with the given `data`, if any.
+    pub fn get_by_slot_and_root(
+        &self,
+        slot: Slot,
+        root: &AttestationDataRoot,
+    ) -> Option<Attestation<E>> {
         self.maps
-            .iter()
-            .find(|(slot, _map)| **slot == data.slot)
-            .map(|(_slot, map)| map.get(data))
-            .unwrap_or_else(|| Ok(None))
+            .get(&slot)
+            .and_then(|map| map.get_by_root(root).cloned())
     }
 
     /// Iterate all attestations in all slots of `self`.
@@ -338,8 +354,7 @@ mod tests {
 
         let retrieved = pool
             .get(&a.data)
-            .expect("should not error while getting attestation")
-            .expect("should get an attestation");
+            .expect("should not error while getting attestation");
         assert_eq!(
             retrieved, a,
             "retrieved attestation should equal the one inserted"
@@ -378,8 +393,7 @@ mod tests {
 
         let retrieved = pool
             .get(&a_0.data)
-            .expect("should not error while getting attestation")
-            .expect("should get an attestation");
+            .expect("should not error while getting attestation");
 
         let mut a_01 = a_0.clone();
         a_01.aggregate(&a_1);
@@ -408,8 +422,7 @@ mod tests {
 
         assert_eq!(
             pool.get(&a_0.data)
-                .expect("should not error while getting attestation")
-                .expect("should get an attestation"),
+                .expect("should not error while getting attestation"),
             retrieved,
             "should not have aggregated different attestation data"
         );

beacon_node/beacon_chain/src/observed_attestations.rs

@@ -1,7 +1,6 @@
 //! Provides an `ObservedAttestations` struct which allows us to reject aggregated attestations if
 //! we've already seen the aggregated attestation.
 
-use parking_lot::RwLock;
 use std::collections::HashSet;
 use std::marker::PhantomData;
 use tree_hash::TreeHash;
@@ -116,16 +115,16 @@ impl SlotHashSet {
 /// Stores the roots of `Attestation` objects for some number of `Slots`, so we can determine if
 /// these have previously been seen on the network.
 pub struct ObservedAttestations<E: EthSpec> {
-    lowest_permissible_slot: RwLock<Slot>,
-    sets: RwLock<Vec<SlotHashSet>>,
+    lowest_permissible_slot: Slot,
+    sets: Vec<SlotHashSet>,
     _phantom: PhantomData<E>,
 }
 
 impl<E: EthSpec> Default for ObservedAttestations<E> {
     fn default() -> Self {
         Self {
-            lowest_permissible_slot: RwLock::new(Slot::new(0)),
-            sets: RwLock::new(vec![]),
+            lowest_permissible_slot: Slot::new(0),
+            sets: vec![],
             _phantom: PhantomData,
         }
     }
@@ -136,7 +135,7 @@ impl<E: EthSpec> ObservedAttestations<E> {
     ///
     /// `root` must equal `a.tree_hash_root()`.
     pub fn observe_attestation(
-        &self,
+        &mut self,
         a: &Attestation<E>,
         root_opt: Option<Hash256>,
     ) -> Result<ObserveOutcome, Error> {
@@ -144,22 +143,20 @@ impl<E: EthSpec> ObservedAttestations<E> {
         let root = root_opt.unwrap_or_else(|| a.tree_hash_root());
 
         self.sets
-            .write()
             .get_mut(index)
-            .ok_or_else(|| Error::InvalidSetIndex(index))
+            .ok_or(Error::InvalidSetIndex(index))
             .and_then(|set| set.observe_attestation(a, root))
     }
 
     /// Check to see if the `root` of `a` is in self.
     ///
     /// `root` must equal `a.tree_hash_root()`.
-    pub fn is_known(&self, a: &Attestation<E>, root: Hash256) -> Result<bool, Error> {
+    pub fn is_known(&mut self, a: &Attestation<E>, root: Hash256) -> Result<bool, Error> {
         let index = self.get_set_index(a.data.slot)?;
 
         self.sets
-            .read()
             .get(index)
-            .ok_or_else(|| Error::InvalidSetIndex(index))
+            .ok_or(Error::InvalidSetIndex(index))
             .and_then(|set| set.is_known(a, root))
     }
 
@@ -172,23 +169,21 @@ impl<E: EthSpec> ObservedAttestations<E> {
 
     /// Removes any attestations with a slot lower than `current_slot` and bars any future
     /// attestations with a slot lower than `current_slot - SLOTS_RETAINED`.
-    pub fn prune(&self, current_slot: Slot) {
+    pub fn prune(&mut self, current_slot: Slot) {
         // Taking advantage of saturating subtraction on `Slot`.
         let lowest_permissible_slot = current_slot - (self.max_capacity() - 1);
 
-        self.sets
-            .write()
-            .retain(|set| set.slot >= lowest_permissible_slot);
+        self.sets.retain(|set| set.slot >= lowest_permissible_slot);
 
-        *self.lowest_permissible_slot.write() = lowest_permissible_slot;
+        self.lowest_permissible_slot = lowest_permissible_slot;
     }
 
     /// Returns the index of `self.set` that matches `slot`.
     ///
     /// If there is no existing set for this slot one will be created. If `self.sets.len() >=
     /// Self::max_capacity()`, the set with the lowest slot will be replaced.
-    fn get_set_index(&self, slot: Slot) -> Result<usize, Error> {
-        let lowest_permissible_slot: Slot = *self.lowest_permissible_slot.read();
+    fn get_set_index(&mut self, slot: Slot) -> Result<usize, Error> {
+        let lowest_permissible_slot = self.lowest_permissible_slot;
 
         if slot < lowest_permissible_slot {
             return Err(Error::SlotTooLow {
@@ -202,15 +197,14 @@ impl<E: EthSpec> ObservedAttestations<E> {
             self.prune(slot)
         }
 
-        let mut sets = self.sets.write();
-
-        if let Some(index) = sets.iter().position(|set| set.slot == slot) {
+        if let Some(index) = self.sets.iter().position(|set| set.slot == slot) {
             return Ok(index);
         }
 
         // To avoid re-allocations, try and determine a rough initial capacity for the new set
         // by obtaining the mean size of all items in earlier epoch.
-        let (count, sum) = sets
+        let (count, sum) = self
+            .sets
             .iter()
             // Only include slots that are less than the given slot in the average. This should
             // generally avoid including recent slots that are still "filling up".
@@ -222,20 +216,21 @@ impl<E: EthSpec> ObservedAttestations<E> {
         // but considering it's approx. 128 * 32 bytes we're not wasting much.
         let initial_capacity = sum.checked_div(count).unwrap_or(128);
 
-        if sets.len() < self.max_capacity() as usize || sets.is_empty() {
-            let index = sets.len();
-            sets.push(SlotHashSet::new(slot, initial_capacity));
+        if self.sets.len() < self.max_capacity() as usize || self.sets.is_empty() {
+            let index = self.sets.len();
+            self.sets.push(SlotHashSet::new(slot, initial_capacity));
             return Ok(index);
         }
 
-        let index = sets
+        let index = self
+            .sets
             .iter()
             .enumerate()
             .min_by_key(|(_i, set)| set.slot)
             .map(|(i, _set)| i)
             .expect("sets cannot be empty due to previous .is_empty() check");
 
-        sets[index] = SlotHashSet::new(slot, initial_capacity);
+        self.sets[index] = SlotHashSet::new(slot, initial_capacity);
 
         Ok(index)
     }
@@ -259,7 +254,7 @@ mod tests {
         a
     }
 
-    fn single_slot_test(store: &ObservedAttestations<E>, slot: Slot) {
+    fn single_slot_test(store: &mut ObservedAttestations<E>, slot: Slot) {
         let attestations = (0..NUM_ELEMENTS as u64)
             .map(|i| get_attestation(slot, i))
             .collect::<Vec<_>>();
@@ -293,17 +288,13 @@ mod tests {
 
     #[test]
     fn single_slot() {
-        let store = ObservedAttestations::default();
+        let mut store = ObservedAttestations::default();
 
-        single_slot_test(&store, Slot::new(0));
+        single_slot_test(&mut store, Slot::new(0));
 
+        assert_eq!(store.sets.len(), 1, "should have a single set stored");
         assert_eq!(
-            store.sets.read().len(),
-            1,
-            "should have a single set stored"
-        );
-        assert_eq!(
-            store.sets.read()[0].len(),
+            store.sets[0].len(),
             NUM_ELEMENTS,
             "set should have NUM_ELEMENTS elements"
         );
@@ -311,13 +302,13 @@ mod tests {
 
     #[test]
     fn mulitple_contiguous_slots() {
-        let store = ObservedAttestations::default();
+        let mut store = ObservedAttestations::default();
         let max_cap = store.max_capacity();
 
         for i in 0..max_cap * 3 {
             let slot = Slot::new(i);
 
-            single_slot_test(&store, slot);
+            single_slot_test(&mut store, slot);
 
             /*
              * Ensure that the number of sets is correct.
@@ -325,14 +316,14 @@ mod tests {
 
             if i < max_cap {
                 assert_eq!(
-                    store.sets.read().len(),
+                    store.sets.len(),
                     i as usize + 1,
                     "should have a {} sets stored",
                     i + 1
                 );
             } else {
                 assert_eq!(
-                    store.sets.read().len(),
+                    store.sets.len(),
                     max_cap as usize,
                     "should have max_capacity sets stored"
                 );
@@ -342,7 +333,7 @@ mod tests {
              * Ensure that each set contains the correct number of elements.
              */
 
-            for set in &store.sets.read()[..] {
+            for set in &store.sets[..] {
                 assert_eq!(
                     set.len(),
                     NUM_ELEMENTS,
@@ -354,12 +345,7 @@ mod tests {
              *  Ensure that all the sets have the expected slots
              */
 
-            let mut store_slots = store
-                .sets
-                .read()
-                .iter()
-                .map(|set| set.slot)
-                .collect::<Vec<_>>();
+            let mut store_slots = store.sets.iter().map(|set| set.slot).collect::<Vec<_>>();
 
             assert!(
                 store_slots.len() <= store.max_capacity() as usize,
@@ -378,7 +364,7 @@ mod tests {
 
     #[test]
     fn mulitple_non_contiguous_slots() {
-        let store = ObservedAttestations::default();
+        let mut store = ObservedAttestations::default();
         let max_cap = store.max_capacity();
 
         let to_skip = vec![1_u64, 2, 3, 5, 6, 29, 30, 31, 32, 64];
@@ -394,13 +380,13 @@ mod tests {
 
             let slot = Slot::from(i);
 
-            single_slot_test(&store, slot);
+            single_slot_test(&mut store, slot);
 
             /*
              * Ensure that each set contains the correct number of elements.
              */
 
-            for set in &store.sets.read()[..] {
+            for set in &store.sets[..] {
                 assert_eq!(
                     set.len(),
                     NUM_ELEMENTS,
@@ -412,12 +398,7 @@ mod tests {
              *  Ensure that all the sets have the expected slots
              */
 
-            let mut store_slots = store
-                .sets
-                .read()
-                .iter()
-                .map(|set| set.slot)
-                .collect::<Vec<_>>();
+            let mut store_slots = store.sets.iter().map(|set| set.slot).collect::<Vec<_>>();
 
             store_slots.sort_unstable();
 
@@ -426,7 +407,7 @@ mod tests {
                 "store size should not exceed max"
             );
 
-            let lowest = store.lowest_permissible_slot.read().as_u64();
+            let lowest = store.lowest_permissible_slot.as_u64();
             let highest = slot.as_u64();
             let expected_slots = (lowest..=highest)
                 .filter(|i| !to_skip.contains(i))

beacon_node/beacon_chain/src/observed_attesters.rs

@@ -7,7 +7,6 @@
 //!   the same epoch
 
 use bitvec::vec::BitVec;
-use parking_lot::RwLock;
 use std::collections::{HashMap, HashSet};
 use std::marker::PhantomData;
 use types::{Attestation, Epoch, EthSpec, Unsigned};
@@ -148,16 +147,16 @@ impl Item for EpochHashSet {
 ///
 /// `T` should be set to a `EpochBitfield` or `EpochHashSet`.
 pub struct AutoPruningContainer<T, E: EthSpec> {
-    lowest_permissible_epoch: RwLock<Epoch>,
-    items: RwLock<HashMap<Epoch, T>>,
+    lowest_permissible_epoch: Epoch,
+    items: HashMap<Epoch, T>,
     _phantom: PhantomData<E>,
 }
 
 impl<T, E: EthSpec> Default for AutoPruningContainer<T, E> {
     fn default() -> Self {
         Self {
-            lowest_permissible_epoch: RwLock::new(Epoch::new(0)),
-            items: RwLock::new(HashMap::new()),
+            lowest_permissible_epoch: Epoch::new(0),
+            items: HashMap::new(),
             _phantom: PhantomData,
         }
     }
@@ -172,7 +171,7 @@ impl<T: Item, E: EthSpec> AutoPruningContainer<T, E> {
     /// - `validator_index` is higher than `VALIDATOR_REGISTRY_LIMIT`.
     /// - `a.data.target.slot` is earlier than `self.earliest_permissible_slot`.
     pub fn observe_validator(
-        &self,
+        &mut self,
         a: &Attestation<E>,
         validator_index: usize,
     ) -> Result<bool, Error> {
@@ -182,14 +181,13 @@ impl<T: Item, E: EthSpec> AutoPruningContainer<T, E> {
 
         self.prune(epoch);
 
-        let mut items = self.items.write();
-
-        if let Some(item) = items.get_mut(&epoch) {
+        if let Some(item) = self.items.get_mut(&epoch) {
             Ok(item.insert(validator_index))
         } else {
             // To avoid re-allocations, try and determine a rough initial capacity for the new item
             // by obtaining the mean size of all items in earlier epoch.
-            let (count, sum) = items
+            let (count, sum) = self
+                .items
                 .iter()
                 // Only include epochs that are less than the given slot in the average. This should
                 // generally avoid including recent epochs that are still "filling up".
@@ -201,7 +199,7 @@ impl<T: Item, E: EthSpec> AutoPruningContainer<T, E> {
 
             let mut item = T::with_capacity(initial_capacity);
             item.insert(validator_index);
-            items.insert(epoch, item);
+            self.items.insert(epoch, item);
 
             Ok(false)
         }
@@ -223,7 +221,6 @@ impl<T: Item, E: EthSpec> AutoPruningContainer<T, E> {
 
         let exists = self
             .items
-            .read()
             .get(&a.data.target.epoch)
             .map_or(false, |item| item.contains(validator_index));
 
@@ -233,10 +230,7 @@ impl<T: Item, E: EthSpec> AutoPruningContainer<T, E> {
     /// Returns the number of validators that have been observed at the given `epoch`. Returns
     /// `None` if `self` does not have a cache for that epoch.
     pub fn observed_validator_count(&self, epoch: Epoch) -> Option<usize> {
-        self.items
-            .read()
-            .get(&epoch)
-            .map(|item| item.validator_count())
+        self.items.get(&epoch).map(|item| item.validator_count())
     }
 
     fn sanitize_request(&self, a: &Attestation<E>, validator_index: usize) -> Result<(), Error> {
@@ -245,7 +239,7 @@ impl<T: Item, E: EthSpec> AutoPruningContainer<T, E> {
         }
 
         let epoch = a.data.target.epoch;
-        let lowest_permissible_epoch: Epoch = *self.lowest_permissible_epoch.read();
+        let lowest_permissible_epoch = self.lowest_permissible_epoch;
         if epoch < lowest_permissible_epoch {
             return Err(Error::EpochTooLow {
                 epoch,
@@ -270,14 +264,13 @@ impl<T: Item, E: EthSpec> AutoPruningContainer<T, E> {
     ///
     /// Also sets `self.lowest_permissible_epoch` with relation to `current_epoch` and
     /// `Self::max_capacity`.
-    pub fn prune(&self, current_epoch: Epoch) {
+    pub fn prune(&mut self, current_epoch: Epoch) {
         // Taking advantage of saturating subtraction on `Slot`.
         let lowest_permissible_epoch = current_epoch - (self.max_capacity().saturating_sub(1));
 
-        *self.lowest_permissible_epoch.write() = lowest_permissible_epoch;
+        self.lowest_permissible_epoch = lowest_permissible_epoch;
 
         self.items
-            .write()
             .retain(|epoch, _item| *epoch >= lowest_permissible_epoch);
     }
 }
@@ -301,7 +294,7 @@ mod tests {
                     a
                 }
 
-                fn single_epoch_test(store: &$type<E>, epoch: Epoch) {
+                fn single_epoch_test(store: &mut $type<E>, epoch: Epoch) {
                     let attesters = [0, 1, 2, 3, 5, 6, 7, 18, 22];
                     let a = &get_attestation(epoch);
 
@@ -334,26 +327,22 @@ mod tests {
 
                 #[test]
                 fn single_epoch() {
-                    let store = $type::default();
+                    let mut store = $type::default();
 
-                    single_epoch_test(&store, Epoch::new(0));
+                    single_epoch_test(&mut store, Epoch::new(0));
 
-                    assert_eq!(
-                        store.items.read().len(),
-                        1,
-                        "should have a single bitfield stored"
-                    );
+                    assert_eq!(store.items.len(), 1, "should have a single bitfield stored");
                 }
 
                 #[test]
                 fn mulitple_contiguous_epochs() {
-                    let store = $type::default();
+                    let mut store = $type::default();
                     let max_cap = store.max_capacity();
 
                     for i in 0..max_cap * 3 {
                         let epoch = Epoch::new(i);
 
-                        single_epoch_test(&store, epoch);
+                        single_epoch_test(&mut store, epoch);
 
                         /*
                          * Ensure that the number of sets is correct.
@@ -361,14 +350,14 @@ mod tests {
 
                         if i < max_cap {
                             assert_eq!(
-                                store.items.read().len(),
+                                store.items.len(),
                                 i as usize + 1,
                                 "should have a {} items stored",
                                 i + 1
                             );
                         } else {
                             assert_eq!(
-                                store.items.read().len(),
+                                store.items.len(),
                                 max_cap as usize,
                                 "should have max_capacity items stored"
                             );
@@ -380,7 +369,6 @@ mod tests {
 
                         let mut store_epochs = store
                             .items
-                            .read()
                             .iter()
                             .map(|(epoch, _set)| *epoch)
                             .collect::<Vec<_>>();
@@ -402,7 +390,7 @@ mod tests {
 
                 #[test]
                 fn mulitple_non_contiguous_epochs() {
-                    let store = $type::default();
+                    let mut store = $type::default();
                     let max_cap = store.max_capacity();
 
                     let to_skip = vec![1_u64, 3, 4, 5];
@@ -418,7 +406,7 @@ mod tests {
 
                         let epoch = Epoch::from(i);
 
-                        single_epoch_test(&store, epoch);
+                        single_epoch_test(&mut store, epoch);
 
                         /*
                          *  Ensure that all the sets have the expected slots
@@ -426,7 +414,6 @@ mod tests {
 
                         let mut store_epochs = store
                             .items
-                            .read()
                             .iter()
                             .map(|(epoch, _)| *epoch)
                             .collect::<Vec<_>>();
@@ -438,7 +425,7 @@ mod tests {
                             "store size should not exceed max"
                         );
 
-                        let lowest = store.lowest_permissible_epoch.read().as_u64();
+                        let lowest = store.lowest_permissible_epoch.as_u64();
                         let highest = epoch.as_u64();
                         let expected_epochs = (lowest..=highest)
                             .filter(|i| !to_skip.contains(i))

beacon_node/beacon_chain/src/observed_block_producers.rs

@@ -1,7 +1,6 @@
 //! Provides the `ObservedBlockProducers` struct which allows for rejecting gossip blocks from
 //! validators that have already produced a block.
 
-use parking_lot::RwLock;
 use std::collections::{HashMap, HashSet};
 use std::marker::PhantomData;
 use types::{BeaconBlock, EthSpec, Slot, Unsigned};
@@ -27,8 +26,8 @@ pub enum Error {
 /// active_validator_count`, however in reality that is more like `slots_since_finality *
 /// known_distinct_shufflings` which is much smaller.
 pub struct ObservedBlockProducers<E: EthSpec> {
-    finalized_slot: RwLock<Slot>,
-    items: RwLock<HashMap<Slot, HashSet<u64>>>,
+    finalized_slot: Slot,
+    items: HashMap<Slot, HashSet<u64>>,
     _phantom: PhantomData<E>,
 }
 
@@ -36,8 +35,8 @@ impl<E: EthSpec> Default for ObservedBlockProducers<E> {
     /// Instantiates `Self` with `finalized_slot == 0`.
     fn default() -> Self {
         Self {
-            finalized_slot: RwLock::new(Slot::new(0)),
-            items: RwLock::new(HashMap::new()),
+            finalized_slot: Slot::new(0),
+            items: HashMap::new(),
             _phantom: PhantomData,
         }
     }
@@ -53,12 +52,11 @@ impl<E: EthSpec> ObservedBlockProducers<E> {
     ///
     /// - `block.proposer_index` is greater than `VALIDATOR_REGISTRY_LIMIT`.
     /// - `block.slot` is equal to or less than the latest pruned `finalized_slot`.
-    pub fn observe_proposer(&self, block: &BeaconBlock<E>) -> Result<bool, Error> {
+    pub fn observe_proposer(&mut self, block: &BeaconBlock<E>) -> Result<bool, Error> {
         self.sanitize_block(block)?;
 
         let did_not_exist = self
             .items
-            .write()
             .entry(block.slot)
             .or_insert_with(|| HashSet::with_capacity(E::SlotsPerEpoch::to_usize()))
             .insert(block.proposer_index);
@@ -79,7 +77,6 @@ impl<E: EthSpec> ObservedBlockProducers<E> {
 
         let exists = self
             .items
-            .read()
             .get(&block.slot)
             .map_or(false, |set| set.contains(&block.proposer_index));
 
@@ -92,7 +89,7 @@ impl<E: EthSpec> ObservedBlockProducers<E> {
             return Err(Error::ValidatorIndexTooHigh(block.proposer_index));
         }
 
-        let finalized_slot = *self.finalized_slot.read();
+        let finalized_slot = self.finalized_slot;
         if finalized_slot > 0 && block.slot <= finalized_slot {
             return Err(Error::FinalizedBlock {
                 slot: block.slot,
@@ -109,15 +106,13 @@ impl<E: EthSpec> ObservedBlockProducers<E> {
     /// equal to or less than `finalized_slot`.
     ///
     /// No-op if `finalized_slot == 0`.
-    pub fn prune(&self, finalized_slot: Slot) {
+    pub fn prune(&mut self, finalized_slot: Slot) {
         if finalized_slot == 0 {
             return;
         }
 
-        *self.finalized_slot.write() = finalized_slot;
-        self.items
-            .write()
-            .retain(|slot, _set| *slot > finalized_slot);
+        self.finalized_slot = finalized_slot;
+        self.items.retain(|slot, _set| *slot > finalized_slot);
     }
 }
 
@@ -137,10 +132,10 @@ mod tests {
 
     #[test]
     fn pruning() {
-        let cache = ObservedBlockProducers::default();
+        let mut cache = ObservedBlockProducers::default();
 
-        assert_eq!(*cache.finalized_slot.read(), 0, "finalized slot is zero");
-        assert_eq!(cache.items.read().len(), 0, "no slots should be present");
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 0, "no slots should be present");
 
         // Slot 0, proposer 0
         let block_a = &get_block(0, 0);
@@ -155,16 +150,11 @@ mod tests {
          * Preconditions.
          */
 
-        assert_eq!(*cache.finalized_slot.read(), 0, "finalized slot is zero");
-        assert_eq!(
-            cache.items.read().len(),
-            1,
-            "only one slot should be present"
-        );
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 1, "only one slot should be present");
         assert_eq!(
             cache
                 .items
-                .read()
                 .get(&Slot::new(0))
                 .expect("slot zero should be present")
                 .len(),
@@ -178,16 +168,11 @@ mod tests {
 
         cache.prune(Slot::new(0));
 
-        assert_eq!(*cache.finalized_slot.read(), 0, "finalized slot is zero");
-        assert_eq!(
-            cache.items.read().len(),
-            1,
-            "only one slot should be present"
-        );
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 1, "only one slot should be present");
         assert_eq!(
             cache
                 .items
-                .read()
                 .get(&Slot::new(0))
                 .expect("slot zero should be present")
                 .len(),
@@ -201,11 +186,11 @@ mod tests {
 
         cache.prune(E::slots_per_epoch().into());
         assert_eq!(
-            *cache.finalized_slot.read(),
+            cache.finalized_slot,
             Slot::from(E::slots_per_epoch()),
             "finalized slot is updated"
         );
-        assert_eq!(cache.items.read().len(), 0, "no items left");
+        assert_eq!(cache.items.len(), 0, "no items left");
 
         /*
          * Check that we can't insert a finalized block
@@ -223,7 +208,7 @@ mod tests {
             "cant insert finalized block"
         );
 
-        assert_eq!(cache.items.read().len(), 0, "block was not added");
+        assert_eq!(cache.items.len(), 0, "block was not added");
 
         /*
          * Check that we _can_ insert a non-finalized block
@@ -240,15 +225,10 @@ mod tests {
             "can insert non-finalized block"
         );
 
-        assert_eq!(
-            cache.items.read().len(),
-            1,
-            "only one slot should be present"
-        );
+        assert_eq!(cache.items.len(), 1, "only one slot should be present");
         assert_eq!(
             cache
                 .items
-                .read()
                 .get(&Slot::new(three_epochs))
                 .expect("the three epochs slot should be present")
                 .len(),
@@ -264,20 +244,15 @@ mod tests {
         cache.prune(two_epochs.into());
 
         assert_eq!(
-            *cache.finalized_slot.read(),
+            cache.finalized_slot,
             Slot::from(two_epochs),
             "finalized slot is updated"
         );
 
-        assert_eq!(
-            cache.items.read().len(),
-            1,
-            "only one slot should be present"
-        );
+        assert_eq!(cache.items.len(), 1, "only one slot should be present");
         assert_eq!(
             cache
                 .items
-                .read()
                 .get(&Slot::new(three_epochs))
                 .expect("the three epochs slot should be present")
                 .len(),
@@ -288,7 +263,7 @@ mod tests {
 
     #[test]
     fn simple_observations() {
-        let cache = ObservedBlockProducers::default();
+        let mut cache = ObservedBlockProducers::default();
 
         // Slot 0, proposer 0
         let block_a = &get_block(0, 0);
@@ -314,16 +289,11 @@ mod tests {
             "observing again indicates true"
         );
 
-        assert_eq!(*cache.finalized_slot.read(), 0, "finalized slot is zero");
-        assert_eq!(
-            cache.items.read().len(),
-            1,
-            "only one slot should be present"
-        );
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 1, "only one slot should be present");
         assert_eq!(
             cache
                 .items
-                .read()
                 .get(&Slot::new(0))
                 .expect("slot zero should be present")
                 .len(),
@@ -355,12 +325,11 @@ mod tests {
             "observing slot 1 again indicates true"
         );
 
-        assert_eq!(*cache.finalized_slot.read(), 0, "finalized slot is zero");
-        assert_eq!(cache.items.read().len(), 2, "two slots should be present");
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 2, "two slots should be present");
         assert_eq!(
             cache
                 .items
-                .read()
                 .get(&Slot::new(0))
                 .expect("slot zero should be present")
                 .len(),
@@ -370,7 +339,6 @@ mod tests {
         assert_eq!(
             cache
                 .items
-                .read()
                 .get(&Slot::new(1))
                 .expect("slot zero should be present")
                 .len(),
@@ -402,12 +370,11 @@ mod tests {
             "observing new proposer again indicates true"
         );
 
-        assert_eq!(*cache.finalized_slot.read(), 0, "finalized slot is zero");
-        assert_eq!(cache.items.read().len(), 2, "two slots should be present");
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 2, "two slots should be present");
         assert_eq!(
             cache
                 .items
-                .read()
                 .get(&Slot::new(0))
                 .expect("slot zero should be present")
                 .len(),
@@ -417,7 +384,6 @@ mod tests {
         assert_eq!(
             cache
                 .items
-                .read()
                 .get(&Slot::new(1))
                 .expect("slot zero should be present")
                 .len(),

beacon_node/beacon_chain/src/observed_operations.rs

@@ -1,9 +1,7 @@
 use derivative::Derivative;
-use parking_lot::Mutex;
 use smallvec::SmallVec;
 use state_processing::{SigVerifiedOp, VerifyOperation};
 use std::collections::HashSet;
-use std::iter::FromIterator;
 use std::marker::PhantomData;
 use types::{
     AttesterSlashing, BeaconState, ChainSpec, EthSpec, ProposerSlashing, SignedVoluntaryExit,
@@ -25,7 +23,7 @@ pub struct ObservedOperations<T: ObservableOperation<E>, E: EthSpec> {
     /// For attester slashings, this is the set of all validators who would be slashed by
     /// previously seen attester slashings, i.e. those validators in the intersection of
     /// `attestation_1.attester_indices` and `attestation_2.attester_indices`.
-    observed_validator_indices: Mutex<HashSet<u64>>,
+    observed_validator_indices: HashSet<u64>,
     _phantom: PhantomData<(T, E)>,
 }
 
@@ -58,10 +56,18 @@ impl<E: EthSpec> ObservableOperation<E> for ProposerSlashing {
 
 impl<E: EthSpec> ObservableOperation<E> for AttesterSlashing<E> {
     fn observed_validators(&self) -> SmallVec<[u64; SMALL_VEC_SIZE]> {
-        let attestation_1_indices =
-            HashSet::<u64>::from_iter(self.attestation_1.attesting_indices.iter().copied());
-        let attestation_2_indices =
-            HashSet::<u64>::from_iter(self.attestation_2.attesting_indices.iter().copied());
+        let attestation_1_indices = self
+            .attestation_1
+            .attesting_indices
+            .iter()
+            .copied()
+            .collect::<HashSet<u64>>();
+        let attestation_2_indices = self
+            .attestation_2
+            .attesting_indices
+            .iter()
+            .copied()
+            .collect::<HashSet<u64>>();
         attestation_1_indices
             .intersection(&attestation_2_indices)
             .copied()
@@ -71,12 +77,12 @@ impl<E: EthSpec> ObservableOperation<E> for AttesterSlashing<E> {
 
 impl<T: ObservableOperation<E>, E: EthSpec> ObservedOperations<T, E> {
     pub fn verify_and_observe(
-        &self,
+        &mut self,
         op: T,
         head_state: &BeaconState<E>,
         spec: &ChainSpec,
     ) -> Result<ObservationOutcome<T>, T::Error> {
-        let mut observed_validator_indices = self.observed_validator_indices.lock();
+        let observed_validator_indices = &mut self.observed_validator_indices;
         let new_validator_indices = op.observed_validators();
 
         // If all of the new validator indices have been previously observed, short-circuit

beacon_node/beacon_chain/src/persisted_beacon_chain.rs

@@ -4,9 +4,19 @@ use ssz_derive::{Decode, Encode};
 use store::{DBColumn, Error as StoreError, StoreItem};
 use types::Hash256;
 
+/// Dummy value to use for the canonical head block root, see below.
+pub const DUMMY_CANONICAL_HEAD_BLOCK_ROOT: Hash256 = Hash256::repeat_byte(0xff);
+
 #[derive(Clone, Encode, Decode)]
 pub struct PersistedBeaconChain {
-    pub canonical_head_block_root: Hash256,
+    /// This value is ignored to resolve the issue described here:
+    ///
+    /// https://github.com/sigp/lighthouse/pull/1639
+    ///
+    /// Its removal is tracked here:
+    ///
+    /// https://github.com/sigp/lighthouse/issues/1784
+    pub _canonical_head_block_root: Hash256,
     pub genesis_block_root: Hash256,
     pub ssz_head_tracker: SszHeadTracker,
 }

beacon_node/beacon_chain/src/shuffling_cache.rs

@@ -1,6 +1,6 @@
 use crate::metrics;
 use lru::LruCache;
-use types::{beacon_state::CommitteeCache, Epoch, Hash256};
+use types::{beacon_state::CommitteeCache, Epoch, Hash256, ShufflingId};
 
 /// The size of the LRU cache that stores committee caches for quicker verification.
 ///
@@ -14,7 +14,7 @@ const CACHE_SIZE: usize = 16;
 /// It has been named `ShufflingCache` because `CommitteeCacheCache` is a bit weird and looks like
 /// a find/replace error.
 pub struct ShufflingCache {
-    cache: LruCache<(Epoch, Hash256), CommitteeCache>,
+    cache: LruCache<ShufflingId, CommitteeCache>,
 }
 
 impl ShufflingCache {
@@ -24,8 +24,8 @@ impl ShufflingCache {
         }
     }
 
-    pub fn get(&mut self, epoch: Epoch, root: Hash256) -> Option<&CommitteeCache> {
-        let opt = self.cache.get(&(epoch, root));
+    pub fn get(&mut self, key: &ShufflingId) -> Option<&CommitteeCache> {
+        let opt = self.cache.get(key);
 
         if opt.is_some() {
             metrics::inc_counter(&metrics::SHUFFLING_CACHE_HITS);
@@ -36,11 +36,37 @@ impl ShufflingCache {
         opt
     }
 
-    pub fn insert(&mut self, epoch: Epoch, root: Hash256, committee_cache: &CommitteeCache) {
-        let key = (epoch, root);
+    pub fn contains(&self, key: &ShufflingId) -> bool {
+        self.cache.contains(key)
+    }
 
+    pub fn insert(&mut self, key: ShufflingId, committee_cache: &CommitteeCache) {
         if !self.cache.contains(&key) {
             self.cache.put(key, committee_cache.clone());
         }
     }
 }
+
+/// Contains the shuffling IDs for a beacon block.
+pub struct BlockShufflingIds {
+    pub current: ShufflingId,
+    pub next: ShufflingId,
+    pub block_root: Hash256,
+}
+
+impl BlockShufflingIds {
+    /// Returns the shuffling ID for the given epoch.
+    ///
+    /// Returns `None` if `epoch` is prior to `self.current.shuffling_epoch`.
+    pub fn id_for_epoch(&self, epoch: Epoch) -> Option<ShufflingId> {
+        if epoch == self.current.shuffling_epoch {
+            Some(self.current.clone())
+        } else if epoch == self.next.shuffling_epoch {
+            Some(self.next.clone())
+        } else if epoch > self.next.shuffling_epoch {
+            Some(ShufflingId::from_components(epoch, self.block_root))
+        } else {
+            None
+        }
+    }
+}

beacon_node/beacon_chain/src/snapshot_cache.rs

@@ -1,6 +1,6 @@
 use crate::BeaconSnapshot;
 use std::cmp;
-use types::{Epoch, EthSpec, Hash256};
+use types::{beacon_state::CloneConfig, Epoch, EthSpec, Hash256};
 
 /// The default size of the cache.
 pub const DEFAULT_SNAPSHOT_CACHE_SIZE: usize = 4;
@@ -69,13 +69,16 @@ impl<T: EthSpec> SnapshotCache<T> {
             .map(|i| self.snapshots.remove(i))
     }
 
-    /// If there is a snapshot with `block_root`, clone it (with only the committee caches) and
-    /// return the clone.
-    pub fn get_cloned(&self, block_root: Hash256) -> Option<BeaconSnapshot<T>> {
+    /// If there is a snapshot with `block_root`, clone it and return the clone.
+    pub fn get_cloned(
+        &self,
+        block_root: Hash256,
+        clone_config: CloneConfig,
+    ) -> Option<BeaconSnapshot<T>> {
         self.snapshots
             .iter()
             .find(|snapshot| snapshot.beacon_block_root == block_root)
-            .map(|snapshot| snapshot.clone_with_only_committee_caches())
+            .map(|snapshot| snapshot.clone_with(clone_config))
     }
 
     /// Removes all snapshots from the queue that are less than or equal to the finalized epoch.
@@ -165,11 +168,13 @@ mod test {
             cache.try_remove(Hash256::from_low_u64_be(1)).is_none(),
             "the snapshot with the lowest slot should have been removed during the insert function"
         );
-        assert!(cache.get_cloned(Hash256::from_low_u64_be(1)).is_none());
+        assert!(cache
+            .get_cloned(Hash256::from_low_u64_be(1), CloneConfig::none())
+            .is_none());
 
         assert!(
             cache
-                .get_cloned(Hash256::from_low_u64_be(0))
+                .get_cloned(Hash256::from_low_u64_be(0), CloneConfig::none())
                 .expect("the head should still be in the cache")
                 .beacon_block_root
                 == Hash256::from_low_u64_be(0),

beacon_node/beacon_chain/src/validator_monitor.rs

@@ -0,0 +1,953 @@
+//! Provides detailed logging and metrics for a set of registered validators.
+//!
+//! This component should not affect consensus.
+
+use crate::metrics;
+use parking_lot::RwLock;
+use slog::{crit, info, Logger};
+use slot_clock::SlotClock;
+use std::collections::{HashMap, HashSet};
+use std::convert::TryFrom;
+use std::io;
+use std::marker::PhantomData;
+use std::str::Utf8Error;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use types::{
+    AttestationData, AttesterSlashing, BeaconBlock, BeaconState, ChainSpec, Epoch, EthSpec,
+    Hash256, IndexedAttestation, ProposerSlashing, PublicKeyBytes, SignedAggregateAndProof, Slot,
+    VoluntaryExit,
+};
+
+/// The validator monitor collects per-epoch data about each monitored validator. Historical data
+/// will be kept around for `HISTORIC_EPOCHS` before it is pruned.
+pub const HISTORIC_EPOCHS: usize = 4;
+
+#[derive(Debug)]
+pub enum Error {
+    InvalidPubkey(String),
+    FileError(io::Error),
+    InvalidUtf8(Utf8Error),
+}
+
+/// Contains data pertaining to one validator for one epoch.
+#[derive(Default)]
+struct EpochSummary {
+    /*
+     * Attestations with a target in the current epoch.
+     */
+    /// The number of attestations seen.
+    pub attestations: usize,
+    /// The delay between when the attestation should have been produced and when it was observed.
+    pub attestation_min_delay: Option<Duration>,
+    /// The number of times a validators attestation was seen in an aggregate.
+    pub attestation_aggregate_incusions: usize,
+    /// The number of times a validators attestation was seen in a block.
+    pub attestation_block_inclusions: usize,
+    /// The minimum observed inclusion distance for an attestation for this epoch..
+    pub attestation_min_block_inclusion_distance: Option<Slot>,
+    /*
+     * Blocks with a slot in the current epoch.
+     */
+    /// The number of blocks observed.
+    pub blocks: usize,
+    /// The delay between when the block should have been produced and when it was observed.
+    pub block_min_delay: Option<Duration>,
+    /*
+     * Aggregates with a target in the current epoch
+     */
+    /// The number of signed aggregate and proofs observed.
+    pub aggregates: usize,
+    /// The delay between when the aggregate should have been produced and when it was observed.
+    pub aggregate_min_delay: Option<Duration>,
+    /*
+     * Others pertaining to this epoch.
+     */
+    /// The number of voluntary exists observed.
+    pub exits: usize,
+    /// The number of proposer slashings observed.
+    pub proposer_slashings: usize,
+    /// The number of attester slashings observed.
+    pub attester_slashings: usize,
+}
+
+impl EpochSummary {
+    /// Update `current` if:
+    ///
+    /// - It is `None`.
+    /// - `new` is greater than its current value.
+    fn update_if_lt<T: Ord>(current: &mut Option<T>, new: T) {
+        if let Some(ref mut current) = current {
+            if new < *current {
+                *current = new
+            }
+        } else {
+            *current = Some(new)
+        }
+    }
+
+    pub fn register_unaggregated_attestation(&mut self, delay: Duration) {
+        self.attestations += 1;
+        Self::update_if_lt(&mut self.attestation_min_delay, delay);
+    }
+
+    pub fn register_aggregated_attestation(&mut self, delay: Duration) {
+        self.aggregates += 1;
+        Self::update_if_lt(&mut self.aggregate_min_delay, delay);
+    }
+
+    pub fn register_aggregate_attestation_inclusion(&mut self) {
+        self.attestation_aggregate_incusions += 1;
+    }
+
+    pub fn register_attestation_block_inclusion(&mut self, delay: Slot) {
+        self.attestation_block_inclusions += 1;
+        Self::update_if_lt(&mut self.attestation_min_block_inclusion_distance, delay);
+    }
+
+    pub fn register_exit(&mut self) {
+        self.exits += 1;
+    }
+
+    pub fn register_proposer_slashing(&mut self) {
+        self.proposer_slashings += 1;
+    }
+
+    pub fn register_attester_slashing(&mut self) {
+        self.attester_slashings += 1;
+    }
+}
+
+type SummaryMap = HashMap<Epoch, EpochSummary>;
+
+/// A validator that is being monitored by the `ValidatorMonitor`.
+struct MonitoredValidator {
+    /// A human-readable identifier for the validator.
+    pub id: String,
+    /// The validator voting pubkey.
+    pub pubkey: PublicKeyBytes,
+    /// The validator index in the state.
+    pub index: Option<u64>,
+    /// A history of the validator over time.
+    pub summaries: RwLock<SummaryMap>,
+}
+
+impl MonitoredValidator {
+    fn new(pubkey: PublicKeyBytes, index: Option<u64>) -> Self {
+        Self {
+            id: index
+                .map(|i| i.to_string())
+                .unwrap_or_else(|| pubkey.to_string()),
+            pubkey,
+            index,
+            summaries: <_>::default(),
+        }
+    }
+
+    fn set_index(&mut self, validator_index: u64) {
+        if self.index.is_none() {
+            self.index = Some(validator_index);
+            self.id = validator_index.to_string();
+        }
+    }
+
+    /// Maps `func` across the `self.summaries`.
+    ///
+    /// ## Warning
+    ///
+    /// It is possible to deadlock this function by trying to obtain a lock on
+    /// `self.summary` inside `func`.
+    ///
+    /// ## Notes
+    ///
+    /// - If `epoch` doesn't exist in `self.summaries`, it is created.
+    /// - `self.summaries` may be pruned after `func` is run.
+    fn with_epoch_summary<F>(&self, epoch: Epoch, func: F)
+    where
+        F: Fn(&mut EpochSummary),
+    {
+        let mut summaries = self.summaries.write();
+
+        func(summaries.entry(epoch).or_default());
+
+        // Prune
+        while summaries.len() > HISTORIC_EPOCHS {
+            if let Some(key) = summaries.iter().map(|(epoch, _)| *epoch).min() {
+                summaries.remove(&key);
+            }
+        }
+    }
+}
+
+/// Holds a collection of `MonitoredValidator` and is notified about a variety of events on the P2P
+/// network, HTTP API and `BeaconChain`.
+///
+/// If any of the events pertain to a `MonitoredValidator`, additional logging and metrics will be
+/// performed.
+///
+/// The intention of this struct is to provide users with more logging and Prometheus metrics around
+/// validators that they are interested in.
+pub struct ValidatorMonitor<T> {
+    /// The validators that require additional monitoring.
+    validators: HashMap<PublicKeyBytes, MonitoredValidator>,
+    /// A map of validator index (state.validators) to a validator public key.
+    indices: HashMap<u64, PublicKeyBytes>,
+    /// If true, allow the automatic registration of validators.
+    auto_register: bool,
+    log: Logger,
+    _phantom: PhantomData<T>,
+}
+
+impl<T: EthSpec> ValidatorMonitor<T> {
+    pub fn new(pubkeys: Vec<PublicKeyBytes>, auto_register: bool, log: Logger) -> Self {
+        let mut s = Self {
+            validators: <_>::default(),
+            indices: <_>::default(),
+            auto_register,
+            log,
+            _phantom: PhantomData,
+        };
+        for pubkey in pubkeys {
+            s.add_validator_pubkey(pubkey)
+        }
+        s
+    }
+
+    /// Add some validators to `self` for additional monitoring.
+    fn add_validator_pubkey(&mut self, pubkey: PublicKeyBytes) {
+        let index_opt = self
+            .indices
+            .iter()
+            .find(|(_, candidate_pk)| **candidate_pk == pubkey)
+            .map(|(index, _)| *index);
+
+        let log = self.log.clone();
+        self.validators.entry(pubkey).or_insert_with(|| {
+            info!(
+                log,
+                "Started monitoring validator";
+                "pubkey" => %pubkey,
+            );
+            MonitoredValidator::new(pubkey, index_opt)
+        });
+    }
+
+    /// Reads information from the given `state`. The `state` *must* be valid (i.e, able to be
+    /// imported).
+    pub fn process_valid_state(&mut self, current_epoch: Epoch, state: &BeaconState<T>) {
+        // Add any new validator indices.
+        state
+            .validators
+            .iter()
+            .enumerate()
+            .skip(self.indices.len())
+            .for_each(|(i, validator)| {
+                let i = i as u64;
+                if let Some(validator) = self.validators.get_mut(&validator.pubkey) {
+                    validator.set_index(i)
+                }
+                self.indices.insert(i, validator.pubkey);
+            });
+
+        // Update metrics for individual validators.
+        for monitored_validator in self.validators.values() {
+            if let Some(i) = monitored_validator.index {
+                let i = i as usize;
+                let id = &monitored_validator.id;
+
+                if let Some(balance) = state.balances.get(i) {
+                    metrics::set_int_gauge(
+                        &metrics::VALIDATOR_MONITOR_BALANCE_GWEI,
+                        &[id],
+                        *balance as i64,
+                    );
+                }
+
+                if let Some(validator) = state.validators.get(i) {
+                    metrics::set_int_gauge(
+                        &metrics::VALIDATOR_MONITOR_EFFECTIVE_BALANCE_GWEI,
+                        &[id],
+                        u64_to_i64(validator.effective_balance),
+                    );
+                    metrics::set_int_gauge(
+                        &metrics::VALIDATOR_MONITOR_SLASHED,
+                        &[id],
+                        if validator.slashed { 1 } else { 0 },
+                    );
+                    metrics::set_int_gauge(
+                        &metrics::VALIDATOR_MONITOR_ACTIVE,
+                        &[id],
+                        if validator.is_active_at(current_epoch) {
+                            1
+                        } else {
+                            0
+                        },
+                    );
+                    metrics::set_int_gauge(
+                        &metrics::VALIDATOR_MONITOR_EXITED,
+                        &[id],
+                        if validator.is_exited_at(current_epoch) {
+                            1
+                        } else {
+                            0
+                        },
+                    );
+                    metrics::set_int_gauge(
+                        &metrics::VALIDATOR_MONITOR_WITHDRAWABLE,
+                        &[id],
+                        if validator.is_withdrawable_at(current_epoch) {
+                            1
+                        } else {
+                            0
+                        },
+                    );
+                    metrics::set_int_gauge(
+                        &metrics::VALIDATOR_ACTIVATION_ELIGIBILITY_EPOCH,
+                        &[id],
+                        u64_to_i64(validator.activation_eligibility_epoch),
+                    );
+                    metrics::set_int_gauge(
+                        &metrics::VALIDATOR_ACTIVATION_EPOCH,
+                        &[id],
+                        u64_to_i64(validator.activation_epoch),
+                    );
+                    metrics::set_int_gauge(
+                        &metrics::VALIDATOR_EXIT_EPOCH,
+                        &[id],
+                        u64_to_i64(validator.exit_epoch),
+                    );
+                    metrics::set_int_gauge(
+                        &metrics::VALIDATOR_WITHDRAWABLE_EPOCH,
+                        &[id],
+                        u64_to_i64(validator.withdrawable_epoch),
+                    );
+                }
+            }
+        }
+    }
+
+    fn get_validator_id(&self, validator_index: u64) -> Option<&str> {
+        self.indices
+            .get(&validator_index)
+            .and_then(|pubkey| self.validators.get(pubkey))
+            .map(|validator| validator.id.as_str())
+    }
+
+    fn get_validator(&self, validator_index: u64) -> Option<&MonitoredValidator> {
+        self.indices
+            .get(&validator_index)
+            .and_then(|pubkey| self.validators.get(pubkey))
+    }
+
+    /// Returns the number of validators monitored by `self`.
+    pub fn num_validators(&self) -> usize {
+        self.validators.len()
+    }
+
+    /// If `self.auto_register == true`, add the `validator_index` to `self.monitored_validators`.
+    /// Otherwise, do nothing.
+    pub fn auto_register_local_validator(&mut self, validator_index: u64) {
+        if !self.auto_register {
+            return;
+        }
+
+        if let Some(pubkey) = self.indices.get(&validator_index) {
+            if !self.validators.contains_key(pubkey) {
+                info!(
+                    self.log,
+                    "Started monitoring validator";
+                    "pubkey" => %pubkey,
+                    "validator" => %validator_index,
+                );
+
+                self.validators.insert(
+                    *pubkey,
+                    MonitoredValidator::new(*pubkey, Some(validator_index)),
+                );
+            }
+        }
+    }
+
+    /// Returns the delay between the start of `block.slot` and `seen_timestamp`.
+    fn get_block_delay_ms<S: SlotClock>(
+        seen_timestamp: Duration,
+        block: &BeaconBlock<T>,
+        slot_clock: &S,
+    ) -> Duration {
+        slot_clock
+            .start_of(block.slot)
+            .and_then(|slot_start| seen_timestamp.checked_sub(slot_start))
+            .unwrap_or_else(|| Duration::from_secs(0))
+    }
+
+    /// Process a block received on gossip.
+    pub fn register_gossip_block<S: SlotClock>(
+        &self,
+        seen_timestamp: Duration,
+        block: &BeaconBlock<T>,
+        block_root: Hash256,
+        slot_clock: &S,
+    ) {
+        self.register_beacon_block("gossip", seen_timestamp, block, block_root, slot_clock)
+    }
+
+    /// Process a block received on the HTTP API from a local validator.
+    pub fn register_api_block<S: SlotClock>(
+        &self,
+        seen_timestamp: Duration,
+        block: &BeaconBlock<T>,
+        block_root: Hash256,
+        slot_clock: &S,
+    ) {
+        self.register_beacon_block("api", seen_timestamp, block, block_root, slot_clock)
+    }
+
+    fn register_beacon_block<S: SlotClock>(
+        &self,
+        src: &str,
+        seen_timestamp: Duration,
+        block: &BeaconBlock<T>,
+        block_root: Hash256,
+        slot_clock: &S,
+    ) {
+        if let Some(id) = self.get_validator_id(block.proposer_index) {
+            let delay = Self::get_block_delay_ms(seen_timestamp, block, slot_clock);
+
+            metrics::inc_counter_vec(&metrics::VALIDATOR_MONITOR_BEACON_BLOCK_TOTAL, &[src, id]);
+            metrics::observe_timer_vec(
+                &metrics::VALIDATOR_MONITOR_BEACON_BLOCK_DELAY_SECONDS,
+                &[src, id],
+                delay,
+            );
+
+            info!(
+                self.log,
+                "Block from API";
+                "root" => ?block_root,
+                "delay" => %delay.as_millis(),
+                "slot" => %block.slot,
+                "src" => src,
+                "validator" => %id,
+            );
+        }
+    }
+
+    /// Returns the duration between when the attestation `data` could be produced (1/3rd through
+    /// the slot) and `seen_timestamp`.
+    fn get_unaggregated_attestation_delay_ms<S: SlotClock>(
+        seen_timestamp: Duration,
+        data: &AttestationData,
+        slot_clock: &S,
+    ) -> Duration {
+        slot_clock
+            .start_of(data.slot)
+            .and_then(|slot_start| seen_timestamp.checked_sub(slot_start))
+            .and_then(|gross_delay| {
+                let production_delay = slot_clock.slot_duration() / 3;
+                gross_delay.checked_sub(production_delay)
+            })
+            .unwrap_or_else(|| Duration::from_secs(0))
+    }
+
+    /// Register an attestation seen on the gossip network.
+    pub fn register_gossip_unaggregated_attestation<S: SlotClock>(
+        &self,
+        seen_timestamp: Duration,
+        indexed_attestation: &IndexedAttestation<T>,
+        slot_clock: &S,
+    ) {
+        self.register_unaggregated_attestation(
+            "gossip",
+            seen_timestamp,
+            indexed_attestation,
+            slot_clock,
+        )
+    }
+
+    /// Register an attestation seen on the HTTP API.
+    pub fn register_api_unaggregated_attestation<S: SlotClock>(
+        &self,
+        seen_timestamp: Duration,
+        indexed_attestation: &IndexedAttestation<T>,
+        slot_clock: &S,
+    ) {
+        self.register_unaggregated_attestation(
+            "api",
+            seen_timestamp,
+            indexed_attestation,
+            slot_clock,
+        )
+    }
+
+    fn register_unaggregated_attestation<S: SlotClock>(
+        &self,
+        src: &str,
+        seen_timestamp: Duration,
+        indexed_attestation: &IndexedAttestation<T>,
+        slot_clock: &S,
+    ) {
+        let data = &indexed_attestation.data;
+        let epoch = data.slot.epoch(T::slots_per_epoch());
+        let delay = Self::get_unaggregated_attestation_delay_ms(seen_timestamp, data, slot_clock);
+
+        indexed_attestation.attesting_indices.iter().for_each(|i| {
+            if let Some(validator) = self.get_validator(*i) {
+                let id = &validator.id;
+
+                metrics::inc_counter_vec(
+                    &metrics::VALIDATOR_MONITOR_UNAGGREGATED_ATTESTATION_TOTAL,
+                    &[src, id],
+                );
+                metrics::observe_timer_vec(
+                    &metrics::VALIDATOR_MONITOR_UNAGGREGATED_ATTESTATION_DELAY_SECONDS,
+                    &[src, id],
+                    delay,
+                );
+
+                info!(
+                    self.log,
+                    "Unaggregated attestation";
+                    "head" => ?data.beacon_block_root,
+                    "index" => %data.index,
+                    "delay_ms" => %delay.as_millis(),
+                    "epoch" => %epoch,
+                    "slot" => %data.slot,
+                    "src" => src,
+                    "validator" => %id,
+                );
+
+                validator.with_epoch_summary(epoch, |summary| {
+                    summary.register_unaggregated_attestation(delay)
+                });
+            }
+        })
+    }
+
+    /// Returns the duration between when a `AggregateAndproof` with `data` could be produced (2/3rd
+    /// through the slot) and `seen_timestamp`.
+    fn get_aggregated_attestation_delay_ms<S: SlotClock>(
+        seen_timestamp: Duration,
+        data: &AttestationData,
+        slot_clock: &S,
+    ) -> Duration {
+        slot_clock
+            .start_of(data.slot)
+            .and_then(|slot_start| seen_timestamp.checked_sub(slot_start))
+            .and_then(|gross_delay| {
+                let production_delay = slot_clock.slot_duration() / 2;
+                gross_delay.checked_sub(production_delay)
+            })
+            .unwrap_or_else(|| Duration::from_secs(0))
+    }
+
+    /// Register a `signed_aggregate_and_proof` seen on the gossip network.
+    pub fn register_gossip_aggregated_attestation<S: SlotClock>(
+        &self,
+        seen_timestamp: Duration,
+        signed_aggregate_and_proof: &SignedAggregateAndProof<T>,
+        indexed_attestation: &IndexedAttestation<T>,
+        slot_clock: &S,
+    ) {
+        self.register_aggregated_attestation(
+            "gossip",
+            seen_timestamp,
+            signed_aggregate_and_proof,
+            indexed_attestation,
+            slot_clock,
+        )
+    }
+
+    /// Register a `signed_aggregate_and_proof` seen on the HTTP API.
+    pub fn register_api_aggregated_attestation<S: SlotClock>(
+        &self,
+        seen_timestamp: Duration,
+        signed_aggregate_and_proof: &SignedAggregateAndProof<T>,
+        indexed_attestation: &IndexedAttestation<T>,
+        slot_clock: &S,
+    ) {
+        self.register_aggregated_attestation(
+            "api",
+            seen_timestamp,
+            signed_aggregate_and_proof,
+            indexed_attestation,
+            slot_clock,
+        )
+    }
+
+    fn register_aggregated_attestation<S: SlotClock>(
+        &self,
+        src: &str,
+        seen_timestamp: Duration,
+        signed_aggregate_and_proof: &SignedAggregateAndProof<T>,
+        indexed_attestation: &IndexedAttestation<T>,
+        slot_clock: &S,
+    ) {
+        let data = &indexed_attestation.data;
+        let epoch = data.slot.epoch(T::slots_per_epoch());
+        let delay = Self::get_aggregated_attestation_delay_ms(seen_timestamp, data, slot_clock);
+
+        let aggregator_index = signed_aggregate_and_proof.message.aggregator_index;
+        if let Some(validator) = self.get_validator(aggregator_index) {
+            let id = &validator.id;
+
+            metrics::inc_counter_vec(
+                &metrics::VALIDATOR_MONITOR_AGGREGATED_ATTESTATION_TOTAL,
+                &[src, id],
+            );
+            metrics::observe_timer_vec(
+                &metrics::VALIDATOR_MONITOR_AGGREGATED_ATTESTATION_DELAY_SECONDS,
+                &[src, id],
+                delay,
+            );
+
+            info!(
+                self.log,
+                "Aggregated attestation";
+                "head" => ?data.beacon_block_root,
+                "index" => %data.index,
+                "delay_ms" => %delay.as_millis(),
+                "epoch" => %epoch,
+                "slot" => %data.slot,
+                "src" => src,
+                "validator" => %id,
+            );
+
+            validator.with_epoch_summary(epoch, |summary| {
+                summary.register_aggregated_attestation(delay)
+            });
+        }
+
+        indexed_attestation.attesting_indices.iter().for_each(|i| {
+            if let Some(validator) = self.get_validator(*i) {
+                let id = &validator.id;
+
+                metrics::inc_counter_vec(
+                    &metrics::VALIDATOR_MONITOR_ATTESTATION_IN_AGGREGATE_TOTAL,
+                    &[src, id],
+                );
+                metrics::observe_timer_vec(
+                    &metrics::VALIDATOR_MONITOR_ATTESTATION_IN_AGGREGATE_DELAY_SECONDS,
+                    &[src, id],
+                    delay,
+                );
+
+                info!(
+                    self.log,
+                    "Attestation included in aggregate";
+                    "head" => ?data.beacon_block_root,
+                    "index" => %data.index,
+                    "delay_ms" => %delay.as_millis(),
+                    "epoch" => %epoch,
+                    "slot" => %data.slot,
+                    "src" => src,
+                    "validator" => %id,
+                );
+
+                validator.with_epoch_summary(epoch, |summary| {
+                    summary.register_aggregate_attestation_inclusion()
+                });
+            }
+        })
+    }
+
+    /// Register that the `indexed_attestation` was included in a *valid* `BeaconBlock`.
+    pub fn register_attestation_in_block(
+        &self,
+        indexed_attestation: &IndexedAttestation<T>,
+        block: &BeaconBlock<T>,
+        spec: &ChainSpec,
+    ) {
+        let data = &indexed_attestation.data;
+        let delay = (block.slot - data.slot) - spec.min_attestation_inclusion_delay;
+        let epoch = data.slot.epoch(T::slots_per_epoch());
+
+        indexed_attestation.attesting_indices.iter().for_each(|i| {
+            if let Some(validator) = self.get_validator(*i) {
+                let id = &validator.id;
+
+                metrics::inc_counter_vec(
+                    &metrics::VALIDATOR_MONITOR_ATTESTATION_IN_BLOCK_TOTAL,
+                    &["block", id],
+                );
+                metrics::set_int_gauge(
+                    &metrics::VALIDATOR_MONITOR_ATTESTATION_IN_BLOCK_DELAY_SLOTS,
+                    &["block", id],
+                    delay.as_u64() as i64,
+                );
+
+                info!(
+                    self.log,
+                    "Attestation included in block";
+                    "head" => ?data.beacon_block_root,
+                    "index" => %data.index,
+                    "inclusion_lag" => format!("{} slot(s)", delay),
+                    "epoch" => %epoch,
+                    "slot" => %data.slot,
+                    "validator" => %id,
+                );
+
+                validator.with_epoch_summary(epoch, |summary| {
+                    summary.register_attestation_block_inclusion(delay)
+                });
+            }
+        })
+    }
+
+    /// Register an exit from the gossip network.
+    pub fn register_gossip_voluntary_exit(&self, exit: &VoluntaryExit) {
+        self.register_voluntary_exit("gossip", exit)
+    }
+
+    /// Register an exit from the HTTP API.
+    pub fn register_api_voluntary_exit(&self, exit: &VoluntaryExit) {
+        self.register_voluntary_exit("api", exit)
+    }
+
+    /// Register an exit included in a *valid* beacon block.
+    pub fn register_block_voluntary_exit(&self, exit: &VoluntaryExit) {
+        self.register_voluntary_exit("block", exit)
+    }
+
+    fn register_voluntary_exit(&self, src: &str, exit: &VoluntaryExit) {
+        if let Some(validator) = self.get_validator(exit.validator_index) {
+            let id = &validator.id;
+            let epoch = exit.epoch;
+
+            metrics::inc_counter_vec(&metrics::VALIDATOR_MONITOR_EXIT_TOTAL, &[src, id]);
+
+            info!(
+                self.log,
+                "Voluntary exit";
+                "epoch" => %epoch,
+                "validator" => %id,
+                "src" => src,
+            );
+
+            validator.with_epoch_summary(epoch, |summary| summary.register_exit());
+        }
+    }
+
+    /// Register a proposer slashing from the gossip network.
+    pub fn register_gossip_proposer_slashing(&self, slashing: &ProposerSlashing) {
+        self.register_proposer_slashing("gossip", slashing)
+    }
+
+    /// Register a proposer slashing from the HTTP API.
+    pub fn register_api_proposer_slashing(&self, slashing: &ProposerSlashing) {
+        self.register_proposer_slashing("api", slashing)
+    }
+
+    /// Register a proposer slashing included in a *valid* `BeaconBlock`.
+    pub fn register_block_proposer_slashing(&self, slashing: &ProposerSlashing) {
+        self.register_proposer_slashing("block", slashing)
+    }
+
+    fn register_proposer_slashing(&self, src: &str, slashing: &ProposerSlashing) {
+        let proposer = slashing.signed_header_1.message.proposer_index;
+        let slot = slashing.signed_header_1.message.slot;
+        let epoch = slot.epoch(T::slots_per_epoch());
+        let root_1 = slashing.signed_header_1.message.canonical_root();
+        let root_2 = slashing.signed_header_2.message.canonical_root();
+
+        if let Some(validator) = self.get_validator(proposer) {
+            let id = &validator.id;
+
+            metrics::inc_counter_vec(
+                &metrics::VALIDATOR_MONITOR_PROPOSER_SLASHING_TOTAL,
+                &[src, id],
+            );
+
+            crit!(
+                self.log,
+                "Proposer slashing";
+                "root_2" => %root_2,
+                "root_1" => %root_1,
+                "slot" => %slot,
+                "validator" => %id,
+                "src" => src,
+            );
+
+            validator.with_epoch_summary(epoch, |summary| summary.register_proposer_slashing());
+        }
+    }
+
+    /// Register an attester slashing from the gossip network.
+    pub fn register_gossip_attester_slashing(&self, slashing: &AttesterSlashing<T>) {
+        self.register_attester_slashing("gossip", slashing)
+    }
+
+    /// Register an attester slashing from the HTTP API.
+    pub fn register_api_attester_slashing(&self, slashing: &AttesterSlashing<T>) {
+        self.register_attester_slashing("api", slashing)
+    }
+
+    /// Register an attester slashing included in a *valid* `BeaconBlock`.
+    pub fn register_block_attester_slashing(&self, slashing: &AttesterSlashing<T>) {
+        self.register_attester_slashing("block", slashing)
+    }
+
+    fn register_attester_slashing(&self, src: &str, slashing: &AttesterSlashing<T>) {
+        let data = &slashing.attestation_1.data;
+        let attestation_1_indices: HashSet<u64> = slashing
+            .attestation_1
+            .attesting_indices
+            .iter()
+            .copied()
+            .collect();
+
+        slashing
+            .attestation_2
+            .attesting_indices
+            .iter()
+            .filter(|index| attestation_1_indices.contains(index))
+            .filter_map(|index| self.get_validator(*index))
+            .for_each(|validator| {
+                let id = &validator.id;
+                let epoch = data.slot.epoch(T::slots_per_epoch());
+
+                metrics::inc_counter_vec(
+                    &metrics::VALIDATOR_MONITOR_ATTESTER_SLASHING_TOTAL,
+                    &[src, id],
+                );
+
+                crit!(
+                    self.log,
+                    "Attester slashing";
+                    "epoch" => %epoch,
+                    "slot" => %data.slot,
+                    "validator" => %id,
+                    "src" => src,
+                );
+
+                validator.with_epoch_summary(epoch, |summary| summary.register_attester_slashing());
+            })
+    }
+
+    /// Scrape `self` for metrics.
+    ///
+    /// Should be called whenever Prometheus is scraping Lighthouse.
+    pub fn scrape_metrics<S: SlotClock>(&self, slot_clock: &S, spec: &ChainSpec) {
+        metrics::set_gauge(
+            &metrics::VALIDATOR_MONITOR_VALIDATORS_TOTAL,
+            self.num_validators() as i64,
+        );
+
+        if let Some(slot) = slot_clock.now() {
+            let epoch = slot.epoch(T::slots_per_epoch());
+            let slot_in_epoch = slot % T::slots_per_epoch();
+
+            // Only start to report on the current epoch once we've progressed past the point where
+            // all attestation should be included in a block.
+            //
+            // This allows us to set alarms on Grafana to detect when an attestation has been
+            // missed. If we didn't delay beyond the attestation inclusion period then we could
+            // expect some occasional false-positives on attestation misses.
+            //
+            // I have chosen 3 as an arbitrary number where we *probably* shouldn't see that many
+            // skip slots on mainnet.
+            let previous_epoch = if slot_in_epoch > spec.min_attestation_inclusion_delay + 3 {
+                epoch - 1
+            } else {
+                epoch - 2
+            };
+
+            for (_, validator) in self.validators.iter() {
+                let id = &validator.id;
+                let summaries = validator.summaries.read();
+
+                if let Some(summary) = summaries.get(&previous_epoch) {
+                    /*
+                     * Attestations
+                     */
+                    metrics::set_gauge_vec(
+                        &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATIONS_TOTAL,
+                        &[id],
+                        summary.attestations as i64,
+                    );
+                    if let Some(delay) = summary.attestation_min_delay {
+                        metrics::observe_timer_vec(
+                            &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATIONS_MIN_DELAY_SECONDS,
+                            &[id],
+                            delay,
+                        );
+                    }
+                    metrics::set_gauge_vec(
+                        &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATION_AGGREGATE_INCLUSIONS,
+                        &[id],
+                        summary.attestation_aggregate_incusions as i64,
+                    );
+                    metrics::set_gauge_vec(
+                        &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATION_BLOCK_INCLUSIONS,
+                        &[id],
+                        summary.attestation_block_inclusions as i64,
+                    );
+                    if let Some(distance) = summary.attestation_min_block_inclusion_distance {
+                        metrics::set_gauge_vec(
+                            &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATION_BLOCK_MIN_INCLUSION_DISTANCE,
+                            &[id],
+                            distance.as_u64() as i64,
+                        );
+                    }
+                    /*
+                     * Blocks
+                     */
+                    metrics::set_gauge_vec(
+                        &metrics::VALIDATOR_MONITOR_PREV_EPOCH_BEACON_BLOCKS_TOTAL,
+                        &[id],
+                        summary.blocks as i64,
+                    );
+                    if let Some(delay) = summary.block_min_delay {
+                        metrics::observe_timer_vec(
+                            &metrics::VALIDATOR_MONITOR_PREV_EPOCH_BEACON_BLOCKS_MIN_DELAY_SECONDS,
+                            &[id],
+                            delay,
+                        );
+                    }
+                    /*
+                     * Aggregates
+                     */
+                    metrics::set_gauge_vec(
+                        &metrics::VALIDATOR_MONITOR_PREV_EPOCH_AGGREGATES_TOTAL,
+                        &[id],
+                        summary.aggregates as i64,
+                    );
+                    if let Some(delay) = summary.aggregate_min_delay {
+                        metrics::observe_timer_vec(
+                            &metrics::VALIDATOR_MONITOR_PREV_EPOCH_AGGREGATES_MIN_DELAY_SECONDS,
+                            &[id],
+                            delay,
+                        );
+                    }
+                    /*
+                     * Other
+                     */
+                    metrics::set_gauge_vec(
+                        &metrics::VALIDATOR_MONITOR_PREV_EPOCH_EXITS_TOTAL,
+                        &[id],
+                        summary.exits as i64,
+                    );
+                    metrics::set_gauge_vec(
+                        &metrics::VALIDATOR_MONITOR_PREV_EPOCH_PROPOSER_SLASHINGS_TOTAL,
+                        &[id],
+                        summary.proposer_slashings as i64,
+                    );
+                    metrics::set_gauge_vec(
+                        &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTER_SLASHINGS_TOTAL,
+                        &[id],
+                        summary.attester_slashings as i64,
+                    );
+                }
+            }
+        }
+    }
+}
+
+/// Returns the duration since the unix epoch.
+pub fn timestamp_now() -> Duration {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap_or_else(|_| Duration::from_secs(0))
+}
+
+fn u64_to_i64(n: impl Into<u64>) -> i64 {
+    i64::try_from(n.into()).unwrap_or(i64::max_value())
+}

beacon_node/beacon_chain/src/validator_pubkey_cache.rs

@@ -39,13 +39,6 @@ impl ValidatorPubkeyCache {
         state: &BeaconState<T>,
         persistence_path: P,
     ) -> Result<Self, BeaconChainError> {
-        if persistence_path.as_ref().exists() {
-            return Err(BeaconChainError::ValidatorPubkeyCacheFileError(format!(
-                "Persistence file already exists: {:?}",
-                persistence_path.as_ref()
-            )));
-        }
-
         let mut cache = Self {
             persitence_file: ValidatorPubkeyCacheFile::create(persistence_path)?,
             pubkeys: vec![],
@@ -100,7 +93,7 @@ impl ValidatorPubkeyCache {
                     .map_err(BeaconChainError::InvalidValidatorPubkeyBytes)?,
             );
 
-            self.indices.insert(v.pubkey.clone(), i);
+            self.indices.insert(v.pubkey, i);
         }
 
         Ok(())
@@ -159,8 +152,9 @@ impl ValidatorPubkeyCacheFile {
     /// Creates a file for reading and writing.
     pub fn create<P: AsRef<Path>>(path: P) -> Result<Self, Error> {
         OpenOptions::new()
-            .create_new(true)
+            .create(true)
             .write(true)
+            .truncate(true)
             .open(path)
             .map(Self)
             .map_err(Error::Io)

beacon_node/beacon_chain/tests/attestation_production.rs

@@ -25,45 +25,40 @@ lazy_static! {
 #[test]
 fn produces_attestations() {
     let num_blocks_produced = MainnetEthSpec::slots_per_epoch() * 4;
+    let additional_slots_tested = MainnetEthSpec::slots_per_epoch() * 3;
 
-    let harness = BeaconChainHarness::new(
+    let harness = BeaconChainHarness::new_with_store_config(
         MainnetEthSpec,
         KEYPAIRS[..].to_vec(),
         StoreConfig::default(),
     );
 
-    // Skip past the genesis slot.
-    harness.advance_slot();
-
-    harness.extend_chain(
-        num_blocks_produced as usize,
-        BlockStrategy::OnCanonicalHead,
-        AttestationStrategy::AllValidators,
-    );
-
     let chain = &harness.chain;
 
-    let state = &harness.chain.head().expect("should get head").beacon_state;
-    assert_eq!(state.slot, num_blocks_produced, "head should have updated");
-    assert_ne!(
-        state.finalized_checkpoint.epoch, 0,
-        "head should have updated"
-    );
-
-    let current_slot = chain.slot().expect("should get slot");
-
     // Test all valid committee indices for all slots in the chain.
-    for slot in 0..=current_slot.as_u64() + MainnetEthSpec::slots_per_epoch() * 3 {
+    // for slot in 0..=current_slot.as_u64() + MainnetEthSpec::slots_per_epoch() * 3 {
+    for slot in 0..=num_blocks_produced + additional_slots_tested {
+        if slot > 0 && slot <= num_blocks_produced {
+            harness.advance_slot();
+
+            harness.extend_chain(
+                1,
+                BlockStrategy::OnCanonicalHead,
+                AttestationStrategy::AllValidators,
+            );
+        }
+
         let slot = Slot::from(slot);
-        let state = chain
+        let mut state = chain
             .state_at_slot(slot, StateSkipConfig::WithStateRoots)
             .expect("should get state");
 
-        let block_slot = if slot > current_slot {
-            current_slot
-        } else {
+        let block_slot = if slot <= num_blocks_produced {
             slot
+        } else {
+            Slot::from(num_blocks_produced)
         };
+
         let block = chain
             .block_at_slot(block_slot)
             .expect("should get block")
@@ -81,6 +76,9 @@ fn produces_attestations() {
                 .expect("should get target block root")
         };
 
+        state
+            .build_committee_cache(RelativeEpoch::Current, &harness.chain.spec)
+            .unwrap();
         let committee_cache = state
             .committee_cache(RelativeEpoch::Current)
             .expect("should get committee_cache");

beacon_node/beacon_chain/tests/attestation_verification.rs

@@ -5,17 +5,19 @@ extern crate lazy_static;
 
 use beacon_chain::{
     attestation_verification::Error as AttnError,
-    test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy, HarnessType},
+    test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType},
     BeaconChain, BeaconChainTypes,
 };
 use int_to_bytes::int_to_bytes32;
-use state_processing::per_slot_processing;
+use state_processing::{
+    per_block_processing::errors::AttestationValidationError, per_slot_processing,
+};
 use store::config::StoreConfig;
 use tree_hash::TreeHash;
 use types::{
-    test_utils::generate_deterministic_keypair, AggregateSignature, Attestation, EthSpec, Hash256,
-    Keypair, MainnetEthSpec, SecretKey, SelectionProof, SignedAggregateAndProof, SignedBeaconBlock,
-    SubnetId, Unsigned,
+    test_utils::generate_deterministic_keypair, AggregateSignature, Attestation, BeaconStateError,
+    BitList, EthSpec, Hash256, Keypair, MainnetEthSpec, SecretKey, SelectionProof,
+    SignedAggregateAndProof, SignedBeaconBlock, SubnetId, Unsigned,
 };
 
 pub type E = MainnetEthSpec;
@@ -30,7 +32,7 @@ lazy_static! {
 }
 
 /// Returns a beacon chain harness.
-fn get_harness(validator_count: usize) -> BeaconChainHarness<HarnessType<E>> {
+fn get_harness(validator_count: usize) -> BeaconChainHarness<EphemeralHarnessType<E>> {
     let harness = BeaconChainHarness::new_with_target_aggregators(
         MainnetEthSpec,
         KEYPAIRS[0..validator_count].to_vec(),
@@ -185,7 +187,6 @@ fn get_non_aggregator<T: BeaconChainTypes>(
 #[test]
 fn aggregated_gossip_verification() {
     let harness = get_harness(VALIDATOR_COUNT);
-    let chain = &harness.chain;
 
     // Extend the chain out a few epochs so we have some chain depth to play with.
     harness.extend_chain(
@@ -197,7 +198,7 @@ fn aggregated_gossip_verification() {
     // Advance into a slot where there have not been blocks or attestations produced.
     harness.advance_slot();
 
-    let current_slot = chain.slot().expect("should get slot");
+    let current_slot = harness.chain.slot().expect("should get slot");
 
     assert_eq!(
         current_slot % E::slots_per_epoch(),
@@ -275,6 +276,21 @@ fn aggregated_gossip_verification() {
             && earliest_permissible_slot == current_slot - E::slots_per_epoch() - 1
     );
 
+    /*
+     * This is not in the specification for aggregate attestations (only unaggregates), but we
+     * check it anyway to avoid weird edge cases.
+     */
+    let unknown_root = Hash256::from_low_u64_le(424242);
+    assert_invalid!(
+        "attestation with invalid target root",
+        {
+            let mut a = valid_aggregate.clone();
+            a.message.aggregate.data.target.root = unknown_root;
+            a
+        },
+        AttnError::InvalidTargetRoot { .. }
+    );
+
     /*
      * The following test ensures:
      *
@@ -533,7 +549,6 @@ fn aggregated_gossip_verification() {
 #[test]
 fn unaggregated_gossip_verification() {
     let harness = get_harness(VALIDATOR_COUNT);
-    let chain = &harness.chain;
 
     // Extend the chain out a few epochs so we have some chain depth to play with.
     harness.extend_chain(
@@ -545,8 +560,8 @@ fn unaggregated_gossip_verification() {
     // Advance into a slot where there have not been blocks or attestations produced.
     harness.advance_slot();
 
-    let current_slot = chain.slot().expect("should get slot");
-    let current_epoch = chain.epoch().expect("should get epoch");
+    let current_slot = harness.chain.slot().expect("should get slot");
+    let current_epoch = harness.chain.epoch().expect("should get epoch");
 
     assert_eq!(
         current_slot % E::slots_per_epoch(),
@@ -568,7 +583,7 @@ fn unaggregated_gossip_verification() {
                 matches!(
                     harness
                         .chain
-                        .verify_unaggregated_attestation_for_gossip($attn_getter, $subnet_getter)
+                        .verify_unaggregated_attestation_for_gossip($attn_getter, Some($subnet_getter))
                         .err()
                         .expect(&format!(
                             "{} should error during verify_unaggregated_attestation_for_gossip",
@@ -582,6 +597,31 @@ fn unaggregated_gossip_verification() {
         };
     }
 
+    /*
+     * The following test ensures:
+     *
+     * Spec v0.12.3
+     *
+     * The committee index is within the expected range -- i.e. `data.index <
+     * get_committee_count_per_slot(state, data.target.epoch)`.
+     */
+    assert_invalid!(
+        "attestation with invalid committee index",
+        {
+            let mut a = valid_attestation.clone();
+            a.data.index = harness
+                .chain
+                .head()
+                .unwrap()
+                .beacon_state
+                .get_committee_count_at_slot(a.data.slot)
+                .unwrap();
+            a
+        },
+        subnet_id,
+        AttnError::NoCommitteeForSlotAndIndex { .. }
+    );
+
     /*
      * The following test ensures:
      *
@@ -642,6 +682,7 @@ fn unaggregated_gossip_verification() {
         {
             let mut a = valid_attestation.clone();
             a.data.slot = early_slot;
+            a.data.target.epoch = early_slot.epoch(E::slots_per_epoch());
             a
         },
         subnet_id,
@@ -654,6 +695,27 @@ fn unaggregated_gossip_verification() {
         if attestation_slot == early_slot && earliest_permissible_slot == current_slot - E::slots_per_epoch() - 1
     );
 
+    /*
+     * The following test ensures:
+     *
+     * Spec v0.12.3
+     *
+     * The attestation's epoch matches its target -- i.e. `attestation.data.target.epoch ==
+     *   compute_epoch_at_slot(attestation.data.slot)`
+     *
+     */
+
+    assert_invalid!(
+        "attestation with invalid target epoch",
+        {
+            let mut a = valid_attestation.clone();
+            a.data.target.epoch += 1;
+            a
+        },
+        subnet_id,
+        AttnError::InvalidTargetEpoch { .. }
+    );
+
     /*
      * The following two tests ensure:
      *
@@ -694,6 +756,32 @@ fn unaggregated_gossip_verification() {
         AttnError::NotExactlyOneAggregationBitSet(2)
     );
 
+    /*
+     * The following test ensures:
+     *
+     * Spec v0.12.3
+     *
+     * The number of aggregation bits matches the committee size -- i.e.
+     *   `len(attestation.aggregation_bits) == len(get_beacon_committee(state, data.slot,
+     *   data.index))`.
+     */
+    assert_invalid!(
+        "attestation with invalid bitfield",
+        {
+            let mut a = valid_attestation.clone();
+            let bits = a.aggregation_bits.iter().collect::<Vec<_>>();
+            a.aggregation_bits = BitList::with_capacity(bits.len() + 1).unwrap();
+            for (i, bit) in bits.into_iter().enumerate() {
+                a.aggregation_bits.set(i, bit).unwrap();
+            }
+            a
+        },
+        subnet_id,
+        AttnError::Invalid(AttestationValidationError::BeaconStateError(
+            BeaconStateError::InvalidBitfield
+        ))
+    );
+
     /*
      * The following test ensures that:
      *
@@ -717,6 +805,26 @@ fn unaggregated_gossip_verification() {
         if beacon_block_root == unknown_root
     );
 
+    /*
+     * The following test ensures that:
+     *
+     * Spec v0.12.3
+     *
+     * The attestation's target block is an ancestor of the block named in the LMD vote
+     */
+
+    let unknown_root = Hash256::from_low_u64_le(424242);
+    assert_invalid!(
+        "attestation with invalid target root",
+        {
+            let mut a = valid_attestation.clone();
+            a.data.target.root = unknown_root;
+            a
+        },
+        subnet_id,
+        AttnError::InvalidTargetRoot { .. }
+    );
+
     /*
      * The following test ensures that:
      *
@@ -742,7 +850,7 @@ fn unaggregated_gossip_verification() {
 
     harness
         .chain
-        .verify_unaggregated_attestation_for_gossip(valid_attestation.clone(), subnet_id)
+        .verify_unaggregated_attestation_for_gossip(valid_attestation.clone(), Some(subnet_id))
         .expect("valid attestation should be verified");
 
     /*
@@ -773,7 +881,6 @@ fn unaggregated_gossip_verification() {
 #[test]
 fn attestation_that_skips_epochs() {
     let harness = get_harness(VALIDATOR_COUNT);
-    let chain = &harness.chain;
 
     // Extend the chain out a few epochs so we have some chain depth to play with.
     harness.extend_chain(
@@ -782,16 +889,18 @@ fn attestation_that_skips_epochs() {
         AttestationStrategy::SomeValidators(vec![]),
     );
 
-    let current_slot = chain.slot().expect("should get slot");
-    let current_epoch = chain.epoch().expect("should get epoch");
+    let current_slot = harness.chain.slot().expect("should get slot");
+    let current_epoch = harness.chain.epoch().expect("should get epoch");
 
     let earlier_slot = (current_epoch - 2).start_slot(MainnetEthSpec::slots_per_epoch());
-    let earlier_block = chain
+    let earlier_block = harness
+        .chain
         .block_at_slot(earlier_slot)
         .expect("should not error getting block at slot")
         .expect("should find block at slot");
 
-    let mut state = chain
+    let mut state = harness
+        .chain
         .get_state(&earlier_block.state_root(), Some(earlier_slot))
         .expect("should not error getting state")
         .expect("should find state");
@@ -830,6 +939,6 @@ fn attestation_that_skips_epochs() {
 
     harness
         .chain
-        .verify_unaggregated_attestation_for_gossip(attestation, subnet_id)
+        .verify_unaggregated_attestation_for_gossip(attestation, Some(subnet_id))
         .expect("should gossip verify attestation that skips slots");
 }

beacon_node/beacon_chain/tests/block_verification.rs

@@ -4,10 +4,13 @@
 extern crate lazy_static;
 
 use beacon_chain::{
-    test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy, HarnessType},
+    test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType},
     BeaconSnapshot, BlockError,
 };
+use slasher::{Config as SlasherConfig, Slasher};
+use std::sync::Arc;
 use store::config::StoreConfig;
+use tempfile::tempdir;
 use types::{
     test_utils::generate_deterministic_keypair, AggregateSignature, AttestationData,
     AttesterSlashing, Checkpoint, Deposit, DepositData, Epoch, EthSpec, Hash256,
@@ -18,8 +21,9 @@ use types::{
 type E = MainnetEthSpec;
 
 // Should ideally be divisible by 3.
-pub const VALIDATOR_COUNT: usize = 24;
-pub const CHAIN_SEGMENT_LENGTH: usize = 64 * 5;
+const VALIDATOR_COUNT: usize = 24;
+const CHAIN_SEGMENT_LENGTH: usize = 64 * 5;
+const BLOCK_INDICES: &[usize] = &[0, 1, 32, 64, 68 + 1, 129, CHAIN_SEGMENT_LENGTH - 1];
 
 lazy_static! {
     /// A cached set of keys.
@@ -47,8 +51,8 @@ fn get_chain_segment() -> Vec<BeaconSnapshot<E>> {
         .collect()
 }
 
-fn get_harness(validator_count: usize) -> BeaconChainHarness<HarnessType<E>> {
-    let harness = BeaconChainHarness::new(
+fn get_harness(validator_count: usize) -> BeaconChainHarness<EphemeralHarnessType<E>> {
+    let harness = BeaconChainHarness::new_with_store_config(
         MainnetEthSpec,
         KEYPAIRS[0..validator_count].to_vec(),
         StoreConfig::default(),
@@ -80,7 +84,7 @@ fn junk_aggregate_signature() -> AggregateSignature {
 
 fn update_proposal_signatures(
     snapshots: &mut [BeaconSnapshot<E>],
-    harness: &BeaconChainHarness<HarnessType<E>>,
+    harness: &BeaconChainHarness<EphemeralHarnessType<E>>,
 ) {
     for snapshot in snapshots {
         let spec = &harness.chain.spec;
@@ -90,7 +94,7 @@ fn update_proposal_signatures(
             .get_beacon_proposer_index(slot, spec)
             .expect("should find proposer index");
         let keypair = harness
-            .keypairs
+            .validator_keypairs
             .get(proposer_index)
             .expect("proposer keypair should be available");
 
@@ -272,17 +276,73 @@ fn chain_segment_non_linear_slots() {
     );
 }
 
+fn assert_invalid_signature(
+    harness: &BeaconChainHarness<EphemeralHarnessType<E>>,
+    block_index: usize,
+    snapshots: &[BeaconSnapshot<E>],
+    item: &str,
+) {
+    let blocks = snapshots
+        .iter()
+        .map(|snapshot| snapshot.beacon_block.clone())
+        .collect();
+
+    // Ensure the block will be rejected if imported in a chain segment.
+    assert!(
+        matches!(
+            harness
+                .chain
+                .process_chain_segment(blocks)
+                .into_block_error(),
+            Err(BlockError::InvalidSignature)
+        ),
+        "should not import chain segment with an invalid {} signature",
+        item
+    );
+
+    // Ensure the block will be rejected if imported on its own (without gossip checking).
+    let ancestor_blocks = CHAIN_SEGMENT
+        .iter()
+        .take(block_index)
+        .map(|snapshot| snapshot.beacon_block.clone())
+        .collect();
+    // We don't care if this fails, we just call this to ensure that all prior blocks have been
+    // imported prior to this test.
+    let _ = harness.chain.process_chain_segment(ancestor_blocks);
+    assert!(
+        matches!(
+            harness
+                .chain
+                .process_block(snapshots[block_index].beacon_block.clone()),
+            Err(BlockError::InvalidSignature)
+        ),
+        "should not import individual block with an invalid {} signature",
+        item
+    );
+
+    // NOTE: we choose not to check gossip verification here. It only checks one signature
+    // (proposal) and that is already tested elsewhere in this file.
+    //
+    // It's not trivial to just check gossip verification since it will start refusing
+    // blocks as soon as it has seen one valid proposal signature for a given (validator,
+    // slot) tuple.
+}
+
+fn get_invalid_sigs_harness() -> BeaconChainHarness<EphemeralHarnessType<E>> {
+    let harness = get_harness(VALIDATOR_COUNT);
+    harness
+        .chain
+        .slot_clock
+        .set_slot(CHAIN_SEGMENT.last().unwrap().beacon_block.slot().as_u64());
+    harness
+}
 #[test]
-fn invalid_signatures() {
-    let mut checked_attestation = false;
-
-    for &block_index in &[0, 1, 32, 64, 68 + 1, 129, CHAIN_SEGMENT.len() - 1] {
-        let harness = get_harness(VALIDATOR_COUNT);
-        harness
-            .chain
-            .slot_clock
-            .set_slot(CHAIN_SEGMENT.last().unwrap().beacon_block.slot().as_u64());
-
+fn invalid_signature_gossip_block() {
+    for &block_index in BLOCK_INDICES {
+        // Ensure the block will be rejected if imported on its own (without gossip checking).
+        let harness = get_invalid_sigs_harness();
+        let mut snapshots = CHAIN_SEGMENT.clone();
+        snapshots[block_index].beacon_block.signature = junk_signature();
         // Import all the ancestors before the `block_index` block.
         let ancestor_blocks = CHAIN_SEGMENT
             .iter()
@@ -294,75 +354,6 @@ fn invalid_signatures() {
             .process_chain_segment(ancestor_blocks)
             .into_block_error()
             .expect("should import all blocks prior to the one being tested");
-
-        // For the given snapshots, test the following:
-        //
-        // - The `process_chain_segment` function returns `InvalidSignature`.
-        // - The `process_block` function returns `InvalidSignature` when importing the
-        //    `SignedBeaconBlock` directly.
-        // - The `verify_block_for_gossip` function does _not_ return an error.
-        // - The `process_block` function returns `InvalidSignature` when verifying the
-        //    `GossipVerifiedBlock`.
-        let assert_invalid_signature = |snapshots: &[BeaconSnapshot<E>], item: &str| {
-            let blocks = snapshots
-                .iter()
-                .map(|snapshot| snapshot.beacon_block.clone())
-                .collect();
-
-            // Ensure the block will be rejected if imported in a chain segment.
-            assert!(
-                matches!(
-                    harness
-                        .chain
-                        .process_chain_segment(blocks)
-                        .into_block_error(),
-                    Err(BlockError::InvalidSignature)
-                ),
-                "should not import chain segment with an invalid {} signature",
-                item
-            );
-
-            // Ensure the block will be rejected if imported on its own (without gossip checking).
-            assert!(
-                matches!(
-                    harness
-                        .chain
-                        .process_block(snapshots[block_index].beacon_block.clone()),
-                    Err(BlockError::InvalidSignature)
-                ),
-                "should not import individual block with an invalid {} signature",
-                item
-            );
-
-            // NOTE: we choose not to check gossip verification here. It only checks one signature
-            // (proposal) and that is already tested elsewhere in this file.
-            //
-            // It's not trivial to just check gossip verification since it will start refusing
-            // blocks as soon as it has seen one valid proposal signature for a given (validator,
-            // slot) tuple.
-        };
-
-        /*
-         * Block proposal
-         */
-        let mut snapshots = CHAIN_SEGMENT.clone();
-        snapshots[block_index].beacon_block.signature = junk_signature();
-        let blocks = snapshots
-            .iter()
-            .map(|snapshot| snapshot.beacon_block.clone())
-            .collect();
-        // Ensure the block will be rejected if imported in a chain segment.
-        assert!(
-            matches!(
-                harness
-                    .chain
-                    .process_chain_segment(blocks)
-                    .into_block_error(),
-                Err(BlockError::InvalidSignature)
-            ),
-            "should not import chain segment with an invalid gossip signature",
-        );
-        // Ensure the block will be rejected if imported on its own (without gossip checking).
         assert!(
             matches!(
                 harness
@@ -372,10 +363,37 @@ fn invalid_signatures() {
             ),
             "should not import individual block with an invalid gossip signature",
         );
+    }
+}
 
-        /*
-         * Randao reveal
-         */
+#[test]
+fn invalid_signature_block_proposal() {
+    for &block_index in BLOCK_INDICES {
+        let harness = get_invalid_sigs_harness();
+        let mut snapshots = CHAIN_SEGMENT.clone();
+        snapshots[block_index].beacon_block.signature = junk_signature();
+        let blocks = snapshots
+            .iter()
+            .map(|snapshot| snapshot.beacon_block.clone())
+            .collect::<Vec<_>>();
+        // Ensure the block will be rejected if imported in a chain segment.
+        assert!(
+            matches!(
+                harness
+                    .chain
+                    .process_chain_segment(blocks)
+                    .into_block_error(),
+                Err(BlockError::InvalidSignature)
+            ),
+            "should not import chain segment with an invalid block signature",
+        );
+    }
+}
+
+#[test]
+fn invalid_signature_randao_reveal() {
+    for &block_index in BLOCK_INDICES {
+        let harness = get_invalid_sigs_harness();
         let mut snapshots = CHAIN_SEGMENT.clone();
         snapshots[block_index]
             .beacon_block
@@ -384,11 +402,14 @@ fn invalid_signatures() {
             .randao_reveal = junk_signature();
         update_parent_roots(&mut snapshots);
         update_proposal_signatures(&mut snapshots, &harness);
-        assert_invalid_signature(&snapshots, "randao");
+        assert_invalid_signature(&harness, block_index, &snapshots, "randao");
+    }
+}
 
-        /*
-         * Proposer slashing
-         */
+#[test]
+fn invalid_signature_proposer_slashing() {
+    for &block_index in BLOCK_INDICES {
+        let harness = get_invalid_sigs_harness();
         let mut snapshots = CHAIN_SEGMENT.clone();
         let proposer_slashing = ProposerSlashing {
             signed_header_1: SignedBeaconBlockHeader {
@@ -409,11 +430,14 @@ fn invalid_signatures() {
             .expect("should update proposer slashing");
         update_parent_roots(&mut snapshots);
         update_proposal_signatures(&mut snapshots, &harness);
-        assert_invalid_signature(&snapshots, "proposer slashing");
+        assert_invalid_signature(&harness, block_index, &snapshots, "proposer slashing");
+    }
+}
 
-        /*
-         * Attester slashing
-         */
+#[test]
+fn invalid_signature_attester_slashing() {
+    for &block_index in BLOCK_INDICES {
+        let harness = get_invalid_sigs_harness();
         let mut snapshots = CHAIN_SEGMENT.clone();
         let indexed_attestation = IndexedAttestation {
             attesting_indices: vec![0].into(),
@@ -445,11 +469,16 @@ fn invalid_signatures() {
             .expect("should update attester slashing");
         update_parent_roots(&mut snapshots);
         update_proposal_signatures(&mut snapshots, &harness);
-        assert_invalid_signature(&snapshots, "attester slashing");
+        assert_invalid_signature(&harness, block_index, &snapshots, "attester slashing");
+    }
+}
 
-        /*
-         * Attestation
-         */
+#[test]
+fn invalid_signature_attestation() {
+    let mut checked_attestation = false;
+
+    for &block_index in BLOCK_INDICES {
+        let harness = get_invalid_sigs_harness();
         let mut snapshots = CHAIN_SEGMENT.clone();
         if let Some(attestation) = snapshots[block_index]
             .beacon_block
@@ -461,15 +490,22 @@ fn invalid_signatures() {
             attestation.signature = junk_aggregate_signature();
             update_parent_roots(&mut snapshots);
             update_proposal_signatures(&mut snapshots, &harness);
-            assert_invalid_signature(&snapshots, "attestation");
+            assert_invalid_signature(&harness, block_index, &snapshots, "attestation");
             checked_attestation = true;
         }
+    }
 
-        /*
-         * Deposit
-         *
-         * Note: an invalid deposit signature is permitted!
-         */
+    assert!(
+        checked_attestation,
+        "the test should check an attestation signature"
+    )
+}
+
+#[test]
+fn invalid_signature_deposit() {
+    for &block_index in BLOCK_INDICES {
+        // Note: an invalid deposit signature is permitted!
+        let harness = get_invalid_sigs_harness();
         let mut snapshots = CHAIN_SEGMENT.clone();
         let deposit = Deposit {
             proof: vec![Hash256::zero(); DEPOSIT_TREE_DEPTH + 1].into(),
@@ -503,10 +539,13 @@ fn invalid_signatures() {
             ),
             "should not throw an invalid signature error for a bad deposit signature"
         );
+    }
+}
 
-        /*
-         * Voluntary exit
-         */
+#[test]
+fn invalid_signature_exit() {
+    for &block_index in BLOCK_INDICES {
+        let harness = get_invalid_sigs_harness();
         let mut snapshots = CHAIN_SEGMENT.clone();
         let epoch = snapshots[block_index].beacon_state.current_epoch();
         snapshots[block_index]
@@ -524,13 +563,8 @@ fn invalid_signatures() {
             .expect("should update deposit");
         update_parent_roots(&mut snapshots);
         update_proposal_signatures(&mut snapshots, &harness);
-        assert_invalid_signature(&snapshots, "voluntary exit");
+        assert_invalid_signature(&harness, block_index, &snapshots, "voluntary exit");
     }
-
-    assert!(
-        checked_attestation,
-        "the test should check an attestation signature"
-    )
 }
 
 fn unwrap_err<T, E>(result: Result<T, E>) -> E {
@@ -641,6 +675,48 @@ fn block_gossip_verification() {
         "should not import a block with an invalid proposal signature"
     );
 
+    /*
+     * This test ensures that:
+     *
+     * Spec v0.12.2
+     *
+     * The block's parent (defined by block.parent_root) passes validation.
+     */
+
+    let mut block = CHAIN_SEGMENT[block_index].beacon_block.clone();
+    let parent_root = Hash256::from_low_u64_be(42);
+    block.message.parent_root = parent_root;
+    assert!(
+        matches!(
+            unwrap_err(harness.chain.verify_block_for_gossip(block)),
+            BlockError::ParentUnknown(block)
+            if block.parent_root() == parent_root
+        ),
+        "should not import a block for an unknown parent"
+    );
+
+    /*
+     * This test ensures that:
+     *
+     * Spec v0.12.2
+     *
+     * The current finalized_checkpoint is an ancestor of block -- i.e. get_ancestor(store,
+     * block.parent_root, compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)) ==
+     * store.finalized_checkpoint.root
+     */
+
+    let mut block = CHAIN_SEGMENT[block_index].beacon_block.clone();
+    let parent_root = CHAIN_SEGMENT[0].beacon_block_root;
+    block.message.parent_root = parent_root;
+    assert!(
+        matches!(
+            unwrap_err(harness.chain.verify_block_for_gossip(block)),
+            BlockError::NotFinalizedDescendant { block_parent_root }
+            if block_parent_root == parent_root
+        ),
+        "should not import a block that conflicts with finality"
+    );
+
     /*
      * This test ensures that:
      *
@@ -721,3 +797,31 @@ fn block_gossip_verification() {
         "the second proposal by this validator should be rejected"
     );
 }
+
+#[test]
+fn verify_block_for_gossip_slashing_detection() {
+    let mut harness = get_harness(VALIDATOR_COUNT);
+
+    let slasher_dir = tempdir().unwrap();
+    let slasher = Arc::new(
+        Slasher::open(
+            SlasherConfig::new(slasher_dir.path().into()),
+            harness.logger().clone(),
+        )
+        .unwrap(),
+    );
+    harness.chain.slasher = Some(slasher.clone());
+
+    let state = harness.get_current_state();
+    let (block1, _) = harness.make_block(state.clone(), Slot::new(1));
+    let (block2, _) = harness.make_block(state, Slot::new(1));
+
+    let verified_block = harness.chain.verify_block_for_gossip(block1).unwrap();
+    harness.chain.process_block(verified_block).unwrap();
+    unwrap_err(harness.chain.verify_block_for_gossip(block2));
+
+    // Slasher should have been handed the two conflicting blocks and crafted a slashing.
+    slasher.process_queued(Epoch::new(0)).unwrap();
+    let proposer_slashings = slasher.get_proposer_slashings();
+    assert_eq!(proposer_slashings.len(), 1);
+}

beacon_node/beacon_chain/tests/op_verification.rs

@@ -58,7 +58,7 @@ fn voluntary_exit() {
     let db_path = tempdir().unwrap();
     let store = get_store(&db_path);
     let harness = get_harness(store.clone(), VALIDATOR_COUNT);
-    let spec = &harness.chain.spec;
+    let spec = &harness.chain.spec.clone();
 
     harness.extend_chain(
         (E::slots_per_epoch() * (spec.shard_committee_period + 1)) as usize,

beacon_node/beacon_chain/tests/persistence_tests.rs

@@ -153,8 +153,11 @@ fn assert_chains_pretty_much_the_same<T: BeaconChainTypes>(a: &BeaconChain<T>, b
         a.genesis_block_root, b.genesis_block_root,
         "genesis_block_root should be equal"
     );
+
+    let slot = a.slot().unwrap();
     assert!(
-        *a.fork_choice.read() == *b.fork_choice.read(),
-        "fork_choice should be equal"
+        a.fork_choice.write().get_head(slot).unwrap()
+            == b.fork_choice.write().get_head(slot).unwrap(),
+        "fork_choice heads should be equal"
     );
 }

beacon_node/beacon_chain/tests/tests.rs

@@ -6,7 +6,8 @@ extern crate lazy_static;
 use beacon_chain::{
     attestation_verification::Error as AttnError,
     test_utils::{
-        AttestationStrategy, BeaconChainHarness, BlockStrategy, HarnessType, OP_POOL_DB_KEY,
+        AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType,
+        OP_POOL_DB_KEY,
     },
 };
 use operation_pool::PersistedOperationPool;
@@ -24,8 +25,8 @@ lazy_static! {
     static ref KEYPAIRS: Vec<Keypair> = types::test_utils::generate_deterministic_keypairs(VALIDATOR_COUNT);
 }
 
-fn get_harness(validator_count: usize) -> BeaconChainHarness<HarnessType<MinimalEthSpec>> {
-    let harness = BeaconChainHarness::new(
+fn get_harness(validator_count: usize) -> BeaconChainHarness<EphemeralHarnessType<MinimalEthSpec>> {
+    let harness = BeaconChainHarness::new_with_store_config(
         MinimalEthSpec,
         KEYPAIRS[0..validator_count].to_vec(),
         StoreConfig::default(),
@@ -354,11 +355,10 @@ fn roundtrip_operation_pool() {
         .persist_op_pool()
         .expect("should persist op pool");
 
-    let key = Hash256::from_slice(&OP_POOL_DB_KEY);
     let restored_op_pool = harness
         .chain
         .store
-        .get_item::<PersistedOperationPool<MinimalEthSpec>>(&key)
+        .get_item::<PersistedOperationPool<MinimalEthSpec>>(&OP_POOL_DB_KEY)
         .expect("should read db")
         .expect("should find op pool")
         .into_operation_pool();
@@ -436,8 +436,8 @@ fn attestations_with_increasing_slots() {
             AttestationStrategy::SomeValidators(vec![]),
         );
 
-        attestations.append(
-            &mut harness.get_unaggregated_attestations(
+        attestations.extend(
+            harness.get_unaggregated_attestations(
                 &AttestationStrategy::AllValidators,
                 &harness.chain.head().expect("should get head").beacon_state,
                 harness
@@ -460,7 +460,7 @@ fn attestations_with_increasing_slots() {
     for (attestation, subnet_id) in attestations.into_iter().flatten() {
         let res = harness
             .chain
-            .verify_unaggregated_attestation_for_gossip(attestation.clone(), subnet_id);
+            .verify_unaggregated_attestation_for_gossip(attestation.clone(), Some(subnet_id));
 
         let current_slot = harness.chain.slot().expect("should get slot");
         let expected_attestation_slot = attestation.data.slot;

beacon_node/client/Cargo.toml

@@ -5,7 +5,7 @@ authors = ["Sigma Prime <contact@sigmaprime.io>"]
 edition = "2018"
 
 [dev-dependencies]
-sloggers = "1.0.0"
+sloggers = "1.0.1"
 toml = "0.5.6"
 
 [dependencies]
@@ -14,30 +14,33 @@ store = { path = "../store" }
 network = { path = "../network" }
 timer = { path = "../timer" }
 eth2_libp2p = { path = "../eth2_libp2p" }
-rest_api = { path = "../rest_api" }
 parking_lot = "0.11.0"
-websocket_server = { path = "../websocket_server" }
-prometheus = "0.9.0"
+prometheus = "0.11.0"
 types = { path = "../../consensus/types" }
-tree_hash = "0.1.0"
+tree_hash = "0.1.1"
 eth2_config = { path = "../../common/eth2_config" }
 slot_clock = { path = "../../common/slot_clock" }
-serde = "1.0.110"
-serde_derive = "1.0.110"
-error-chain = "0.12.2"
-serde_yaml = "0.8.11"
+serde = "1.0.116"
+serde_derive = "1.0.116"
+error-chain = "0.12.4"
+serde_yaml = "0.8.13"
 slog = { version = "2.5.2", features = ["max_level_trace"] }
 slog-async = "2.5.0"
-tokio = "0.2.21"
-dirs = "2.0.2"
-futures = "0.3.5"
-reqwest = "0.10.4"
+tokio = "0.3.2"
+dirs = "3.0.1"
+futures = "0.3.7"
+reqwest = { version = "0.10.8", features = ["native-tls-vendored"] }
 url = "2.1.1"
 eth1 = { path = "../eth1" }
 genesis = { path = "../genesis" }
+task_executor = { path = "../../common/task_executor" }
 environment = { path = "../../lighthouse/environment" }
 eth2_ssz = "0.1.2"
 lazy_static = "1.4.0"
 lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
-time = "0.2.16"
-bus = "2.2.3"
+time = "0.2.22"
+directory = {path = "../../common/directory"}
+http_api = { path = "../http_api" }
+http_metrics = { path = "../http_metrics" }
+slasher = { path = "../../slasher" }
+slasher_service = { path = "../../slasher/service" }

beacon_node/client/src/builder.rs

@@ -1,36 +1,29 @@
 use crate::config::{ClientGenesis, Config as ClientConfig};
 use crate::notifier::spawn_notifier;
 use crate::Client;
-use beacon_chain::events::TeeEventHandler;
 use beacon_chain::{
     builder::{BeaconChainBuilder, Witness},
     eth1_chain::{CachingEth1Backend, Eth1Chain},
-    migrate::{BackgroundMigrator, Migrate},
     slot_clock::{SlotClock, SystemTimeSlotClock},
     store::{HotColdDB, ItemStore, LevelDB, StoreConfig},
-    BeaconChain, BeaconChainTypes, Eth1ChainBackend, EventHandler,
+    BeaconChain, BeaconChainTypes, Eth1ChainBackend, ServerSentEventHandler,
 };
-use bus::Bus;
 use environment::RuntimeContext;
 use eth1::{Config as Eth1Config, Service as Eth1Service};
-use eth2_config::Eth2Config;
 use eth2_libp2p::NetworkGlobals;
 use genesis::{interop_genesis_state, Eth1GenesisService};
 use network::{NetworkConfig, NetworkMessage, NetworkService};
-use parking_lot::Mutex;
-use slog::info;
+use slasher::Slasher;
+use slasher_service::SlasherService;
+use slog::{debug, info, warn};
 use ssz::Decode;
-use std::net::SocketAddr;
-use std::path::Path;
+use std::net::TcpListener;
+use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::Duration;
 use timer::spawn_timer;
-use tokio::sync::mpsc::UnboundedSender;
-use types::{
-    test_utils::generate_deterministic_keypairs, BeaconState, ChainSpec, EthSpec,
-    SignedBeaconBlockHash,
-};
-use websocket_server::{Config as WebSocketConfig, WebSocketSender};
+use tokio::sync::{mpsc::UnboundedSender, oneshot};
+use types::{test_utils::generate_deterministic_keypairs, BeaconState, ChainSpec, EthSpec};
 
 /// Interval between polling the eth1 node for genesis information.
 pub const ETH1_GENESIS_UPDATE_INTERVAL_MILLIS: u64 = 7_000;
@@ -52,38 +45,27 @@ pub struct ClientBuilder<T: BeaconChainTypes> {
     slot_clock: Option<T::SlotClock>,
     #[allow(clippy::type_complexity)]
     store: Option<Arc<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>>,
-    store_migrator: Option<T::StoreMigrator>,
     runtime_context: Option<RuntimeContext<T::EthSpec>>,
     chain_spec: Option<ChainSpec>,
     beacon_chain_builder: Option<BeaconChainBuilder<T>>,
     beacon_chain: Option<Arc<BeaconChain<T>>>,
     eth1_service: Option<Eth1Service>,
-    event_handler: Option<T::EventHandler>,
     network_globals: Option<Arc<NetworkGlobals<T::EthSpec>>>,
     network_send: Option<UnboundedSender<NetworkMessage<T::EthSpec>>>,
-    http_listen_addr: Option<SocketAddr>,
-    websocket_listen_addr: Option<SocketAddr>,
+    db_path: Option<PathBuf>,
+    freezer_db_path: Option<PathBuf>,
+    http_api_config: http_api::Config,
+    http_metrics_config: http_metrics::Config,
+    slasher: Option<Arc<Slasher<T::EthSpec>>>,
     eth_spec_instance: T::EthSpec,
 }
 
-impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
-    ClientBuilder<
-        Witness<
-            TStoreMigrator,
-            TSlotClock,
-            TEth1Backend,
-            TEthSpec,
-            TEventHandler,
-            THotStore,
-            TColdStore,
-        >,
-    >
+impl<TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>
+    ClientBuilder<Witness<TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>>
 where
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore>,
     TSlotClock: SlotClock + Clone + 'static,
     TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
     TEthSpec: EthSpec + 'static,
-    TEventHandler: EventHandler<TEthSpec> + 'static,
     THotStore: ItemStore<TEthSpec> + 'static,
     TColdStore: ItemStore<TEthSpec> + 'static,
 {
@@ -94,17 +76,18 @@ where
         Self {
             slot_clock: None,
             store: None,
-            store_migrator: None,
             runtime_context: None,
             chain_spec: None,
             beacon_chain_builder: None,
             beacon_chain: None,
             eth1_service: None,
-            event_handler: None,
             network_globals: None,
             network_send: None,
-            http_listen_addr: None,
-            websocket_listen_addr: None,
+            db_path: None,
+            freezer_db_path: None,
+            http_api_config: <_>::default(),
+            http_metrics_config: <_>::default(),
+            slasher: None,
             eth_spec_instance,
         }
     }
@@ -121,6 +104,11 @@ where
         self
     }
 
+    pub fn slasher(mut self, slasher: Arc<Slasher<TEthSpec>>) -> Self {
+        self.slasher = Some(slasher);
+        self
+    }
+
     /// Initializes the `BeaconChainBuilder`. The `build_beacon_chain` method will need to be
     /// called later in order to actually instantiate the `BeaconChain`.
     pub async fn beacon_chain_builder(
@@ -129,36 +117,50 @@ where
         config: ClientConfig,
     ) -> Result<Self, String> {
         let store = self.store.clone();
-        let store_migrator = self.store_migrator.take();
         let chain_spec = self.chain_spec.clone();
         let runtime_context = self.runtime_context.clone();
         let eth_spec_instance = self.eth_spec_instance.clone();
         let data_dir = config.data_dir.clone();
         let disabled_forks = config.disabled_forks.clone();
+        let chain_config = config.chain.clone();
         let graffiti = config.graffiti;
 
-        let store =
-            store.ok_or_else(|| "beacon_chain_start_method requires a store".to_string())?;
-        let store_migrator = store_migrator
-            .ok_or_else(|| "beacon_chain_start_method requires a store migrator".to_string())?;
-        let context = runtime_context
-            .ok_or_else(|| "beacon_chain_start_method requires a runtime context".to_string())?
-            .service_context("beacon".into());
-        let spec = chain_spec
-            .ok_or_else(|| "beacon_chain_start_method requires a chain spec".to_string())?;
+        let store = store.ok_or("beacon_chain_start_method requires a store")?;
+        let runtime_context =
+            runtime_context.ok_or("beacon_chain_start_method requires a runtime context")?;
+        let context = runtime_context.service_context("beacon".into());
+        let spec = chain_spec.ok_or("beacon_chain_start_method requires a chain spec")?;
+        let event_handler = if self.http_api_config.enabled {
+            Some(ServerSentEventHandler::new(context.log().clone()))
+        } else {
+            None
+        };
 
         let builder = BeaconChainBuilder::new(eth_spec_instance)
             .logger(context.log().clone())
             .store(store)
-            .store_migrator(store_migrator)
             .data_dir(data_dir)
             .custom_spec(spec.clone())
+            .chain_config(chain_config)
             .disabled_forks(disabled_forks)
-            .graffiti(graffiti);
+            .graffiti(graffiti)
+            .event_handler(event_handler)
+            .monitor_validators(
+                config.validator_monitor_auto,
+                config.validator_monitor_pubkeys.clone(),
+                runtime_context
+                    .service_context("val_mon".to_string())
+                    .log()
+                    .clone(),
+            );
 
-        let chain_exists = builder
-            .store_contains_beacon_chain()
-            .unwrap_or_else(|_| false);
+        let builder = if let Some(slasher) = self.slasher.clone() {
+            builder.slasher(slasher)
+        } else {
+            builder
+        };
+
+        let chain_exists = builder.store_contains_beacon_chain().unwrap_or(false);
 
         // If the client is expect to resume but there's no beacon chain in the database,
         // use the `DepositContract` method. This scenario is quite common when the client
@@ -202,7 +204,7 @@ where
                 info!(
                     context.log(),
                     "Waiting for eth2 genesis from eth1";
-                    "eth1_endpoint" => &config.eth1.endpoint,
+                    "eth1_endpoints" => format!("{:?}", &config.eth1.endpoints),
                     "contract_deploy_block" => config.eth1.deposit_contract_deploy_block,
                     "deposit_contract" => &config.eth1.deposit_contract_address
                 );
@@ -213,6 +215,52 @@ where
                     context.eth2_config().spec.clone(),
                 );
 
+                // If the HTTP API server is enabled, start an instance of it where it only
+                // contains a reference to the eth1 service (all non-eth1 endpoints will fail
+                // gracefully).
+                //
+                // Later in this function we will shutdown this temporary "waiting for genesis"
+                // server so the real one can be started later.
+                let (exit_tx, exit_rx) = oneshot::channel::<()>();
+                let http_listen_opt = if self.http_api_config.enabled {
+                    #[allow(clippy::type_complexity)]
+                    let ctx: Arc<
+                        http_api::Context<
+                            Witness<TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>,
+                        >,
+                    > = Arc::new(http_api::Context {
+                        config: self.http_api_config.clone(),
+                        chain: None,
+                        network_tx: None,
+                        network_globals: None,
+                        eth1_service: Some(genesis_service.eth1_service.clone()),
+                        log: context.log().clone(),
+                    });
+
+                    // Discard the error from the oneshot.
+                    let exit_future = async {
+                        let _ = exit_rx.await;
+                    };
+
+                    let (listen_addr, server) = http_api::serve(ctx, exit_future)
+                        .map_err(|e| format!("Unable to start HTTP API server: {:?}", e))?;
+
+                    let log_clone = context.log().clone();
+                    let http_api_task = async move {
+                        server.await;
+                        debug!(log_clone, "HTTP API server task ended");
+                    };
+
+                    context
+                        .clone()
+                        .executor
+                        .spawn_without_exit(http_api_task, "http-api");
+
+                    Some(listen_addr)
+                } else {
+                    None
+                };
+
                 let genesis_state = genesis_service
                     .wait_for_genesis_state(
                         Duration::from_millis(ETH1_GENESIS_UPDATE_INTERVAL_MILLIS),
@@ -220,6 +268,22 @@ where
                     )
                     .await?;
 
+                let _ = exit_tx.send(());
+
+                if let Some(http_listen) = http_listen_opt {
+                    // This is a bit of a hack to ensure that the HTTP server has indeed shutdown.
+                    //
+                    // We will restart it again after we've finished setting up for genesis.
+                    while TcpListener::bind(http_listen).is_err() {
+                        warn!(
+                            context.log(),
+                            "Waiting for HTTP server port to open";
+                            "port" => http_listen
+                        );
+                        tokio::time::sleep(Duration::from_secs(1)).await;
+                    }
+                }
+
                 builder
                     .genesis_state(genesis_state)
                     .map(|v| (v, Some(genesis_service.into_core_service())))?
@@ -232,20 +296,21 @@ where
         Ok(self)
     }
 
-    /// Immediately starts the networking stack.
-    pub fn network(mut self, config: &NetworkConfig) -> Result<Self, String> {
+    /// Starts the networking stack.
+    pub async fn network(mut self, config: &NetworkConfig) -> Result<Self, String> {
         let beacon_chain = self
             .beacon_chain
             .clone()
-            .ok_or_else(|| "network requires a beacon chain")?;
+            .ok_or("network requires a beacon chain")?;
         let context = self
             .runtime_context
             .as_ref()
-            .ok_or_else(|| "network requires a runtime_context")?
+            .ok_or("network requires a runtime_context")?
             .clone();
 
         let (network_globals, network_send) =
             NetworkService::start(beacon_chain, config, context.executor)
+                .await
                 .map_err(|e| format!("Failed to start network: {:?}", e))?;
 
         self.network_globals = Some(network_globals);
@@ -259,73 +324,54 @@ where
         let context = self
             .runtime_context
             .as_ref()
-            .ok_or_else(|| "node timer requires a runtime_context")?
+            .ok_or("node timer requires a runtime_context")?
             .service_context("node_timer".into());
         let beacon_chain = self
             .beacon_chain
             .clone()
-            .ok_or_else(|| "node timer requires a beacon chain")?;
-        let milliseconds_per_slot = self
+            .ok_or("node timer requires a beacon chain")?;
+        let seconds_per_slot = self
             .chain_spec
             .as_ref()
-            .ok_or_else(|| "node timer requires a chain spec".to_string())?
-            .milliseconds_per_slot;
+            .ok_or("node timer requires a chain spec")?
+            .seconds_per_slot;
 
-        spawn_timer(context.executor, beacon_chain, milliseconds_per_slot)
+        spawn_timer(context.executor, beacon_chain, seconds_per_slot)
             .map_err(|e| format!("Unable to start node timer: {}", e))?;
 
         Ok(self)
     }
 
-    /// Immediately starts the beacon node REST API http server.
-    pub fn http_server(
-        mut self,
-        client_config: &ClientConfig,
-        eth2_config: &Eth2Config,
-        events: Arc<Mutex<Bus<SignedBeaconBlockHash>>>,
-    ) -> Result<Self, String> {
+    /// Provides configuration for the HTTP API.
+    pub fn http_api_config(mut self, config: http_api::Config) -> Self {
+        self.http_api_config = config;
+        self
+    }
+
+    /// Provides configuration for the HTTP server that serves Prometheus metrics.
+    pub fn http_metrics_config(mut self, config: http_metrics::Config) -> Self {
+        self.http_metrics_config = config;
+        self
+    }
+
+    /// Immediately start the slasher service.
+    ///
+    /// Error if no slasher is configured.
+    pub fn start_slasher_service(&self) -> Result<(), String> {
         let beacon_chain = self
             .beacon_chain
             .clone()
-            .ok_or_else(|| "http_server requires a beacon chain")?;
-        let context = self
-            .runtime_context
-            .as_ref()
-            .ok_or_else(|| "http_server requires a runtime_context")?
-            .service_context("http".into());
-        let network_globals = self
-            .network_globals
-            .clone()
-            .ok_or_else(|| "http_server requires a libp2p network")?;
+            .ok_or("slasher service requires a beacon chain")?;
         let network_send = self
             .network_send
             .clone()
-            .ok_or_else(|| "http_server requires a libp2p network sender")?;
-
-        let network_info = rest_api::NetworkInfo {
-            network_globals,
-            network_chan: network_send,
-        };
-
-        let listening_addr = rest_api::start_server(
-            context.executor,
-            &client_config.rest_api,
-            beacon_chain,
-            network_info,
-            client_config
-                .create_db_path()
-                .map_err(|_| "unable to read data dir")?,
-            client_config
-                .create_freezer_db_path()
-                .map_err(|_| "unable to read freezer DB dir")?,
-            eth2_config.clone(),
-            events,
-        )
-        .map_err(|e| format!("Failed to start HTTP API: {:?}", e))?;
-
-        self.http_listen_addr = Some(listening_addr);
-
-        Ok(self)
+            .ok_or("slasher service requires a network sender")?;
+        let context = self
+            .runtime_context
+            .as_ref()
+            .ok_or("slasher requires a runtime_context")?
+            .service_context("slasher_service_ctxt".into());
+        SlasherService::new(beacon_chain, network_send).run(&context.executor)
     }
 
     /// Immediately starts the service that periodically logs information each slot.
@@ -333,27 +379,27 @@ where
         let context = self
             .runtime_context
             .as_ref()
-            .ok_or_else(|| "slot_notifier requires a runtime_context")?
+            .ok_or("slot_notifier requires a runtime_context")?
             .service_context("slot_notifier".into());
         let beacon_chain = self
             .beacon_chain
             .clone()
-            .ok_or_else(|| "slot_notifier requires a beacon chain")?;
+            .ok_or("slot_notifier requires a beacon chain")?;
         let network_globals = self
             .network_globals
             .clone()
-            .ok_or_else(|| "slot_notifier requires a libp2p network")?;
-        let milliseconds_per_slot = self
+            .ok_or("slot_notifier requires a libp2p network")?;
+        let seconds_per_slot = self
             .chain_spec
             .as_ref()
-            .ok_or_else(|| "slot_notifier requires a chain spec".to_string())?
-            .milliseconds_per_slot;
+            .ok_or("slot_notifier requires a chain spec")?
+            .seconds_per_slot;
 
         spawn_notifier(
             context.executor,
             beacon_chain,
             network_globals,
-            milliseconds_per_slot,
+            seconds_per_slot,
         )
         .map_err(|e| format!("Unable to start slot notifier: {}", e))?;
 
@@ -364,141 +410,129 @@ where
     /// specified.
     ///
     /// If type inference errors are being raised, see the comment on the definition of `Self`.
+    #[allow(clippy::type_complexity)]
     pub fn build(
         self,
-    ) -> Client<
-        Witness<
-            TStoreMigrator,
-            TSlotClock,
-            TEth1Backend,
-            TEthSpec,
-            TEventHandler,
-            THotStore,
-            TColdStore,
-        >,
-    > {
-        Client {
+    ) -> Result<Client<Witness<TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>>, String>
+    {
+        let runtime_context = self
+            .runtime_context
+            .as_ref()
+            .ok_or("build requires a runtime context")?;
+        let log = runtime_context.log().clone();
+
+        let http_api_listen_addr = if self.http_api_config.enabled {
+            let ctx = Arc::new(http_api::Context {
+                config: self.http_api_config.clone(),
+                chain: self.beacon_chain.clone(),
+                network_tx: self.network_send.clone(),
+                network_globals: self.network_globals.clone(),
+                eth1_service: self.eth1_service.clone(),
+                log: log.clone(),
+            });
+
+            let exit = runtime_context.executor.exit();
+
+            let (listen_addr, server) = http_api::serve(ctx, exit)
+                .map_err(|e| format!("Unable to start HTTP API server: {:?}", e))?;
+
+            let http_log = runtime_context.log().clone();
+            let http_api_task = async move {
+                server.await;
+                debug!(http_log, "HTTP API server task ended");
+            };
+
+            runtime_context
+                .clone()
+                .executor
+                .spawn_without_exit(http_api_task, "http-api");
+
+            Some(listen_addr)
+        } else {
+            info!(log, "HTTP server is disabled");
+            None
+        };
+
+        let http_metrics_listen_addr = if self.http_metrics_config.enabled {
+            let ctx = Arc::new(http_metrics::Context {
+                config: self.http_metrics_config.clone(),
+                chain: self.beacon_chain.clone(),
+                db_path: self.db_path.clone(),
+                freezer_db_path: self.freezer_db_path.clone(),
+                log: log.clone(),
+            });
+
+            let exit = runtime_context.executor.exit();
+
+            let (listen_addr, server) = http_metrics::serve(ctx, exit)
+                .map_err(|e| format!("Unable to start HTTP metrics server: {:?}", e))?;
+
+            runtime_context
+                .executor
+                .spawn_without_exit(async move { server.await }, "http-metrics");
+
+            Some(listen_addr)
+        } else {
+            debug!(log, "Metrics server is disabled");
+            None
+        };
+
+        if self.slasher.is_some() {
+            self.start_slasher_service()?;
+        }
+
+        Ok(Client {
             beacon_chain: self.beacon_chain,
             network_globals: self.network_globals,
-            http_listen_addr: self.http_listen_addr,
-            websocket_listen_addr: self.websocket_listen_addr,
-        }
+            http_api_listen_addr,
+            http_metrics_listen_addr,
+        })
     }
 }
 
-impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
-    ClientBuilder<
-        Witness<
-            TStoreMigrator,
-            TSlotClock,
-            TEth1Backend,
-            TEthSpec,
-            TEventHandler,
-            THotStore,
-            TColdStore,
-        >,
-    >
+impl<TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>
+    ClientBuilder<Witness<TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>>
 where
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore>,
     TSlotClock: SlotClock + Clone + 'static,
     TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
     TEthSpec: EthSpec + 'static,
-    TEventHandler: EventHandler<TEthSpec> + 'static,
     THotStore: ItemStore<TEthSpec> + 'static,
     TColdStore: ItemStore<TEthSpec> + 'static,
 {
     /// Consumes the internal `BeaconChainBuilder`, attaching the resulting `BeaconChain` to self.
     pub fn build_beacon_chain(mut self) -> Result<Self, String> {
+        let context = self
+            .runtime_context
+            .as_ref()
+            .ok_or("beacon_chain requires a runtime context")?
+            .clone();
+
         let chain = self
             .beacon_chain_builder
-            .ok_or_else(|| "beacon_chain requires a beacon_chain_builder")?
-            .event_handler(
-                self.event_handler
-                    .ok_or_else(|| "beacon_chain requires an event handler")?,
-            )
+            .ok_or("beacon_chain requires a beacon_chain_builder")?
             .slot_clock(
                 self.slot_clock
                     .clone()
-                    .ok_or_else(|| "beacon_chain requires a slot clock")?,
+                    .ok_or("beacon_chain requires a slot clock")?,
             )
+            .shutdown_sender(context.executor.shutdown_sender())
             .build()
             .map_err(|e| format!("Failed to build beacon chain: {}", e))?;
 
         self.beacon_chain = Some(Arc::new(chain));
         self.beacon_chain_builder = None;
-        self.event_handler = None;
 
         // a beacon chain requires a timer
         self.timer()
     }
 }
 
-impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>
-    ClientBuilder<
-        Witness<
-            TStoreMigrator,
-            TSlotClock,
-            TEth1Backend,
-            TEthSpec,
-            TeeEventHandler<TEthSpec>,
-            THotStore,
-            TColdStore,
-        >,
-    >
+impl<TSlotClock, TEth1Backend, TEthSpec>
+    ClientBuilder<Witness<TSlotClock, TEth1Backend, TEthSpec, LevelDB<TEthSpec>, LevelDB<TEthSpec>>>
 where
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore>,
     TSlotClock: SlotClock + 'static,
     TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
     TEthSpec: EthSpec + 'static,
-    THotStore: ItemStore<TEthSpec> + 'static,
-    TColdStore: ItemStore<TEthSpec> + 'static,
-{
-    #[allow(clippy::type_complexity)]
-    /// Specifies that the `BeaconChain` should publish events using the WebSocket server.
-    pub fn tee_event_handler(
-        mut self,
-        config: WebSocketConfig,
-    ) -> Result<(Self, Arc<Mutex<Bus<SignedBeaconBlockHash>>>), String> {
-        let context = self
-            .runtime_context
-            .as_ref()
-            .ok_or_else(|| "tee_event_handler requires a runtime_context")?
-            .service_context("ws".into());
-
-        let log = context.log().clone();
-        let (sender, listening_addr): (WebSocketSender<TEthSpec>, Option<_>) = if config.enabled {
-            let (sender, listening_addr) =
-                websocket_server::start_server(context.executor, &config)?;
-            (sender, Some(listening_addr))
-        } else {
-            (WebSocketSender::dummy(), None)
-        };
-
-        self.websocket_listen_addr = listening_addr;
-        let (tee_event_handler, bus) = TeeEventHandler::new(log, sender)?;
-        self.event_handler = Some(tee_event_handler);
-        Ok((self, bus))
-    }
-}
-
-impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler>
-    ClientBuilder<
-        Witness<
-            TStoreMigrator,
-            TSlotClock,
-            TEth1Backend,
-            TEthSpec,
-            TEventHandler,
-            LevelDB<TEthSpec>,
-            LevelDB<TEthSpec>,
-        >,
-    >
-where
-    TSlotClock: SlotClock + 'static,
-    TStoreMigrator: Migrate<TEthSpec, LevelDB<TEthSpec>, LevelDB<TEthSpec>> + 'static,
-    TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
-    TEthSpec: EthSpec + 'static,
-    TEventHandler: EventHandler<TEthSpec> + 'static,
 {
     /// Specifies that the `Client` should use a `HotColdDB` database.
     pub fn disk_store(
@@ -510,12 +544,15 @@ where
         let context = self
             .runtime_context
             .as_ref()
-            .ok_or_else(|| "disk_store requires a log".to_string())?
+            .ok_or("disk_store requires a log")?
             .service_context("freezer_db".into());
         let spec = self
             .chain_spec
             .clone()
-            .ok_or_else(|| "disk_store requires a chain spec".to_string())?;
+            .ok_or("disk_store requires a chain spec")?;
+
+        self.db_path = Some(hot_path.into());
+        self.freezer_db_path = Some(cold_path.into());
 
         let store = HotColdDB::open(hot_path, cold_path, config, spec, context.log().clone())
             .map_err(|e| format!("Unable to open database: {:?}", e))?;
@@ -524,76 +561,32 @@ where
     }
 }
 
-impl<TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
+impl<TSlotClock, TEthSpec, THotStore, TColdStore>
     ClientBuilder<
-        Witness<
-            BackgroundMigrator<TEthSpec, THotStore, TColdStore>,
-            TSlotClock,
-            TEth1Backend,
-            TEthSpec,
-            TEventHandler,
-            THotStore,
-            TColdStore,
-        >,
+        Witness<TSlotClock, CachingEth1Backend<TEthSpec>, TEthSpec, THotStore, TColdStore>,
     >
 where
     TSlotClock: SlotClock + 'static,
-    TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
     TEthSpec: EthSpec + 'static,
-    TEventHandler: EventHandler<TEthSpec> + 'static,
-    THotStore: ItemStore<TEthSpec> + 'static,
-    TColdStore: ItemStore<TEthSpec> + 'static,
-{
-    pub fn background_migrator(mut self) -> Result<Self, String> {
-        let context = self
-            .runtime_context
-            .as_ref()
-            .ok_or_else(|| "disk_store requires a log".to_string())?
-            .service_context("freezer_db".into());
-        let store = self.store.clone().ok_or_else(|| {
-            "background_migrator requires the store to be initialized".to_string()
-        })?;
-        self.store_migrator = Some(BackgroundMigrator::new(store, context.log().clone()));
-        Ok(self)
-    }
-}
-
-impl<TStoreMigrator, TSlotClock, TEthSpec, TEventHandler, THotStore, TColdStore>
-    ClientBuilder<
-        Witness<
-            TStoreMigrator,
-            TSlotClock,
-            CachingEth1Backend<TEthSpec>,
-            TEthSpec,
-            TEventHandler,
-            THotStore,
-            TColdStore,
-        >,
-    >
-where
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore>,
-    TSlotClock: SlotClock + 'static,
-    TEthSpec: EthSpec + 'static,
-    TEventHandler: EventHandler<TEthSpec> + 'static,
     THotStore: ItemStore<TEthSpec> + 'static,
     TColdStore: ItemStore<TEthSpec> + 'static,
 {
     /// Specifies that the `BeaconChain` should cache eth1 blocks/logs from a remote eth1 node
     /// (e.g., Parity/Geth) and refer to that cache when collecting deposits or eth1 votes during
     /// block production.
-    pub fn caching_eth1_backend(mut self, config: Eth1Config) -> Result<Self, String> {
+    pub async fn caching_eth1_backend(mut self, config: Eth1Config) -> Result<Self, String> {
         let context = self
             .runtime_context
             .as_ref()
-            .ok_or_else(|| "caching_eth1_backend requires a runtime_context")?
+            .ok_or("caching_eth1_backend requires a runtime_context")?
             .service_context("eth1_rpc".into());
         let beacon_chain_builder = self
             .beacon_chain_builder
-            .ok_or_else(|| "caching_eth1_backend requires a beacon_chain_builder")?;
+            .ok_or("caching_eth1_backend requires a beacon_chain_builder")?;
         let spec = self
             .chain_spec
             .clone()
-            .ok_or_else(|| "caching_eth1_backend requires a chain spec".to_string())?;
+            .ok_or("caching_eth1_backend requires a chain spec")?;
 
         let backend = if let Some(eth1_service_from_genesis) = self.eth1_service {
             eth1_service_from_genesis.update_config(config)?;
@@ -609,6 +602,8 @@ where
             eth1_service_from_genesis.drop_block_cache();
 
             CachingEth1Backend::from_service(eth1_service_from_genesis)
+        } else if config.purge_cache {
+            CachingEth1Backend::new(config, context.log().clone(), spec)
         } else {
             beacon_chain_builder
                 .get_persisted_eth1_backend()?
@@ -630,7 +625,7 @@ where
                 })?
         };
 
-        self.eth1_service = None;
+        self.eth1_service = Some(backend.core.clone());
 
         // Starts the service that connects to an eth1 node and periodically updates caches.
         backend.start(context.executor);
@@ -644,7 +639,7 @@ where
     pub fn no_eth1_backend(mut self) -> Result<Self, String> {
         let beacon_chain_builder = self
             .beacon_chain_builder
-            .ok_or_else(|| "caching_eth1_backend requires a beacon_chain_builder")?;
+            .ok_or("caching_eth1_backend requires a beacon_chain_builder")?;
 
         self.beacon_chain_builder = Some(beacon_chain_builder.no_eth1_backend());
 
@@ -663,7 +658,7 @@ where
     pub fn dummy_eth1_backend(mut self) -> Result<Self, String> {
         let beacon_chain_builder = self
             .beacon_chain_builder
-            .ok_or_else(|| "caching_eth1_backend requires a beacon_chain_builder")?;
+            .ok_or("caching_eth1_backend requires a beacon_chain_builder")?;
 
         self.beacon_chain_builder = Some(beacon_chain_builder.dummy_eth1_backend()?);
 
@@ -671,23 +666,11 @@ where
     }
 }
 
-impl<TStoreMigrator, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
-    ClientBuilder<
-        Witness<
-            TStoreMigrator,
-            SystemTimeSlotClock,
-            TEth1Backend,
-            TEthSpec,
-            TEventHandler,
-            THotStore,
-            TColdStore,
-        >,
-    >
+impl<TEth1Backend, TEthSpec, THotStore, TColdStore>
+    ClientBuilder<Witness<SystemTimeSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>>
 where
-    TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore>,
     TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
     TEthSpec: EthSpec + 'static,
-    TEventHandler: EventHandler<TEthSpec> + 'static,
     THotStore: ItemStore<TEthSpec> + 'static,
     TColdStore: ItemStore<TEthSpec> + 'static,
 {
@@ -696,24 +679,21 @@ where
         let beacon_chain_builder = self
             .beacon_chain_builder
             .as_ref()
-            .ok_or_else(|| "system_time_slot_clock requires a beacon_chain_builder")?;
+            .ok_or("system_time_slot_clock requires a beacon_chain_builder")?;
 
         let genesis_time = beacon_chain_builder
-            .finalized_snapshot
-            .as_ref()
-            .ok_or_else(|| "system_time_slot_clock requires an initialized beacon state")?
-            .beacon_state
-            .genesis_time;
+            .genesis_time
+            .ok_or("system_time_slot_clock requires an initialized beacon state")?;
 
         let spec = self
             .chain_spec
             .clone()
-            .ok_or_else(|| "system_time_slot_clock requires a chain spec".to_string())?;
+            .ok_or("system_time_slot_clock requires a chain spec")?;
 
         let slot_clock = SystemTimeSlotClock::new(
             spec.genesis_slot,
             Duration::from_secs(genesis_time),
-            Duration::from_millis(spec.milliseconds_per_slot),
+            Duration::from_secs(spec.seconds_per_slot),
         );
 
         self.slot_clock = Some(slot_clock);

beacon_node/client/src/config.rs

@@ -1,13 +1,9 @@
+use directory::DEFAULT_ROOT_DIR;
 use network::NetworkConfig;
 use serde_derive::{Deserialize, Serialize};
 use std::fs;
 use std::path::PathBuf;
-use types::Graffiti;
-
-pub const DEFAULT_DATADIR: &str = ".lighthouse";
-
-/// The number initial validators when starting the `Minimal`.
-const TESTNET_SPEC_CONSTANTS: &str = "minimal";
+use types::{Graffiti, PublicKeyBytes};
 
 /// Default directory name for the freezer database under the top-level data dir.
 const DEFAULT_FREEZER_DB_DIR: &str = "freezer_db";
@@ -47,7 +43,6 @@ pub struct Config {
     /// Path where the freezer database will be located.
     pub freezer_db_path: Option<PathBuf>,
     pub log_file: PathBuf,
-    pub spec_constants: String,
     /// If true, the node will use co-ordinated junk for eth1 values.
     ///
     /// This is the method used for the 2019 client interop in Canada.
@@ -57,35 +52,44 @@ pub struct Config {
     pub disabled_forks: Vec<String>,
     /// Graffiti to be inserted everytime we create a block.
     pub graffiti: Graffiti,
+    /// When true, automatically monitor validators using the HTTP API.
+    pub validator_monitor_auto: bool,
+    /// A list of validator pubkeys to monitor.
+    pub validator_monitor_pubkeys: Vec<PublicKeyBytes>,
     #[serde(skip)]
     /// The `genesis` field is not serialized or deserialized by `serde` to ensure it is defined
     /// via the CLI at runtime, instead of from a configuration file saved to disk.
     pub genesis: ClientGenesis,
     pub store: store::StoreConfig,
     pub network: network::NetworkConfig,
-    pub rest_api: rest_api::Config,
-    pub websocket_server: websocket_server::Config,
+    pub chain: beacon_chain::ChainConfig,
     pub eth1: eth1::Config,
+    pub http_api: http_api::Config,
+    pub http_metrics: http_metrics::Config,
+    pub slasher: Option<slasher::Config>,
 }
 
 impl Default for Config {
     fn default() -> Self {
         Self {
-            data_dir: PathBuf::from(DEFAULT_DATADIR),
+            data_dir: PathBuf::from(DEFAULT_ROOT_DIR),
             db_name: "chain_db".to_string(),
             freezer_db_path: None,
             log_file: PathBuf::from(""),
             genesis: <_>::default(),
             store: <_>::default(),
             network: NetworkConfig::default(),
-            rest_api: <_>::default(),
-            websocket_server: <_>::default(),
-            spec_constants: TESTNET_SPEC_CONSTANTS.into(),
+            chain: <_>::default(),
             dummy_eth1_backend: false,
             sync_eth1_chain: false,
             eth1: <_>::default(),
             disabled_forks: Vec::new(),
             graffiti: Graffiti::default(),
+            http_api: <_>::default(),
+            http_metrics: <_>::default(),
+            slasher: None,
+            validator_monitor_auto: false,
+            validator_monitor_pubkeys: vec![],
         }
     }
 }
@@ -101,7 +105,7 @@ impl Config {
     pub fn create_db_path(&self) -> Result<PathBuf, String> {
         let db_path = self
             .get_db_path()
-            .ok_or_else(|| "Unable to locate user home directory")?;
+            .ok_or("Unable to locate user home directory")?;
         ensure_dir_exists(db_path)
     }
 
@@ -125,7 +129,7 @@ impl Config {
     pub fn create_freezer_db_path(&self) -> Result<PathBuf, String> {
         let freezer_db_path = self
             .get_freezer_db_path()
-            .ok_or_else(|| "Unable to locate user home directory")?;
+            .ok_or("Unable to locate user home directory")?;
         ensure_dir_exists(freezer_db_path)
     }
 
@@ -142,7 +146,7 @@ impl Config {
     pub fn create_data_dir(&self) -> Result<PathBuf, String> {
         let path = self
             .get_data_dir()
-            .ok_or_else(|| "Unable to locate user home directory".to_string())?;
+            .ok_or("Unable to locate user home directory")?;
         ensure_dir_exists(path)
     }
 }

beacon_node/client/src/lib.rs

@@ -23,8 +23,10 @@ pub use eth2_config::Eth2Config;
 pub struct Client<T: BeaconChainTypes> {
     beacon_chain: Option<Arc<BeaconChain<T>>>,
     network_globals: Option<Arc<NetworkGlobals<T::EthSpec>>>,
-    http_listen_addr: Option<SocketAddr>,
-    websocket_listen_addr: Option<SocketAddr>,
+    /// Listen address for the standard eth2.0 API, if the service was started.
+    http_api_listen_addr: Option<SocketAddr>,
+    /// Listen address for the HTTP server which serves Prometheus metrics.
+    http_metrics_listen_addr: Option<SocketAddr>,
 }
 
 impl<T: BeaconChainTypes> Client<T> {
@@ -33,14 +35,14 @@ impl<T: BeaconChainTypes> Client<T> {
         self.beacon_chain.clone()
     }
 
-    /// Returns the address of the client's HTTP API server, if it was started.
-    pub fn http_listen_addr(&self) -> Option<SocketAddr> {
-        self.http_listen_addr
+    /// Returns the address of the client's standard eth2.0 API server, if it was started.
+    pub fn http_api_listen_addr(&self) -> Option<SocketAddr> {
+        self.http_api_listen_addr
     }
 
-    /// Returns the address of the client's WebSocket API server, if it was started.
-    pub fn websocket_listen_addr(&self) -> Option<SocketAddr> {
-        self.websocket_listen_addr
+    /// Returns the address of the client's HTTP Prometheus metrics server, if it was started.
+    pub fn http_metrics_listen_addr(&self) -> Option<SocketAddr> {
+        self.http_metrics_listen_addr
     }
 
     /// Returns the port of the client's libp2p stack, if it was started.

beacon_node/client/src/notifier.rs

@@ -3,11 +3,11 @@ use beacon_chain::{BeaconChain, BeaconChainTypes};
 use eth2_libp2p::NetworkGlobals;
 use futures::prelude::*;
 use parking_lot::Mutex;
-use slog::{debug, error, info, warn};
+use slog::{debug, error, info, warn, Logger};
 use slot_clock::SlotClock;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
-use tokio::time::delay_for;
+use tokio::time::sleep;
 use types::{EthSpec, Slot};
 
 /// Create a warning log whenever the peer count is at or below this value.
@@ -22,16 +22,16 @@ const SPEEDO_OBSERVATIONS: usize = 4;
 
 /// Spawns a notifier service which periodically logs information about the node.
 pub fn spawn_notifier<T: BeaconChainTypes>(
-    executor: environment::TaskExecutor,
+    executor: task_executor::TaskExecutor,
     beacon_chain: Arc<BeaconChain<T>>,
     network: Arc<NetworkGlobals<T::EthSpec>>,
-    milliseconds_per_slot: u64,
+    seconds_per_slot: u64,
 ) -> Result<(), String> {
-    let slot_duration = Duration::from_millis(milliseconds_per_slot);
+    let slot_duration = Duration::from_secs(seconds_per_slot);
     let duration_to_next_slot = beacon_chain
         .slot_clock
         .duration_to_next_slot()
-        .ok_or_else(|| "slot_notifier unable to determine time to next slot")?;
+        .ok_or("slot_notifier unable to determine time to next slot")?;
 
     // Run this half way through each slot.
     let start_instant = tokio::time::Instant::now() + duration_to_next_slot + (slot_duration / 2);
@@ -56,7 +56,8 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
                         "peers" => peer_count_pretty(network.connected_peers()),
                         "wait_time" => estimated_time_pretty(Some(next_slot.as_secs() as f64)),
                     );
-                    delay_for(slot_duration).await;
+                    eth1_logging(&beacon_chain, &log);
+                    sleep(slot_duration).await;
                 }
                 _ => break,
             }
@@ -93,7 +94,7 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
 
             metrics::set_gauge(
                 &metrics::SYNC_SLOTS_PER_SECOND,
-                speedo.slots_per_second().unwrap_or_else(|| 0_f64) as i64,
+                speedo.slots_per_second().unwrap_or(0_f64) as i64,
             );
 
             // The next two lines take advantage of saturating subtraction on `Slot`.
@@ -122,14 +123,28 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
                     head_distance.as_u64(),
                     slot_distance_pretty(head_distance, slot_duration)
                 );
-                info!(
-                    log,
-                    "Syncing";
-                    "peers" => peer_count_pretty(connected_peer_count),
-                    "distance" => distance,
-                    "speed" => sync_speed_pretty(speedo.slots_per_second()),
-                    "est_time" => estimated_time_pretty(speedo.estimated_time_till_slot(current_slot)),
-                );
+
+                let speed = speedo.slots_per_second();
+                let display_speed = speed.map_or(false, |speed| speed != 0.0);
+
+                if display_speed {
+                    info!(
+                        log,
+                        "Syncing";
+                        "peers" => peer_count_pretty(connected_peer_count),
+                        "distance" => distance,
+                        "speed" => sync_speed_pretty(speed),
+                        "est_time" => estimated_time_pretty(speedo.estimated_time_till_slot(current_slot)),
+                    );
+                } else {
+                    info!(
+                        log,
+                        "Syncing";
+                        "peers" => peer_count_pretty(connected_peer_count),
+                        "distance" => distance,
+                        "est_time" => estimated_time_pretty(speedo.estimated_time_till_slot(current_slot)),
+                    );
+                }
             } else if sync_state.is_synced() {
                 let block_info = if current_slot > head_slot {
                     "   …  empty".to_string()
@@ -157,6 +172,8 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
                     "current_slot" => current_slot,
                 );
             }
+
+            eth1_logging(&beacon_chain, &log);
         }
         Ok::<(), ()>(())
     };
@@ -167,6 +184,59 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
     Ok(())
 }
 
+fn eth1_logging<T: BeaconChainTypes>(beacon_chain: &BeaconChain<T>, log: &Logger) {
+    let current_slot_opt = beacon_chain.slot().ok();
+
+    if let Ok(head_info) = beacon_chain.head_info() {
+        // Perform some logging about the eth1 chain
+        if let Some(eth1_chain) = beacon_chain.eth1_chain.as_ref() {
+            if let Some(status) =
+                eth1_chain.sync_status(head_info.genesis_time, current_slot_opt, &beacon_chain.spec)
+            {
+                debug!(
+                    log,
+                    "Eth1 cache sync status";
+                    "eth1_head_block" => status.head_block_number,
+                    "latest_cached_block_number" => status.latest_cached_block_number,
+                    "latest_cached_timestamp" => status.latest_cached_block_timestamp,
+                    "voting_target_timestamp" => status.voting_target_timestamp,
+                    "ready" => status.lighthouse_is_cached_and_ready
+                );
+
+                if !status.lighthouse_is_cached_and_ready {
+                    let voting_target_timestamp = status.voting_target_timestamp;
+
+                    let distance = status
+                        .latest_cached_block_timestamp
+                        .map(|latest| {
+                            voting_target_timestamp.saturating_sub(latest)
+                                / beacon_chain.spec.seconds_per_eth1_block
+                        })
+                        .map(|distance| distance.to_string())
+                        .unwrap_or_else(|| "initializing deposits".to_string());
+
+                    warn!(
+                        log,
+                        "Syncing eth1 block cache";
+                        "msg" => "sync can take longer when using remote eth1 nodes",
+                        "est_blocks_remaining" => distance,
+                    );
+                }
+            } else {
+                error!(
+                    log,
+                    "Unable to determine eth1 sync status";
+                );
+            }
+        }
+    } else {
+        error!(
+            log,
+            "Unable to get head info";
+        );
+    }
+}
+
 /// Returns the peer count, returning something helpful if it's `usize::max_value` (effectively a
 /// `None` value).
 fn peer_count_pretty(peer_count: usize) -> String {
@@ -220,14 +290,37 @@ fn seconds_pretty(secs: f64) -> String {
     let hours = d.whole_hours();
     let minutes = d.whole_minutes();
 
+    let week_string = if weeks == 1 { "week" } else { "weeks" };
+    let day_string = if days == 1 { "day" } else { "days" };
+    let hour_string = if hours == 1 { "hr" } else { "hrs" };
+    let min_string = if minutes == 1 { "min" } else { "mins" };
+
     if weeks > 0 {
-        format!("{:.0} weeks {:.0} days", weeks, days % DAYS_PER_WEEK)
+        format!(
+            "{:.0} {} {:.0} {}",
+            weeks,
+            week_string,
+            days % DAYS_PER_WEEK,
+            day_string
+        )
     } else if days > 0 {
-        format!("{:.0} days {:.0} hrs", days, hours % HOURS_PER_DAY)
+        format!(
+            "{:.0} {} {:.0} {}",
+            days,
+            day_string,
+            hours % HOURS_PER_DAY,
+            hour_string
+        )
     } else if hours > 0 {
-        format!("{:.0} hrs {:.0} mins", hours, minutes % MINUTES_PER_HOUR)
+        format!(
+            "{:.0} {} {:.0} {}",
+            hours,
+            hour_string,
+            minutes % MINUTES_PER_HOUR,
+            min_string
+        )
     } else {
-        format!("{:.0} mins", minutes)
+        format!("{:.0} {}", minutes, min_string)
     }
 }
 

beacon_node/eth1/Cargo.toml

@@ -7,26 +7,30 @@ edition = "2018"
 [dev-dependencies]
 eth1_test_rig = { path = "../../testing/eth1_test_rig" }
 toml = "0.5.6"
-web3 = "0.11.0"
-sloggers = "1.0.0"
+web3 = "0.14.0"
+sloggers = "1.0.1"
+environment = { path = "../../lighthouse/environment" }
+tokio-compat-02 = "0.1"
 
 [dependencies]
-reqwest = "0.10.4"
-futures = { version = "0.3.5", features = ["compat"] }
-serde_json = "1.0.52"
-serde = { version = "1.0.110", features = ["derive"] }
+reqwest = { version = "0.10.8", features = ["native-tls-vendored"] }
+futures = "0.3.7"
+serde_json = "1.0.58"
+serde = { version = "1.0.116", features = ["derive"] }
 hex = "0.4.2"
 types = { path = "../../consensus/types"}
 merkle_proof = { path = "../../consensus/merkle_proof"}
 eth2_ssz = "0.1.2"
 eth2_ssz_derive = "0.1.0"
-tree_hash = "0.1.0"
+tree_hash = "0.1.1"
 eth2_hashing = "0.1.0"
 parking_lot = "0.11.0"
 slog = "2.5.2"
-tokio = { version = "0.2.21", features = ["full"] }
+tokio = { version = "0.3.2", features = ["full"] }
 state_processing = { path = "../../consensus/state_processing" }
-libflate = "1.0.0"
+libflate = "1.0.2"
 lighthouse_metrics = { path = "../../common/lighthouse_metrics"}
 lazy_static = "1.4.0"
-environment = { path = "../../lighthouse/environment" }
+task_executor = { path = "../../common/task_executor" }
+eth2 = { path = "../../common/eth2" }
+fallback = { path = "../../common/fallback" }

beacon_node/eth1/src/block_cache.rs

@@ -1,6 +1,7 @@
 use ssz_derive::{Decode, Encode};
 use std::ops::RangeInclusive;
-use types::{Eth1Data, Hash256};
+
+pub use eth2::lighthouse::Eth1Block;
 
 #[derive(Debug, PartialEq, Clone)]
 pub enum Error {
@@ -15,28 +16,6 @@ pub enum Error {
     Internal(String),
 }
 
-/// A block of the eth1 chain.
-///
-/// Contains all information required to add a `BlockCache` entry.
-#[derive(Debug, PartialEq, Clone, Eq, Hash, Encode, Decode)]
-pub struct Eth1Block {
-    pub hash: Hash256,
-    pub timestamp: u64,
-    pub number: u64,
-    pub deposit_root: Option<Hash256>,
-    pub deposit_count: Option<u64>,
-}
-
-impl Eth1Block {
-    pub fn eth1_data(self) -> Option<Eth1Data> {
-        Some(Eth1Data {
-            deposit_root: self.deposit_root?,
-            deposit_count: self.deposit_count?,
-            block_hash: self.hash,
-        })
-    }
-}
-
 /// Stores block and deposit contract information and provides queries based upon the block
 /// timestamp.
 #[derive(Debug, PartialEq, Clone, Default, Encode, Decode)]
@@ -55,6 +34,16 @@ impl BlockCache {
         self.blocks.is_empty()
     }
 
+    /// Returns the earliest (lowest timestamp) block, if any.
+    pub fn earliest_block(&self) -> Option<&Eth1Block> {
+        self.blocks.first()
+    }
+
+    /// Returns the latest (highest timestamp) block, if any.
+    pub fn latest_block(&self) -> Option<&Eth1Block> {
+        self.blocks.last()
+    }
+
     /// Returns the timestamp of the earliest block in the cache (if any).
     pub fn earliest_block_timestamp(&self) -> Option<u64> {
         self.blocks.first().map(|block| block.timestamp)
@@ -181,6 +170,7 @@ impl BlockCache {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use types::Hash256;
 
     fn get_block(i: u64, interval_secs: u64) -> Eth1Block {
         Eth1Block {

beacon_node/eth1/src/deposit_cache.rs

@@ -102,6 +102,12 @@ impl Default for DepositCache {
     }
 }
 
+#[derive(Debug, PartialEq)]
+pub enum DepositCacheInsertOutcome {
+    Inserted,
+    Duplicate,
+}
+
 impl DepositCache {
     /// Create new `DepositCache` given block number at which deposit
     /// contract was deployed.
@@ -146,7 +152,7 @@ impl DepositCache {
     ///
     /// - If a log with index `log.index - 1` is not already present in `self` (ignored when empty).
     /// - If a log with `log.index` is already known, but the given `log` is distinct to it.
-    pub fn insert_log(&mut self, log: DepositLog) -> Result<(), Error> {
+    pub fn insert_log(&mut self, log: DepositLog) -> Result<DepositCacheInsertOutcome, Error> {
         match log.index.cmp(&(self.logs.len() as u64)) {
             Ordering::Equal => {
                 let deposit = log.deposit_data.tree_hash_root();
@@ -156,11 +162,11 @@ impl DepositCache {
                     .push_leaf(deposit)
                     .map_err(Error::DepositTreeError)?;
                 self.deposit_roots.push(self.deposit_tree.root());
-                Ok(())
+                Ok(DepositCacheInsertOutcome::Inserted)
             }
             Ordering::Less => {
                 if self.logs[log.index as usize] == log {
-                    Ok(())
+                    Ok(DepositCacheInsertOutcome::Duplicate)
                 } else {
                     Err(Error::DuplicateDistinctLog(log.index))
                 }
@@ -304,7 +310,7 @@ pub mod tests {
             block_number: 42,
             data: EXAMPLE_LOG.to_vec(),
         };
-        DepositLog::from_log(&log, &spec).expect("should decode log")
+        log.to_deposit_log(&spec).expect("should decode log")
     }
 
     #[test]
@@ -314,7 +320,7 @@ pub mod tests {
         for i in 0..16 {
             let mut log = example_log();
             log.index = i;
-            tree.insert_log(log).expect("should add consecutive logs")
+            tree.insert_log(log).expect("should add consecutive logs");
         }
     }
 
@@ -325,13 +331,16 @@ pub mod tests {
         for i in 0..4 {
             let mut log = example_log();
             log.index = i;
-            tree.insert_log(log).expect("should add consecutive logs")
+            tree.insert_log(log).expect("should add consecutive logs");
         }
 
         // Add duplicate, when given is the same as the one known.
         let mut log = example_log();
         log.index = 3;
-        assert!(tree.insert_log(log).is_ok());
+        assert_eq!(
+            tree.insert_log(log).unwrap(),
+            DepositCacheInsertOutcome::Duplicate
+        );
 
         // Add duplicate, when given is different to the one known.
         let mut log = example_log();
@@ -355,7 +364,7 @@ pub mod tests {
             log.index = i;
             log.block_number = i;
             log.deposit_data.withdrawal_credentials = Hash256::from_low_u64_be(i);
-            tree.insert_log(log).expect("should add consecutive logs")
+            tree.insert_log(log).expect("should add consecutive logs");
         }
 
         // Get 0 deposits, with max deposit count.
@@ -432,7 +441,7 @@ pub mod tests {
             log.index = i;
             log.block_number = i;
             log.deposit_data.withdrawal_credentials = Hash256::from_low_u64_be(i);
-            tree.insert_log(log).expect("should add consecutive logs")
+            tree.insert_log(log).expect("should add consecutive logs");
         }
 
         // Range too high.

beacon_node/eth1/src/deposit_log.rs

@@ -1,11 +1,10 @@
 use super::http::Log;
 use ssz::Decode;
-use ssz_derive::{Decode, Encode};
-use state_processing::per_block_processing::signature_sets::{
-    deposit_pubkey_signature_message, deposit_signature_set,
-};
+use state_processing::per_block_processing::signature_sets::deposit_pubkey_signature_message;
 use types::{ChainSpec, DepositData, Hash256, PublicKeyBytes, SignatureBytes};
 
+pub use eth2::lighthouse::DepositLog;
+
 /// The following constants define the layout of bytes in the deposit contract `DepositEvent`. The
 /// event bytes are formatted according to the  Ethereum ABI.
 const PUBKEY_START: usize = 192;
@@ -19,38 +18,26 @@ const SIG_LEN: usize = 96;
 const INDEX_START: usize = SIG_START + 96 + 32;
 const INDEX_LEN: usize = 8;
 
-/// A fully parsed eth1 deposit contract log.
-#[derive(Debug, PartialEq, Clone, Encode, Decode)]
-pub struct DepositLog {
-    pub deposit_data: DepositData,
-    /// The block number of the log that included this `DepositData`.
-    pub block_number: u64,
-    /// The index included with the deposit log.
-    pub index: u64,
-    /// True if the signature is valid.
-    pub signature_is_valid: bool,
-}
-
-impl DepositLog {
+impl Log {
     /// Attempts to parse a raw `Log` from the deposit contract into a `DepositLog`.
-    pub fn from_log(log: &Log, spec: &ChainSpec) -> Result<Self, String> {
-        let bytes = &log.data;
+    pub fn to_deposit_log(&self, spec: &ChainSpec) -> Result<DepositLog, String> {
+        let bytes = &self.data;
 
         let pubkey = bytes
             .get(PUBKEY_START..PUBKEY_START + PUBKEY_LEN)
-            .ok_or_else(|| "Insufficient bytes for pubkey".to_string())?;
+            .ok_or("Insufficient bytes for pubkey")?;
         let withdrawal_credentials = bytes
             .get(CREDS_START..CREDS_START + CREDS_LEN)
-            .ok_or_else(|| "Insufficient bytes for withdrawal credential".to_string())?;
+            .ok_or("Insufficient bytes for withdrawal credential")?;
         let amount = bytes
             .get(AMOUNT_START..AMOUNT_START + AMOUNT_LEN)
-            .ok_or_else(|| "Insufficient bytes for amount".to_string())?;
+            .ok_or("Insufficient bytes for amount")?;
         let signature = bytes
             .get(SIG_START..SIG_START + SIG_LEN)
-            .ok_or_else(|| "Insufficient bytes for signature".to_string())?;
+            .ok_or("Insufficient bytes for signature")?;
         let index = bytes
             .get(INDEX_START..INDEX_START + INDEX_LEN)
-            .ok_or_else(|| "Insufficient bytes for index".to_string())?;
+            .ok_or("Insufficient bytes for index")?;
 
         let deposit_data = DepositData {
             pubkey: PublicKeyBytes::from_ssz_bytes(pubkey)
@@ -64,11 +51,13 @@ impl DepositLog {
         };
 
         let signature_is_valid = deposit_pubkey_signature_message(&deposit_data, spec)
-            .map_or(false, |msg| deposit_signature_set(&msg).verify());
+            .map_or(false, |(public_key, signature, msg)| {
+                signature.verify(&public_key, msg)
+            });
 
         Ok(DepositLog {
             deposit_data,
-            block_number: log.block_number,
+            block_number: self.block_number,
             index: u64::from_ssz_bytes(index).map_err(|e| format!("Invalid index ssz: {:?}", e))?,
             signature_is_valid,
         })
@@ -77,7 +66,6 @@ impl DepositLog {
 
 #[cfg(test)]
 pub mod tests {
-    use super::*;
     use crate::http::Log;
     use types::{EthSpec, MainnetEthSpec};
 
@@ -113,6 +101,7 @@ pub mod tests {
             block_number: 42,
             data: EXAMPLE_LOG.to_vec(),
         };
-        DepositLog::from_log(&log, &MainnetEthSpec::default_spec()).expect("should decode log");
+        log.to_deposit_log(&MainnetEthSpec::default_spec())
+            .expect("should decode log");
     }
 }

beacon_node/eth1/src/http.rs

@@ -12,8 +12,10 @@
 
 use futures::future::TryFutureExt;
 use reqwest::{header::CONTENT_TYPE, ClientBuilder, StatusCode};
+use serde::{Deserialize, Serialize};
 use serde_json::{json, Value};
 use std::ops::Range;
+use std::str::FromStr;
 use std::time::Duration;
 use types::Hash256;
 
@@ -30,6 +32,75 @@ pub const DEPOSIT_COUNT_RESPONSE_BYTES: usize = 96;
 /// Number of bytes in deposit contract deposit root (value only).
 pub const DEPOSIT_ROOT_BYTES: usize = 32;
 
+/// Represents an eth1 chain/network id.
+#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
+pub enum Eth1Id {
+    Goerli,
+    Mainnet,
+    Custom(u64),
+}
+
+/// Used to identify a block when querying the Eth1 node.
+#[derive(Clone, Copy)]
+pub enum BlockQuery {
+    Number(u64),
+    Latest,
+}
+
+impl Into<u64> for Eth1Id {
+    fn into(self) -> u64 {
+        match self {
+            Eth1Id::Mainnet => 1,
+            Eth1Id::Goerli => 5,
+            Eth1Id::Custom(id) => id,
+        }
+    }
+}
+
+impl From<u64> for Eth1Id {
+    fn from(id: u64) -> Self {
+        let into = |x: Eth1Id| -> u64 { x.into() };
+        match id {
+            id if id == into(Eth1Id::Mainnet) => Eth1Id::Mainnet,
+            id if id == into(Eth1Id::Goerli) => Eth1Id::Goerli,
+            id => Eth1Id::Custom(id),
+        }
+    }
+}
+
+impl FromStr for Eth1Id {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        u64::from_str_radix(s, 10)
+            .map(Into::into)
+            .map_err(|e| format!("Failed to parse eth1 network id {}", e))
+    }
+}
+
+/// Get the eth1 network id of the given endpoint.
+pub async fn get_network_id(endpoint: &str, timeout: Duration) -> Result<Eth1Id, String> {
+    let response_body = send_rpc_request(endpoint, "net_version", json!([]), timeout).await?;
+    Eth1Id::from_str(
+        response_result(&response_body)?
+            .ok_or("No result was returned for network id")?
+            .as_str()
+            .ok_or("Data was not string")?,
+    )
+}
+
+/// Get the eth1 chain id of the given endpoint.
+pub async fn get_chain_id(endpoint: &str, timeout: Duration) -> Result<Eth1Id, String> {
+    let response_body = send_rpc_request(endpoint, "eth_chainId", json!([]), timeout).await?;
+    hex_to_u64_be(
+        response_result(&response_body)?
+            .ok_or("No result was returned for chain id")?
+            .as_str()
+            .ok_or("Data was not string")?,
+    )
+    .map(Into::into)
+}
+
 #[derive(Debug, PartialEq, Clone)]
 pub struct Block {
     pub hash: Hash256,
@@ -44,9 +115,9 @@ pub async fn get_block_number(endpoint: &str, timeout: Duration) -> Result<u64,
     let response_body = send_rpc_request(endpoint, "eth_blockNumber", json!([]), timeout).await?;
     hex_to_u64_be(
         response_result(&response_body)?
-            .ok_or_else(|| "No result field was returned for block number".to_string())?
+            .ok_or("No result field was returned for block number")?
             .as_str()
-            .ok_or_else(|| "Data was not string")?,
+            .ok_or("Data was not string")?,
     )
     .map_err(|e| format!("Failed to get block number: {}", e))
 }
@@ -56,22 +127,26 @@ pub async fn get_block_number(endpoint: &str, timeout: Duration) -> Result<u64,
 /// Uses HTTP JSON RPC at `endpoint`. E.g., `http://localhost:8545`.
 pub async fn get_block(
     endpoint: &str,
-    block_number: u64,
+    query: BlockQuery,
     timeout: Duration,
 ) -> Result<Block, String> {
+    let query_param = match query {
+        BlockQuery::Number(block_number) => format!("0x{:x}", block_number),
+        BlockQuery::Latest => "latest".to_string(),
+    };
     let params = json!([
-        format!("0x{:x}", block_number),
+        query_param,
         false // do not return full tx objects.
     ]);
 
     let response_body = send_rpc_request(endpoint, "eth_getBlockByNumber", params, timeout).await?;
     let hash = hex_to_bytes(
         response_result(&response_body)?
-            .ok_or_else(|| "No result field was returned for block".to_string())?
+            .ok_or("No result field was returned for block")?
             .get("hash")
-            .ok_or_else(|| "No hash for block")?
+            .ok_or("No hash for block")?
             .as_str()
-            .ok_or_else(|| "Block hash was not string")?,
+            .ok_or("Block hash was not string")?,
     )?;
     let hash = if hash.len() == 32 {
         Ok(Hash256::from_slice(&hash))
@@ -81,20 +156,20 @@ pub async fn get_block(
 
     let timestamp = hex_to_u64_be(
         response_result(&response_body)?
-            .ok_or_else(|| "No result field was returned for timestamp".to_string())?
+            .ok_or("No result field was returned for timestamp")?
             .get("timestamp")
-            .ok_or_else(|| "No timestamp for block")?
+            .ok_or("No timestamp for block")?
             .as_str()
-            .ok_or_else(|| "Block timestamp was not string")?,
+            .ok_or("Block timestamp was not string")?,
     )?;
 
     let number = hex_to_u64_be(
         response_result(&response_body)?
-            .ok_or_else(|| "No result field was returned for number".to_string())?
+            .ok_or("No result field was returned for number")?
             .get("number")
-            .ok_or_else(|| "No number for block")?
+            .ok_or("No number for block")?
             .as_str()
-            .ok_or_else(|| "Block number was not string")?,
+            .ok_or("Block number was not string")?,
     )?;
 
     if number <= usize::max_value() as u64 {
@@ -212,7 +287,7 @@ async fn call(
             let hex = result
                 .as_str()
                 .map(|s| s.to_string())
-                .ok_or_else(|| "'result' value was not a string".to_string())?;
+                .ok_or("'result' value was not a string")?;
 
             Ok(Some(hex_to_bytes(&hex)?))
         }
@@ -247,23 +322,23 @@ pub async fn get_deposit_logs_in_range(
 
     let response_body = send_rpc_request(endpoint, "eth_getLogs", params, timeout).await?;
     response_result(&response_body)?
-        .ok_or_else(|| "No result field was returned for deposit logs".to_string())?
+        .ok_or("No result field was returned for deposit logs")?
         .as_array()
         .cloned()
-        .ok_or_else(|| "'result' value was not an array".to_string())?
+        .ok_or("'result' value was not an array")?
         .into_iter()
         .map(|value| {
             let block_number = value
                 .get("blockNumber")
-                .ok_or_else(|| "No block number field in log")?
+                .ok_or("No block number field in log")?
                 .as_str()
-                .ok_or_else(|| "Block number was not string")?;
+                .ok_or("Block number was not string")?;
 
             let data = value
                 .get("data")
-                .ok_or_else(|| "No block number field in log")?
+                .ok_or("No block number field in log")?
                 .as_str()
-                .ok_or_else(|| "Data was not string")?;
+                .ok_or("Data was not string")?;
 
             Ok(Log {
                 block_number: hex_to_u64_be(&block_number)?,
@@ -314,7 +389,7 @@ pub async fn send_rpc_request(
     let encoding = response
         .headers()
         .get(CONTENT_TYPE)
-        .ok_or_else(|| "No content-type header in response".to_string())?
+        .ok_or("No content-type header in response")?
         .to_str()
         .map(|s| s.to_string())
         .map_err(|e| format!("Failed to parse content-type header: {}", e))?;
@@ -368,8 +443,8 @@ fn hex_to_bytes(hex: &str) -> Result<Vec<u8>, String> {
 
 /// Removes the `0x` prefix from some bytes. Returns an error if the prefix is not present.
 fn strip_prefix(hex: &str) -> Result<&str, String> {
-    if hex.starts_with("0x") {
-        Ok(&hex[2..])
+    if let Some(stripped) = hex.strip_prefix("0x") {
+        Ok(stripped)
     } else {
         Err("Hex string did not start with `0x`".to_string())
     }

beacon_node/eth1/src/inner.rs

@@ -1,6 +1,6 @@
 use crate::Config;
 use crate::{
-    block_cache::BlockCache,
+    block_cache::{BlockCache, Eth1Block},
     deposit_cache::{DepositCache, SszDepositCache},
 };
 use parking_lot::RwLock;
@@ -29,6 +29,7 @@ pub struct Inner {
     pub block_cache: RwLock<BlockCache>,
     pub deposit_cache: RwLock<DepositUpdater>,
     pub config: RwLock<Config>,
+    pub remote_head_block: RwLock<Option<Eth1Block>>,
     pub spec: ChainSpec,
 }
 
@@ -86,6 +87,9 @@ impl SszEth1Cache {
                 cache: self.deposit_cache.to_deposit_cache()?,
                 last_processed_block: self.last_processed_block,
             }),
+            // Set the remote head_block zero when creating a new instance. We only care about
+            // present and future eth1 nodes.
+            remote_head_block: RwLock::new(None),
             config: RwLock::new(config),
             spec,
         })

beacon_node/eth1/src/lib.rs

@@ -13,4 +13,7 @@ pub use block_cache::{BlockCache, Eth1Block};
 pub use deposit_cache::DepositCache;
 pub use deposit_log::DepositLog;
 pub use inner::SszEth1Cache;
-pub use service::{BlockCacheUpdateOutcome, Config, DepositCacheUpdateOutcome, Error, Service};
+pub use service::{
+    BlockCacheUpdateOutcome, Config, DepositCacheUpdateOutcome, Error, Service, DEFAULT_CHAIN_ID,
+    DEFAULT_NETWORK_ID,
+};

beacon_node/eth1/src/metrics.rs

@@ -16,4 +16,14 @@ lazy_static! {
         try_create_int_gauge("eth1_deposit_cache_len", "Number of deposits in the eth1 cache");
     pub static ref HIGHEST_PROCESSED_DEPOSIT_BLOCK: Result<IntGauge> =
         try_create_int_gauge("eth1_highest_processed_deposit_block", "Number of the last block checked for deposits");
+
+    /*
+     * Eth1 endpoint errors
+     */
+    pub static ref ENDPOINT_ERRORS: Result<IntCounterVec> = try_create_int_counter_vec(
+        "eth1_endpoint_errors", "The number of eth1 request errors for each endpoint", &["endpoint"]
+    );
+    pub static ref ENDPOINT_REQUESTS: Result<IntCounterVec> = try_create_int_counter_vec(
+        "eth1_endpoint_requests", "The number of eth1 requests for each endpoint", &["endpoint"]
+    );
 }

beacon_node/eth2_libp2p/Cargo.toml

@@ -5,52 +5,53 @@ authors = ["Sigma Prime <contact@sigmaprime.io>"]
 edition = "2018"
 
 [dependencies]
-hex = "0.4.2"
+discv5 = { version = "0.1.0-beta.2", features = ["libp2p"] }
+unsigned-varint = { git = "https://github.com/sigp/unsigned-varint", branch = "dep-update", features = ["codec"] }
 types = { path =  "../../consensus/types" }
 hashset_delay = { path = "../../common/hashset_delay" }
 eth2_ssz_types = { path =  "../../consensus/ssz_types" }
-serde = { version = "1.0.110", features = ["derive"] }
-serde_derive = "1.0.110"
+serde = { version = "1.0.116", features = ["derive"] }
+serde_derive = "1.0.116"
 eth2_ssz = "0.1.2"
 eth2_ssz_derive = "0.1.0"
 slog = { version = "2.5.2", features = ["max_level_trace"] }
 lighthouse_version = { path = "../../common/lighthouse_version" }
-tokio = { version = "0.2.21", features = ["time", "macros"] }
-futures = "0.3.5"
-error-chain = "0.12.2"
-dirs = "2.0.2"
+tokio = { version = "0.3.2", features = ["time", "macros"] }
+futures = "0.3.7"
+error-chain = "0.12.4"
+dirs = "3.0.1"
 fnv = "1.0.7"
-unsigned-varint = { git = "https://github.com/sigp/unsigned-varint", branch = "latest-codecs", features = ["codec"] }
 lazy_static = "1.4.0"
 lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
-smallvec = "1.4.1"
-lru = "0.5.1"
+smallvec = "1.6.1"
+tokio-io-timeout = "0.5.0"
+lru = "0.6.0"
 parking_lot = "0.11.0"
 sha2 = "0.9.1"
-base64 = "0.12.1"
-snap = "1.0.0"
+base64 = "0.13.0"
+snap = "1.0.1"
 void = "1.0.2"
-tokio-io-timeout = "0.4.0"
-tokio-util = { version = "0.3.1", features = ["codec", "compat"] }
-discv5 = { version = "0.1.0-alpha.8", features = ["libp2p"] }
+hex = "0.4.2"
+tokio-util = { version = "0.4.0", features = ["codec", "compat", "time"] }
 tiny-keccak = "2.0.2"
-environment = { path = "../../lighthouse/environment" }
-# TODO: Remove rand crate for mainnet
+task_executor = { path = "../../common/task_executor" }
 rand = "0.7.3"
+directory = { path = "../../common/directory" }
+regex = "1.3.9"
+strum = { version = "0.20", features = ["derive"] }
 
 [dependencies.libp2p]
 #version = "0.23.0"
 git = "https://github.com/sigp/rust-libp2p"
-rev = "3096cb6b89b2883a79ce5ffcb03d41778a09b695"
+rev = "97000533e4710183124abde017c6c3d68287c1ae"
 default-features = false
 features = ["websocket", "identify", "mplex", "yamux", "noise", "gossipsub", "dns", "tcp-tokio"]
 
 [dev-dependencies]
-tokio = { version = "0.2.21", features = ["full"] }
-slog-stdlog = "4.0.0"
-slog-term = "2.5.0"
+tokio = { version = "0.3.2", features = ["full"] }
+slog-term = "2.6.0"
 slog-async = "2.5.0"
-tempdir = "0.3.7"
+tempfile = "3.1.0"
 exit-future = "0.2.0"
 
 [features]

beacon_node/eth2_libp2p/src/behaviour/gossipsub_scoring_parameters.rs

@@ -0,0 +1,342 @@
+use crate::types::{GossipEncoding, GossipKind, GossipTopic};
+use crate::{error, TopicHash};
+use libp2p::gossipsub::{
+    GossipsubConfig, IdentTopic as Topic, PeerScoreParams, PeerScoreThresholds, TopicScoreParams,
+};
+use std::cmp::max;
+use std::collections::HashMap;
+use std::marker::PhantomData;
+use std::time::Duration;
+use types::{ChainSpec, EnrForkId, EthSpec, Slot, SubnetId};
+
+const MAX_IN_MESH_SCORE: f64 = 10.0;
+const MAX_FIRST_MESSAGE_DELIVERIES_SCORE: f64 = 40.0;
+const BEACON_BLOCK_WEIGHT: f64 = 0.5;
+const BEACON_AGGREGATE_PROOF_WEIGHT: f64 = 0.5;
+const VOLUNTARY_EXIT_WEIGHT: f64 = 0.05;
+const PROPOSER_SLASHING_WEIGHT: f64 = 0.05;
+const ATTESTER_SLASHING_WEIGHT: f64 = 0.05;
+
+pub struct PeerScoreSettings<TSpec: EthSpec> {
+    slot: Duration,
+    epoch: Duration,
+
+    beacon_attestation_subnet_weight: f64,
+    max_positive_score: f64,
+
+    decay_interval: Duration,
+    decay_to_zero: f64,
+
+    mesh_n: usize,
+    max_committees_per_slot: usize,
+    target_committee_size: usize,
+    target_aggregators_per_committee: usize,
+    attestation_subnet_count: u64,
+    phantom: PhantomData<TSpec>,
+}
+
+impl<TSpec: EthSpec> PeerScoreSettings<TSpec> {
+    pub fn new(chain_spec: &ChainSpec, gs_config: &GossipsubConfig) -> PeerScoreSettings<TSpec> {
+        let slot = Duration::from_secs(chain_spec.seconds_per_slot);
+        let beacon_attestation_subnet_weight = 1.0 / chain_spec.attestation_subnet_count as f64;
+        let max_positive_score = (MAX_IN_MESH_SCORE + MAX_FIRST_MESSAGE_DELIVERIES_SCORE)
+            * (BEACON_BLOCK_WEIGHT
+                + BEACON_AGGREGATE_PROOF_WEIGHT
+                + beacon_attestation_subnet_weight * chain_spec.attestation_subnet_count as f64
+                + VOLUNTARY_EXIT_WEIGHT
+                + PROPOSER_SLASHING_WEIGHT
+                + ATTESTER_SLASHING_WEIGHT);
+
+        PeerScoreSettings {
+            slot,
+            epoch: slot * TSpec::slots_per_epoch() as u32,
+            beacon_attestation_subnet_weight,
+            max_positive_score,
+            decay_interval: max(Duration::from_secs(1), slot),
+            decay_to_zero: 0.01,
+            mesh_n: gs_config.mesh_n(),
+            max_committees_per_slot: chain_spec.max_committees_per_slot,
+            target_committee_size: chain_spec.target_committee_size,
+            target_aggregators_per_committee: chain_spec.target_aggregators_per_committee as usize,
+            attestation_subnet_count: chain_spec.attestation_subnet_count,
+            phantom: PhantomData,
+        }
+    }
+
+    pub fn get_peer_score_params(
+        &self,
+        active_validators: usize,
+        thresholds: &PeerScoreThresholds,
+        enr_fork_id: &EnrForkId,
+        current_slot: Slot,
+    ) -> error::Result<PeerScoreParams> {
+        let mut params = PeerScoreParams {
+            decay_interval: self.decay_interval,
+            decay_to_zero: self.decay_to_zero,
+            retain_score: self.epoch * 100,
+            app_specific_weight: 1.0,
+            ip_colocation_factor_threshold: 3.0,
+            behaviour_penalty_threshold: 6.0,
+            behaviour_penalty_decay: self.score_parameter_decay(self.epoch * 10),
+            ..Default::default()
+        };
+
+        let target_value = Self::decay_convergence(
+            params.behaviour_penalty_decay,
+            10.0 / TSpec::slots_per_epoch() as f64,
+        ) - params.behaviour_penalty_threshold;
+        params.behaviour_penalty_weight = thresholds.gossip_threshold / target_value.powi(2);
+
+        params.topic_score_cap = self.max_positive_score * 0.5;
+        params.ip_colocation_factor_weight = -params.topic_score_cap;
+
+        params.topics = HashMap::new();
+
+        let get_hash = |kind: GossipKind| -> TopicHash {
+            let topic: Topic =
+                GossipTopic::new(kind, GossipEncoding::default(), enr_fork_id.fork_digest).into();
+            topic.hash()
+        };
+
+        //first all fixed topics
+        params.topics.insert(
+            get_hash(GossipKind::VoluntaryExit),
+            Self::get_topic_params(
+                self,
+                VOLUNTARY_EXIT_WEIGHT,
+                4.0 / TSpec::slots_per_epoch() as f64,
+                self.epoch * 100,
+                None,
+            ),
+        );
+        params.topics.insert(
+            get_hash(GossipKind::AttesterSlashing),
+            Self::get_topic_params(
+                self,
+                ATTESTER_SLASHING_WEIGHT,
+                1.0 / 5.0 / TSpec::slots_per_epoch() as f64,
+                self.epoch * 100,
+                None,
+            ),
+        );
+        params.topics.insert(
+            get_hash(GossipKind::ProposerSlashing),
+            Self::get_topic_params(
+                self,
+                PROPOSER_SLASHING_WEIGHT,
+                1.0 / 5.0 / TSpec::slots_per_epoch() as f64,
+                self.epoch * 100,
+                None,
+            ),
+        );
+
+        //dynamic topics
+        let (beacon_block_params, beacon_aggregate_proof_params, beacon_attestation_subnet_params) =
+            self.get_dynamic_topic_params(active_validators, current_slot)?;
+
+        params
+            .topics
+            .insert(get_hash(GossipKind::BeaconBlock), beacon_block_params);
+
+        params.topics.insert(
+            get_hash(GossipKind::BeaconAggregateAndProof),
+            beacon_aggregate_proof_params,
+        );
+
+        for i in 0..self.attestation_subnet_count {
+            params.topics.insert(
+                get_hash(GossipKind::Attestation(SubnetId::new(i))),
+                beacon_attestation_subnet_params.clone(),
+            );
+        }
+
+        Ok(params)
+    }
+
+    pub fn get_dynamic_topic_params(
+        &self,
+        active_validators: usize,
+        current_slot: Slot,
+    ) -> error::Result<(TopicScoreParams, TopicScoreParams, TopicScoreParams)> {
+        let (aggregators_per_slot, committees_per_slot) =
+            self.expected_aggregator_count_per_slot(active_validators)?;
+        let multiple_bursts_per_subnet_per_epoch = committees_per_slot as u64
+            >= 2 * self.attestation_subnet_count / TSpec::slots_per_epoch();
+
+        let beacon_block_params = Self::get_topic_params(
+            self,
+            BEACON_BLOCK_WEIGHT,
+            1.0,
+            self.epoch * 20,
+            Some((TSpec::slots_per_epoch() * 5, 3.0, self.epoch, current_slot)),
+        );
+
+        let beacon_aggregate_proof_params = Self::get_topic_params(
+            self,
+            BEACON_AGGREGATE_PROOF_WEIGHT,
+            aggregators_per_slot,
+            self.epoch,
+            Some((TSpec::slots_per_epoch() * 2, 4.0, self.epoch, current_slot)),
+        );
+        let beacon_attestation_subnet_params = Self::get_topic_params(
+            self,
+            self.beacon_attestation_subnet_weight,
+            active_validators as f64
+                / self.attestation_subnet_count as f64
+                / TSpec::slots_per_epoch() as f64,
+            self.epoch
+                * (if multiple_bursts_per_subnet_per_epoch {
+                    1
+                } else {
+                    4
+                }),
+            Some((
+                TSpec::slots_per_epoch()
+                    * (if multiple_bursts_per_subnet_per_epoch {
+                        4
+                    } else {
+                        16
+                    }),
+                16.0,
+                if multiple_bursts_per_subnet_per_epoch {
+                    self.slot * (TSpec::slots_per_epoch() as u32 / 2 + 1)
+                } else {
+                    self.epoch * 3
+                },
+                current_slot,
+            )),
+        );
+
+        Ok((
+            beacon_block_params,
+            beacon_aggregate_proof_params,
+            beacon_attestation_subnet_params,
+        ))
+    }
+
+    pub fn attestation_subnet_count(&self) -> u64 {
+        self.attestation_subnet_count
+    }
+
+    fn score_parameter_decay_with_base(
+        decay_time: Duration,
+        decay_interval: Duration,
+        decay_to_zero: f64,
+    ) -> f64 {
+        let ticks = decay_time.as_secs_f64() / decay_interval.as_secs_f64();
+        decay_to_zero.powf(1.0 / ticks)
+    }
+
+    fn decay_convergence(decay: f64, rate: f64) -> f64 {
+        rate / (1.0 - decay)
+    }
+
+    fn threshold(decay: f64, rate: f64) -> f64 {
+        Self::decay_convergence(decay, rate) * decay
+    }
+
+    fn expected_aggregator_count_per_slot(
+        &self,
+        active_validators: usize,
+    ) -> error::Result<(f64, usize)> {
+        let committees_per_slot = TSpec::get_committee_count_per_slot_with(
+            active_validators,
+            self.max_committees_per_slot,
+            self.target_committee_size,
+        )
+        .map_err(|e| format!("Could not get committee count from spec: {:?}", e))?;
+
+        let committees = committees_per_slot * TSpec::slots_per_epoch() as usize;
+
+        let smaller_committee_size = active_validators / committees;
+        let num_larger_committees = active_validators - smaller_committee_size * committees;
+
+        let modulo_smaller = max(
+            1,
+            smaller_committee_size / self.target_aggregators_per_committee as usize,
+        );
+        let modulo_larger = max(
+            1,
+            (smaller_committee_size + 1) / self.target_aggregators_per_committee as usize,
+        );
+
+        Ok((
+            (((committees - num_larger_committees) * smaller_committee_size) as f64
+                / modulo_smaller as f64
+                + (num_larger_committees * (smaller_committee_size + 1)) as f64
+                    / modulo_larger as f64)
+                / TSpec::slots_per_epoch() as f64,
+            committees_per_slot,
+        ))
+    }
+
+    fn score_parameter_decay(&self, decay_time: Duration) -> f64 {
+        Self::score_parameter_decay_with_base(decay_time, self.decay_interval, self.decay_to_zero)
+    }
+
+    fn get_topic_params(
+        &self,
+        topic_weight: f64,
+        expected_message_rate: f64,
+        first_message_decay_time: Duration,
+        // decay slots (decay time in slots), cap factor, activation window, current slot
+        mesh_message_info: Option<(u64, f64, Duration, Slot)>,
+    ) -> TopicScoreParams {
+        let mut t_params = TopicScoreParams::default();
+
+        t_params.topic_weight = topic_weight;
+
+        t_params.time_in_mesh_quantum = self.slot;
+        t_params.time_in_mesh_cap = 3600.0 / t_params.time_in_mesh_quantum.as_secs_f64();
+        t_params.time_in_mesh_weight = 10.0 / t_params.time_in_mesh_cap;
+
+        t_params.first_message_deliveries_decay =
+            self.score_parameter_decay(first_message_decay_time);
+        t_params.first_message_deliveries_cap = Self::decay_convergence(
+            t_params.first_message_deliveries_decay,
+            2.0 * expected_message_rate / self.mesh_n as f64,
+        );
+        t_params.first_message_deliveries_weight = 40.0 / t_params.first_message_deliveries_cap;
+
+        if let Some((decay_slots, cap_factor, activation_window, current_slot)) = mesh_message_info
+        {
+            let decay_time = self.slot * decay_slots as u32;
+            t_params.mesh_message_deliveries_decay = self.score_parameter_decay(decay_time);
+            t_params.mesh_message_deliveries_threshold = Self::threshold(
+                t_params.mesh_message_deliveries_decay,
+                expected_message_rate / 50.0,
+            );
+            t_params.mesh_message_deliveries_cap =
+                if cap_factor * t_params.mesh_message_deliveries_threshold < 2.0 {
+                    2.0
+                } else {
+                    cap_factor * t_params.mesh_message_deliveries_threshold
+                };
+            t_params.mesh_message_deliveries_activation = activation_window;
+            t_params.mesh_message_deliveries_window = Duration::from_secs(2);
+            t_params.mesh_failure_penalty_decay = t_params.mesh_message_deliveries_decay;
+            t_params.mesh_message_deliveries_weight = -self.max_positive_score
+                / (t_params.topic_weight * t_params.mesh_message_deliveries_threshold.powi(2));
+            t_params.mesh_failure_penalty_weight = t_params.mesh_message_deliveries_weight;
+            if decay_slots >= current_slot.as_u64() {
+                t_params.mesh_message_deliveries_threshold = 0.0;
+                t_params.mesh_message_deliveries_weight = 0.0;
+            }
+        } else {
+            t_params.mesh_message_deliveries_weight = 0.0;
+            t_params.mesh_message_deliveries_threshold = 0.0;
+            t_params.mesh_message_deliveries_decay = 0.0;
+            t_params.mesh_message_deliveries_cap = 0.0;
+            t_params.mesh_message_deliveries_window = Duration::from_secs(0);
+            t_params.mesh_message_deliveries_activation = Duration::from_secs(0);
+            t_params.mesh_failure_penalty_decay = 0.0;
+            t_params.mesh_failure_penalty_weight = 0.0;
+        }
+
+        t_params.invalid_message_deliveries_weight =
+            -self.max_positive_score / t_params.topic_weight;
+        t_params.invalid_message_deliveries_decay = self.score_parameter_decay(self.epoch * 50);
+
+        t_params
+    }
+}

beacon_node/eth2_libp2p/src/behaviour/handler/delegate.rs

@@ -1,8 +1,8 @@
+use crate::behaviour::Gossipsub;
 use crate::rpc::*;
 use libp2p::{
     core::either::{EitherError, EitherOutput},
     core::upgrade::{EitherUpgrade, InboundUpgrade, OutboundUpgrade, SelectUpgrade, UpgradeError},
-    gossipsub::Gossipsub,
     identify::Identify,
     swarm::{
         protocols_handler::{
@@ -54,8 +54,6 @@ impl<TSpec: EthSpec> DelegatingHandler<TSpec> {
     }
 }
 
-// TODO: this can all be created with macros
-
 /// Wrapper around the `ProtocolsHandler::InEvent` types of the handlers.
 /// Simply delegated to the corresponding behaviour's handler.
 #[derive(Debug, Clone)]
@@ -115,7 +113,6 @@ pub type DelegateOutProto<TSpec> = EitherUpgrade<
     >,
 >;
 
-// TODO: prob make this an enum
 pub type DelegateOutInfo<TSpec> = EitherOutput<
     <GossipHandler as ProtocolsHandler>::OutboundOpenInfo,
     EitherOutput<
@@ -131,8 +128,9 @@ impl<TSpec: EthSpec> ProtocolsHandler for DelegatingHandler<TSpec> {
     type InboundProtocol = DelegateInProto<TSpec>;
     type OutboundProtocol = DelegateOutProto<TSpec>;
     type OutboundOpenInfo = DelegateOutInfo<TSpec>;
+    type InboundOpenInfo = ();
 
-    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol> {
+    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol, ()> {
         let gossip_proto = self.gossip_handler.listen_protocol();
         let rpc_proto = self.rpc_handler.listen_protocol();
         let identify_proto = self.identify_handler.listen_protocol();
@@ -147,24 +145,27 @@ impl<TSpec: EthSpec> ProtocolsHandler for DelegatingHandler<TSpec> {
             SelectUpgrade::new(rpc_proto.into_upgrade().1, identify_proto.into_upgrade().1),
         );
 
-        SubstreamProtocol::new(select).with_timeout(timeout)
+        SubstreamProtocol::new(select, ()).with_timeout(timeout)
     }
 
     fn inject_fully_negotiated_inbound(
         &mut self,
         out: <Self::InboundProtocol as InboundUpgrade<NegotiatedSubstream>>::Output,
+        _info: Self::InboundOpenInfo,
     ) {
         match out {
             // Gossipsub
-            EitherOutput::First(out) => self.gossip_handler.inject_fully_negotiated_inbound(out),
+            EitherOutput::First(out) => {
+                self.gossip_handler.inject_fully_negotiated_inbound(out, ())
+            }
             // RPC
             EitherOutput::Second(EitherOutput::First(out)) => {
-                self.rpc_handler.inject_fully_negotiated_inbound(out)
+                self.rpc_handler.inject_fully_negotiated_inbound(out, ())
             }
             // Identify
-            EitherOutput::Second(EitherOutput::Second(out)) => {
-                self.identify_handler.inject_fully_negotiated_inbound(out)
-            }
+            EitherOutput::Second(EitherOutput::Second(out)) => self
+                .identify_handler
+                .inject_fully_negotiated_inbound(out, ()),
         }
     }
 
@@ -212,7 +213,6 @@ impl<TSpec: EthSpec> ProtocolsHandler for DelegatingHandler<TSpec> {
             <Self::OutboundProtocol as OutboundUpgrade<NegotiatedSubstream>>::Error,
         >,
     ) {
-        // TODO: find how to clean up
         match info {
             // Gossipsub
             EitherOutput::First(info) => match error {
@@ -317,10 +317,11 @@ impl<TSpec: EthSpec> ProtocolsHandler for DelegatingHandler<TSpec> {
                     event,
                 )));
             }
-            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info }) => {
+            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol }) => {
                 return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-                    protocol: protocol.map_upgrade(EitherUpgrade::A),
-                    info: EitherOutput::First(info),
+                    protocol: protocol
+                        .map_upgrade(EitherUpgrade::A)
+                        .map_info(EitherOutput::First),
                 });
             }
             Poll::Pending => (),
@@ -333,10 +334,11 @@ impl<TSpec: EthSpec> ProtocolsHandler for DelegatingHandler<TSpec> {
             Poll::Ready(ProtocolsHandlerEvent::Close(event)) => {
                 return Poll::Ready(ProtocolsHandlerEvent::Close(DelegateError::RPC(event)));
             }
-            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info }) => {
+            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol }) => {
                 return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-                    protocol: protocol.map_upgrade(|u| EitherUpgrade::B(EitherUpgrade::A(u))),
-                    info: EitherOutput::Second(EitherOutput::First(info)),
+                    protocol: protocol
+                        .map_upgrade(|u| EitherUpgrade::B(EitherUpgrade::A(u)))
+                        .map_info(|info| EitherOutput::Second(EitherOutput::First(info))),
                 });
             }
             Poll::Pending => (),
@@ -351,10 +353,11 @@ impl<TSpec: EthSpec> ProtocolsHandler for DelegatingHandler<TSpec> {
             Poll::Ready(ProtocolsHandlerEvent::Close(event)) => {
                 return Poll::Ready(ProtocolsHandlerEvent::Close(DelegateError::Identify(event)));
             }
-            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info: () }) => {
+            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol }) => {
                 return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-                    protocol: protocol.map_upgrade(|u| EitherUpgrade::B(EitherUpgrade::B(u))),
-                    info: EitherOutput::Second(EitherOutput::Second(())),
+                    protocol: protocol
+                        .map_upgrade(|u| EitherUpgrade::B(EitherUpgrade::B(u)))
+                        .map_info(|_| EitherOutput::Second(EitherOutput::Second(()))),
                 });
             }
             Poll::Pending => (),

beacon_node/eth2_libp2p/src/behaviour/handler/mod.rs

@@ -1,3 +1,4 @@
+use crate::behaviour::Gossipsub;
 use crate::rpc::*;
 use delegate::DelegatingHandler;
 pub(super) use delegate::{
@@ -5,7 +6,6 @@ pub(super) use delegate::{
 };
 use libp2p::{
     core::upgrade::{InboundUpgrade, OutboundUpgrade},
-    gossipsub::Gossipsub,
     identify::Identify,
     swarm::protocols_handler::{
         KeepAlive, ProtocolsHandlerEvent, ProtocolsHandlerUpgrErr, SubstreamProtocol,
@@ -41,29 +41,25 @@ pub enum BehaviourHandlerIn<TSpec: EthSpec> {
     Shutdown(Option<(RequestId, RPCRequest<TSpec>)>),
 }
 
-pub enum BehaviourHandlerOut<TSpec: EthSpec> {
-    Delegate(Box<DelegateOut<TSpec>>),
-    // TODO: replace custom with events to send
-    Custom,
-}
-
 impl<TSpec: EthSpec> ProtocolsHandler for BehaviourHandler<TSpec> {
     type InEvent = BehaviourHandlerIn<TSpec>;
-    type OutEvent = BehaviourHandlerOut<TSpec>;
+    type OutEvent = DelegateOut<TSpec>;
     type Error = DelegateError<TSpec>;
     type InboundProtocol = DelegateInProto<TSpec>;
     type OutboundProtocol = DelegateOutProto<TSpec>;
     type OutboundOpenInfo = DelegateOutInfo<TSpec>;
+    type InboundOpenInfo = ();
 
-    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol> {
+    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol, ()> {
         self.delegate.listen_protocol()
     }
 
     fn inject_fully_negotiated_inbound(
         &mut self,
         out: <Self::InboundProtocol as InboundUpgrade<NegotiatedSubstream>>::Output,
+        _info: Self::InboundOpenInfo,
     ) {
-        self.delegate.inject_fully_negotiated_inbound(out)
+        self.delegate.inject_fully_negotiated_inbound(out, ())
     }
 
     fn inject_fully_negotiated_outbound(
@@ -120,18 +116,13 @@ impl<TSpec: EthSpec> ProtocolsHandler for BehaviourHandler<TSpec> {
 
         match self.delegate.poll(cx) {
             Poll::Ready(ProtocolsHandlerEvent::Custom(event)) => {
-                return Poll::Ready(ProtocolsHandlerEvent::Custom(
-                    BehaviourHandlerOut::Delegate(Box::new(event)),
-                ))
+                return Poll::Ready(ProtocolsHandlerEvent::Custom(event))
             }
             Poll::Ready(ProtocolsHandlerEvent::Close(err)) => {
                 return Poll::Ready(ProtocolsHandlerEvent::Close(err))
             }
-            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info }) => {
-                return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-                    protocol,
-                    info,
-                });
+            Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol }) => {
+                return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol });
             }
             Poll::Pending => (),
         }

beacon_node/eth2_libp2p/src/config.rs

@@ -1,8 +1,12 @@
 use crate::types::GossipKind;
-use crate::Enr;
+use crate::{Enr, PeerIdSerialized};
+use directory::{
+    DEFAULT_BEACON_NODE_DIR, DEFAULT_HARDCODED_NETWORK, DEFAULT_NETWORK_DIR, DEFAULT_ROOT_DIR,
+};
 use discv5::{Discv5Config, Discv5ConfigBuilder};
 use libp2p::gossipsub::{
-    GossipsubConfig, GossipsubConfigBuilder, GossipsubMessage, MessageId, ValidationMode,
+    FastMessageId, GossipsubConfig, GossipsubConfigBuilder, GossipsubMessage, MessageId,
+    RawGossipsubMessage, ValidationMode,
 };
 use libp2p::Multiaddr;
 use serde_derive::{Deserialize, Serialize};
@@ -12,6 +16,12 @@ use std::time::Duration;
 
 pub const GOSSIP_MAX_SIZE: usize = 1_048_576;
 
+// We treat uncompressed messages as invalid and never use the INVALID_SNAPPY_DOMAIN as in the
+// specification. We leave it here for posterity.
+// const MESSAGE_DOMAIN_INVALID_SNAPPY: [u8; 4] = [0, 0, 0, 0];
+const MESSAGE_DOMAIN_VALID_SNAPPY: [u8; 4] = [1, 0, 0, 0];
+pub const MESH_N_LOW: usize = 6;
+
 #[derive(Clone, Debug, Serialize, Deserialize)]
 #[serde(default)]
 /// Network configuration for lighthouse.
@@ -50,17 +60,37 @@ pub struct Config {
     pub discv5_config: Discv5Config,
 
     /// List of nodes to initially connect to.
-    pub boot_nodes: Vec<Enr>,
+    pub boot_nodes_enr: Vec<Enr>,
+
+    /// List of nodes to initially connect to, on Multiaddr format.
+    pub boot_nodes_multiaddr: Vec<Multiaddr>,
 
     /// List of libp2p nodes to initially connect to.
     pub libp2p_nodes: Vec<Multiaddr>,
 
+    /// List of trusted libp2p nodes which are not scored.
+    pub trusted_peers: Vec<PeerIdSerialized>,
+
     /// Client version
     pub client_version: String,
 
     /// Disables the discovery protocol from starting.
     pub disable_discovery: bool,
 
+    /// Attempt to construct external port mappings with UPnP.
+    pub upnp_enabled: bool,
+
+    /// Subscribe to all subnets for the duration of the runtime.
+    pub subscribe_all_subnets: bool,
+
+    /// Import/aggregate all attestations recieved on subscribed subnets for the duration of the
+    /// runtime.
+    pub import_all_attestations: bool,
+
+    /// Indicates if the user has set the network to be in private mode. Currently this
+    /// prevents sending client identifying information over identify.
+    pub private: bool,
+
     /// List of extra topics to initially subscribe to as strings.
     pub topics: Vec<GossipKind>,
 }
@@ -68,58 +98,70 @@ pub struct Config {
 impl Default for Config {
     /// Generate a default network configuration.
     fn default() -> Self {
-        let mut network_dir = dirs::home_dir().unwrap_or_else(|| PathBuf::from("."));
-        network_dir.push(".lighthouse");
-        network_dir.push("network");
-
-        // The default topics that we will initially subscribe to
-        let topics = vec![
-            GossipKind::BeaconBlock,
-            GossipKind::BeaconAggregateAndProof,
-            GossipKind::VoluntaryExit,
-            GossipKind::ProposerSlashing,
-            GossipKind::AttesterSlashing,
-        ];
+        // WARNING: this directory default should be always overwritten with parameters
+        // from cli for specific networks.
+        let network_dir = dirs::home_dir()
+            .unwrap_or_else(|| PathBuf::from("."))
+            .join(DEFAULT_ROOT_DIR)
+            .join(DEFAULT_HARDCODED_NETWORK)
+            .join(DEFAULT_BEACON_NODE_DIR)
+            .join(DEFAULT_NETWORK_DIR);
 
         // The function used to generate a gossipsub message id
-        // We use base64(SHA256(data)) for content addressing
+        // We use the first 8 bytes of SHA256(data) for content addressing
+        let fast_gossip_message_id = |message: &RawGossipsubMessage| {
+            FastMessageId::from(&Sha256::digest(&message.data)[..8])
+        };
+
+        fn prefix(prefix: [u8; 4], data: &[u8]) -> Vec<u8> {
+            let mut vec = Vec::with_capacity(prefix.len() + data.len());
+            vec.extend_from_slice(&prefix);
+            vec.extend_from_slice(data);
+            vec
+        }
+
         let gossip_message_id = |message: &GossipsubMessage| {
-            MessageId::from(base64::encode_config(
-                &Sha256::digest(&message.data),
-                base64::URL_SAFE_NO_PAD,
-            ))
+            MessageId::from(
+                &Sha256::digest(prefix(MESSAGE_DOMAIN_VALID_SNAPPY, &message.data).as_slice())
+                    [..20],
+            )
         };
 
         // gossipsub configuration
         // Note: The topics by default are sent as plain strings. Hashes are an optional
         // parameter.
-        let gs_config = GossipsubConfigBuilder::new()
+        let gs_config = GossipsubConfigBuilder::default()
             .max_transmit_size(GOSSIP_MAX_SIZE)
             .heartbeat_interval(Duration::from_millis(700))
-            .mesh_n(6)
-            .mesh_n_low(5)
+            .mesh_n(8)
+            .mesh_n_low(MESH_N_LOW)
             .mesh_n_high(12)
             .gossip_lazy(6)
             .fanout_ttl(Duration::from_secs(60))
             .history_length(6)
+            .max_messages_per_rpc(Some(10))
             .history_gossip(3)
             .validate_messages() // require validation before propagation
-            .validation_mode(ValidationMode::Permissive)
+            .validation_mode(ValidationMode::Anonymous)
             // prevent duplicates for 550 heartbeats(700millis * 550) = 385 secs
             .duplicate_cache_time(Duration::from_secs(385))
             .message_id_fn(gossip_message_id)
-            .build();
+            .fast_message_id_fn(fast_gossip_message_id)
+            .allow_self_origin(true)
+            .build()
+            .expect("valid gossipsub configuration");
 
         // discv5 configuration
         let discv5_config = Discv5ConfigBuilder::new()
             .enable_packet_filter()
             .session_cache_capacity(1000)
-            .request_timeout(Duration::from_secs(4))
+            .request_timeout(Duration::from_secs(1))
+            .query_peer_timeout(Duration::from_secs(2))
+            .query_timeout(Duration::from_secs(30))
             .request_retries(1)
             .enr_peer_update_min(10)
             .query_parallelism(5)
-            .query_timeout(Duration::from_secs(30))
-            .query_peer_timeout(Duration::from_secs(2))
+            .disable_report_discovered_peers()
             .ip_limit() // limits /24 IP's in buckets.
             .ping_interval(Duration::from_secs(300))
             .build();
@@ -136,11 +178,17 @@ impl Default for Config {
             target_peers: 50,
             gs_config,
             discv5_config,
-            boot_nodes: vec![],
+            boot_nodes_enr: vec![],
+            boot_nodes_multiaddr: vec![],
             libp2p_nodes: vec![],
+            trusted_peers: vec![],
             client_version: lighthouse_version::version_with_platform(),
             disable_discovery: false,
-            topics,
+            upnp_enabled: true,
+            private: false,
+            subscribe_all_subnets: false,
+            import_all_attestations: false,
+            topics: Vec::new(),
         }
     }
 }

beacon_node/eth2_libp2p/src/discovery/enr.rs

@@ -6,6 +6,7 @@ use super::enr_ext::CombinedKeyExt;
 use super::ENR_FILENAME;
 use crate::types::{Enr, EnrBitfield};
 use crate::NetworkConfig;
+use discv5::enr::EnrKey;
 use libp2p::core::identity::Keypair;
 use slog::{debug, warn};
 use ssz::{Decode, Encode};
@@ -33,21 +34,69 @@ impl Eth2Enr for Enr {
     fn bitfield<TSpec: EthSpec>(&self) -> Result<EnrBitfield<TSpec>, &'static str> {
         let bitfield_bytes = self
             .get(BITFIELD_ENR_KEY)
-            .ok_or_else(|| "ENR bitfield non-existent")?;
+            .ok_or("ENR bitfield non-existent")?;
 
         BitVector::<TSpec::SubnetBitfieldLength>::from_ssz_bytes(bitfield_bytes)
             .map_err(|_| "Could not decode the ENR SSZ bitfield")
     }
 
     fn eth2(&self) -> Result<EnrForkId, &'static str> {
-        let eth2_bytes = self
-            .get(ETH2_ENR_KEY)
-            .ok_or_else(|| "ENR has no eth2 field")?;
+        let eth2_bytes = self.get(ETH2_ENR_KEY).ok_or("ENR has no eth2 field")?;
 
         EnrForkId::from_ssz_bytes(eth2_bytes).map_err(|_| "Could not decode EnrForkId")
     }
 }
 
+/// Either use the given ENR or load an ENR from file if it exists and matches the current NodeId
+/// and sequence number.
+/// If an ENR exists, with the same NodeId, this function checks to see if the loaded ENR from
+/// disk is suitable to use, otherwise we increment the given ENR's sequence number.
+pub fn use_or_load_enr(
+    enr_key: &CombinedKey,
+    local_enr: &mut Enr,
+    config: &NetworkConfig,
+    log: &slog::Logger,
+) -> Result<(), String> {
+    let enr_f = config.network_dir.join(ENR_FILENAME);
+    if let Ok(mut enr_file) = File::open(enr_f.clone()) {
+        let mut enr_string = String::new();
+        match enr_file.read_to_string(&mut enr_string) {
+            Err(_) => debug!(log, "Could not read ENR from file"),
+            Ok(_) => {
+                match Enr::from_str(&enr_string) {
+                    Ok(disk_enr) => {
+                        // if the same node id, then we may need to update our sequence number
+                        if local_enr.node_id() == disk_enr.node_id() {
+                            if compare_enr(&local_enr, &disk_enr) {
+                                debug!(log, "ENR loaded from disk"; "file" => ?enr_f);
+                                // the stored ENR has the same configuration, use it
+                                *local_enr = disk_enr;
+                                return Ok(());
+                            }
+
+                            // same node id, different configuration - update the sequence number
+                            // Note: local_enr is generated with default(0) attnets value,
+                            // so a non default value in persisted enr will also update sequence number.
+                            let new_seq_no = disk_enr.seq().checked_add(1).ok_or("ENR sequence number on file is too large. Remove it to generate a new NodeId")?;
+                            local_enr.set_seq(new_seq_no, enr_key).map_err(|e| {
+                                format!("Could not update ENR sequence number: {:?}", e)
+                            })?;
+                            debug!(log, "ENR sequence number increased"; "seq" =>  new_seq_no);
+                        }
+                    }
+                    Err(e) => {
+                        warn!(log, "ENR from file could not be decoded"; "error" => ?e);
+                    }
+                }
+            }
+        }
+    }
+
+    save_enr_to_disk(&config.network_dir, &local_enr, log);
+
+    Ok(())
+}
+
 /// Loads an ENR from file if it exists and matches the current NodeId and sequence number. If none
 /// exists, generates a new one.
 ///
@@ -65,49 +114,14 @@ pub fn build_or_load_enr<T: EthSpec>(
     let enr_key = CombinedKey::from_libp2p(&local_key)?;
     let mut local_enr = build_enr::<T>(&enr_key, config, enr_fork_id)?;
 
-    let enr_f = config.network_dir.join(ENR_FILENAME);
-    if let Ok(mut enr_file) = File::open(enr_f.clone()) {
-        let mut enr_string = String::new();
-        match enr_file.read_to_string(&mut enr_string) {
-            Err(_) => debug!(log, "Could not read ENR from file"),
-            Ok(_) => {
-                match Enr::from_str(&enr_string) {
-                    Ok(disk_enr) => {
-                        // if the same node id, then we may need to update our sequence number
-                        if local_enr.node_id() == disk_enr.node_id() {
-                            if compare_enr(&local_enr, &disk_enr) {
-                                debug!(log, "ENR loaded from disk"; "file" => format!("{:?}", enr_f));
-                                // the stored ENR has the same configuration, use it
-                                return Ok(disk_enr);
-                            }
-
-                            // same node id, different configuration - update the sequence number
-                            let new_seq_no = disk_enr.seq().checked_add(1).ok_or_else(|| "ENR sequence number on file is too large. Remove it to generate a new NodeId")?;
-                            local_enr.set_seq(new_seq_no, &enr_key).map_err(|e| {
-                                format!("Could not update ENR sequence number: {:?}", e)
-                            })?;
-                            debug!(log, "ENR sequence number increased"; "seq" =>  new_seq_no);
-                        }
-                    }
-                    Err(e) => {
-                        warn!(log, "ENR from file could not be decoded"; "error" => format!("{:?}", e));
-                    }
-                }
-            }
-        }
-    }
-
-    save_enr_to_disk(&config.network_dir, &local_enr, log);
-
+    use_or_load_enr(&enr_key, &mut local_enr, config, log)?;
     Ok(local_enr)
 }
 
-/// Builds a lighthouse ENR given a `NetworkConfig`.
-pub fn build_enr<T: EthSpec>(
-    enr_key: &CombinedKey,
+pub fn create_enr_builder_from_config<T: EnrKey>(
     config: &NetworkConfig,
-    enr_fork_id: EnrForkId,
-) -> Result<Enr, String> {
+    enable_tcp: bool,
+) -> EnrBuilder<T> {
     let mut builder = EnrBuilder::new("v4");
     if let Some(enr_address) = config.enr_address {
         builder.ip(enr_address);
@@ -116,20 +130,30 @@ pub fn build_enr<T: EthSpec>(
         builder.udp(udp_port);
     }
     // we always give it our listening tcp port
-    // TODO: Add uPnP support to map udp and tcp ports
-    let tcp_port = config.enr_tcp_port.unwrap_or_else(|| config.libp2p_port);
-    builder.tcp(tcp_port);
+    if enable_tcp {
+        let tcp_port = config.enr_tcp_port.unwrap_or(config.libp2p_port);
+        builder.tcp(tcp_port);
+    }
+    builder
+}
+
+/// Builds a lighthouse ENR given a `NetworkConfig`.
+pub fn build_enr<T: EthSpec>(
+    enr_key: &CombinedKey,
+    config: &NetworkConfig,
+    enr_fork_id: EnrForkId,
+) -> Result<Enr, String> {
+    let mut builder = create_enr_builder_from_config(config, true);
 
     // set the `eth2` field on our ENR
-    builder.add_value(ETH2_ENR_KEY.into(), enr_fork_id.as_ssz_bytes());
+    builder.add_value(ETH2_ENR_KEY, &enr_fork_id.as_ssz_bytes());
 
     // set the "attnets" field on our ENR
     let bitfield = BitVector::<T::SubnetBitfieldLength>::new();
 
-    builder.add_value(BITFIELD_ENR_KEY.into(), bitfield.as_ssz_bytes());
+    builder.add_value(BITFIELD_ENR_KEY, &bitfield.as_ssz_bytes());
 
     builder
-        .tcp(config.libp2p_port)
         .build(enr_key)
         .map_err(|e| format!("Could not build Local ENR: {:?}", e))
 }
@@ -162,7 +186,7 @@ pub fn save_enr_to_disk(dir: &Path, enr: &Enr, log: &slog::Logger) {
         Err(e) => {
             warn!(
                 log,
-                "Could not write ENR to file"; "file" => format!("{:?}{:?}",dir, ENR_FILENAME),  "error" => format!("{}", e)
+                "Could not write ENR to file"; "file" => format!("{:?}{:?}",dir, ENR_FILENAME),  "error" => %e
             );
         }
     }

beacon_node/eth2_libp2p/src/discovery/enr_ext.rs

@@ -13,9 +13,15 @@ pub trait EnrExt {
     /// The vector remains empty if these fields are not defined.
     fn multiaddr(&self) -> Vec<Multiaddr>;
 
-    /// Returns the multiaddr with the `PeerId` prepended.
+    /// Returns a list of multiaddrs with the `PeerId` prepended.
     fn multiaddr_p2p(&self) -> Vec<Multiaddr>;
 
+    /// Returns any multiaddrs that contain the TCP protocol with the `PeerId` prepended.
+    fn multiaddr_p2p_tcp(&self) -> Vec<Multiaddr>;
+
+    /// Returns any multiaddrs that contain the UDP protocol with the `PeerId` prepended.
+    fn multiaddr_p2p_udp(&self) -> Vec<Multiaddr>;
+
     /// Returns any multiaddrs that contain the TCP protocol.
     fn multiaddr_tcp(&self) -> Vec<Multiaddr>;
 }
@@ -111,6 +117,58 @@ impl EnrExt for Enr {
         multiaddrs
     }
 
+    /// Returns a list of multiaddrs if the ENR has an `ip` and a `tcp` key **or** an `ip6` and a `tcp6`.
+    /// The vector remains empty if these fields are not defined.
+    ///
+    /// This also prepends the `PeerId` into each multiaddr with the `P2p` protocol.
+    fn multiaddr_p2p_tcp(&self) -> Vec<Multiaddr> {
+        let peer_id = self.peer_id();
+        let mut multiaddrs: Vec<Multiaddr> = Vec::new();
+        if let Some(ip) = self.ip() {
+            if let Some(tcp) = self.tcp() {
+                let mut multiaddr: Multiaddr = ip.into();
+                multiaddr.push(Protocol::Tcp(tcp));
+                multiaddr.push(Protocol::P2p(peer_id.clone().into()));
+                multiaddrs.push(multiaddr);
+            }
+        }
+        if let Some(ip6) = self.ip6() {
+            if let Some(tcp6) = self.tcp6() {
+                let mut multiaddr: Multiaddr = ip6.into();
+                multiaddr.push(Protocol::Tcp(tcp6));
+                multiaddr.push(Protocol::P2p(peer_id.into()));
+                multiaddrs.push(multiaddr);
+            }
+        }
+        multiaddrs
+    }
+
+    /// Returns a list of multiaddrs if the ENR has an `ip` and a `udp` key **or** an `ip6` and a `udp6`.
+    /// The vector remains empty if these fields are not defined.
+    ///
+    /// This also prepends the `PeerId` into each multiaddr with the `P2p` protocol.
+    fn multiaddr_p2p_udp(&self) -> Vec<Multiaddr> {
+        let peer_id = self.peer_id();
+        let mut multiaddrs: Vec<Multiaddr> = Vec::new();
+        if let Some(ip) = self.ip() {
+            if let Some(udp) = self.udp() {
+                let mut multiaddr: Multiaddr = ip.into();
+                multiaddr.push(Protocol::Udp(udp));
+                multiaddr.push(Protocol::P2p(peer_id.clone().into()));
+                multiaddrs.push(multiaddr);
+            }
+        }
+        if let Some(ip6) = self.ip6() {
+            if let Some(udp6) = self.udp6() {
+                let mut multiaddr: Multiaddr = ip6.into();
+                multiaddr.push(Protocol::Udp(udp6));
+                multiaddr.push(Protocol::P2p(peer_id.into()));
+                multiaddrs.push(multiaddr);
+            }
+        }
+        multiaddrs
+    }
+
     /// Returns a list of multiaddrs if the ENR has an `ip` and either a `tcp` or `udp` key **or** an `ip6` and either a `tcp6` or `udp6`.
     /// The vector remains empty if these fields are not defined.
     fn multiaddr_tcp(&self) -> Vec<Multiaddr> {
@@ -140,7 +198,7 @@ impl CombinedKeyPublicExt for CombinedPublicKey {
     fn into_peer_id(&self) -> PeerId {
         match self {
             Self::Secp256k1(pk) => {
-                let pk_bytes = pk.serialize_compressed();
+                let pk_bytes = pk.to_bytes();
                 let libp2p_pk = libp2p::core::PublicKey::Secp256k1(
                     libp2p::core::identity::secp256k1::PublicKey::decode(&pk_bytes)
                         .expect("valid public key"),
@@ -163,7 +221,7 @@ impl CombinedKeyExt for CombinedKey {
     fn from_libp2p(key: &libp2p::core::identity::Keypair) -> Result<CombinedKey, &'static str> {
         match key {
             Keypair::Secp256k1(key) => {
-                let secret = discv5::enr::secp256k1::SecretKey::parse(&key.secret().to_bytes())
+                let secret = discv5::enr::k256::ecdsa::SigningKey::new(&key.secret().to_bytes())
                     .expect("libp2p key must be valid");
                 Ok(CombinedKey::Secp256k1(secret))
             }
@@ -183,7 +241,7 @@ impl CombinedKeyExt for CombinedKey {
 pub fn peer_id_to_node_id(peer_id: &PeerId) -> Result<discv5::enr::NodeId, String> {
     // A libp2p peer id byte representation should be 2 length bytes + 4 protobuf bytes + compressed pk bytes
     // if generated from a PublicKey with Identity multihash.
-    let pk_bytes = &peer_id.as_bytes()[2..];
+    let pk_bytes = &peer_id.to_bytes()[2..];
 
     match PublicKey::from_protobuf_encoding(pk_bytes).map_err(|e| {
         format!(
@@ -219,7 +277,7 @@ mod tests {
     fn test_secp256k1_peer_id_conversion() {
         let sk_hex = "df94a73d528434ce2309abb19c16aedb535322797dbd59c157b1e04095900f48";
         let sk_bytes = hex::decode(sk_hex).unwrap();
-        let secret_key = discv5::enr::secp256k1::SecretKey::parse_slice(&sk_bytes).unwrap();
+        let secret_key = discv5::enr::k256::ecdsa::SigningKey::new(&sk_bytes).unwrap();
 
         let libp2p_sk = libp2p::identity::secp256k1::SecretKey::from_bytes(sk_bytes).unwrap();
         let secp256k1_kp: libp2p::identity::secp256k1::Keypair = libp2p_sk.into();

beacon_node/eth2_libp2p/src/discovery/mod.rs

@@ -3,24 +3,24 @@ pub(crate) mod enr;
 pub mod enr_ext;
 
 // Allow external use of the lighthouse ENR builder
-pub use enr::{build_enr, CombinedKey, Eth2Enr};
-pub use enr_ext::{CombinedKeyExt, EnrExt};
-pub use libp2p::core::identity::Keypair;
+pub use enr::{build_enr, create_enr_builder_from_config, use_or_load_enr, CombinedKey, Eth2Enr};
+pub use enr_ext::{peer_id_to_node_id, CombinedKeyExt, EnrExt};
+pub use libp2p::core::identity::{Keypair, PublicKey};
 
-use crate::metrics;
-use crate::{error, Enr, NetworkConfig, NetworkGlobals};
+use crate::{config, metrics};
+use crate::{error, Enr, NetworkConfig, NetworkGlobals, SubnetDiscovery};
 use discv5::{enr::NodeId, Discv5, Discv5Event};
 use enr::{BITFIELD_ENR_KEY, ETH2_ENR_KEY};
 use futures::prelude::*;
 use futures::stream::FuturesUnordered;
 use libp2p::core::PeerId;
 use lru::LruCache;
-use slog::{crit, debug, info, warn};
+use slog::{crit, debug, error, info, warn};
 use ssz::{Decode, Encode};
 use ssz_types::BitVector;
 use std::{
     collections::{HashMap, VecDeque},
-    net::SocketAddr,
+    net::{IpAddr, SocketAddr},
     path::Path,
     pin::Pin,
     sync::Arc,
@@ -31,12 +31,12 @@ use tokio::sync::mpsc;
 use types::{EnrForkId, EthSpec, SubnetId};
 
 mod subnet_predicate;
-use subnet_predicate::subnet_predicate;
+pub use subnet_predicate::subnet_predicate;
 
 /// Local ENR storage filename.
 pub const ENR_FILENAME: &str = "enr.dat";
 /// Target number of peers we'd like to have connected to a given long-lived subnet.
-const TARGET_SUBNET_PEERS: usize = 3;
+pub const TARGET_SUBNET_PEERS: usize = config::MESH_N_LOW;
 /// Target number of peers to search for given a grouped subnet query.
 const TARGET_PEERS_FOR_GROUPED_QUERY: usize = 6;
 /// Number of times to attempt a discovery request.
@@ -155,7 +155,7 @@ pub struct Discovery<TSpec: EthSpec> {
 
     /// Indicates if the discovery service has been started. When the service is disabled, this is
     /// always false.
-    started: bool,
+    pub started: bool,
 
     /// Logger for the discovery behaviour.
     log: slog::Logger,
@@ -163,7 +163,7 @@ pub struct Discovery<TSpec: EthSpec> {
 
 impl<TSpec: EthSpec> Discovery<TSpec> {
     /// NOTE: Creating discovery requires running within a tokio execution environment.
-    pub fn new(
+    pub async fn new(
         local_key: &Keypair,
         config: &NetworkConfig,
         network_globals: Arc<NetworkGlobals<TSpec>>,
@@ -178,7 +178,7 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
 
         let local_enr = network_globals.local_enr.read().clone();
 
-        info!(log, "ENR Initialised"; "enr" => local_enr.to_base64(), "seq" => local_enr.seq(), "id"=> format!("{}",local_enr.node_id()), "ip" => format!("{:?}", local_enr.ip()), "udp"=> format!("{:?}", local_enr.udp()), "tcp" => format!("{:?}", local_enr.tcp()));
+        info!(log, "ENR Initialised"; "enr" => local_enr.to_base64(), "seq" => local_enr.seq(), "id"=> %local_enr.node_id(), "ip" => ?local_enr.ip(), "udp"=> ?local_enr.udp(), "tcp" => ?local_enr.tcp());
 
         let listen_socket = SocketAddr::new(config.listen_address, config.discovery_port);
 
@@ -189,35 +189,84 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
             .map_err(|e| format!("Discv5 service failed. Error: {:?}", e))?;
 
         // Add bootnodes to routing table
-        for bootnode_enr in config.boot_nodes.clone() {
+        for bootnode_enr in config.boot_nodes_enr.clone() {
             debug!(
                 log,
                 "Adding node to routing table";
-                "node_id" => format!("{}", bootnode_enr.node_id()),
-                "peer_id" => format!("{}", bootnode_enr.peer_id()),
-                "ip" => format!("{:?}", bootnode_enr.ip()),
-                "udp" => format!("{:?}", bootnode_enr.udp()),
-                "tcp" => format!("{:?}", bootnode_enr.tcp())
+                "node_id" => %bootnode_enr.node_id(),
+                "peer_id" => %bootnode_enr.peer_id(),
+                "ip" => ?bootnode_enr.ip(),
+                "udp" => ?bootnode_enr.udp(),
+                "tcp" => ?bootnode_enr.tcp()
             );
+            let repr = bootnode_enr.to_string();
             let _ = discv5.add_enr(bootnode_enr).map_err(|e| {
-                debug!(
+                error!(
                     log,
                     "Could not add peer to the local routing table";
-                    "error" => e.to_string()
+                    "addr" => repr,
+                    "error" => e.to_string(),
                 )
             });
         }
 
         // Start the discv5 service and obtain an event stream
         let event_stream = if !config.disable_discovery {
-            discv5.start(listen_socket).map_err(|e| e.to_string())?;
+            discv5
+                .start(listen_socket)
+                .map_err(|e| e.to_string())
+                .await?;
             debug!(log, "Discovery service started");
             EventStream::Awaiting(Box::pin(discv5.event_stream()))
         } else {
             EventStream::InActive
         };
 
-        // Obtain the event stream
+        if !config.boot_nodes_multiaddr.is_empty() {
+            info!(log, "Contacting Multiaddr boot-nodes for their ENR");
+        }
+
+        // get futures for requesting the Enrs associated to these multiaddr and wait for their
+        // completion
+        let mut fut_coll = config
+            .boot_nodes_multiaddr
+            .iter()
+            .map(|addr| addr.to_string())
+            // request the ENR for this multiaddr and keep the original for logging
+            .map(|addr| {
+                futures::future::join(
+                    discv5.request_enr(addr.clone()),
+                    futures::future::ready(addr),
+                )
+            })
+            .collect::<FuturesUnordered<_>>();
+
+        while let Some((result, original_addr)) = fut_coll.next().await {
+            match result {
+                Ok(enr) => {
+                    debug!(
+                        log,
+                        "Adding node to routing table";
+                        "node_id" => %enr.node_id(),
+                        "peer_id" => %enr.peer_id(),
+                        "ip" => ?enr.ip(),
+                        "udp" => ?enr.udp(),
+                        "tcp" => ?enr.tcp()
+                    );
+                    let _ = discv5.add_enr(enr).map_err(|e| {
+                        error!(
+                            log,
+                            "Could not add peer to the local routing table";
+                            "addr" => original_addr.to_string(),
+                            "error" => e.to_string(),
+                        )
+                    });
+                }
+                Err(e) => {
+                    error!(log, "Error getting mapping to ENR"; "multiaddr" => original_addr.to_string(), "error" => e.to_string())
+                }
+            }
+        }
 
         Ok(Self {
             cached_enrs: LruCache::new(50),
@@ -238,6 +287,11 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
         self.discv5.local_enr()
     }
 
+    /// Return the cached enrs.
+    pub fn cached_enrs(&self) -> impl Iterator<Item = (&PeerId, &Enr)> {
+        self.cached_enrs.iter()
+    }
+
     /// This adds a new `FindPeers` query to the queue if one doesn't already exist.
     pub fn discover_peers(&mut self) {
         // If the discv5 service isn't running or we are in the process of a query, don't bother queuing a new one.
@@ -256,12 +310,19 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
     }
 
     /// Processes a request to search for more peers on a subnet.
-    pub fn discover_subnet_peers(&mut self, subnet_id: SubnetId, min_ttl: Option<Instant>) {
+    pub fn discover_subnet_peers(&mut self, subnets_to_discover: Vec<SubnetDiscovery>) {
         // If the discv5 service isn't running, ignore queries
         if !self.started {
             return;
         }
-        self.add_subnet_query(subnet_id, min_ttl, 0);
+        debug!(
+            self.log,
+            "Making discovery query for subnets";
+            "subnets" => ?subnets_to_discover.iter().map(|s| s.subnet_id).collect::<Vec<_>>()
+        );
+        for subnet in subnets_to_discover {
+            self.add_subnet_query(subnet.subnet_id, subnet.min_ttl, 0);
+        }
     }
 
     /// Add an ENR to the routing table of the discovery mechanism.
@@ -273,7 +334,7 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
             debug!(
                 self.log,
                 "Could not add peer to the local routing table";
-                "error" => e.to_string()
+                "error" => %e
             )
         }
     }
@@ -297,6 +358,54 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
         }
     }
 
+    /// Updates the local ENR TCP port.
+    /// There currently isn't a case to update the address here. We opt for discovery to
+    /// automatically update the external address.
+    ///
+    /// If the external address needs to be modified, use `update_enr_udp_socket.
+    pub fn update_enr_tcp_port(&mut self, port: u16) -> Result<(), String> {
+        self.discv5
+            .enr_insert("tcp", &port.to_be_bytes())
+            .map_err(|e| format!("{:?}", e))?;
+
+        // replace the global version
+        *self.network_globals.local_enr.write() = self.discv5.local_enr();
+        // persist modified enr to disk
+        enr::save_enr_to_disk(Path::new(&self.enr_dir), &self.local_enr(), &self.log);
+        Ok(())
+    }
+
+    /// Updates the local ENR UDP socket.
+    ///
+    /// This is with caution. Discovery should automatically maintain this. This should only be
+    /// used when automatic discovery is disabled.
+    pub fn update_enr_udp_socket(&mut self, socket_addr: SocketAddr) -> Result<(), String> {
+        match socket_addr {
+            SocketAddr::V4(socket) => {
+                self.discv5
+                    .enr_insert("ip", &socket.ip().octets())
+                    .map_err(|e| format!("{:?}", e))?;
+                self.discv5
+                    .enr_insert("udp", &socket.port().to_be_bytes())
+                    .map_err(|e| format!("{:?}", e))?;
+            }
+            SocketAddr::V6(socket) => {
+                self.discv5
+                    .enr_insert("ip6", &socket.ip().octets())
+                    .map_err(|e| format!("{:?}", e))?;
+                self.discv5
+                    .enr_insert("udp6", &socket.port().to_be_bytes())
+                    .map_err(|e| format!("{:?}", e))?;
+            }
+        }
+
+        // replace the global version
+        *self.network_globals.local_enr.write() = self.discv5.local_enr();
+        // persist modified enr to disk
+        enr::save_enr_to_disk(Path::new(&self.enr_dir), &self.local_enr(), &self.log);
+        Ok(())
+    }
+
     /// Adds/Removes a subnet from the ENR Bitfield
     pub fn update_enr_bitfield(&mut self, subnet_id: SubnetId, value: bool) -> Result<(), String> {
         let id = *subnet_id as usize;
@@ -329,12 +438,15 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
             .map_err(|_| String::from("Subnet ID out of bounds, could not set subnet ID"))?;
 
         // insert the bitfield into the ENR record
-        let _ = self
-            .discv5
-            .enr_insert(BITFIELD_ENR_KEY, current_bitfield.as_ssz_bytes());
+        self.discv5
+            .enr_insert(BITFIELD_ENR_KEY, &current_bitfield.as_ssz_bytes())
+            .map_err(|e| format!("{:?}", e))?;
 
         // replace the global version
         *self.network_globals.local_enr.write() = self.discv5.local_enr();
+
+        // persist modified enr to disk
+        enr::save_enr_to_disk(Path::new(&self.enr_dir), &self.local_enr(), &self.log);
         Ok(())
     }
 
@@ -349,24 +461,54 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
         };
 
         info!(self.log, "Updating the ENR fork version";
-            "fork_digest" => format!("{:?}", enr_fork_id.fork_digest),
-            "next_fork_version" => format!("{:?}", enr_fork_id.next_fork_version),
+            "fork_digest" => ?enr_fork_id.fork_digest,
+            "next_fork_version" => ?enr_fork_id.next_fork_version,
             "next_fork_epoch" => next_fork_epoch_log,
         );
 
         let _ = self
             .discv5
-            .enr_insert(ETH2_ENR_KEY, enr_fork_id.as_ssz_bytes())
+            .enr_insert(ETH2_ENR_KEY, &enr_fork_id.as_ssz_bytes())
             .map_err(|e| {
                 warn!(
                     self.log,
                     "Could not update eth2 ENR field";
-                    "error" => format!("{:?}", e)
+                    "error" => ?e
                 )
             });
 
         // replace the global version with discovery version
         *self.network_globals.local_enr.write() = self.discv5.local_enr();
+
+        // persist modified enr to disk
+        enr::save_enr_to_disk(Path::new(&self.enr_dir), &self.local_enr(), &self.log);
+    }
+
+    // Bans a peer and it's associated seen IP addresses.
+    pub fn ban_peer(&mut self, peer_id: &PeerId, ip_addresses: Vec<IpAddr>) {
+        // first try and convert the peer_id to a node_id.
+        if let Ok(node_id) = peer_id_to_node_id(peer_id) {
+            // If we could convert this peer id, remove it from the DHT and ban it from discovery.
+            self.discv5.ban_node(&node_id);
+            // Remove the node from the routing table.
+            self.discv5.remove_node(&node_id);
+        }
+
+        for ip_address in ip_addresses {
+            self.discv5.ban_ip(ip_address);
+        }
+    }
+
+    pub fn unban_peer(&mut self, peer_id: &PeerId, ip_addresses: Vec<IpAddr>) {
+        // first try and convert the peer_id to a node_id.
+        if let Ok(node_id) = peer_id_to_node_id(peer_id) {
+            // If we could convert this peer id, remove it from the DHT and ban it from discovery.
+            self.discv5.permit_node(&node_id);
+        }
+
+        for ip_address in ip_addresses {
+            self.discv5.permit_ip(ip_address);
+        }
     }
 
     /* Internal Functions */
@@ -417,13 +559,15 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
     /// Consume the discovery queue and initiate queries when applicable.
     ///
     /// This also sanitizes the queue removing out-dated queries.
-    fn process_queue(&mut self) {
+    /// Returns `true` if any of the queued queries is processed and a discovery
+    /// query (Subnet or FindPeers) is started.
+    fn process_queue(&mut self) -> bool {
         // Sanitize the queue, removing any out-dated subnet queries
         self.queued_queries.retain(|query| !query.expired());
 
         // use this to group subnet queries together for a single discovery request
         let mut subnet_queries: Vec<SubnetQuery> = Vec::new();
-
+        let mut processed = false;
         // Check that we are within our query concurrency limit
         while !self.at_capacity() && !self.queued_queries.is_empty() {
             // consume and process the query queue
@@ -440,6 +584,7 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
                             FIND_NODE_QUERY_CLOSEST_PEERS,
                             |_| true,
                         );
+                        processed = true;
                     } else {
                         self.queued_queries.push_back(QueryType::FindPeers);
                     }
@@ -459,7 +604,13 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
                         // This query is for searching for peers of a particular subnet
                         // Drain subnet_queries so we can re-use it as we continue to process the queue
                         let grouped_queries: Vec<SubnetQuery> = subnet_queries.drain(..).collect();
+                        debug!(
+                            self.log,
+                            "Starting grouped subnet query";
+                            "subnets" => ?grouped_queries.iter().map(|q| q.subnet_id).collect::<Vec<_>>(),
+                        );
                         self.start_subnet_query(grouped_queries);
+                        processed = true;
                     }
                 }
                 None => {} // Queue is empty
@@ -467,6 +618,7 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
         }
         // Update the queue metric
         metrics::set_gauge(&metrics::DISCOVERY_QUEUE, self.queued_queries.len() as i64);
+        processed
     }
 
     // Returns a boolean indicating if we are currently processing the maximum number of
@@ -488,10 +640,10 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
                     .network_globals
                     .peers
                     .read()
-                    .peers_on_subnet(subnet_query.subnet_id)
+                    .good_peers_on_subnet(subnet_query.subnet_id)
                     .count();
 
-                if peers_on_subnet > TARGET_SUBNET_PEERS {
+                if peers_on_subnet >= TARGET_SUBNET_PEERS {
                     debug!(self.log, "Discovery ignored";
                         "reason" => "Already connected to desired peers",
                         "connected_peers_on_subnet" => peers_on_subnet,
@@ -507,7 +659,7 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
                     "target_subnet_peers" => TARGET_SUBNET_PEERS,
                     "peers_to_find" => target_peers,
                     "attempt" => subnet_query.retries,
-                    "min_ttl" => format!("{:?}", subnet_query.min_ttl),
+                    "min_ttl" => ?subnet_query.min_ttl,
                 );
 
                 filtered_subnet_ids.push(subnet_query.subnet_id);
@@ -563,8 +715,10 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
                 return;
             }
         };
-        // predicate for finding nodes with a matching fork
-        let eth2_fork_predicate = move |enr: &Enr| enr.eth2() == Ok(enr_fork_id.clone());
+        // predicate for finding nodes with a matching fork and valid tcp port
+        let eth2_fork_predicate = move |enr: &Enr| {
+            enr.eth2() == Ok(enr_fork_id.clone()) && (enr.tcp().is_some() || enr.tcp6().is_some())
+        };
 
         // General predicate
         let predicate: Box<dyn Fn(&Enr) -> bool + Send> =
@@ -580,111 +734,129 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
         self.active_queries.push(Box::pin(query_future));
     }
 
+    /// Process the completed QueryResult returned from discv5.
+    fn process_completed_queries(
+        &mut self,
+        query_result: QueryResult,
+    ) -> Option<HashMap<PeerId, Option<Instant>>> {
+        match query_result.0 {
+            GroupedQueryType::FindPeers => {
+                self.find_peer_active = false;
+                match query_result.1 {
+                    Ok(r) if r.is_empty() => {
+                        debug!(self.log, "Discovery query yielded no results.");
+                    }
+                    Ok(r) => {
+                        debug!(self.log, "Discovery query completed"; "peers_found" => r.len());
+                        let mut results: HashMap<_, Option<Instant>> = HashMap::new();
+                        r.iter().for_each(|enr| {
+                            // cache the found ENR's
+                            self.cached_enrs.put(enr.peer_id(), enr.clone());
+                            results.insert(enr.peer_id(), None);
+                        });
+                        return Some(results);
+                    }
+                    Err(e) => {
+                        warn!(self.log, "Discovery query failed"; "error" => %e);
+                    }
+                }
+            }
+            GroupedQueryType::Subnet(queries) => {
+                let subnets_searched_for: Vec<SubnetId> =
+                    queries.iter().map(|query| query.subnet_id).collect();
+                match query_result.1 {
+                    Ok(r) if r.is_empty() => {
+                        debug!(self.log, "Grouped subnet discovery query yielded no results."; "subnets_searched_for" => ?subnets_searched_for);
+                        queries.iter().for_each(|query| {
+                            self.add_subnet_query(
+                                query.subnet_id,
+                                query.min_ttl,
+                                query.retries + 1,
+                            );
+                        })
+                    }
+                    Ok(r) => {
+                        debug!(self.log, "Peer grouped subnet discovery request completed"; "peers_found" => r.len(), "subnets_searched_for" => ?subnets_searched_for);
+
+                        let mut mapped_results = HashMap::new();
+
+                        // cache the found ENR's
+                        for enr in r.iter().cloned() {
+                            self.cached_enrs.put(enr.peer_id(), enr);
+                        }
+
+                        // Map each subnet query's min_ttl to the set of ENR's returned for that subnet.
+                        queries.iter().for_each(|query| {
+                            // A subnet query has completed. Add back to the queue, incrementing retries.
+                            self.add_subnet_query(
+                                query.subnet_id,
+                                query.min_ttl,
+                                query.retries + 1,
+                            );
+
+                            // Check the specific subnet against the enr
+                            let subnet_predicate =
+                                subnet_predicate::<TSpec>(vec![query.subnet_id], &self.log);
+
+                            r.iter()
+                                .filter(|enr| subnet_predicate(enr))
+                                .map(|enr| enr.peer_id())
+                                .for_each(|peer_id| {
+                                    let other_min_ttl = mapped_results.get_mut(&peer_id);
+
+                                    // map peer IDs to the min_ttl furthest in the future
+                                    match (query.min_ttl, other_min_ttl) {
+                                        // update the mapping if the min_ttl is greater
+                                        (
+                                            Some(min_ttl_instant),
+                                            Some(Some(other_min_ttl_instant)),
+                                        ) => {
+                                            if min_ttl_instant
+                                                .saturating_duration_since(*other_min_ttl_instant)
+                                                > DURATION_DIFFERENCE
+                                            {
+                                                *other_min_ttl_instant = min_ttl_instant;
+                                            }
+                                        }
+                                        // update the mapping if we have a specified min_ttl
+                                        (Some(min_ttl), Some(None)) => {
+                                            mapped_results.insert(peer_id, Some(min_ttl));
+                                        }
+                                        // first seen min_ttl for this enr
+                                        (Some(min_ttl), None) => {
+                                            mapped_results.insert(peer_id, Some(min_ttl));
+                                        }
+                                        // first seen min_ttl for this enr
+                                        (None, None) => {
+                                            mapped_results.insert(peer_id, None);
+                                        }
+                                        (None, Some(Some(_))) => {} // Don't replace the existing specific min_ttl
+                                        (None, Some(None)) => {} // No-op because this is a duplicate
+                                    }
+                                });
+                        });
+
+                        if mapped_results.is_empty() {
+                            return None;
+                        } else {
+                            return Some(mapped_results);
+                        }
+                    }
+                    Err(e) => {
+                        warn!(self.log,"Grouped subnet discovery query failed"; "subnets_searched_for" => ?subnets_searched_for, "error" => %e);
+                    }
+                }
+            }
+        }
+        None
+    }
+
     /// Drives the queries returning any results from completed queries.
     fn poll_queries(&mut self, cx: &mut Context) -> Option<HashMap<PeerId, Option<Instant>>> {
-        while let Poll::Ready(Some(query_future)) = self.active_queries.poll_next_unpin(cx) {
-            match query_future.0 {
-                GroupedQueryType::FindPeers => {
-                    self.find_peer_active = false;
-                    match query_future.1 {
-                        Ok(r) if r.is_empty() => {
-                            debug!(self.log, "Discovery query yielded no results.");
-                        }
-                        Ok(r) => {
-                            debug!(self.log, "Discovery query completed"; "peers_found" => r.len());
-                            let mut results: HashMap<PeerId, Option<Instant>> = HashMap::new();
-                            r.iter().for_each(|enr| {
-                                // cache the found ENR's
-                                self.cached_enrs.put(enr.peer_id(), enr.clone());
-                                results.insert(enr.peer_id(), None);
-                            });
-                            return Some(results);
-                        }
-                        Err(e) => {
-                            warn!(self.log, "Discovery query failed"; "error" => e.to_string());
-                        }
-                    }
-                }
-                GroupedQueryType::Subnet(queries) => {
-                    let subnets_searched_for: Vec<SubnetId> =
-                        queries.iter().map(|query| query.subnet_id).collect();
-                    match query_future.1 {
-                        Ok(r) if r.is_empty() => {
-                            debug!(self.log, "Grouped subnet discovery query yielded no results."; "subnets_searched_for" => format!("{:?}",subnets_searched_for));
-                        }
-                        Ok(r) => {
-                            debug!(self.log, "Peer grouped subnet discovery request completed"; "peers_found" => r.len(), "subnets_searched_for" => format!("{:?}",subnets_searched_for));
-
-                            let mut mapped_results: HashMap<PeerId, Option<Instant>> =
-                                HashMap::new();
-
-                            // cache the found ENR's
-                            for enr in r.iter().cloned() {
-                                self.cached_enrs.put(enr.peer_id(), enr);
-                            }
-
-                            // Map each subnet query's min_ttl to the set of ENR's returned for that subnet.
-                            queries.iter().for_each(|query| {
-                                // A subnet query has completed. Add back to the queue, incrementing retries.
-                                self.add_subnet_query(
-                                    query.subnet_id,
-                                    query.min_ttl,
-                                    query.retries + 1,
-                                );
-
-                                // Check the specific subnet against the enr
-                                let subnet_predicate =
-                                    subnet_predicate::<TSpec>(vec![query.subnet_id], &self.log);
-
-                                r.iter()
-                                    .filter(|enr| subnet_predicate(enr))
-                                    .map(|enr| enr.peer_id())
-                                    .for_each(|peer_id| {
-                                        let other_min_ttl = mapped_results.get_mut(&peer_id);
-
-                                        // map peer IDs to the min_ttl furthest in the future
-                                        match (query.min_ttl, other_min_ttl) {
-                                            // update the mapping if the min_ttl is greater
-                                            (
-                                                Some(min_ttl_instant),
-                                                Some(Some(other_min_ttl_instant)),
-                                            ) => {
-                                                if min_ttl_instant.saturating_duration_since(
-                                                    *other_min_ttl_instant,
-                                                ) > DURATION_DIFFERENCE
-                                                {
-                                                    *other_min_ttl_instant = min_ttl_instant;
-                                                }
-                                            }
-                                            // update the mapping if we have a specified min_ttl
-                                            (Some(min_ttl), Some(None)) => {
-                                                mapped_results.insert(peer_id, Some(min_ttl));
-                                            }
-                                            // first seen min_ttl for this enr
-                                            (Some(min_ttl), None) => {
-                                                mapped_results.insert(peer_id, Some(min_ttl));
-                                            }
-                                            // first seen min_ttl for this enr
-                                            (None, None) => {
-                                                mapped_results.insert(peer_id, None);
-                                            }
-                                            (None, Some(Some(_))) => {} // Don't replace the existing specific min_ttl
-                                            (None, Some(None)) => {} // No-op because this is a duplicate
-                                        }
-                                    });
-                            });
-
-                            if mapped_results.is_empty() {
-                                return None;
-                            } else {
-                                return Some(mapped_results);
-                            }
-                        }
-                        Err(e) => {
-                            warn!(self.log,"Grouped subnet discovery query failed"; "subnets_searched_for" => format!("{:?}",subnets_searched_for), "error" => e.to_string());
-                        }
-                    }
-                }
+        while let Poll::Ready(Some(query_result)) = self.active_queries.poll_next_unpin(cx) {
+            let result = self.process_completed_queries(query_result);
+            if result.is_some() {
+                return result;
             }
         }
         None
@@ -711,9 +883,12 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
                 // Still awaiting the event stream, poll it
                 if let Poll::Ready(event_stream) = fut.poll_unpin(cx) {
                     match event_stream {
-                        Ok(stream) => self.event_stream = EventStream::Present(stream),
+                        Ok(stream) => {
+                            debug!(self.log, "Discv5 event stream ready");
+                            self.event_stream = EventStream::Present(stream);
+                        }
                         Err(e) => {
-                            slog::crit!(self.log, "Discv5 event stream failed"; "error" => e.to_string());
+                            slog::crit!(self.log, "Discv5 event stream failed"; "error" => %e);
                             self.event_stream = EventStream::InActive;
                         }
                     }
@@ -733,13 +908,13 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
                             if enr.eth2() == self.local_enr().eth2() {
                                 trace!(self.log, "Peer found in process of query"; "peer_id" => format!("{}", enr.peer_id()), "tcp_socket" => enr.tcp_socket());
                             } else {
-                                // this is temporary warning for debugging the DHT
-                                warn!(self.log, "Found peer during discovery not on correct fork"; "peer_id" => format!("{}", enr.peer_id()), "tcp_socket" => enr.tcp_socket());
+                            // this is temporary warning for debugging the DHT
+                            warn!(self.log, "Found peer during discovery not on correct fork"; "peer_id" => format!("{}", enr.peer_id()), "tcp_socket" => enr.tcp_socket());
                             }
                             */
                         }
                         Discv5Event::SocketUpdated(socket) => {
-                            info!(self.log, "Address updated"; "ip" => format!("{}",socket.ip()), "udp_port" => format!("{}", socket.port()));
+                            info!(self.log, "Address updated"; "ip" => %socket.ip(), "udp_port" => %socket.port());
                             metrics::inc_counter(&metrics::ADDRESS_UPDATE_COUNT);
                             // Discv5 will have updated our local ENR. We save the updated version
                             // to disk.
@@ -757,3 +932,184 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
         Poll::Pending
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::rpc::methods::MetaData;
+    use enr::EnrBuilder;
+    use slog::{o, Drain};
+    use std::net::UdpSocket;
+    use types::MinimalEthSpec;
+
+    type E = MinimalEthSpec;
+
+    pub fn unused_port() -> u16 {
+        let socket = UdpSocket::bind("127.0.0.1:0").expect("should create udp socket");
+        let local_addr = socket.local_addr().expect("should read udp socket");
+        local_addr.port()
+    }
+
+    pub fn build_log(level: slog::Level, enabled: bool) -> slog::Logger {
+        let decorator = slog_term::TermDecorator::new().build();
+        let drain = slog_term::FullFormat::new(decorator).build().fuse();
+        let drain = slog_async::Async::new(drain).build().fuse();
+
+        if enabled {
+            slog::Logger::root(drain.filter_level(level).fuse(), o!())
+        } else {
+            slog::Logger::root(drain.filter(|_| false).fuse(), o!())
+        }
+    }
+
+    async fn build_discovery() -> Discovery<E> {
+        let keypair = libp2p::identity::Keypair::generate_secp256k1();
+        let mut config = NetworkConfig::default();
+        config.discovery_port = unused_port();
+        let enr_key: CombinedKey = CombinedKey::from_libp2p(&keypair).unwrap();
+        let enr: Enr = build_enr::<E>(&enr_key, &config, EnrForkId::default()).unwrap();
+        let log = build_log(slog::Level::Debug, false);
+        let globals = NetworkGlobals::new(
+            enr,
+            9000,
+            9000,
+            MetaData {
+                seq_number: 0,
+                attnets: Default::default(),
+            },
+            vec![],
+            &log,
+        );
+        Discovery::new(&keypair, &config, Arc::new(globals), &log)
+            .await
+            .unwrap()
+    }
+
+    #[tokio::test]
+    async fn test_add_subnet_query() {
+        let mut discovery = build_discovery().await;
+        let now = Instant::now();
+        let mut subnet_query = SubnetQuery {
+            subnet_id: SubnetId::new(1),
+            min_ttl: Some(now),
+            retries: 0,
+        };
+        discovery.add_subnet_query(
+            subnet_query.subnet_id,
+            subnet_query.min_ttl,
+            subnet_query.retries,
+        );
+        assert_eq!(
+            discovery.queued_queries.back(),
+            Some(&QueryType::Subnet(subnet_query.clone()))
+        );
+
+        // New query should replace old query
+        subnet_query.min_ttl = Some(now + Duration::from_secs(1));
+        discovery.add_subnet_query(subnet_query.subnet_id, subnet_query.min_ttl, 1);
+
+        subnet_query.retries += 1;
+
+        assert_eq!(discovery.queued_queries.len(), 1);
+        assert_eq!(
+            discovery.queued_queries.pop_back(),
+            Some(QueryType::Subnet(subnet_query.clone()))
+        );
+
+        // Retries > MAX_DISCOVERY_RETRY must return immediately without adding
+        // anything.
+        discovery.add_subnet_query(
+            subnet_query.subnet_id,
+            subnet_query.min_ttl,
+            MAX_DISCOVERY_RETRY + 1,
+        );
+
+        assert_eq!(discovery.queued_queries.len(), 0);
+    }
+
+    #[tokio::test]
+    async fn test_process_queue() {
+        let mut discovery = build_discovery().await;
+
+        // FindPeers query is processed if there is no subnet query
+        discovery.queued_queries.push_back(QueryType::FindPeers);
+        assert!(discovery.process_queue());
+
+        let now = Instant::now();
+        let subnet_query = SubnetQuery {
+            subnet_id: SubnetId::new(1),
+            min_ttl: Some(now + Duration::from_secs(10)),
+            retries: 0,
+        };
+
+        // Refresh active queries
+        discovery.active_queries = Default::default();
+
+        // SubnetQuery is processed if it's the only queued query
+        discovery
+            .queued_queries
+            .push_back(QueryType::Subnet(subnet_query.clone()));
+        assert!(discovery.process_queue());
+
+        // SubnetQuery is processed if it's there is also 1 queued discovery query
+        discovery.queued_queries.push_back(QueryType::FindPeers);
+        discovery
+            .queued_queries
+            .push_back(QueryType::Subnet(subnet_query.clone()));
+        // Process Subnet query and FindPeers afterwards.
+        assert!(discovery.process_queue());
+    }
+
+    fn make_enr(subnet_ids: Vec<usize>) -> Enr {
+        let mut builder = EnrBuilder::new("v4");
+        let keypair = libp2p::identity::Keypair::generate_secp256k1();
+        let enr_key: CombinedKey = CombinedKey::from_libp2p(&keypair).unwrap();
+
+        // set the "attnets" field on our ENR
+        let mut bitfield = BitVector::<ssz_types::typenum::U64>::new();
+        for id in subnet_ids {
+            bitfield.set(id, true).unwrap();
+        }
+
+        builder.add_value(BITFIELD_ENR_KEY, &bitfield.as_ssz_bytes());
+        builder.build(&enr_key).unwrap()
+    }
+
+    #[tokio::test]
+    async fn test_completed_subnet_queries() {
+        let mut discovery = build_discovery().await;
+        let now = Instant::now();
+        let instant1 = Some(now + Duration::from_secs(10));
+        let instant2 = Some(now + Duration::from_secs(5));
+
+        let query = GroupedQueryType::Subnet(vec![
+            SubnetQuery {
+                subnet_id: SubnetId::new(1),
+                min_ttl: instant1,
+                retries: 0,
+            },
+            SubnetQuery {
+                subnet_id: SubnetId::new(2),
+                min_ttl: instant2,
+                retries: 0,
+            },
+        ]);
+
+        // Create enr which is subscribed to subnets 1 and 2
+        let enr1 = make_enr(vec![1, 2]);
+        let enr2 = make_enr(vec![2]);
+        // Unwanted enr for the given grouped query
+        let enr3 = make_enr(vec![3]);
+
+        let enrs: Vec<Enr> = vec![enr1.clone(), enr2.clone(), enr3.clone()];
+        let results = discovery
+            .process_completed_queries(QueryResult(query, Ok(enrs)))
+            .unwrap();
+
+        // enr1 and enr2 are required peers based on the requested subnet ids
+        assert_eq!(results.len(), 2);
+
+        // when a peer belongs to multiple subnet ids, we use the highest ttl.
+        assert_eq!(results.get(&enr1.peer_id()).unwrap(), &instant1);
+    }
+}

beacon_node/eth2_libp2p/src/discovery/subnet_predicate.rs

@@ -1,5 +1,6 @@
 ///! The subnet predicate used for searching for a particular subnet.
 use super::*;
+use slog::trace;
 use std::ops::Deref;
 
 /// Returns the predicate for a given subnet.
@@ -30,18 +31,18 @@ where
                 .collect();
 
             if matches.is_empty() {
-                debug!(
+                trace!(
                     log_clone,
                     "Peer found but not on any of the desired subnets";
-                    "peer_id" => format!("{}", enr.peer_id())
+                    "peer_id" => %enr.peer_id()
                 );
                 return false;
             } else {
-                debug!(
+                trace!(
                    log_clone,
                    "Peer found on desired subnet(s)";
-                   "peer_id" => format!("{}", enr.peer_id()),
-                   "subnets" => format!("{:?}", matches.as_slice())
+                   "peer_id" => %enr.peer_id(),
+                   "subnets" => ?matches.as_slice()
                 );
                 return true;
             }

beacon_node/eth2_libp2p/src/lib.rs

@@ -7,6 +7,8 @@ extern crate lazy_static;
 
 pub mod behaviour;
 mod config;
+
+#[allow(clippy::mutable_key_type)] // PeerId in hashmaps are no longer permitted by clippy
 pub mod discovery;
 mod metrics;
 mod peer_manager;
@@ -14,16 +16,63 @@ pub mod rpc;
 mod service;
 pub mod types;
 
-pub use crate::types::{error, Enr, GossipTopic, NetworkGlobals, PubsubMessage};
-pub use behaviour::{BehaviourEvent, PeerRequestId, Request, Response};
+use serde::{de, Deserialize, Deserializer, Serialize, Serializer};
+use std::str::FromStr;
+
+/// Wrapper over a libp2p `PeerId` which implements `Serialize` and `Deserialize`
+#[derive(Clone, Debug)]
+pub struct PeerIdSerialized(libp2p::PeerId);
+
+impl From<PeerIdSerialized> for PeerId {
+    fn from(peer_id: PeerIdSerialized) -> Self {
+        peer_id.0
+    }
+}
+
+impl FromStr for PeerIdSerialized {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(Self(
+            PeerId::from_str(s).map_err(|e| format!("Invalid peer id: {}", e))?,
+        ))
+    }
+}
+
+impl Serialize for PeerIdSerialized {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_str(&self.0.to_string())
+    }
+}
+
+impl<'de> Deserialize<'de> for PeerIdSerialized {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s: String = Deserialize::deserialize(deserializer)?;
+        Ok(Self(PeerId::from_str(&s).map_err(|e| {
+            de::Error::custom(format!("Failed to deserialise peer id: {:?}", e))
+        })?))
+    }
+}
+
+pub use crate::types::{error, Enr, GossipTopic, NetworkGlobals, PubsubMessage, SubnetDiscovery};
+pub use behaviour::{BehaviourEvent, Gossipsub, PeerRequestId, Request, Response};
 pub use config::Config as NetworkConfig;
 pub use discovery::{CombinedKeyExt, EnrExt, Eth2Enr};
 pub use discv5;
-pub use libp2p::gossipsub::{MessageId, Topic, TopicHash};
+pub use libp2p::bandwidth::BandwidthSinks;
+pub use libp2p::gossipsub::{MessageAcceptance, MessageId, Topic, TopicHash};
 pub use libp2p::{core::ConnectedPoint, PeerId, Swarm};
 pub use libp2p::{multiaddr, Multiaddr};
 pub use metrics::scrape_discovery_metrics;
 pub use peer_manager::{
-    client::Client, score::PeerAction, PeerDB, PeerInfo, PeerSyncStatus, SyncInfo,
+    client::Client,
+    score::{PeerAction, ReportSource},
+    ConnectionDirection, PeerConnectionStatus, PeerDB, PeerInfo, PeerSyncStatus, SyncInfo,
 };
-pub use service::{Libp2pEvent, Service, NETWORK_KEY_FILENAME};
+pub use service::{load_private_key, Libp2pEvent, Service, NETWORK_KEY_FILENAME};

beacon_node/eth2_libp2p/src/metrics.rs

@@ -34,11 +34,39 @@ lazy_static! {
         "Unsolicited discovery requests per ip per second",
         &["Addresses"]
     );
-    pub static ref GOSSIPSUB_SUBSCRIBED_PEERS_COUNT: Result<IntGaugeVec> = try_create_int_gauge_vec(
-        "gossipsub_peers_per_topic_count",
-        "Peers subscribed per topic",
+    pub static ref PEERS_PER_CLIENT: Result<IntGaugeVec> = try_create_int_gauge_vec(
+        "libp2p_peers_per_client",
+        "The connected peers via client implementation",
+        &["Client"]
+    );
+    pub static ref FAILED_ATTESTATION_PUBLISHES_PER_SUBNET: Result<IntGaugeVec> =
+        try_create_int_gauge_vec(
+            "gossipsub_failed_attestation_publishes_per_subnet",
+            "Failed attestation publishes per subnet",
+            &["subnet"]
+        );
+    pub static ref FAILED_PUBLISHES_PER_MAIN_TOPIC: Result<IntGaugeVec> = try_create_int_gauge_vec(
+        "gossipsub_failed_publishes_per_main_topic",
+        "Failed gossip publishes",
         &["topic_hash"]
     );
+    pub static ref TOTAL_RPC_ERRORS_PER_CLIENT: Result<IntCounterVec> = try_create_int_counter_vec(
+        "libp2p_rpc_errors_per_client",
+        "RPC errors per client",
+        &["client", "rpc_error", "direction"]
+    );
+    pub static ref PEER_ACTION_EVENTS_PER_CLIENT: Result<IntCounterVec> =
+        try_create_int_counter_vec(
+            "libp2p_peer_actions_per_client",
+            "Score reports per client",
+            &["client", "action", "source"]
+        );
+    pub static ref GOSSIP_UNACCEPTED_MESSAGES_PER_CLIENT: Result<IntCounterVec> =
+        try_create_int_counter_vec(
+            "gossipsub_unaccepted_messages_per_client",
+            "Gossipsub messages that we did not accept, per client",
+            &["client", "validation_result"]
+        );
 }
 
 pub fn scrape_discovery_metrics() {

beacon_node/eth2_libp2p/src/peer_manager/client.rs

@@ -4,6 +4,7 @@
 
 use libp2p::identify::IdentifyInfo;
 use serde::Serialize;
+use strum::{AsRefStr, AsStaticStr};
 
 /// Various client and protocol information related to a node.
 #[derive(Clone, Debug, Serialize)]
@@ -20,7 +21,7 @@ pub struct Client {
     pub agent_string: Option<String>,
 }
 
-#[derive(Clone, Debug, Serialize)]
+#[derive(Clone, Debug, Serialize, PartialEq, AsRefStr, AsStaticStr)]
 pub enum ClientKind {
     /// A lighthouse node (the best kind).
     Lighthouse,
@@ -30,6 +31,8 @@ pub enum ClientKind {
     Teku,
     /// A Prysm node.
     Prysm,
+    /// A lodestar node.
+    Lodestar,
     /// An unknown client.
     Unknown,
 }
@@ -84,6 +87,7 @@ impl std::fmt::Display for Client {
                 "Prysm: version: {}, os_version: {}",
                 self.version, self.os_version
             ),
+            ClientKind::Lodestar => write!(f, "Lodestar: version: {}", self.version),
             ClientKind::Unknown => {
                 if let Some(agent_string) = &self.agent_string {
                     write!(f, "Unknown: {}", agent_string)
@@ -95,6 +99,12 @@ impl std::fmt::Display for Client {
     }
 }
 
+impl std::fmt::Display for ClientKind {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(self.as_ref())
+    }
+}
+
 // helper function to identify clients from their agent_version. Returns the client
 // kind and it's associated version and the OS kind.
 fn client_from_agent_version(agent_version: &str) -> (ClientKind, String, String) {
@@ -145,6 +155,20 @@ fn client_from_agent_version(agent_version: &str) -> (ClientKind, String, String
             }
             (kind, version, os_version)
         }
+        Some("nimbus") => {
+            let kind = ClientKind::Nimbus;
+            let mut version = String::from("unknown");
+            let mut os_version = version.clone();
+            if agent_split.next().is_some() {
+                if let Some(agent_version) = agent_split.next() {
+                    version = agent_version.into();
+                    if let Some(agent_os_version) = agent_split.next() {
+                        os_version = agent_os_version.into();
+                    }
+                }
+            }
+            (kind, version, os_version)
+        }
         Some("nim-libp2p") => {
             let kind = ClientKind::Nimbus;
             let mut version = String::from("unknown");
@@ -157,6 +181,18 @@ fn client_from_agent_version(agent_version: &str) -> (ClientKind, String, String
             }
             (kind, version, os_version)
         }
+        Some("js-libp2p") => {
+            let kind = ClientKind::Lodestar;
+            let mut version = String::from("unknown");
+            let mut os_version = version.clone();
+            if let Some(agent_version) = agent_split.next() {
+                version = agent_version.into();
+                if let Some(agent_os_version) = agent_split.next() {
+                    os_version = agent_os_version.into();
+                }
+            }
+            (kind, version, os_version)
+        }
         _ => {
             let unknown = String::from("unknown");
             (ClientKind::Unknown, unknown.clone(), unknown)

beacon_node/eth2_libp2p/src/peer_manager/peer_info.rs

@@ -1,13 +1,17 @@
 use super::client::Client;
-use super::score::Score;
+use super::score::{PeerAction, Score, ScoreState};
 use super::PeerSyncStatus;
 use crate::rpc::MetaData;
 use crate::Multiaddr;
+use discv5::Enr;
 use serde::{
-    ser::{SerializeStructVariant, Serializer},
+    ser::{SerializeStruct, Serializer},
     Serialize,
 };
+use std::collections::HashSet;
+use std::net::{IpAddr, SocketAddr};
 use std::time::Instant;
+use strum::AsRefStr;
 use types::{EthSpec, SubnetId};
 use PeerConnectionStatus::*;
 
@@ -18,23 +22,36 @@ pub struct PeerInfo<T: EthSpec> {
     /// The connection status of the peer
     _status: PeerStatus,
     /// The peers reputation
-    pub score: Score,
+    score: Score,
     /// Client managing this peer
     pub client: Client,
     /// Connection status of this peer
-    pub connection_status: PeerConnectionStatus,
-    /// The known listening addresses of this peer.
+    connection_status: PeerConnectionStatus,
+    /// The known listening addresses of this peer. This is given by identify and can be arbitrary
+    /// (including local IPs).
     pub listening_addresses: Vec<Multiaddr>,
+    /// This is addresses we have physically seen and this is what we use for banning/un-banning
+    /// peers.
+    pub seen_addresses: HashSet<SocketAddr>,
     /// The current syncing state of the peer. The state may be determined after it's initial
     /// connection.
     pub sync_status: PeerSyncStatus,
     /// The ENR subnet bitfield of the peer. This may be determined after it's initial
     /// connection.
     pub meta_data: Option<MetaData<T>>,
+    /// Subnets the peer is connected to.
+    pub subnets: HashSet<SubnetId>,
     /// The time we would like to retain this peer. After this time, the peer is no longer
     /// necessary.
     #[serde(skip)]
     pub min_ttl: Option<Instant>,
+    /// Is the peer a trusted peer.
+    pub is_trusted: bool,
+    /// Direction of the first connection of the last (or current) connected session with this peer.
+    /// None if this peer was never connected.
+    pub connection_direction: Option<ConnectionDirection>,
+    /// The enr of the peer, if known.
+    pub enr: Option<Enr>,
 }
 
 impl<TSpec: EthSpec> Default for PeerInfo<TSpec> {
@@ -44,30 +61,248 @@ impl<TSpec: EthSpec> Default for PeerInfo<TSpec> {
             score: Score::default(),
             client: Client::default(),
             connection_status: Default::default(),
-            listening_addresses: vec![],
+            listening_addresses: Vec::new(),
+            seen_addresses: HashSet::new(),
+            subnets: HashSet::new(),
             sync_status: PeerSyncStatus::Unknown,
             meta_data: None,
             min_ttl: None,
+            is_trusted: false,
+            connection_direction: None,
+            enr: None,
         }
     }
 }
 
 impl<T: EthSpec> PeerInfo<T> {
-    /// Returns if the peer is subscribed to a given `SubnetId`
-    pub fn on_subnet(&self, subnet_id: SubnetId) -> bool {
+    /// Return a PeerInfo struct for a trusted peer.
+    pub fn trusted_peer_info() -> Self {
+        PeerInfo {
+            score: Score::max_score(),
+            is_trusted: true,
+            ..Default::default()
+        }
+    }
+
+    /// Returns if the peer is subscribed to a given `SubnetId` from the metadata attnets field.
+    pub fn on_subnet_metadata(&self, subnet_id: SubnetId) -> bool {
         if let Some(meta_data) = &self.meta_data {
-            return meta_data
-                .attnets
-                .get(*subnet_id as usize)
-                .unwrap_or_else(|_| false);
+            return meta_data.attnets.get(*subnet_id as usize).unwrap_or(false);
         }
         false
     }
 
+    /// Returns if the peer is subscribed to a given `SubnetId` from the gossipsub subscriptions.
+    pub fn on_subnet_gossipsub(&self, subnet_id: SubnetId) -> bool {
+        self.subnets.contains(&subnet_id)
+    }
+
+    /// Returns the seen IP addresses of the peer.
+    pub fn seen_addresses(&self) -> impl Iterator<Item = IpAddr> + '_ {
+        self.seen_addresses
+            .iter()
+            .map(|socket_addr| socket_addr.ip())
+    }
+
+    /// Returns the connection status of the peer.
+    pub fn connection_status(&self) -> &PeerConnectionStatus {
+        &self.connection_status
+    }
+
     /// Reports if this peer has some future validator duty in which case it is valuable to keep it.
     pub fn has_future_duty(&self) -> bool {
         self.min_ttl.map_or(false, |i| i >= Instant::now())
     }
+
+    /// Returns score of the peer.
+    pub fn score(&self) -> &Score {
+        &self.score
+    }
+
+    /// Returns the state of the peer based on the score.
+    pub(crate) fn score_state(&self) -> ScoreState {
+        self.score.state()
+    }
+
+    /// Applies decay rates to a non-trusted peer's score.
+    pub fn score_update(&mut self) {
+        if !self.is_trusted {
+            self.score.update()
+        }
+    }
+
+    /// Apply peer action to a non-trusted peer's score.
+    pub fn apply_peer_action_to_score(&mut self, peer_action: PeerAction) {
+        if !self.is_trusted {
+            self.score.apply_peer_action(peer_action)
+        }
+    }
+
+    pub(crate) fn update_gossipsub_score(&mut self, new_score: f64, ignore: bool) {
+        self.score.update_gossipsub_score(new_score, ignore);
+    }
+
+    pub fn is_good_gossipsub_peer(&self) -> bool {
+        self.score.is_good_gossipsub_peer()
+    }
+
+    #[cfg(test)]
+    /// Resets the peers score.
+    pub fn reset_score(&mut self) {
+        self.score.test_reset();
+    }
+
+    /* Peer connection status API */
+
+    /// Checks if the status is connected.
+    pub fn is_connected(&self) -> bool {
+        matches!(self.connection_status, PeerConnectionStatus::Connected { .. })
+    }
+
+    /// Checks if the status is connected.
+    pub fn is_dialing(&self) -> bool {
+        matches!(self.connection_status, PeerConnectionStatus::Dialing { .. })
+    }
+
+    /// The peer is either connected or in the process of being dialed.
+    pub fn is_connected_or_dialing(&self) -> bool {
+        self.is_connected() || self.is_dialing()
+    }
+
+    /// Checks if the status is banned.
+    pub fn is_banned(&self) -> bool {
+        matches!(self.connection_status, PeerConnectionStatus::Banned { .. })
+    }
+
+    /// Checks if the status is disconnected.
+    pub fn is_disconnected(&self) -> bool {
+        matches!(self.connection_status, Disconnected { .. })
+    }
+
+    /// Returns the number of connections with this peer.
+    pub fn connections(&self) -> (u8, u8) {
+        match self.connection_status {
+            Connected { n_in, n_out } => (n_in, n_out),
+            _ => (0, 0),
+        }
+    }
+
+    // Setters
+
+    /// Modifies the status to Disconnected and sets the last seen instant to now. Returns None if
+    /// no changes were made. Returns Some(bool) where the bool represents if peer became banned or
+    /// simply just disconnected.
+    pub fn notify_disconnect(&mut self) -> Option<bool> {
+        match self.connection_status {
+            Banned { .. } | Disconnected { .. } => None,
+            Disconnecting { to_ban } => {
+                // If we are disconnecting this peer in the process of banning, we now ban the
+                // peer.
+                if to_ban {
+                    self.connection_status = Banned {
+                        since: Instant::now(),
+                    };
+                    Some(true)
+                } else {
+                    self.connection_status = Disconnected {
+                        since: Instant::now(),
+                    };
+                    Some(false)
+                }
+            }
+            Connected { .. } | Dialing { .. } | Unknown => {
+                self.connection_status = Disconnected {
+                    since: Instant::now(),
+                };
+                Some(false)
+            }
+        }
+    }
+
+    /// Notify the we are currently disconnecting this peer, after which the peer will be
+    /// considered banned.
+    // This intermediate state is required to inform the network behaviours that the sub-protocols
+    // are aware this peer exists and it is in the process of being banned. Compared to nodes that
+    // try to connect to us and are already banned (sub protocols do not know of these peers).
+    pub fn disconnecting(&mut self, to_ban: bool) {
+        self.connection_status = Disconnecting { to_ban }
+    }
+
+    /// Modifies the status to Banned
+    pub fn ban(&mut self) {
+        self.connection_status = Banned {
+            since: Instant::now(),
+        };
+    }
+
+    /// The score system has unbanned the peer. Update the connection status
+    pub fn unban(&mut self) {
+        if let PeerConnectionStatus::Banned { since, .. } = self.connection_status {
+            self.connection_status = PeerConnectionStatus::Disconnected { since };
+        }
+    }
+
+    /// Modifies the status to Dialing
+    /// Returns an error if the current state is unexpected.
+    pub(crate) fn dialing_peer(&mut self) -> Result<(), &'static str> {
+        match &mut self.connection_status {
+            Connected { .. } => return Err("Dialing connected peer"),
+            Dialing { .. } => return Err("Dialing an already dialing peer"),
+            Disconnecting { .. } => return Err("Dialing a disconnecting peer"),
+            Disconnected { .. } | Banned { .. } | Unknown => {}
+        }
+        self.connection_status = Dialing {
+            since: Instant::now(),
+        };
+        Ok(())
+    }
+
+    /// Modifies the status to Connected and increases the number of ingoing
+    /// connections by one
+    pub(crate) fn connect_ingoing(&mut self, seen_address: Option<SocketAddr>) {
+        match &mut self.connection_status {
+            Connected { n_in, .. } => *n_in += 1,
+            Disconnected { .. }
+            | Banned { .. }
+            | Dialing { .. }
+            | Disconnecting { .. }
+            | Unknown => {
+                self.connection_status = Connected { n_in: 1, n_out: 0 };
+                self.connection_direction = Some(ConnectionDirection::Incoming);
+            }
+        }
+
+        if let Some(socket_addr) = seen_address {
+            self.seen_addresses.insert(socket_addr);
+        }
+    }
+
+    /// Modifies the status to Connected and increases the number of outgoing
+    /// connections by one
+    pub(crate) fn connect_outgoing(&mut self, seen_address: Option<SocketAddr>) {
+        match &mut self.connection_status {
+            Connected { n_out, .. } => *n_out += 1,
+            Disconnected { .. }
+            | Banned { .. }
+            | Dialing { .. }
+            | Disconnecting { .. }
+            | Unknown => {
+                self.connection_status = Connected { n_in: 0, n_out: 1 };
+                self.connection_direction = Some(ConnectionDirection::Outgoing);
+            }
+        }
+        if let Some(ip_addr) = seen_address {
+            self.seen_addresses.insert(ip_addr);
+        }
+    }
+
+    #[cfg(test)]
+    /// Add an f64 to a non-trusted peer's score abiding by the limits.
+    pub fn add_to_score(&mut self, score: f64) {
+        if !self.is_trusted {
+            self.score.test_add(score)
+        }
+    }
 }
 
 #[derive(Clone, Debug, Serialize)]
@@ -85,6 +320,14 @@ impl Default for PeerStatus {
     }
 }
 
+/// Connection Direction of connection.
+#[derive(Debug, Clone, Serialize, AsRefStr)]
+#[strum(serialize_all = "snake_case")]
+pub enum ConnectionDirection {
+    Incoming,
+    Outgoing,
+}
+
 /// Connection Status of the peer.
 #[derive(Debug, Clone)]
 pub enum PeerConnectionStatus {
@@ -95,11 +338,17 @@ pub enum PeerConnectionStatus {
         /// number of outgoing connections.
         n_out: u8,
     },
+    /// The peer is being disconnected.
+    Disconnecting {
+        // After the disconnection the peer will be considered banned.
+        to_ban: bool,
+    },
     /// The peer has disconnected.
     Disconnected {
         /// last time the peer was connected or discovered.
         since: Instant,
     },
+
     /// The peer has been banned and is disconnected.
     Banned {
         /// moment when the peer was banned.
@@ -117,29 +366,51 @@ pub enum PeerConnectionStatus {
 /// Serialization for http requests.
 impl Serialize for PeerConnectionStatus {
     fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+        let mut s = serializer.serialize_struct("connection_status", 6)?;
         match self {
             Connected { n_in, n_out } => {
-                let mut s = serializer.serialize_struct_variant("", 0, "Connected", 2)?;
-                s.serialize_field("in", n_in)?;
-                s.serialize_field("out", n_out)?;
+                s.serialize_field("status", "connected")?;
+                s.serialize_field("connections_in", n_in)?;
+                s.serialize_field("connections_out", n_out)?;
+                s.serialize_field("last_seen", &0)?;
+                s.end()
+            }
+            Disconnecting { .. } => {
+                s.serialize_field("status", "disconnecting")?;
+                s.serialize_field("connections_in", &0)?;
+                s.serialize_field("connections_out", &0)?;
+                s.serialize_field("last_seen", &0)?;
                 s.end()
             }
             Disconnected { since } => {
-                let mut s = serializer.serialize_struct_variant("", 1, "Disconnected", 1)?;
-                s.serialize_field("since", &since.elapsed().as_secs())?;
+                s.serialize_field("status", "disconnected")?;
+                s.serialize_field("connections_in", &0)?;
+                s.serialize_field("connections_out", &0)?;
+                s.serialize_field("last_seen", &since.elapsed().as_secs())?;
+                s.serialize_field("banned_ips", &Vec::<IpAddr>::new())?;
                 s.end()
             }
             Banned { since } => {
-                let mut s = serializer.serialize_struct_variant("", 2, "Banned", 1)?;
-                s.serialize_field("since", &since.elapsed().as_secs())?;
+                s.serialize_field("status", "banned")?;
+                s.serialize_field("connections_in", &0)?;
+                s.serialize_field("connections_out", &0)?;
+                s.serialize_field("last_seen", &since.elapsed().as_secs())?;
                 s.end()
             }
             Dialing { since } => {
-                let mut s = serializer.serialize_struct_variant("", 3, "Dialing", 1)?;
-                s.serialize_field("since", &since.elapsed().as_secs())?;
+                s.serialize_field("status", "dialing")?;
+                s.serialize_field("connections_in", &0)?;
+                s.serialize_field("connections_out", &0)?;
+                s.serialize_field("last_seen", &since.elapsed().as_secs())?;
+                s.end()
+            }
+            Unknown => {
+                s.serialize_field("status", "unknown")?;
+                s.serialize_field("connections_in", &0)?;
+                s.serialize_field("connections_out", &0)?;
+                s.serialize_field("last_seen", &0)?;
                 s.end()
             }
-            Unknown => serializer.serialize_unit_variant("", 4, "Unknown"),
         }
     }
 }
@@ -149,92 +420,3 @@ impl Default for PeerConnectionStatus {
         PeerConnectionStatus::Unknown
     }
 }
-
-impl PeerConnectionStatus {
-    /// Checks if the status is connected.
-    pub fn is_connected(&self) -> bool {
-        match self {
-            PeerConnectionStatus::Connected { .. } => true,
-            _ => false,
-        }
-    }
-
-    /// Checks if the status is connected.
-    pub fn is_dialing(&self) -> bool {
-        match self {
-            PeerConnectionStatus::Dialing { .. } => true,
-            _ => false,
-        }
-    }
-
-    /// The peer is either connected or in the process of being dialed.
-    pub fn is_connected_or_dialing(&self) -> bool {
-        self.is_connected() || self.is_dialing()
-    }
-
-    /// Checks if the status is banned.
-    pub fn is_banned(&self) -> bool {
-        match self {
-            PeerConnectionStatus::Banned { .. } => true,
-            _ => false,
-        }
-    }
-
-    /// Checks if the status is disconnected.
-    pub fn is_disconnected(&self) -> bool {
-        match self {
-            Disconnected { .. } => true,
-            _ => false,
-        }
-    }
-
-    /// Modifies the status to Connected and increases the number of ingoing
-    /// connections by one
-    pub fn connect_ingoing(&mut self) {
-        match self {
-            Connected { n_in, .. } => *n_in += 1,
-            Disconnected { .. } | Banned { .. } | Dialing { .. } | Unknown => {
-                *self = Connected { n_in: 1, n_out: 0 }
-            }
-        }
-    }
-
-    /// Modifies the status to Connected and increases the number of outgoing
-    /// connections by one
-    pub fn connect_outgoing(&mut self) {
-        match self {
-            Connected { n_out, .. } => *n_out += 1,
-            Disconnected { .. } | Banned { .. } | Dialing { .. } | Unknown => {
-                *self = Connected { n_in: 0, n_out: 1 }
-            }
-        }
-    }
-
-    /// Modifies the status to Disconnected and sets the last seen instant to now
-    pub fn disconnect(&mut self) {
-        *self = Disconnected {
-            since: Instant::now(),
-        };
-    }
-
-    /// Modifies the status to Banned
-    pub fn ban(&mut self) {
-        *self = Banned {
-            since: Instant::now(),
-        };
-    }
-
-    /// The score system has unbanned the peer. Update the connection status
-    pub fn unban(&mut self) {
-        if let PeerConnectionStatus::Banned { since } = self {
-            *self = PeerConnectionStatus::Disconnected { since: *since }
-        }
-    }
-
-    pub fn connections(&self) -> (u8, u8) {
-        match self {
-            Connected { n_in, n_out } => (*n_in, *n_out),
-            _ => (0, 0),
-        }
-    }
-}

beacon_node/eth2_libp2p/src/peer_manager/peer_sync_status.rs

@@ -12,93 +12,71 @@ pub enum PeerSyncStatus {
     Advanced { info: SyncInfo },
     /// Is behind our current head and not useful for block downloads.
     Behind { info: SyncInfo },
+    /// This peer is in an incompatible network.
+    IrrelevantPeer,
     /// Not currently known as a STATUS handshake has not occurred.
     Unknown,
 }
 
-/// This is stored inside the PeerSyncStatus and is very similar to `PeerSyncInfo` in the
-/// `Network` crate.
+/// A relevant peer's sync information.
 #[derive(Clone, Debug, Serialize)]
 pub struct SyncInfo {
-    pub status_head_slot: Slot,
-    pub status_head_root: Hash256,
-    pub status_finalized_epoch: Epoch,
-    pub status_finalized_root: Hash256,
+    pub head_slot: Slot,
+    pub head_root: Hash256,
+    pub finalized_epoch: Epoch,
+    pub finalized_root: Hash256,
+}
+
+impl std::cmp::PartialEq for PeerSyncStatus {
+    fn eq(&self, other: &Self) -> bool {
+        matches!((self, other),
+            (PeerSyncStatus::Synced { .. }, PeerSyncStatus::Synced { .. }) |
+            (PeerSyncStatus::Advanced { .. }, PeerSyncStatus::Advanced { .. }) |
+            (PeerSyncStatus::Behind { .. }, PeerSyncStatus::Behind { .. }) |
+            (PeerSyncStatus::IrrelevantPeer, PeerSyncStatus::IrrelevantPeer) |
+            (PeerSyncStatus::Unknown, PeerSyncStatus::Unknown))
+    }
 }
 
 impl PeerSyncStatus {
     /// Returns true if the peer has advanced knowledge of the chain.
     pub fn is_advanced(&self) -> bool {
-        match self {
-            PeerSyncStatus::Advanced { .. } => true,
-            _ => false,
-        }
+        matches!(self, PeerSyncStatus::Advanced { .. })
     }
 
     /// Returns true if the peer is up to date with the current chain.
     pub fn is_synced(&self) -> bool {
-        match self {
-            PeerSyncStatus::Synced { .. } => true,
-            _ => false,
-        }
+        matches!(self, PeerSyncStatus::Synced { .. })
     }
 
     /// Returns true if the peer is behind the current chain.
     pub fn is_behind(&self) -> bool {
-        match self {
-            PeerSyncStatus::Behind { .. } => true,
-            _ => false,
+        matches!(self, PeerSyncStatus::Behind { .. })
+    }
+
+    pub fn update(&mut self, new_state: PeerSyncStatus) -> bool {
+        if *self == new_state {
+            *self = new_state;
+            false // state was not updated
+        } else {
+            *self = new_state;
+            true
         }
     }
 
-    /// Updates the sync state given a fully synced peer.
-    /// Returns true if the state has changed.
-    pub fn update_synced(&mut self, info: SyncInfo) -> bool {
-        let new_state = PeerSyncStatus::Synced { info };
-
+    pub fn as_str(&self) -> &'static str {
         match self {
-            PeerSyncStatus::Synced { .. } | PeerSyncStatus::Unknown => {
-                *self = new_state;
-                false // state was not updated
-            }
-            _ => {
-                *self = new_state;
-                true
-            }
-        }
-    }
-
-    /// Updates the sync state given a peer that is further ahead in the chain than us.
-    /// Returns true if the state has changed.
-    pub fn update_advanced(&mut self, info: SyncInfo) -> bool {
-        let new_state = PeerSyncStatus::Advanced { info };
-
-        match self {
-            PeerSyncStatus::Advanced { .. } | PeerSyncStatus::Unknown => {
-                *self = new_state;
-                false // state was not updated
-            }
-            _ => {
-                *self = new_state;
-                true
-            }
-        }
-    }
-
-    /// Updates the sync state given a peer that is behind us in the chain.
-    /// Returns true if the state has changed.
-    pub fn update_behind(&mut self, info: SyncInfo) -> bool {
-        let new_state = PeerSyncStatus::Behind { info };
-
-        match self {
-            PeerSyncStatus::Behind { .. } | PeerSyncStatus::Unknown => {
-                *self = new_state;
-                false // state was not updated
-            }
-            _ => {
-                *self = new_state;
-                true
-            }
+            PeerSyncStatus::Advanced { .. } => "Advanced",
+            PeerSyncStatus::Behind { .. } => "Behind",
+            PeerSyncStatus::Synced { .. } => "Synced",
+            PeerSyncStatus::Unknown => "Unknown",
+            PeerSyncStatus::IrrelevantPeer => "Irrelevant",
         }
     }
 }
+
+impl std::fmt::Display for PeerSyncStatus {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(self.as_str())
+    }
+}

beacon_node/eth2_libp2p/src/peer_manager/score.rs

@@ -5,8 +5,11 @@
 //! As the logic develops this documentation will advance.
 //!
 //! The scoring algorithms are currently experimental.
+use crate::behaviour::GOSSIPSUB_GREYLIST_THRESHOLD;
 use serde::Serialize;
 use std::time::Instant;
+use strum::AsRefStr;
+use tokio::time::Duration;
 
 lazy_static! {
     static ref HALFLIFE_DECAY: f64 = -(2.0f64.ln()) / SCORE_HALFLIFE;
@@ -18,6 +21,9 @@ pub(crate) const DEFAULT_SCORE: f64 = 0.0;
 const MIN_SCORE_BEFORE_DISCONNECT: f64 = -20.0;
 /// The minimum reputation before a peer is banned.
 const MIN_SCORE_BEFORE_BAN: f64 = -50.0;
+/// If a peer has a lighthouse score below this constant all other score parts will get ignored and
+/// the peer will get banned regardless of the other parts.
+const MIN_LIGHTHOUSE_SCORE_BEFORE_BAN: f64 = -60.0;
 /// The maximum score a peer can obtain.
 const MAX_SCORE: f64 = 100.0;
 /// The minimum score a peer can obtain.
@@ -25,13 +31,20 @@ const MIN_SCORE: f64 = -100.0;
 /// The halflife of a peer's score. I.e the number of seconds it takes for the score to decay to half its value.
 const SCORE_HALFLIFE: f64 = 600.0;
 /// The number of seconds we ban a peer for before their score begins to decay.
-const BANNED_BEFORE_DECAY: u64 = 1800;
+const BANNED_BEFORE_DECAY: Duration = Duration::from_secs(1800);
+
+/// We weight negative gossipsub scores in such a way that they never result in a disconnect by
+/// themselves. This "solves" the problem of non-decaying gossipsub scores for disconnected peers.
+const GOSSIPSUB_NEGATIVE_SCORE_WEIGHT: f64 =
+    (MIN_SCORE_BEFORE_DISCONNECT + 1.0) / GOSSIPSUB_GREYLIST_THRESHOLD;
+const GOSSIPSUB_POSITIVE_SCORE_WEIGHT: f64 = GOSSIPSUB_NEGATIVE_SCORE_WEIGHT;
 
 /// A collection of actions a peer can perform which will adjust its score.
 /// Each variant has an associated score change.
 // To easily assess the behaviour of scores changes the number of variants should stay low, and
 // somewhat generic.
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone, Copy, AsRefStr)]
+#[strum(serialize_all = "snake_case")]
 pub enum PeerAction {
     /// We should not communicate more with this peer.
     /// This action will cause the peer to get banned.
@@ -48,80 +61,30 @@ pub enum PeerAction {
     /// An error occurred with this peer but it is not necessarily malicious.
     /// We have high tolerance for this actions: several occurrences are needed for a peer to get
     /// kicked.
-    /// NOTE: ~15 occurrences will get the peer banned
+    /// NOTE: ~50 occurrences will get the peer banned
     HighToleranceError,
-    /// Received an expected message.
-    _ValidMessage,
 }
 
-/// The expected state of the peer given the peer's score.
-#[derive(Debug, PartialEq)]
-pub(crate) enum ScoreState {
-    /// We are content with the peers performance. We permit connections and messages.
-    Healthy,
-    /// The peer should be disconnected. We allow re-connections if the peer is persistent.
-    Disconnected,
-    /// The peer is banned. We disallow new connections until it's score has decayed into a
-    /// tolerable threshold.
-    Banned,
+/// Service reporting a `PeerAction` for a peer.
+#[derive(Debug)]
+pub enum ReportSource {
+    Gossipsub,
+    RPC,
+    Processor,
+    SyncService,
 }
 
-/// A peer's score (perceived potential usefulness).
-///
-/// This simplistic version consists of a global score per peer which decays to 0 over time. The
-/// decay rate applies equally to positive and negative scores.
-#[derive(Copy, PartialEq, Clone, Debug, Serialize)]
-pub struct Score {
-    /// The global score.
-    // NOTE: In the future we may separate this into sub-scores involving the RPC, Gossipsub and
-    // lighthouse.
-    score: f64,
-    /// The time the score was last updated to perform time-based adjustments such as score-decay.
-    #[serde(skip)]
-    last_updated: Instant,
-}
-
-impl Default for Score {
-    fn default() -> Self {
-        Score {
-            score: DEFAULT_SCORE,
-            last_updated: Instant::now(),
+impl From<ReportSource> for &'static str {
+    fn from(report_source: ReportSource) -> &'static str {
+        match report_source {
+            ReportSource::Gossipsub => "gossipsub",
+            ReportSource::RPC => "rpc_error",
+            ReportSource::Processor => "processor",
+            ReportSource::SyncService => "sync",
         }
     }
 }
 
-impl Eq for Score {}
-
-impl PartialOrd for Score {
-    fn partial_cmp(&self, other: &Score) -> Option<std::cmp::Ordering> {
-        self.score
-            .partial_cmp(&other.score)
-            .or_else(|| self.last_updated.partial_cmp(&other.last_updated))
-    }
-}
-
-impl Ord for Score {
-    fn cmp(&self, other: &Score) -> std::cmp::Ordering {
-        self.partial_cmp(other)
-            .unwrap_or_else(|| std::cmp::Ordering::Equal)
-    }
-}
-
-impl From<f64> for Score {
-    fn from(f: f64) -> Self {
-        Score {
-            score: f,
-            last_updated: Instant::now(),
-        }
-    }
-}
-
-impl std::fmt::Display for Score {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{:.2}", self.score)
-    }
-}
-
 impl std::fmt::Display for PeerAction {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
@@ -129,11 +92,22 @@ impl std::fmt::Display for PeerAction {
             PeerAction::LowToleranceError => write!(f, "Low Tolerance Error"),
             PeerAction::MidToleranceError => write!(f, "Mid Tolerance Error"),
             PeerAction::HighToleranceError => write!(f, "High Tolerance Error"),
-            PeerAction::_ValidMessage => write!(f, "Valid Message"),
         }
     }
 }
 
+/// The expected state of the peer given the peer's score.
+#[derive(Debug, PartialEq, Clone, Copy)]
+pub(crate) enum ScoreState {
+    /// We are content with the peers performance. We permit connections and messages.
+    Healthy,
+    /// The peer should be disconnected. We allow re-connections if the peer is persistent.
+    Disconnected,
+    /// The peer is banned. We disallow new connections until it's score has decayed into a
+    /// tolerable threshold.
+    Banned,
+}
+
 impl std::fmt::Display for ScoreState {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
@@ -144,35 +118,73 @@ impl std::fmt::Display for ScoreState {
     }
 }
 
-impl Score {
+/// A peer's score (perceived potential usefulness).
+///
+/// This simplistic version consists of a global score per peer which decays to 0 over time. The
+/// decay rate applies equally to positive and negative scores.
+#[derive(PartialEq, Clone, Debug, Serialize)]
+pub struct RealScore {
+    /// The global score.
+    // NOTE: In the future we may separate this into sub-scores involving the RPC, Gossipsub and
+    // lighthouse.
+    lighthouse_score: f64,
+    gossipsub_score: f64,
+    /// We ignore the negative gossipsub scores of some peers to allow decaying without
+    /// disconnecting.
+    ignore_negative_gossipsub_score: bool,
+    score: f64,
+    /// The time the score was last updated to perform time-based adjustments such as score-decay.
+    #[serde(skip)]
+    last_updated: Instant,
+}
+
+impl Default for RealScore {
+    fn default() -> Self {
+        RealScore {
+            lighthouse_score: DEFAULT_SCORE,
+            gossipsub_score: DEFAULT_SCORE,
+            score: DEFAULT_SCORE,
+            last_updated: Instant::now(),
+            ignore_negative_gossipsub_score: false,
+        }
+    }
+}
+
+impl RealScore {
     /// Access to the underlying score.
-    pub fn score(&self) -> f64 {
+    fn recompute_score(&mut self) {
+        self.score = self.lighthouse_score;
+        if self.lighthouse_score <= MIN_LIGHTHOUSE_SCORE_BEFORE_BAN {
+            //ignore all other scores, i.e. do nothing here
+        } else if self.gossipsub_score >= 0.0 {
+            self.score += self.gossipsub_score * GOSSIPSUB_POSITIVE_SCORE_WEIGHT;
+        } else if !self.ignore_negative_gossipsub_score {
+            self.score += self.gossipsub_score * GOSSIPSUB_NEGATIVE_SCORE_WEIGHT;
+        }
+    }
+
+    fn score(&self) -> f64 {
         self.score
     }
 
     /// Modifies the score based on a peer's action.
     pub fn apply_peer_action(&mut self, peer_action: PeerAction) {
         match peer_action {
-            PeerAction::Fatal => self.score = MIN_SCORE, // The worst possible score
+            PeerAction::Fatal => self.set_lighthouse_score(MIN_SCORE), // The worst possible score
             PeerAction::LowToleranceError => self.add(-10.0),
             PeerAction::MidToleranceError => self.add(-5.0),
             PeerAction::HighToleranceError => self.add(-1.0),
-            PeerAction::_ValidMessage => self.add(0.1),
         }
     }
 
-    /// Returns the expected state of the peer given it's score.
-    pub(crate) fn state(&self) -> ScoreState {
-        match self.score {
-            x if x <= MIN_SCORE_BEFORE_BAN => ScoreState::Banned,
-            x if x <= MIN_SCORE_BEFORE_DISCONNECT => ScoreState::Disconnected,
-            _ => ScoreState::Healthy,
-        }
+    fn set_lighthouse_score(&mut self, new_score: f64) {
+        self.lighthouse_score = new_score;
+        self.update_state();
     }
 
     /// Add an f64 to the score abiding by the limits.
-    pub fn add(&mut self, score: f64) {
-        let mut new_score = self.score + score;
+    fn add(&mut self, score: f64) {
+        let mut new_score = self.lighthouse_score + score;
         if new_score > MAX_SCORE {
             new_score = MAX_SCORE;
         }
@@ -180,42 +192,153 @@ impl Score {
             new_score = MIN_SCORE;
         }
 
-        self.score = new_score;
+        self.set_lighthouse_score(new_score);
+    }
+
+    fn update_state(&mut self) {
+        let was_not_banned = self.score > MIN_SCORE_BEFORE_BAN;
+        self.recompute_score();
+        if was_not_banned && self.score <= MIN_SCORE_BEFORE_BAN {
+            //we ban this peer for at least BANNED_BEFORE_DECAY seconds
+            self.last_updated += BANNED_BEFORE_DECAY;
+        }
+    }
+
+    /// Add an f64 to the score abiding by the limits.
+    #[cfg(test)]
+    pub fn test_add(&mut self, score: f64) {
+        self.add(score);
+    }
+
+    #[cfg(test)]
+    // reset the score
+    pub fn test_reset(&mut self) {
+        self.set_lighthouse_score(0f64);
     }
 
     /// Applies time-based logic such as decay rates to the score.
     /// This function should be called periodically.
     pub fn update(&mut self) {
-        // Apply decay logic
-        //
-        // There is two distinct decay processes. One for banned peers and one for all others. If
-        // the score is below the banning threshold and the duration since it was last update is
-        // shorter than the banning threshold, we do nothing.
-        let now = Instant::now();
-        if self.score <= MIN_SCORE_BEFORE_BAN
-            && now
-                .checked_duration_since(self.last_updated)
-                .map(|d| d.as_secs())
-                <= Some(BANNED_BEFORE_DECAY)
-        {
-            // The peer is banned and still within the ban timeout. Do not update it's score.
-            // Update last_updated so that the decay begins correctly when ready.
-            self.last_updated = now;
-            return;
-        }
+        self.update_at(Instant::now())
+    }
 
+    /// Applies time-based logic such as decay rates to the score with the given now value.
+    /// This private sub function is mainly used for testing.
+    fn update_at(&mut self, now: Instant) {
         // Decay the current score
         // Using exponential decay based on a constant half life.
+
+        // It is important that we use here `checked_duration_since` instead of elapsed, since
+        // we set last_updated to the future when banning peers. Therefore `checked_duration_since`
+        // will return None in this case and the score does not get decayed.
         if let Some(secs_since_update) = now
             .checked_duration_since(self.last_updated)
             .map(|d| d.as_secs())
         {
             // e^(-ln(2)/HL*t)
             let decay_factor = (*HALFLIFE_DECAY * secs_since_update as f64).exp();
-            self.score *= decay_factor;
+            self.lighthouse_score *= decay_factor;
             self.last_updated = now;
+            self.update_state();
         }
     }
+
+    pub fn update_gossipsub_score(&mut self, new_score: f64, ignore: bool) {
+        // we only update gossipsub if last_updated is in the past which means either the peer is
+        // not banned or the BANNED_BEFORE_DECAY time is over.
+        if self.last_updated <= Instant::now() {
+            self.gossipsub_score = new_score;
+            self.ignore_negative_gossipsub_score = ignore;
+            self.update_state();
+        }
+    }
+
+    pub fn is_good_gossipsub_peer(&self) -> bool {
+        self.gossipsub_score >= 0.0
+    }
+}
+
+#[derive(PartialEq, Clone, Debug, Serialize)]
+pub enum Score {
+    Max,
+    Real(RealScore),
+}
+
+impl Default for Score {
+    fn default() -> Self {
+        Self::Real(RealScore::default())
+    }
+}
+
+macro_rules! apply {
+    ( $method:ident $(, $param_name: ident: $param_type: ty)*) => {
+        impl Score {
+            pub fn $method(
+                &mut self, $($param_name: $param_type, )*
+            ) {
+                if let Self::Real(score) = self {
+                    score.$method($($param_name, )*);
+                }
+            }
+        }
+    };
+}
+
+apply!(apply_peer_action, peer_action: PeerAction);
+apply!(update);
+apply!(update_gossipsub_score, new_score: f64, ignore: bool);
+#[cfg(test)]
+apply!(test_add, score: f64);
+#[cfg(test)]
+apply!(test_reset);
+
+impl Score {
+    pub fn score(&self) -> f64 {
+        match self {
+            Self::Max => f64::INFINITY,
+            Self::Real(score) => score.score(),
+        }
+    }
+
+    pub fn max_score() -> Self {
+        Self::Max
+    }
+
+    /// Returns the expected state of the peer given it's score.
+    pub(crate) fn state(&self) -> ScoreState {
+        match self.score() {
+            x if x <= MIN_SCORE_BEFORE_BAN => ScoreState::Banned,
+            x if x <= MIN_SCORE_BEFORE_DISCONNECT => ScoreState::Disconnected,
+            _ => ScoreState::Healthy,
+        }
+    }
+
+    pub fn is_good_gossipsub_peer(&self) -> bool {
+        match self {
+            Self::Max => true,
+            Self::Real(score) => score.is_good_gossipsub_peer(),
+        }
+    }
+}
+
+impl Eq for Score {}
+
+impl PartialOrd for Score {
+    fn partial_cmp(&self, other: &Score) -> Option<std::cmp::Ordering> {
+        self.score().partial_cmp(&other.score())
+    }
+}
+
+impl Ord for Score {
+    fn cmp(&self, other: &Score) -> std::cmp::Ordering {
+        self.partial_cmp(other).unwrap_or(std::cmp::Ordering::Equal)
+    }
+}
+
+impl std::fmt::Display for Score {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{:.2}", self.score())
+    }
 }
 
 #[cfg(test)]
@@ -229,25 +352,60 @@ mod tests {
         // 0 change does not change de reputation
         //
         let change = 0.0;
-        score.add(change);
+        score.test_add(change);
         assert_eq!(score.score(), DEFAULT_SCORE);
 
         // underflowing change is capped
         let mut score = Score::default();
         let change = MIN_SCORE - 50.0;
-        score.add(change);
+        score.test_add(change);
         assert_eq!(score.score(), MIN_SCORE);
 
         // overflowing change is capped
         let mut score = Score::default();
         let change = MAX_SCORE + 50.0;
-        score.add(change);
+        score.test_add(change);
         assert_eq!(score.score(), MAX_SCORE);
 
         // Score adjusts
         let mut score = Score::default();
         let change = 1.32;
-        score.add(change);
+        score.test_add(change);
         assert_eq!(score.score(), DEFAULT_SCORE + change);
     }
+
+    #[test]
+    fn test_ban_time() {
+        let mut score = RealScore::default();
+        let now = Instant::now();
+
+        let change = MIN_SCORE_BEFORE_BAN;
+        score.test_add(change);
+        assert_eq!(score.score(), MIN_SCORE_BEFORE_BAN);
+
+        score.update_at(now + BANNED_BEFORE_DECAY);
+        assert_eq!(score.score(), MIN_SCORE_BEFORE_BAN);
+
+        score.update_at(now + BANNED_BEFORE_DECAY + Duration::from_secs(1));
+        assert!(score.score() > MIN_SCORE_BEFORE_BAN);
+    }
+
+    #[test]
+    fn test_very_negative_gossipsub_score() {
+        let mut score = Score::default();
+        score.update_gossipsub_score(GOSSIPSUB_GREYLIST_THRESHOLD, false);
+        assert!(!score.is_good_gossipsub_peer());
+        assert!(score.score() < 0.0);
+        assert_eq!(score.state(), ScoreState::Healthy);
+        score.test_add(-1.0001);
+        assert_eq!(score.state(), ScoreState::Disconnected);
+    }
+
+    #[test]
+    fn test_ignored_gossipsub_score() {
+        let mut score = Score::default();
+        score.update_gossipsub_score(GOSSIPSUB_GREYLIST_THRESHOLD, true);
+        assert!(!score.is_good_gossipsub_peer());
+        assert_eq!(score.score(), 0.0);
+    }
 }

beacon_node/eth2_libp2p/src/rpc/codec/base.rs

@@ -1,5 +1,6 @@
 //! This handles the various supported encoding mechanism for the Eth 2.0 RPC.
 
+use crate::rpc::methods::ErrorType;
 use crate::rpc::{RPCCodedResponse, RPCRequest, RPCResponse};
 use libp2p::bytes::BufMut;
 use libp2p::bytes::BytesMut;
@@ -8,12 +9,12 @@ use tokio_util::codec::{Decoder, Encoder};
 use types::EthSpec;
 
 pub trait OutboundCodec<TItem>: Encoder<TItem> + Decoder {
-    type ErrorType;
+    type CodecErrorType;
 
     fn decode_error(
         &mut self,
         src: &mut BytesMut,
-    ) -> Result<Option<Self::ErrorType>, <Self as Decoder>::Error>;
+    ) -> Result<Option<Self::CodecErrorType>, <Self as Decoder>::Error>;
 }
 
 /* Global Inbound Codec */
@@ -130,8 +131,8 @@ where
 impl<TCodec, TSpec> Decoder for BaseOutboundCodec<TCodec, TSpec>
 where
     TSpec: EthSpec,
-    TCodec:
-        OutboundCodec<RPCRequest<TSpec>, ErrorType = String> + Decoder<Item = RPCResponse<TSpec>>,
+    TCodec: OutboundCodec<RPCRequest<TSpec>, CodecErrorType = ErrorType>
+        + Decoder<Item = RPCResponse<TSpec>>,
 {
     type Item = RPCCodedResponse<TSpec>;
     type Error = <TCodec as Decoder>::Error;
@@ -176,34 +177,159 @@ where
 mod tests {
     use super::super::ssz_snappy::*;
     use super::*;
+    use crate::rpc::methods::StatusMessage;
     use crate::rpc::protocol::*;
+    use snap::write::FrameEncoder;
+    use ssz::Encode;
+    use std::io::Write;
+    use types::{Epoch, Hash256, Slot};
+    use unsigned_varint::codec::Uvi;
+
+    type Spec = types::MainnetEthSpec;
 
     #[test]
     fn test_decode_status_message() {
-        let message = hex::decode("ff060000734e615070590032000006e71e7b54989925efd6c9cbcb8ceb9b5f71216f5137282bf6a1e3b50f64e42d6c7fb347abe07eb0db8200000005029e2800").unwrap();
+        let message = hex::decode("0054ff060000734e615070590032000006e71e7b54989925efd6c9cbcb8ceb9b5f71216f5137282bf6a1e3b50f64e42d6c7fb347abe07eb0db8200000005029e2800").unwrap();
         let mut buf = BytesMut::new();
         buf.extend_from_slice(&message);
 
-        type Spec = types::MainnetEthSpec;
-
         let snappy_protocol_id =
             ProtocolId::new(Protocol::Status, Version::V1, Encoding::SSZSnappy);
 
         let mut snappy_outbound_codec =
             SSZSnappyOutboundCodec::<Spec>::new(snappy_protocol_id, 1_048_576);
 
+        // remove response code
+        let mut snappy_buf = buf.clone();
+        let _ = snappy_buf.split_to(1);
+
         // decode message just as snappy message
-        let snappy_decoded_message = snappy_outbound_codec.decode(&mut buf.clone());
-        // decode message just a ssz message
+        let snappy_decoded_message = snappy_outbound_codec.decode(&mut snappy_buf).unwrap();
 
         // build codecs for entire chunk
         let mut snappy_base_outbound_codec = BaseOutboundCodec::new(snappy_outbound_codec);
 
         // decode message as ssz snappy chunk
-        let snappy_decoded_chunk = snappy_base_outbound_codec.decode(&mut buf.clone());
-        // decode message just a ssz chunk
+        let snappy_decoded_chunk = snappy_base_outbound_codec.decode(&mut buf).unwrap();
 
-        let _ = dbg!(snappy_decoded_message);
-        let _ = dbg!(snappy_decoded_chunk);
+        dbg!(snappy_decoded_message);
+        dbg!(snappy_decoded_chunk);
+    }
+
+    #[test]
+    fn test_invalid_length_prefix() {
+        let mut uvi_codec: Uvi<u128> = Uvi::default();
+        let mut dst = BytesMut::with_capacity(1024);
+
+        // Smallest > 10 byte varint
+        let len: u128 = 2u128.pow(70);
+
+        // Insert length-prefix
+        uvi_codec.encode(len, &mut dst).unwrap();
+
+        let snappy_protocol_id =
+            ProtocolId::new(Protocol::Status, Version::V1, Encoding::SSZSnappy);
+        let mut snappy_outbound_codec =
+            SSZSnappyOutboundCodec::<Spec>::new(snappy_protocol_id, 1_048_576);
+
+        let snappy_decoded_message = snappy_outbound_codec.decode(&mut dst).unwrap_err();
+
+        assert_eq!(
+            snappy_decoded_message,
+            RPCError::IoError("input bytes exceed maximum".to_string()),
+            "length-prefix of > 10 bytes is invalid"
+        );
+    }
+
+    #[test]
+    fn test_length_limits() {
+        fn encode_len(len: usize) -> BytesMut {
+            let mut uvi_codec: Uvi<usize> = Uvi::default();
+            let mut dst = BytesMut::with_capacity(1024);
+            uvi_codec.encode(len, &mut dst).unwrap();
+            dst
+        }
+
+        let protocol_id =
+            ProtocolId::new(Protocol::BlocksByRange, Version::V1, Encoding::SSZSnappy);
+
+        // Response limits
+        let limit = protocol_id.rpc_response_limits::<Spec>();
+        let mut max = encode_len(limit.max + 1);
+        let mut codec = SSZSnappyOutboundCodec::<Spec>::new(protocol_id.clone(), 1_048_576);
+        assert_eq!(codec.decode(&mut max).unwrap_err(), RPCError::InvalidData);
+
+        let mut min = encode_len(limit.min - 1);
+        let mut codec = SSZSnappyOutboundCodec::<Spec>::new(protocol_id.clone(), 1_048_576);
+        assert_eq!(codec.decode(&mut min).unwrap_err(), RPCError::InvalidData);
+
+        // Request limits
+        let limit = protocol_id.rpc_request_limits();
+        let mut max = encode_len(limit.max + 1);
+        let mut codec = SSZSnappyOutboundCodec::<Spec>::new(protocol_id.clone(), 1_048_576);
+        assert_eq!(codec.decode(&mut max).unwrap_err(), RPCError::InvalidData);
+
+        let mut min = encode_len(limit.min - 1);
+        let mut codec = SSZSnappyOutboundCodec::<Spec>::new(protocol_id, 1_048_576);
+        assert_eq!(codec.decode(&mut min).unwrap_err(), RPCError::InvalidData);
+    }
+
+    #[test]
+    fn test_decode_malicious_status_message() {
+        // 10 byte snappy stream identifier
+        let stream_identifier: &'static [u8] = b"\xFF\x06\x00\x00sNaPpY";
+
+        assert_eq!(stream_identifier.len(), 10);
+
+        // byte 0(0xFE) is padding chunk type identifier for snappy messages
+        // byte 1,2,3 are chunk length (little endian)
+        let malicious_padding: &'static [u8] = b"\xFE\x00\x00\x00";
+
+        // Status message is 84 bytes uncompressed. `max_compressed_len` is 32 + 84 + 84/6 = 130.
+        let status_message_bytes = StatusMessage {
+            fork_digest: [0; 4],
+            finalized_root: Hash256::from_low_u64_be(0),
+            finalized_epoch: Epoch::new(1),
+            head_root: Hash256::from_low_u64_be(0),
+            head_slot: Slot::new(1),
+        }
+        .as_ssz_bytes();
+
+        assert_eq!(status_message_bytes.len(), 84);
+        assert_eq!(snap::raw::max_compress_len(status_message_bytes.len()), 130);
+
+        let mut uvi_codec: Uvi<usize> = Uvi::default();
+        let mut dst = BytesMut::with_capacity(1024);
+
+        // Insert length-prefix
+        uvi_codec
+            .encode(status_message_bytes.len(), &mut dst)
+            .unwrap();
+
+        // Insert snappy stream identifier
+        dst.extend_from_slice(stream_identifier);
+
+        // Insert malicious padding of 80 bytes.
+        for _ in 0..20 {
+            dst.extend_from_slice(malicious_padding);
+        }
+
+        // Insert payload (42 bytes compressed)
+        let mut writer = FrameEncoder::new(Vec::new());
+        writer.write_all(&status_message_bytes).unwrap();
+        writer.flush().unwrap();
+        assert_eq!(writer.get_ref().len(), 42);
+        dst.extend_from_slice(writer.get_ref());
+
+        // 10 (for stream identifier) + 80 + 42 = 132 > `max_compressed_len`. Hence, decoding should fail with `InvalidData`.
+
+        let snappy_protocol_id =
+            ProtocolId::new(Protocol::Status, Version::V1, Encoding::SSZSnappy);
+
+        let mut snappy_outbound_codec =
+            SSZSnappyOutboundCodec::<Spec>::new(snappy_protocol_id, 1_048_576);
+
+        let snappy_decoded_message = snappy_outbound_codec.decode(&mut dst).unwrap_err();
+        assert_eq!(snappy_decoded_message, RPCError::InvalidData);
     }
 }

beacon_node/eth2_libp2p/src/rpc/codec/ssz_snappy.rs

@@ -1,10 +1,7 @@
 use crate::rpc::methods::*;
 use crate::rpc::{
     codec::base::OutboundCodec,
-    protocol::{
-        Encoding, Protocol, ProtocolId, RPCError, Version, BLOCKS_BY_ROOT_REQUEST_MAX,
-        BLOCKS_BY_ROOT_REQUEST_MIN, SIGNED_BEACON_BLOCK_MAX, SIGNED_BEACON_BLOCK_MIN,
-    },
+    protocol::{Encoding, Protocol, ProtocolId, RPCError, Version, ERROR_TYPE_MAX, ERROR_TYPE_MIN},
 };
 use crate::rpc::{RPCCodedResponse, RPCRequest, RPCResponse};
 use libp2p::bytes::BytesMut;
@@ -97,91 +94,70 @@ impl<TSpec: EthSpec> Decoder for SSZSnappyInboundCodec<TSpec> {
     type Error = RPCError;
 
     fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
-        if self.len.is_none() {
+        let length = if let Some(length) = self.len {
+            length
+        } else {
             // Decode the length of the uncompressed bytes from an unsigned varint
+            // Note: length-prefix of > 10 bytes(uint64) would be a decoding error
             match self.inner.decode(src).map_err(RPCError::from)? {
                 Some(length) => {
                     self.len = Some(length);
+                    length
                 }
                 None => return Ok(None), // need more bytes to decode length
             }
         };
 
-        let length = self.len.expect("length should be Some");
-
-        // Should not attempt to decode rpc chunks with length > max_packet_size
-        if length > self.max_packet_size {
+        // Should not attempt to decode rpc chunks with `length > max_packet_size` or not within bounds of
+        // packet size for ssz container corresponding to `self.protocol`.
+        let ssz_limits = self.protocol.rpc_request_limits();
+        if length > self.max_packet_size || ssz_limits.is_out_of_bounds(length) {
             return Err(RPCError::InvalidData);
         }
-        let mut reader = FrameDecoder::new(Cursor::new(&src));
+        // Calculate worst case compression length for given uncompressed length
+        let max_compressed_len = snap::raw::max_compress_len(length) as u64;
+
+        // Create a limit reader as a wrapper that reads only upto `max_compressed_len` from `src`.
+        let limit_reader = Cursor::new(src.as_ref()).take(max_compressed_len);
+        let mut reader = FrameDecoder::new(limit_reader);
         let mut decoded_buffer = vec![0; length];
 
         match reader.read_exact(&mut decoded_buffer) {
             Ok(()) => {
                 // `n` is how many bytes the reader read in the compressed stream
-                let n = reader.get_ref().position();
+                let n = reader.get_ref().get_ref().position();
                 self.len = None;
                 let _read_bytes = src.split_to(n as usize);
+
+                // We need not check that decoded_buffer.len() is within bounds here
+                // since we have already checked `length` above.
                 match self.protocol.message_name {
                     Protocol::Status => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len() == <StatusMessage as Encode>::ssz_fixed_len() {
-                                Ok(Some(RPCRequest::Status(StatusMessage::from_ssz_bytes(
-                                    &decoded_buffer,
-                                )?)))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCRequest::Status(StatusMessage::from_ssz_bytes(
+                            &decoded_buffer,
+                        )?))),
                     },
                     Protocol::Goodbye => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len() == <GoodbyeReason as Encode>::ssz_fixed_len() {
-                                Ok(Some(RPCRequest::Goodbye(GoodbyeReason::from_ssz_bytes(
-                                    &decoded_buffer,
-                                )?)))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCRequest::Goodbye(
+                            GoodbyeReason::from_ssz_bytes(&decoded_buffer)?,
+                        ))),
                     },
                     Protocol::BlocksByRange => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len()
-                                == <BlocksByRangeRequest as Encode>::ssz_fixed_len()
-                            {
-                                Ok(Some(RPCRequest::BlocksByRange(
-                                    BlocksByRangeRequest::from_ssz_bytes(&decoded_buffer)?,
-                                )))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCRequest::BlocksByRange(
+                            BlocksByRangeRequest::from_ssz_bytes(&decoded_buffer)?,
+                        ))),
                     },
                     Protocol::BlocksByRoot => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len() >= *BLOCKS_BY_ROOT_REQUEST_MIN
-                                && decoded_buffer.len() <= *BLOCKS_BY_ROOT_REQUEST_MAX
-                            {
-                                Ok(Some(RPCRequest::BlocksByRoot(BlocksByRootRequest {
-                                    block_roots: VariableList::from_ssz_bytes(&decoded_buffer)?,
-                                })))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCRequest::BlocksByRoot(BlocksByRootRequest {
+                            block_roots: VariableList::from_ssz_bytes(&decoded_buffer)?,
+                        }))),
                     },
                     Protocol::Ping => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len() == <Ping as Encode>::ssz_fixed_len() {
-                                Ok(Some(RPCRequest::Ping(Ping {
-                                    data: u64::from_ssz_bytes(&decoded_buffer)?,
-                                })))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCRequest::Ping(Ping {
+                            data: u64::from_ssz_bytes(&decoded_buffer)?,
+                        }))),
                     },
+                    // This case should be unreachable as `MetaData` requests are handled separately in the `InboundUpgrade`
                     Protocol::MetaData => match self.protocol.version {
                         Version::V1 => {
                             if !decoded_buffer.is_empty() {
@@ -193,12 +169,7 @@ impl<TSpec: EthSpec> Decoder for SSZSnappyInboundCodec<TSpec> {
                     },
                 }
             }
-            Err(e) => match e.kind() {
-                // Haven't received enough bytes to decode yet
-                // TODO: check if this is the only Error variant where we return `Ok(None)`
-                ErrorKind::UnexpectedEof => Ok(None),
-                _ => Err(e).map_err(RPCError::from),
-            },
+            Err(e) => handle_error(e, reader.get_ref().get_ref().position(), max_compressed_len),
         }
     }
 }
@@ -275,142 +246,144 @@ impl<TSpec: EthSpec> Decoder for SSZSnappyOutboundCodec<TSpec> {
     type Error = RPCError;
 
     fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
-        if self.len.is_none() {
+        let length = if let Some(length) = self.len {
+            length
+        } else {
             // Decode the length of the uncompressed bytes from an unsigned varint
+            // Note: length-prefix of > 10 bytes(uint64) would be a decoding error
             match self.inner.decode(src).map_err(RPCError::from)? {
                 Some(length) => {
                     self.len = Some(length as usize);
+                    length
                 }
                 None => return Ok(None), // need more bytes to decode length
             }
         };
 
-        let length = self.len.expect("length should be Some");
-
-        // Should not attempt to decode rpc chunks with length > max_packet_size
-        if length > self.max_packet_size {
+        // Should not attempt to decode rpc chunks with `length > max_packet_size` or not within bounds of
+        // packet size for ssz container corresponding to `self.protocol`.
+        let ssz_limits = self.protocol.rpc_response_limits::<TSpec>();
+        if length > self.max_packet_size || ssz_limits.is_out_of_bounds(length) {
             return Err(RPCError::InvalidData);
         }
-        let mut reader = FrameDecoder::new(Cursor::new(&src));
+        // Calculate worst case compression length for given uncompressed length
+        let max_compressed_len = snap::raw::max_compress_len(length) as u64;
+        // Create a limit reader as a wrapper that reads only upto `max_compressed_len` from `src`.
+        let limit_reader = Cursor::new(src.as_ref()).take(max_compressed_len);
+        let mut reader = FrameDecoder::new(limit_reader);
+
         let mut decoded_buffer = vec![0; length];
+
         match reader.read_exact(&mut decoded_buffer) {
             Ok(()) => {
                 // `n` is how many bytes the reader read in the compressed stream
-                let n = reader.get_ref().position();
+                let n = reader.get_ref().get_ref().position();
                 self.len = None;
-                let _read_byts = src.split_to(n as usize);
+                let _read_bytes = src.split_to(n as usize);
+
+                // We need not check that decoded_buffer.len() is within bounds here
+                // since we have already checked `length` above.
                 match self.protocol.message_name {
                     Protocol::Status => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len() == <StatusMessage as Encode>::ssz_fixed_len() {
-                                Ok(Some(RPCResponse::Status(StatusMessage::from_ssz_bytes(
-                                    &decoded_buffer,
-                                )?)))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCResponse::Status(
+                            StatusMessage::from_ssz_bytes(&decoded_buffer)?,
+                        ))),
                     },
+                    // This case should be unreachable as `Goodbye` has no response.
                     Protocol::Goodbye => Err(RPCError::InvalidData),
                     Protocol::BlocksByRange => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len() >= *SIGNED_BEACON_BLOCK_MIN
-                                && decoded_buffer.len() <= *SIGNED_BEACON_BLOCK_MAX
-                            {
-                                Ok(Some(RPCResponse::BlocksByRange(Box::new(
-                                    SignedBeaconBlock::from_ssz_bytes(&decoded_buffer)?,
-                                ))))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCResponse::BlocksByRange(Box::new(
+                            SignedBeaconBlock::from_ssz_bytes(&decoded_buffer)?,
+                        )))),
                     },
                     Protocol::BlocksByRoot => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len() >= *SIGNED_BEACON_BLOCK_MIN
-                                && decoded_buffer.len() <= *SIGNED_BEACON_BLOCK_MAX
-                            {
-                                Ok(Some(RPCResponse::BlocksByRoot(Box::new(
-                                    SignedBeaconBlock::from_ssz_bytes(&decoded_buffer)?,
-                                ))))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCResponse::BlocksByRoot(Box::new(
+                            SignedBeaconBlock::from_ssz_bytes(&decoded_buffer)?,
+                        )))),
                     },
                     Protocol::Ping => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len() == <Ping as Encode>::ssz_fixed_len() {
-                                Ok(Some(RPCResponse::Pong(Ping {
-                                    data: u64::from_ssz_bytes(&decoded_buffer)?,
-                                })))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCResponse::Pong(Ping {
+                            data: u64::from_ssz_bytes(&decoded_buffer)?,
+                        }))),
                     },
                     Protocol::MetaData => match self.protocol.version {
-                        Version::V1 => {
-                            if decoded_buffer.len() == <MetaData<TSpec> as Encode>::ssz_fixed_len()
-                            {
-                                Ok(Some(RPCResponse::MetaData(MetaData::from_ssz_bytes(
-                                    &decoded_buffer,
-                                )?)))
-                            } else {
-                                Err(RPCError::InvalidData)
-                            }
-                        }
+                        Version::V1 => Ok(Some(RPCResponse::MetaData(MetaData::from_ssz_bytes(
+                            &decoded_buffer,
+                        )?))),
                     },
                 }
             }
-            Err(e) => match e.kind() {
-                // Haven't received enough bytes to decode yet
-                // TODO: check if this is the only Error variant where we return `Ok(None)`
-                ErrorKind::UnexpectedEof => Ok(None),
-                _ => Err(e).map_err(RPCError::from),
-            },
+            Err(e) => handle_error(e, reader.get_ref().get_ref().position(), max_compressed_len),
         }
     }
 }
 
 impl<TSpec: EthSpec> OutboundCodec<RPCRequest<TSpec>> for SSZSnappyOutboundCodec<TSpec> {
-    type ErrorType = String;
+    type CodecErrorType = ErrorType;
 
-    fn decode_error(&mut self, src: &mut BytesMut) -> Result<Option<Self::ErrorType>, RPCError> {
-        if self.len.is_none() {
+    fn decode_error(
+        &mut self,
+        src: &mut BytesMut,
+    ) -> Result<Option<Self::CodecErrorType>, RPCError> {
+        let length = if let Some(length) = self.len {
+            length
+        } else {
             // Decode the length of the uncompressed bytes from an unsigned varint
             match self.inner.decode(src).map_err(RPCError::from)? {
                 Some(length) => {
                     self.len = Some(length as usize);
+                    length
                 }
                 None => return Ok(None), // need more bytes to decode length
             }
         };
 
-        let length = self.len.expect("length should be Some");
-
-        // Should not attempt to decode rpc chunks with length > max_packet_size
-        if length > self.max_packet_size {
+        // Should not attempt to decode rpc chunks with `length > max_packet_size` or not within bounds of
+        // packet size for ssz container corresponding to `ErrorType`.
+        if length > self.max_packet_size || length > *ERROR_TYPE_MAX || length < *ERROR_TYPE_MIN {
             return Err(RPCError::InvalidData);
         }
-        let mut reader = FrameDecoder::new(Cursor::new(&src));
+
+        // Calculate worst case compression length for given uncompressed length
+        let max_compressed_len = snap::raw::max_compress_len(length) as u64;
+        // Create a limit reader as a wrapper that reads only upto `max_compressed_len` from `src`.
+        let limit_reader = Cursor::new(src.as_ref()).take(max_compressed_len);
+        let mut reader = FrameDecoder::new(limit_reader);
         let mut decoded_buffer = vec![0; length];
         match reader.read_exact(&mut decoded_buffer) {
             Ok(()) => {
                 // `n` is how many bytes the reader read in the compressed stream
-                let n = reader.get_ref().position();
+                let n = reader.get_ref().get_ref().position();
                 self.len = None;
                 let _read_bytes = src.split_to(n as usize);
-                Ok(Some(
-                    String::from_utf8_lossy(&<Vec<u8>>::from_ssz_bytes(&decoded_buffer)?).into(),
-                ))
+                Ok(Some(ErrorType(VariableList::from_ssz_bytes(
+                    &decoded_buffer,
+                )?)))
             }
-            Err(e) => match e.kind() {
-                // Haven't received enough bytes to decode yet
-                // TODO: check if this is the only Error variant where we return `Ok(None)`
-                ErrorKind::UnexpectedEof => Ok(None),
-                _ => Err(e).map_err(RPCError::from),
-            },
+            Err(e) => handle_error(e, reader.get_ref().get_ref().position(), max_compressed_len),
         }
     }
 }
+
+/// Handle errors that we get from decoding an RPC message from the stream.
+/// `num_bytes_read` is the number of bytes the snappy decoder has read from the underlying stream.
+/// `max_compressed_len` is the maximum compressed size for a given uncompressed size.
+fn handle_error<T>(
+    err: std::io::Error,
+    num_bytes: u64,
+    max_compressed_len: u64,
+) -> Result<Option<T>, RPCError> {
+    match err.kind() {
+        ErrorKind::UnexpectedEof => {
+            // If snappy has read `max_compressed_len` from underlying stream and still can't fill buffer, we have a malicious message.
+            // Report as `InvalidData` so that malicious peer gets banned.
+            if num_bytes >= max_compressed_len {
+                Err(RPCError::InvalidData)
+            } else {
+                // Haven't received enough bytes to decode yet, wait for more
+                Ok(None)
+            }
+        }
+        _ => Err(err).map_err(RPCError::from),
+    }
+}

beacon_node/eth2_libp2p/src/rpc/handler.rs

@@ -14,7 +14,7 @@ use libp2p::swarm::protocols_handler::{
     KeepAlive, ProtocolsHandler, ProtocolsHandlerEvent, ProtocolsHandlerUpgrErr, SubstreamProtocol,
 };
 use libp2p::swarm::NegotiatedSubstream;
-use slog::{crit, debug, warn};
+use slog::{crit, debug, trace, warn};
 use smallvec::SmallVec;
 use std::{
     collections::hash_map::Entry,
@@ -22,11 +22,10 @@ use std::{
     task::{Context, Poll},
     time::Duration,
 };
-use tokio::time::{delay_queue, delay_until, Delay, DelayQueue, Instant as TInstant};
+use tokio::time::{sleep_until, Instant as TInstant, Sleep};
+use tokio_util::time::{delay_queue, DelayQueue};
 use types::EthSpec;
 
-//TODO: Implement check_timeout() on the substream types
-
 /// The time (in seconds) before a substream that is awaiting a response from the user times out.
 pub const RESPONSE_TIMEOUT: u64 = 10;
 
@@ -50,6 +49,9 @@ type InboundProcessingOutput<TSpec> = (
     u64,                     /* Chunks remaining to be sent after this processing finishes */
 );
 
+/// Events the handler emits to the behaviour.
+type HandlerEvent<T> = Result<RPCReceived<T>, HandlerErr>;
+
 /// An error encountered by the handler.
 pub enum HandlerErr {
     /// An error occurred for this peer's request. This can occur during protocol negotiation,
@@ -81,13 +83,10 @@ where
     TSpec: EthSpec,
 {
     /// The upgrade for inbound substreams.
-    listen_protocol: SubstreamProtocol<RPCProtocol<TSpec>>,
-
-    /// Errors occurring on outbound and inbound connections queued for reporting back.
-    pending_errors: Vec<HandlerErr>,
+    listen_protocol: SubstreamProtocol<RPCProtocol<TSpec>, ()>,
 
     /// Queue of events to produce in `poll()`.
-    events_out: SmallVec<[RPCReceived<TSpec>; 4]>,
+    events_out: SmallVec<[HandlerEvent<TSpec>; 4]>,
 
     /// Queue of outbound substreams to open.
     dial_queue: SmallVec<[(RequestId, RPCRequest<TSpec>); 4]>,
@@ -116,9 +115,6 @@ where
     /// Maximum number of concurrent outbound substreams being opened. Value is never modified.
     max_dial_negotiated: u32,
 
-    /// Value to return from `connection_keep_alive`.
-    keep_alive: KeepAlive,
-
     /// State of the handler.
     state: HandlerState,
 
@@ -137,7 +133,7 @@ enum HandlerState {
     ///
     /// While in this state the handler rejects new requests but tries to finish existing ones.
     /// Once the timer expires, all messages are killed.
-    ShuttingDown(Delay),
+    ShuttingDown(Sleep),
     /// The handler is deactivated. A goodbye has been sent and no more messages are sent or
     /// received.
     Deactivated,
@@ -166,8 +162,6 @@ struct OutboundInfo<TSpec: EthSpec> {
     /// Info over the protocol this substream is handling.
     proto: Protocol,
     /// Number of chunks to be seen from the peer's response.
-    // TODO: removing the option could allow clossing the streams after the number of
-    // expected responses is met for all protocols.
     remaining_chunks: Option<u64>,
     /// `RequestId` as given by the application that sent the request.
     req_id: RequestId,
@@ -183,31 +177,6 @@ enum InboundState<TSpec: EthSpec> {
     Poisoned,
 }
 
-impl<TSpec: EthSpec> InboundState<TSpec> {
-    /// Sends the given items over the underlying substream, if the state allows it, and returns the
-    /// final state.
-    fn send_items(
-        self,
-        pending_items: &mut Vec<RPCCodedResponse<TSpec>>,
-        remaining_chunks: u64,
-    ) -> Self {
-        if let InboundState::Idle(substream) = self {
-            // only send on Idle
-            if !pending_items.is_empty() {
-                // take the items that we need to send
-                let to_send = std::mem::replace(pending_items, vec![]);
-                let fut = process_inbound_substream(substream, remaining_chunks, to_send).boxed();
-                InboundState::Busy(Box::pin(fut))
-            } else {
-                // nothing to do, keep waiting for responses
-                InboundState::Idle(substream)
-            }
-        } else {
-            self
-        }
-    }
-}
-
 /// State of an outbound substream. Either waiting for a response, or in the process of sending.
 pub enum OutboundSubstreamState<TSpec: EthSpec> {
     /// A request has been sent, and we are awaiting a response. This future is driven in the
@@ -228,10 +197,12 @@ impl<TSpec> RPCHandler<TSpec>
 where
     TSpec: EthSpec,
 {
-    pub fn new(listen_protocol: SubstreamProtocol<RPCProtocol<TSpec>>, log: &slog::Logger) -> Self {
+    pub fn new(
+        listen_protocol: SubstreamProtocol<RPCProtocol<TSpec>, ()>,
+        log: &slog::Logger,
+    ) -> Self {
         RPCHandler {
             listen_protocol,
-            pending_errors: Vec::new(),
             events_out: SmallVec::new(),
             dial_queue: SmallVec::new(),
             dial_negotiated: 0,
@@ -243,39 +214,24 @@ where
             current_outbound_substream_id: SubstreamId(0),
             state: HandlerState::Active,
             max_dial_negotiated: 8,
-            keep_alive: KeepAlive::Yes,
             outbound_io_error_retries: 0,
             log: log.clone(),
         }
     }
 
-    /// Returns a reference to the listen protocol configuration.
-    ///
-    /// > **Note**: If you modify the protocol, modifications will only applies to future inbound
-    /// >           substreams, not the ones already being negotiated.
-    pub fn listen_protocol_ref(&self) -> &SubstreamProtocol<RPCProtocol<TSpec>> {
-        &self.listen_protocol
-    }
-
-    /// Returns a mutable reference to the listen protocol configuration.
-    ///
-    /// > **Note**: If you modify the protocol, modifications will only apply to future inbound
-    /// >           substreams, not the ones already being negotiated.
-    pub fn listen_protocol_mut(&mut self) -> &mut SubstreamProtocol<RPCProtocol<TSpec>> {
-        &mut self.listen_protocol
-    }
-
     /// Initiates the handler's shutdown process, sending an optional last message to the peer.
     pub fn shutdown(&mut self, final_msg: Option<(RequestId, RPCRequest<TSpec>)>) {
         if matches!(self.state, HandlerState::Active) {
-            debug!(self.log, "Starting handler shutdown"; "unsent_queued_requests" => self.dial_queue.len());
+            if !self.dial_queue.is_empty() {
+                debug!(self.log, "Starting handler shutdown"; "unsent_queued_requests" => self.dial_queue.len());
+            }
             // we now drive to completion communications already dialed/established
             while let Some((id, req)) = self.dial_queue.pop() {
-                self.pending_errors.push(HandlerErr::Outbound {
-                    id,
-                    proto: req.protocol(),
+                self.events_out.push(Err(HandlerErr::Outbound {
                     error: RPCError::HandlerRejected,
-                })
+                    proto: req.protocol(),
+                    id,
+                }));
             }
 
             // Queue our final message, if any
@@ -283,11 +239,10 @@ where
                 self.dial_queue.push((id, req));
             }
 
-            self.state = HandlerState::ShuttingDown(delay_until(
+            self.state = HandlerState::ShuttingDown(sleep_until(
                 TInstant::now() + Duration::from_secs(SHUTDOWN_TIMEOUT_SECS as u64),
             ));
         }
-        self.update_keep_alive();
     }
 
     /// Opens an outbound substream with a request.
@@ -295,15 +250,12 @@ where
         match self.state {
             HandlerState::Active => {
                 self.dial_queue.push((id, req));
-                self.update_keep_alive();
-            }
-            _ => {
-                self.pending_errors.push(HandlerErr::Outbound {
-                    id,
-                    proto: req.protocol(),
-                    error: RPCError::HandlerRejected,
-                });
             }
+            _ => self.events_out.push(Err(HandlerErr::Outbound {
+                error: RPCError::HandlerRejected,
+                proto: req.protocol(),
+                id,
+            })),
         }
     }
 
@@ -315,66 +267,31 @@ where
         let inbound_info = if let Some(info) = self.inbound_substreams.get_mut(&inbound_id) {
             info
         } else {
-            warn!(self.log, "Stream has expired. Response not sent";
-                "response" => response.to_string(), "id" => inbound_id);
+            if !matches!(response, RPCCodedResponse::StreamTermination(..)) {
+                // the stream is closed after sending the expected number of responses
+                trace!(self.log, "Inbound stream has expired, response not sent";
+                    "response" => %response, "id" => inbound_id);
+            }
             return;
         };
 
         // If the response we are sending is an error, report back for handling
         if let RPCCodedResponse::Error(ref code, ref reason) = response {
-            let err = HandlerErr::Inbound {
-                id: inbound_id,
-                proto: inbound_info.protocol,
+            self.events_out.push(Err(HandlerErr::Inbound {
                 error: RPCError::ErrorResponse(*code, reason.to_string()),
-            };
-            self.pending_errors.push(err);
+                proto: inbound_info.protocol,
+                id: inbound_id,
+            }));
         }
 
         if matches!(self.state, HandlerState::Deactivated) {
             // we no longer send responses after the handler is deactivated
             debug!(self.log, "Response not sent. Deactivated handler";
-                "response" => response.to_string(), "id" => inbound_id);
+                "response" => %response, "id" => inbound_id);
             return;
         }
         inbound_info.pending_items.push(response);
     }
-
-    /// Updates the `KeepAlive` returned by `connection_keep_alive`.
-    ///
-    /// The handler stays alive as long as there are inbound/outbound substreams established and no
-    /// items dialing/to be dialed. Otherwise it is given a grace period of inactivity of
-    /// `self.inactive_timeout`.
-    fn update_keep_alive(&mut self) {
-        // Check that we don't have outbound items pending for dialing, nor dialing, nor
-        // established. Also check that there are no established inbound substreams.
-        // Errors and events need to be reported back, so check those too.
-        let should_shutdown = match self.state {
-            HandlerState::ShuttingDown(_) => {
-                self.dial_queue.is_empty()
-                    && self.outbound_substreams.is_empty()
-                    && self.inbound_substreams.is_empty()
-                    && self.pending_errors.is_empty()
-                    && self.events_out.is_empty()
-                    && self.dial_negotiated == 0
-            }
-            HandlerState::Deactivated => {
-                // Regardless of events, the timeout has expired. Force the disconnect.
-                true
-            }
-            _ => false,
-        };
-
-        match self.keep_alive {
-            KeepAlive::Yes if should_shutdown => self.keep_alive = KeepAlive::No,
-            KeepAlive::Yes => {} // We continue being active
-            KeepAlive::Until(_) if should_shutdown => self.keep_alive = KeepAlive::No, // Already deemed inactive
-            KeepAlive::Until(_) => {
-                // No longer idle
-                self.keep_alive = KeepAlive::Yes;
-            }
-            KeepAlive::No => {} // currently not used
-        }
-    }
 }
 
 impl<TSpec> ProtocolsHandler for RPCHandler<TSpec>
@@ -382,19 +299,21 @@ where
     TSpec: EthSpec,
 {
     type InEvent = RPCSend<TSpec>;
-    type OutEvent = Result<RPCReceived<TSpec>, HandlerErr>;
+    type OutEvent = HandlerEvent<TSpec>;
     type Error = RPCError;
     type InboundProtocol = RPCProtocol<TSpec>;
     type OutboundProtocol = RPCRequest<TSpec>;
     type OutboundOpenInfo = (RequestId, RPCRequest<TSpec>); // Keep track of the id and the request
+    type InboundOpenInfo = ();
 
-    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol> {
+    fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol, ()> {
         self.listen_protocol.clone()
     }
 
     fn inject_fully_negotiated_inbound(
         &mut self,
         substream: <Self::InboundProtocol as InboundUpgrade<NegotiatedSubstream>>::Output,
+        _info: Self::InboundOpenInfo,
     ) {
         // only accept new peer requests when active
         if !matches!(self.state, HandlerState::Active) {
@@ -424,11 +343,11 @@ where
             );
         }
 
-        self.events_out
-            .push(RPCReceived::Request(self.current_inbound_substream_id, req));
+        self.events_out.push(Ok(RPCReceived::Request(
+            self.current_inbound_substream_id,
+            req,
+        )));
         self.current_inbound_substream_id.0 += 1;
-
-        self.update_keep_alive();
     }
 
     fn inject_fully_negotiated_outbound(
@@ -442,12 +361,11 @@ where
 
         // accept outbound connections only if the handler is not deactivated
         if matches!(self.state, HandlerState::Deactivated) {
-            self.pending_errors.push(HandlerErr::Outbound {
-                id,
-                proto,
+            self.events_out.push(Err(HandlerErr::Outbound {
                 error: RPCError::HandlerRejected,
-            });
-            return;
+                proto,
+                id,
+            }));
         }
 
         // add the stream to substreams if we expect a response, otherwise drop the stream.
@@ -482,12 +400,10 @@ where
                 )
                 .is_some()
             {
-                crit!(self.log, "Duplicate outbound substream id"; "id" => format!("{:?}", self.current_outbound_substream_id));
+                crit!(self.log, "Duplicate outbound substream id"; "id" => self.current_outbound_substream_id);
             }
             self.current_outbound_substream_id.0 += 1;
         }
-
-        self.update_keep_alive();
     }
 
     fn inject_event(&mut self, rpc_event: Self::InEvent) {
@@ -515,7 +431,6 @@ where
 
         // This dialing is now considered failed
         self.dial_negotiated -= 1;
-        self.update_keep_alive();
 
         self.outbound_io_error_retries = 0;
         // map the error
@@ -540,15 +455,36 @@ where
                 }
             },
         };
-        self.pending_errors.push(HandlerErr::Outbound {
-            id,
-            proto: req.protocol(),
+        self.events_out.push(Err(HandlerErr::Outbound {
             error,
-        });
+            proto: req.protocol(),
+            id,
+        }));
     }
 
     fn connection_keep_alive(&self) -> KeepAlive {
-        self.keep_alive
+        // Check that we don't have outbound items pending for dialing, nor dialing, nor
+        // established. Also check that there are no established inbound substreams.
+        // Errors and events need to be reported back, so check those too.
+        let should_shutdown = match self.state {
+            HandlerState::ShuttingDown(_) => {
+                self.dial_queue.is_empty()
+                    && self.outbound_substreams.is_empty()
+                    && self.inbound_substreams.is_empty()
+                    && self.events_out.is_empty()
+                    && self.dial_negotiated == 0
+            }
+            HandlerState::Deactivated => {
+                // Regardless of events, the timeout has expired. Force the disconnect.
+                true
+            }
+            _ => false,
+        };
+        if should_shutdown {
+            KeepAlive::No
+        } else {
+            KeepAlive::Yes
+        }
     }
 
     fn poll(
@@ -562,15 +498,9 @@ where
             Self::Error,
         >,
     > {
-        // report failures
-        if !self.pending_errors.is_empty() {
-            let err_info = self.pending_errors.remove(0);
-            return Poll::Ready(ProtocolsHandlerEvent::Custom(Err(err_info)));
-        }
-
         // return any events that need to be reported
         if !self.events_out.is_empty() {
-            return Poll::Ready(ProtocolsHandlerEvent::Custom(Ok(self.events_out.remove(0))));
+            return Poll::Ready(ProtocolsHandlerEvent::Custom(self.events_out.remove(0)));
         } else {
             self.events_out.shrink_to_fit();
         }
@@ -585,17 +515,17 @@ where
 
         // purge expired inbound substreams and send an error
         loop {
-            match self.inbound_substreams_delay.poll_next_unpin(cx) {
+            match self.inbound_substreams_delay.poll_expired(cx) {
                 Poll::Ready(Some(Ok(inbound_id))) => {
                     // handle a stream timeout for various states
                     if let Some(info) = self.inbound_substreams.get_mut(inbound_id.get_ref()) {
                         // the delay has been removed
                         info.delay_key = None;
-                        self.pending_errors.push(HandlerErr::Inbound {
-                            id: *inbound_id.get_ref(),
-                            proto: info.protocol,
+                        self.events_out.push(Err(HandlerErr::Inbound {
                             error: RPCError::StreamTimeout,
-                        });
+                            proto: info.protocol,
+                            id: *inbound_id.get_ref(),
+                        }));
 
                         if info.pending_items.last().map(|l| l.close_after()) == Some(false) {
                             // if the last chunk does not close the stream, append an error
@@ -607,7 +537,7 @@ where
                     }
                 }
                 Poll::Ready(Some(Err(e))) => {
-                    warn!(self.log, "Inbound substream poll failed"; "error" => format!("{:?}", e));
+                    warn!(self.log, "Inbound substream poll failed"; "error" => ?e);
                     // drops the peer if we cannot read the delay queue
                     return Poll::Ready(ProtocolsHandlerEvent::Close(RPCError::InternalError(
                         "Could not poll inbound stream timer",
@@ -619,13 +549,11 @@ where
 
         // purge expired outbound substreams
         loop {
-            match self.outbound_substreams_delay.poll_next_unpin(cx) {
+            match self.outbound_substreams_delay.poll_expired(cx) {
                 Poll::Ready(Some(Ok(outbound_id))) => {
                     if let Some(OutboundInfo { proto, req_id, .. }) =
                         self.outbound_substreams.remove(outbound_id.get_ref())
                     {
-                        self.update_keep_alive();
-
                         let outbound_err = HandlerErr::Outbound {
                             id: req_id,
                             proto,
@@ -638,7 +566,7 @@ where
                     }
                 }
                 Poll::Ready(Some(Err(e))) => {
-                    warn!(self.log, "Outbound substream poll failed"; "error" => format!("{:?}", e));
+                    warn!(self.log, "Outbound substream poll failed"; "error" => ?e);
                     return Poll::Ready(ProtocolsHandlerEvent::Close(RPCError::InternalError(
                         "Could not poll outbound stream timer",
                     )));
@@ -653,69 +581,110 @@ where
         // drive inbound streams that need to be processed
         let mut substreams_to_remove = Vec::new(); // Closed substreams that need to be removed
         for (id, info) in self.inbound_substreams.iter_mut() {
-            match std::mem::replace(&mut info.state, InboundState::Poisoned) {
-                state @ InboundState::Idle(..) if !deactivated => {
-                    info.state = state.send_items(&mut info.pending_items, info.remaining_chunks);
-                }
-                InboundState::Idle(mut substream) => {
-                    // handler is deactivated, close the stream and mark it for removal
-                    match substream.close().poll_unpin(cx) {
-                        // if we can't close right now, put the substream back and try again later
-                        Poll::Pending => info.state = InboundState::Idle(substream),
-                        Poll::Ready(res) => {
-                            substreams_to_remove.push(*id);
-                            if let Some(ref delay_key) = info.delay_key {
-                                self.inbound_substreams_delay.remove(delay_key);
-                            }
-                            if let Err(error) = res {
-                                self.pending_errors.push(HandlerErr::Inbound {
-                                    id: *id,
-                                    error,
-                                    proto: info.protocol,
-                                });
-                            }
-                            if info.pending_items.last().map(|l| l.close_after()) == Some(false) {
-                                // if the request was still active, report back to cancel it
-                                self.pending_errors.push(HandlerErr::Inbound {
-                                    id: *id,
-                                    proto: info.protocol,
-                                    error: RPCError::HandlerRejected,
-                                });
-                            }
+            loop {
+                match std::mem::replace(&mut info.state, InboundState::Poisoned) {
+                    InboundState::Idle(substream) if !deactivated => {
+                        if !info.pending_items.is_empty() {
+                            let to_send = std::mem::replace(&mut info.pending_items, vec![]);
+                            let fut = process_inbound_substream(
+                                substream,
+                                info.remaining_chunks,
+                                to_send,
+                            )
+                            .boxed();
+                            info.state = InboundState::Busy(Box::pin(fut));
+                        } else {
+                            info.state = InboundState::Idle(substream);
+                            break;
                         }
                     }
-                }
-                InboundState::Busy(mut fut) => {
-                    // first check if sending finished
-                    let state = match fut.poll_unpin(cx) {
-                        Poll::Ready((substream, errors, remove, new_remaining_chunks)) => {
-                            info.remaining_chunks = new_remaining_chunks;
-                            // report any error
-                            for error in errors {
-                                self.pending_errors.push(HandlerErr::Inbound {
-                                    id: *id,
-                                    error,
-                                    proto: info.protocol,
-                                })
-                            }
-                            if remove {
+                    InboundState::Idle(mut substream) => {
+                        // handler is deactivated, close the stream and mark it for removal
+                        match substream.close().poll_unpin(cx) {
+                            // if we can't close right now, put the substream back and try again later
+                            Poll::Pending => info.state = InboundState::Idle(substream),
+                            Poll::Ready(res) => {
+                                // The substream closed, we remove it
                                 substreams_to_remove.push(*id);
                                 if let Some(ref delay_key) = info.delay_key {
                                     self.inbound_substreams_delay.remove(delay_key);
                                 }
+                                if let Err(error) = res {
+                                    self.events_out.push(Err(HandlerErr::Inbound {
+                                        error,
+                                        proto: info.protocol,
+                                        id: *id,
+                                    }));
+                                }
+                                if info.pending_items.last().map(|l| l.close_after()) == Some(false)
+                                {
+                                    // if the request was still active, report back to cancel it
+                                    self.events_out.push(Err(HandlerErr::Inbound {
+                                        error: RPCError::HandlerRejected,
+                                        proto: info.protocol,
+                                        id: *id,
+                                    }));
+                                }
                             }
-                            InboundState::Idle(substream)
                         }
-                        Poll::Pending => InboundState::Busy(fut),
-                    };
-                    info.state = if !deactivated {
-                        // if the last batch finished, send more.
-                        state.send_items(&mut info.pending_items, info.remaining_chunks)
-                    } else {
-                        state
-                    };
+                        break;
+                    }
+                    InboundState::Busy(mut fut) => {
+                        // first check if sending finished
+                        match fut.poll_unpin(cx) {
+                            Poll::Ready((substream, errors, remove, new_remaining_chunks)) => {
+                                info.remaining_chunks = new_remaining_chunks;
+                                // report any error
+                                for error in errors {
+                                    self.events_out.push(Err(HandlerErr::Inbound {
+                                        error,
+                                        proto: info.protocol,
+                                        id: *id,
+                                    }))
+                                }
+                                if remove {
+                                    substreams_to_remove.push(*id);
+                                    if let Some(ref delay_key) = info.delay_key {
+                                        self.inbound_substreams_delay.remove(delay_key);
+                                    }
+                                    break;
+                                } else {
+                                    // If we are not removing this substream, we reset the timer.
+                                    // Each chunk is allowed RESPONSE_TIMEOUT to be sent.
+                                    if let Some(ref delay_key) = info.delay_key {
+                                        self.inbound_substreams_delay.reset(
+                                            delay_key,
+                                            Duration::from_secs(RESPONSE_TIMEOUT),
+                                        );
+                                    }
+                                }
+
+                                // The stream may be currently idle. Attempt to process more
+                                // elements
+
+                                if !deactivated && !info.pending_items.is_empty() {
+                                    let to_send =
+                                        std::mem::replace(&mut info.pending_items, vec![]);
+                                    let fut = process_inbound_substream(
+                                        substream,
+                                        info.remaining_chunks,
+                                        to_send,
+                                    )
+                                    .boxed();
+                                    info.state = InboundState::Busy(Box::pin(fut));
+                                } else {
+                                    info.state = InboundState::Idle(substream);
+                                    break;
+                                }
+                            }
+                            Poll::Pending => {
+                                info.state = InboundState::Busy(fut);
+                                break;
+                            }
+                        };
+                    }
+                    InboundState::Poisoned => unreachable!("Poisoned inbound substream"),
                 }
-                InboundState::Poisoned => unreachable!("Poisoned inbound substream"),
             }
         }
 
@@ -724,7 +693,6 @@ where
             self.inbound_substreams.remove(&inbound_id);
         }
 
-        self.update_keep_alive();
         // drive outbound streams that need to be processed
         for outbound_id in self.outbound_substreams.keys().copied().collect::<Vec<_>>() {
             // get the state and mark it as poisoned
@@ -746,11 +714,11 @@ where
                 } if deactivated => {
                     // the handler is deactivated. Close the stream
                     entry.get_mut().state = OutboundSubstreamState::Closing(substream);
-                    self.pending_errors.push(HandlerErr::Outbound {
-                        id: entry.get().req_id,
-                        proto: entry.get().proto,
+                    self.events_out.push(Err(HandlerErr::Outbound {
                         error: RPCError::HandlerRejected,
-                    })
+                        proto: entry.get().proto,
+                        id: entry.get().req_id,
+                    }))
                 }
                 OutboundSubstreamState::RequestPendingResponse {
                     mut substream,
@@ -813,7 +781,6 @@ where
                         let request_id = entry.get().req_id;
                         self.outbound_substreams_delay.remove(delay_key);
                         entry.remove_entry();
-                        self.update_keep_alive();
                         // notify the application error
                         if request.expected_responses() > 1 {
                             // return an end of stream result
@@ -844,7 +811,6 @@ where
                             error: e,
                         };
                         entry.remove_entry();
-                        self.update_keep_alive();
                         return Poll::Ready(ProtocolsHandlerEvent::Custom(Err(outbound_err)));
                     }
                 },
@@ -857,7 +823,6 @@ where
                             let request_id = entry.get().req_id;
                             self.outbound_substreams_delay.remove(delay_key);
                             entry.remove_entry();
-                            self.update_keep_alive();
 
                             // report the stream termination to the user
                             //
@@ -894,10 +859,8 @@ where
             self.dial_negotiated += 1;
             let (id, req) = self.dial_queue.remove(0);
             self.dial_queue.shrink_to_fit();
-            self.update_keep_alive();
             return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest {
-                protocol: SubstreamProtocol::new(req.clone()),
-                info: (id, req),
+                protocol: SubstreamProtocol::new(req.clone(), ()).map_info(|()| (id, req)),
             });
         }
         Poll::Pending
@@ -945,6 +908,8 @@ async fn process_inbound_substream<TSpec: EthSpec>(
                     substream_closed = true;
                 }
             }
+        } else if matches!(item, RPCCodedResponse::StreamTermination(_)) {
+            // The sender closed the stream before us, ignore this.
         } else {
             // we have more items after a closed substream, report those as errors
             errors.push(RPCError::InternalError(

beacon_node/eth2_libp2p/src/rpc/methods.rs

@@ -1,6 +1,7 @@
 //! Available RPC methods types and ids.
 
 use crate::types::EnrBitfield;
+use regex::bytes::Regex;
 use serde::Serialize;
 use ssz_derive::{Decode, Encode};
 use ssz_types::{
@@ -8,6 +9,7 @@ use ssz_types::{
     VariableList,
 };
 use std::ops::Deref;
+use strum::AsStaticStr;
 use types::{Epoch, EthSpec, Hash256, SignedBeaconBlock, Slot};
 
 /// Maximum number of blocks in a single request.
@@ -15,11 +17,12 @@ pub type MaxRequestBlocks = U1024;
 pub const MAX_REQUEST_BLOCKS: u64 = 1024;
 
 /// Maximum length of error message.
-type MaxErrorLen = U256;
+pub type MaxErrorLen = U256;
+pub const MAX_ERROR_LEN: u64 = 256;
 
 /// Wrapper over SSZ List to represent error message in rpc responses.
 #[derive(Debug, Clone)]
-pub struct ErrorType(VariableList<u8, MaxErrorLen>);
+pub struct ErrorType(pub VariableList<u8, MaxErrorLen>);
 
 impl From<String> for ErrorType {
     fn from(s: String) -> Self {
@@ -42,10 +45,9 @@ impl Deref for ErrorType {
 
 impl ToString for ErrorType {
     fn to_string(&self) -> String {
-        match std::str::from_utf8(self.0.deref()) {
-            Ok(s) => s.to_string(),
-            Err(_) => format!("{:?}", self.0.deref()), // Display raw bytes if not a UTF-8 string
-        }
+        #[allow(clippy::invalid_regex)]
+        let re = Regex::new("\\p{C}").expect("Regex is valid");
+        String::from_utf8_lossy(&re.replace_all(self.0.deref(), &b""[..])).to_string()
     }
 }
 
@@ -256,11 +258,14 @@ pub enum RPCCodedResponse<T: EthSpec> {
 }
 
 /// The code assigned to an erroneous `RPCResponse`.
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone, Copy, PartialEq, AsStaticStr)]
+#[strum(serialize_all = "snake_case")]
 pub enum RPCResponseErrorCode {
     RateLimited,
     InvalidRequest,
     ServerError,
+    /// Error spec'd to indicate that a peer does not have blocks on a requested range.
+    ResourceUnavailable,
     Unknown,
 }
 
@@ -276,20 +281,19 @@ impl<T: EthSpec> RPCCodedResponse<T> {
 
     /// Tells the codec whether to decode as an RPCResponse or an error.
     pub fn is_response(response_code: u8) -> bool {
-        match response_code {
-            0 => true,
-            _ => false,
-        }
+        matches!(response_code, 0)
     }
 
     /// Builds an RPCCodedResponse from a response code and an ErrorMessage
-    pub fn from_error(response_code: u8, err: String) -> Self {
+    pub fn from_error(response_code: u8, err: ErrorType) -> Self {
         let code = match response_code {
             1 => RPCResponseErrorCode::InvalidRequest,
             2 => RPCResponseErrorCode::ServerError,
+            3 => RPCResponseErrorCode::ResourceUnavailable,
+            139 => RPCResponseErrorCode::RateLimited,
             _ => RPCResponseErrorCode::Unknown,
         };
-        RPCCodedResponse::Error(code, err.into())
+        RPCCodedResponse::Error(code, err)
     }
 
     /// Specifies which response allows for multiple chunks for the stream handler.
@@ -310,10 +314,7 @@ impl<T: EthSpec> RPCCodedResponse<T> {
 
     /// Returns true if this response always terminates the stream.
     pub fn close_after(&self) -> bool {
-        match self {
-            RPCCodedResponse::Success(_) => false,
-            _ => true,
-        }
+        !matches!(self, RPCCodedResponse::Success(_))
     }
 }
 
@@ -322,8 +323,9 @@ impl RPCResponseErrorCode {
         match self {
             RPCResponseErrorCode::InvalidRequest => 1,
             RPCResponseErrorCode::ServerError => 2,
+            RPCResponseErrorCode::ResourceUnavailable => 3,
             RPCResponseErrorCode::Unknown => 255,
-            RPCResponseErrorCode::RateLimited => 128,
+            RPCResponseErrorCode::RateLimited => 139,
         }
     }
 }
@@ -332,6 +334,7 @@ impl std::fmt::Display for RPCResponseErrorCode {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         let repr = match self {
             RPCResponseErrorCode::InvalidRequest => "The request was invalid",
+            RPCResponseErrorCode::ResourceUnavailable => "Resource unavailable",
             RPCResponseErrorCode::ServerError => "Server error occurred",
             RPCResponseErrorCode::Unknown => "Unknown error occurred",
             RPCResponseErrorCode::RateLimited => "Rate limited",
@@ -397,6 +400,22 @@ impl std::fmt::Display for BlocksByRangeRequest {
     }
 }
 
+impl slog::KV for StatusMessage {
+    fn serialize(
+        &self,
+        record: &slog::Record,
+        serializer: &mut dyn slog::Serializer,
+    ) -> slog::Result {
+        use slog::Value;
+        serializer.emit_arguments("fork_digest", &format_args!("{:?}", self.fork_digest))?;
+        Value::serialize(&self.finalized_epoch, record, "finalized_epoch", serializer)?;
+        serializer.emit_arguments("finalized_root", &format_args!("{}", self.finalized_root))?;
+        Value::serialize(&self.head_slot, record, "head_slot", serializer)?;
+        serializer.emit_arguments("head_root", &format_args!("{}", self.head_root))?;
+        slog::Result::Ok(())
+    }
+}
+
 impl slog::Value for RequestId {
     fn serialize(
         &self,

beacon_node/eth2_libp2p/src/rpc/mod.rs

@@ -105,7 +105,7 @@ impl<TSpec: EthSpec> RPC<TSpec> {
         let log = log.new(o!("service" => "libp2p_rpc"));
         let limiter = RPCRateLimiterBuilder::new()
             .n_every(Protocol::MetaData, 2, Duration::from_secs(5))
-            .one_every(Protocol::Ping, Duration::from_secs(5))
+            .n_every(Protocol::Ping, 2, Duration::from_secs(10))
             .n_every(Protocol::Status, 5, Duration::from_secs(15))
             .one_every(Protocol::Goodbye, Duration::from_secs(10))
             .n_every(
@@ -119,7 +119,7 @@ impl<TSpec: EthSpec> RPC<TSpec> {
                 Duration::from_secs(10),
             )
             .build()
-            .unwrap();
+            .expect("Configuration parameters are valid");
         RPC {
             limiter,
             events: Vec::new(),
@@ -169,9 +169,12 @@ where
 
     fn new_handler(&mut self) -> Self::ProtocolsHandler {
         RPCHandler::new(
-            SubstreamProtocol::new(RPCProtocol {
-                phantom: PhantomData,
-            }),
+            SubstreamProtocol::new(
+                RPCProtocol {
+                    phantom: PhantomData,
+                },
+                (),
+            ),
             &self.log,
         )
     }
@@ -184,10 +187,10 @@ where
     // Use connection established/closed instead of these currently
     fn inject_connected(&mut self, peer_id: &PeerId) {
         // find the peer's meta-data
-        debug!(self.log, "Requesting new peer's metadata"; "peer_id" => format!("{}",peer_id));
+        debug!(self.log, "Requesting new peer's metadata"; "peer_id" => %peer_id);
         let rpc_event = RPCSend::Request(RequestId::Behaviour, RPCRequest::MetaData(PhantomData));
         self.events.push(NetworkBehaviourAction::NotifyHandler {
-            peer_id: peer_id.clone(),
+            peer_id: *peer_id,
             handler: NotifyHandler::Any,
             event: rpc_event,
         });
@@ -230,13 +233,13 @@ where
                         }))
                 }
                 Err(RateLimitedErr::TooLarge) => {
-                    // we set the batch sizes, so this is a coding/config err
-                    crit!(self.log, "Batch too large to ever be processed";
-                        "protocol" => format!("{}", req.protocol()));
-                }
-                Err(RateLimitedErr::TooSoon(wait_time)) => {
-                    debug!(self.log, "Request exceeds the rate limit";
-                        "request" => req.to_string(), "peer_id" => peer_id.to_string(), "wait_time_ms" => wait_time.as_millis());
+                    // we set the batch sizes, so this is a coding/config err for most protocols
+                    let protocol = req.protocol();
+                    if matches!(protocol, Protocol::BlocksByRange) {
+                        debug!(self.log, "Blocks by range request will never be processed"; "request" => %req);
+                    } else {
+                        crit!(self.log, "Request size too large to ever be processed"; "protocol" => %protocol);
+                    }
                     // send an error code to the peer.
                     // the handler upon receiving the error code will send it back to the behaviour
                     self.send_response(
@@ -244,7 +247,21 @@ where
                         (conn_id, *id),
                         RPCCodedResponse::Error(
                             RPCResponseErrorCode::RateLimited,
-                            format!("Rate limited: wait {:?}", wait_time).into(),
+                            "Rate limited. Request too large".into(),
+                        ),
+                    );
+                }
+                Err(RateLimitedErr::TooSoon(wait_time)) => {
+                    debug!(self.log, "Request exceeds the rate limit";
+                        "request" => %req, "peer_id" => %peer_id, "wait_time_ms" => wait_time.as_millis());
+                    // send an error code to the peer.
+                    // the handler upon receiving the error code will send it back to the behaviour
+                    self.send_response(
+                        peer_id,
+                        (conn_id, *id),
+                        RPCCodedResponse::Error(
+                            RPCResponseErrorCode::RateLimited,
+                            format!("Wait {:?}", wait_time).into(),
                         ),
                     );
                 }