diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000000..dac0163003 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,4 @@ +[env] +# Set the number of arenas to 16 when using jemalloc. +JEMALLOC_SYS_WITH_MALLOC_CONF = "abort_conf:true,narenas:16" + diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 13b8411695..46896073ad 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -43,7 +43,7 @@ jobs: VERSION: ${{ env.VERSION }} VERSION_SUFFIX: ${{ env.VERSION_SUFFIX }} build-docker-single-arch: - name: build-docker-${{ matrix.binary }} + name: build-docker-${{ matrix.binary }}${{ matrix.features.version_suffix }} runs-on: ubuntu-22.04 strategy: matrix: @@ -51,6 +51,10 @@ jobs: aarch64-portable, x86_64, x86_64-portable] + features: [ + {version_suffix: "", env: "gnosis,slasher-lmdb,slasher-mdbx,jemalloc"}, + {version_suffix: "-dev", env: "gnosis,slasher-lmdb,slasher-mdbx,jemalloc,spec-minimal"} + ] include: - profile: maxperf @@ -60,6 +64,9 @@ jobs: DOCKER_CLI_EXPERIMENTAL: enabled VERSION: ${{ needs.extract-version.outputs.VERSION }} VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }} + FEATURE_SUFFIX: ${{ matrix.features.version_suffix }} + FEATURES: ${{ matrix.features.env }} + CROSS_FEATURES: ${{ matrix.features.env }} steps: - uses: actions/checkout@v3 - name: Update Rust @@ -70,7 +77,7 @@ jobs: - name: Cross build Lighthouse binary run: | cargo install cross - env CROSS_PROFILE=${{ matrix.profile }} make build-${{ matrix.binary }} + env CROSS_PROFILE=${{ matrix.profile }} CROSS_FEATURES=${{ matrix.features.env }} make build-${{ matrix.binary }} - name: Move cross-built binary into Docker scope (if ARM) if: startsWith(matrix.binary, 'aarch64') run: | @@ -98,7 +105,9 @@ jobs: docker buildx build \ --platform=linux/${SHORT_ARCH} \ --file ./Dockerfile.cross . \ - --tag ${IMAGE_NAME}:${VERSION}-${SHORT_ARCH}${VERSION_SUFFIX}${MODERNITY_SUFFIX} \ + --tag ${IMAGE_NAME}:${VERSION}-${SHORT_ARCH}${VERSION_SUFFIX}${MODERNITY_SUFFIX}${FEATURE_SUFFIX} \ + --build-arg FEATURES=${FEATURES} \ + --provenance=false \ --push build-docker-multiarch: name: build-docker-multiarch${{ matrix.modernity }} diff --git a/.github/workflows/local-testnet.yml b/.github/workflows/local-testnet.yml index b916ffee65..a522f2efb9 100644 --- a/.github/workflows/local-testnet.yml +++ b/.github/workflows/local-testnet.yml @@ -21,7 +21,7 @@ jobs: - name: Get latest version of stable Rust run: rustup update stable - name: Install Protoc - uses: arduino/setup-protoc@v1 + uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install ganache diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 957d016dc6..2e63b4d6c2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,8 +8,8 @@ on: env: DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - REPO_NAME: sigp/lighthouse - IMAGE_NAME: sigp/lighthouse + REPO_NAME: ${{ github.repository_owner }}/lighthouse + IMAGE_NAME: ${{ github.repository_owner }}/lighthouse jobs: extract-version: @@ -63,12 +63,8 @@ jobs: steps: - name: Checkout sources uses: actions/checkout@v3 - - name: Build toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - profile: minimal - override: true + - name: Get latest version of stable Rust + run: rustup update stable # ============================== # Windows dependencies @@ -88,7 +84,7 @@ jobs: # ============================== - name: Install Protoc if: contains(matrix.arch, 'darwin') || contains(matrix.arch, 'windows') - uses: arduino/setup-protoc@v1 + uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612 with: repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -138,11 +134,17 @@ jobs: - name: Build Lighthouse for Windows portable if: matrix.arch == 'x86_64-windows-portable' - run: cargo install --path lighthouse --force --locked --features portable,gnosis --profile ${{ matrix.profile }} + # NOTE: profile set to release until this rustc issue is fixed: + # + # https://github.com/rust-lang/rust/issues/107781 + # + # tracked at: https://github.com/sigp/lighthouse/issues/3964 + run: cargo install --path lighthouse --force --locked --features portable,gnosis --profile release - name: Build Lighthouse for Windows modern if: matrix.arch == 'x86_64-windows' - run: cargo install --path lighthouse --force --locked --features modern,gnosis --profile ${{ matrix.profile }} + # NOTE: profile set to release (see above) + run: cargo install --path lighthouse --force --locked --features modern,gnosis --profile release - name: Configure GPG and create artifacts if: startsWith(matrix.arch, 'x86_64-windows') != true @@ -179,13 +181,13 @@ jobs: # ======================================================================= - name: Upload artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz path: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz - name: Upload signature - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc path: lighthouse-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc @@ -208,7 +210,7 @@ jobs: # ============================== - name: Download artifacts - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 # ============================== # Create release draft @@ -216,11 +218,14 @@ jobs: - name: Generate Full Changelog id: changelog - run: echo "CHANGELOG=$(git log --pretty=format:"- %s" $(git describe --tags --abbrev=0 ${{ env.VERSION }}^)..${{ env.VERSION }})" >> $GITHUB_OUTPUT + run: | + echo "CHANGELOG<> $GITHUB_OUTPUT + echo "$(git log --pretty=format:"- %s" $(git describe --tags --abbrev=0 ${{ env.VERSION }}^)..${{ env.VERSION }})" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT - name: Create Release Draft env: - GITHUB_USER: sigp + GITHUB_USER: ${{ github.repository_owner }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # The formatting here is borrowed from OpenEthereum: https://github.com/openethereum/openethereum/blob/main/.github/workflows/build.yml diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index d536869e45..57fee71830 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -12,7 +12,9 @@ env: # Deny warnings in CI RUSTFLAGS: "-D warnings" # The Nightly version used for cargo-udeps, might need updating from time to time. - PINNED_NIGHTLY: nightly-2022-05-20 + PINNED_NIGHTLY: nightly-2022-12-15 + # Prevent Github API rate limiting. + LIGHTHOUSE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: target-branch-check: name: target-branch-check @@ -51,7 +53,7 @@ jobs: - name: Get latest version of stable Rust run: rustup update stable - name: Install Protoc - uses: arduino/setup-protoc@v1 + uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install ganache @@ -95,7 +97,7 @@ jobs: - name: Get latest version of stable Rust run: rustup update stable - name: Install Protoc - uses: arduino/setup-protoc@v1 + uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Run beacon_chain tests for all known forks @@ -109,7 +111,7 @@ jobs: - name: Get latest version of stable Rust run: rustup update stable - name: Install Protoc - uses: arduino/setup-protoc@v1 + uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Run operation_pool tests for all known forks @@ -133,7 +135,7 @@ jobs: - name: Get latest version of stable Rust run: rustup update stable - name: Install Protoc - uses: arduino/setup-protoc@v1 + uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install ganache @@ -149,7 +151,7 @@ jobs: - name: Get latest version of stable Rust run: rustup update stable - name: Install Protoc - uses: arduino/setup-protoc@v1 + uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Run state_transition_vectors in release. @@ -163,7 +165,7 @@ jobs: - name: Get latest version of stable Rust run: rustup update stable - name: Install Protoc - uses: arduino/setup-protoc@v1 + uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Run consensus-spec-tests with blst, milagro and fake_crypto @@ -189,7 +191,7 @@ jobs: - name: Get latest version of stable Rust run: rustup update stable - name: Install Protoc - uses: arduino/setup-protoc@v1 + uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install ganache @@ -205,7 +207,7 @@ jobs: - name: Get latest version of stable Rust run: rustup update stable - name: Install Protoc - uses: arduino/setup-protoc@v1 + uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install ganache @@ -221,7 +223,7 @@ jobs: - name: Get latest version of stable Rust run: rustup update stable - name: Install Protoc - uses: arduino/setup-protoc@v1 + uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install ganache @@ -237,7 +239,7 @@ jobs: - name: Get latest version of stable Rust run: rustup update stable - name: Install Protoc - uses: arduino/setup-protoc@v1 + uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install ganache @@ -253,7 +255,7 @@ jobs: - name: Get latest version of stable Rust run: rustup update stable - name: Install Protoc - uses: arduino/setup-protoc@v1 + uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install ganache @@ -285,7 +287,7 @@ jobs: - name: Get latest version of stable Rust run: rustup update stable - name: Install Protoc - uses: arduino/setup-protoc@v1 + uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Run exec engine integration tests in release @@ -299,21 +301,11 @@ jobs: - name: Get latest version of stable Rust run: rustup update stable - name: Install Protoc - uses: arduino/setup-protoc@v1 + uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Typecheck benchmark code without running it run: make check-benches - check-consensus: - name: check-consensus - runs-on: ubuntu-latest - needs: cargo-fmt - steps: - - uses: actions/checkout@v3 - - name: Get latest version of stable Rust - run: rustup update stable - - name: Typecheck consensus code in strict mode - run: make check-consensus clippy: name: clippy runs-on: ubuntu-latest @@ -323,34 +315,13 @@ jobs: - name: Get latest version of stable Rust run: rustup update stable - name: Install Protoc - uses: arduino/setup-protoc@v1 + uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Lint code for quality and style with Clippy run: make lint - name: Certify Cargo.lock freshness run: git diff --exit-code Cargo.lock - disallowed-from-async-lint: - name: disallowed-from-async-lint - runs-on: ubuntu-latest - needs: cargo-fmt - continue-on-error: true - steps: - - uses: actions/checkout@v3 - - name: Install SigP Clippy fork - run: | - cd .. - git clone https://github.com/michaelsproul/rust-clippy.git - cd rust-clippy - git checkout 31a49666ccfcd7963b63345d6ce757c373f22c2a - cargo build --release --bin cargo-clippy --bin clippy-driver - cargo build --release --bin cargo-clippy --bin clippy-driver -Zunstable-options --out-dir $(rustc --print=sysroot)/bin - - name: Install Protoc - uses: arduino/setup-protoc@v1 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Run Clippy with the disallowed-from-async lint - run: make nightly-lint check-msrv: name: check-msrv runs-on: ubuntu-latest @@ -360,7 +331,7 @@ jobs: - name: Install Rust @ MSRV (${{ needs.extract-msrv.outputs.MSRV }}) run: rustup override set ${{ needs.extract-msrv.outputs.MSRV }} - name: Install Protoc - uses: arduino/setup-protoc@v1 + uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Run cargo check @@ -401,14 +372,12 @@ jobs: - uses: actions/checkout@v3 - name: Install Rust (${{ env.PINNED_NIGHTLY }}) run: rustup toolchain install $PINNED_NIGHTLY - # NOTE: cargo-udeps version is pinned until this issue is resolved: - # https://github.com/est31/cargo-udeps/issues/135 - name: Install Protoc - uses: arduino/setup-protoc@v1 + uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install cargo-udeps - run: cargo install cargo-udeps --locked --force --version 0.1.30 + run: cargo install cargo-udeps --locked --force - name: Create Cargo config dir run: mkdir -p .cargo - name: Install custom Cargo config diff --git a/Cargo.lock b/Cargo.lock index feff2e3305..1449d8d678 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -52,9 +52,9 @@ dependencies = [ [[package]] name = "addr2line" -version = "0.17.0" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b" +checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97" dependencies = [ "gimli", ] @@ -71,6 +71,15 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" +[[package]] +name = "aead" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fc95d1bdb8e6666b2b217308eeeb09f2d6728d104be3e31916cc74d15420331" +dependencies = [ + "generic-array", +] + [[package]] name = "aead" version = "0.4.3" @@ -78,6 +87,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b613b8e1e3cf911a086f53f03bf286f52fd7a7258e4fa606f0ef220d39d8877" dependencies = [ "generic-array", + "rand_core 0.6.4", +] + +[[package]] +name = "aes" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884391ef1066acaa41e766ba8f596341b96e93ce34f9a43e7d24bf0a0eaf0561" +dependencies = [ + "aes-soft", + "aesni", + "cipher 0.2.5", ] [[package]] @@ -87,26 +108,60 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" dependencies = [ "cfg-if", - "cipher", + "cipher 0.3.0", "cpufeatures", - "ctr", + "ctr 0.8.0", "opaque-debug", ] +[[package]] +name = "aes-gcm" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5278b5fabbb9bd46e24aa69b2fdea62c99088e0a950a9be40e3e0101298f88da" +dependencies = [ + "aead 0.3.2", + "aes 0.6.0", + "cipher 0.2.5", + "ctr 0.6.0", + "ghash 0.3.1", + "subtle", +] + [[package]] name = "aes-gcm" version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df5f85a83a7d8b0442b6aa7b504b8212c1733da07b98aae43d4bc21b2cb3cdf6" dependencies = [ - "aead", - "aes", - "cipher", - "ctr", - "ghash", + "aead 0.4.3", + "aes 0.7.5", + "cipher 0.3.0", + "ctr 0.8.0", + "ghash 0.4.4", "subtle", ] +[[package]] +name = "aes-soft" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be14c7498ea50828a38d0e24a765ed2effe92a705885b57d029cd67d45744072" +dependencies = [ + "cipher 0.2.5", + "opaque-debug", +] + +[[package]] +name = "aesni" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2e11f5e94c2f7d386164cc2aa1f97823fed6f259e486940a71c174dd01b0ce" +dependencies = [ + "cipher 0.2.5", + "opaque-debug", +] + [[package]] name = "ahash" version = "0.7.6" @@ -120,9 +175,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "0.7.19" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" dependencies = [ "memchr", ] @@ -152,24 +207,24 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.66" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" +checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" [[package]] name = "arbitrary" -version = "1.2.0" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29d47fbf90d5149a107494b15a7dc8d69b351be2db3bb9691740e88ec17fd880" +checksum = "3e90af4de65aa7b293ef2d09daff88501eb254f58edde2e1ac02c82d873eadad" dependencies = [ "derive_arbitrary", ] [[package]] name = "arc-swap" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "983cd8b9d4b02a6dc6ffa557262eb5858a27a0038ffffe21a0f133eaa819a164" +checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" [[package]] name = "arrayref" @@ -183,12 +238,109 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" +[[package]] +name = "asn1-rs" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ff05a702273012438132f449575dbc804e27b2f3cbe3069aa237d26c98fa33" +dependencies = [ + "asn1-rs-derive 0.1.0", + "asn1-rs-impl", + "displaydoc", + "nom 7.1.3", + "num-traits", + "rusticata-macros", + "thiserror", + "time 0.3.17", +] + +[[package]] +name = "asn1-rs" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf6690c370453db30743b373a60ba498fc0d6d83b11f4abfd87a84a075db5dd4" +dependencies = [ + "asn1-rs-derive 0.4.0", + "asn1-rs-impl", + "displaydoc", + "nom 7.1.3", + "num-traits", + "rusticata-macros", + "thiserror", + "time 0.3.17", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db8b7511298d5b7784b40b092d9e9dcd3a627a5707e4b5e507931ab0d44eeebf" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "asn1_der" version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e22d1f4b888c298a027c99dc9048015fac177587de20fc30232a057dfbe24a21" +[[package]] +name = "async-io" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c374dda1ed3e7d8f0d9ba58715f924862c63eae6849c92d3a18e7fbde9e2794" +dependencies = [ + "async-lock", + "autocfg 1.1.0", + "concurrent-queue", + "futures-lite", + "libc", + "log", + "parking", + "polling", + "slab", + "socket2", + "waker-fn", + "windows-sys 0.42.0", +] + +[[package]] +name = "async-lock" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8101efe8695a6c17e02911402145357e718ac92d3ff88ae8419e84b1707b685" +dependencies = [ + "event-listener", + "futures-lite", +] + [[package]] name = "async-stream" version = "0.3.3" @@ -212,9 +364,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.58" +version = "0.1.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e805d94e6b5001b651426cf4cd446b1ab5f319d27bab5c644f61de0a804360c" +checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" dependencies = [ "proc-macro2", "quote", @@ -234,9 +386,9 @@ dependencies = [ [[package]] name = "asynchronous-codec" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0de5164e5edbf51c45fb8c2d9664ae1c095cce1b265ecf7569093c0d66ef690" +checksum = "06a0daa378f5fd10634e44b0a29b2a87b890657658e072a30d6f26e57ddee182" dependencies = [ "bytes", "futures-sink", @@ -245,6 +397,12 @@ dependencies = [ "pin-project-lite 0.2.9", ] +[[package]] +name = "atomic-waker" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "debc29dde2e69f9e47506b525f639ed42300fc014a3e007832592448fa8e4599" + [[package]] name = "attohttpc" version = "0.10.1" @@ -262,7 +420,7 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ - "hermit-abi", + "hermit-abi 0.1.19", "libc", "winapi", ] @@ -308,7 +466,7 @@ dependencies = [ "http", "http-body", "hyper", - "itoa 1.0.4", + "itoa 1.0.5", "matchit", "memchr", "mime", @@ -343,9 +501,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.66" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cab84319d616cfb654d03394f38ab7e6f0919e181b1b57e1fd15e7fb4077d9a7" +checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca" dependencies = [ "addr2line", "cc", @@ -356,6 +514,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base-x" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" + [[package]] name = "base16ct" version = "0.1.1" @@ -368,6 +532,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" + [[package]] name = "base64ct" version = "1.5.3" @@ -417,7 +587,7 @@ dependencies = [ "lazy_static", "lighthouse_metrics", "logging", - "lru", + "lru 0.7.8", "maplit", "merkle_proof", "oneshot_broadcast", @@ -450,7 +620,7 @@ dependencies = [ [[package]] name = "beacon_node" -version = "3.3.0" +version = "3.4.0" dependencies = [ "beacon_chain", "clap", @@ -537,16 +707,16 @@ dependencies = [ "funty 2.0.0", "radium 0.7.0", "tap", - "wyz 0.5.0", + "wyz 0.5.1", ] [[package]] name = "blake2" -version = "0.10.4" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9cf849ee05b2ee5fba5e36f97ff8ec2533916700fc0758d40d92136a42f3388" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest 0.10.5", + "digest 0.10.6", ] [[package]] @@ -568,6 +738,16 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-modes" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a0e8073e8baa88212fb5823574c02ebccb395136ba9a164ab89379ec6072f0" +dependencies = [ + "block-padding", + "cipher 0.2.5", +] + [[package]] name = "block-padding" version = "0.2.1" @@ -583,7 +763,7 @@ dependencies = [ "eth2_hashing", "eth2_serde_utils", "eth2_ssz", - "ethereum-types 0.12.1", + "ethereum-types 0.14.1", "hex", "milagro_bls", "rand 0.7.3", @@ -608,7 +788,7 @@ dependencies = [ [[package]] name = "boot_node" -version = "3.3.0" +version = "3.4.0" dependencies = [ "beacon_node", "clap", @@ -673,9 +853,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.11.1" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "byte-slice-cast" @@ -691,18 +871,18 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.2.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" dependencies = [ "serde", ] [[package]] name = "bzip2" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6afcd980b5f3a45017c57e57a2fcccbb351cc43a356ce117ef760ef8052b89b0" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" dependencies = [ "bzip2-sys", "libc", @@ -727,7 +907,7 @@ dependencies = [ "eth2_ssz", "eth2_ssz_derive", "eth2_ssz_types", - "ethereum-types 0.12.1", + "ethereum-types 0.14.1", "quickcheck", "quickcheck_macros", "smallvec", @@ -742,9 +922,20 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.74" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581f5dba903aac52ea3feb5ec4810848460ee833876f1f9b0fdeab1f19091574" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + +[[package]] +name = "ccm" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aca1a8fbc20b50ac9673ff014abfb2b5f4085ee1a850d408f14a159c5853ac7" +dependencies = [ + "aead 0.3.2", + "cipher 0.2.5", + "subtle", +] [[package]] name = "cexpr" @@ -752,7 +943,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" dependencies = [ - "nom 7.1.1", + "nom 7.1.3", ] [[package]] @@ -768,7 +959,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c80e5460aa66fe3b91d40bcbdab953a597b60053e34d684ac6903f863b680a6" dependencies = [ "cfg-if", - "cipher", + "cipher 0.3.0", "cpufeatures", "zeroize", ] @@ -779,28 +970,37 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a18446b09be63d457bbec447509e85f662f32952b035ce892290396bc0b0cff5" dependencies = [ - "aead", + "aead 0.4.3", "chacha20", - "cipher", + "cipher 0.3.0", "poly1305", "zeroize", ] [[package]] name = "chrono" -version = "0.4.22" +version = "0.4.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1" +checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" dependencies = [ "iana-time-zone", "js-sys", "num-integer", "num-traits", - "time 0.1.44", + "time 0.1.45", "wasm-bindgen", "winapi", ] +[[package]] +name = "cipher" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f8e7987cbd042a63249497f41aed09f8e65add917ea6566effbc56578d6801" +dependencies = [ + "generic-array", +] + [[package]] name = "cipher" version = "0.3.0" @@ -844,7 +1044,7 @@ dependencies = [ "dirs", "eth2_network_config", "eth2_ssz", - "ethereum-types 0.12.1", + "ethereum-types 0.14.1", "hex", "serde", "serde_json", @@ -884,7 +1084,7 @@ dependencies = [ "slot_clock", "store", "task_executor", - "time 0.3.16", + "time 0.3.17", "timer", "tokio", "types", @@ -924,6 +1124,15 @@ dependencies = [ "syn", ] +[[package]] +name = "concurrent-queue" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c278839b831783b70278b14df4d45e1beb1aad306c07bb796637de9a0e323e8e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "console_error_panic_hook" version = "0.1.7" @@ -936,9 +1145,9 @@ dependencies = [ [[package]] name = "const-oid" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "722e23542a15cea1f65d4a1419c4cfd7a26706c70871a13a04238ca3f40f1661" +checksum = "cec318a675afcb6a1ea1d4340e2d377e56e47c266f28043ceccbf4412ddfdd3b" [[package]] name = "convert_case" @@ -980,6 +1189,27 @@ dependencies = [ "libc", ] +[[package]] +name = "cpuid-bool" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb25d077389e53838a8158c8e99174c5a9d902dee4904320db714f3c653ffba" + +[[package]] +name = "crc" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484" + [[package]] name = "crc32fast" version = "1.3.2" @@ -1048,22 +1278,22 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.11" +version = "0.9.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f916dfc5d356b0ed9dae65f1db9fc9770aa2851d2662b988ccf4fe3516e86348" +checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" dependencies = [ "autocfg 1.1.0", "cfg-if", "crossbeam-utils", - "memoffset", + "memoffset 0.7.1", "scopeguard", ] [[package]] name = "crossbeam-utils" -version = "0.8.12" +version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edbafec5fa1f196ca66527c1b12c2ec4745ca14b50f1ad8f9f6f720b55d11fac" +checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" dependencies = [ "cfg-if", ] @@ -1106,6 +1336,16 @@ dependencies = [ "subtle", ] +[[package]] +name = "crypto-mac" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bff07008ec701e8028e2ceb8f83f0e4274ee62bd2dbdc4fefff2e9a91824081a" +dependencies = [ + "generic-array", + "subtle", +] + [[package]] name = "crypto-mac" version = "0.11.1" @@ -1138,23 +1378,32 @@ dependencies = [ "memchr", ] +[[package]] +name = "ctr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb4a30d54f7443bf3d6191dcd486aca19e67cb3c49fa7a06a319966346707e7f" +dependencies = [ + "cipher 0.2.5", +] + [[package]] name = "ctr" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "049bb91fb4aaf0e3c7efa6cd5ef877dbbbd15b39dad06d9948de4ec8a75761ea" dependencies = [ - "cipher", + "cipher 0.3.0", ] [[package]] name = "ctrlc" -version = "3.2.3" +version = "3.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d91974fbbe88ec1df0c24a4f00f99583667a7e2e6272b2b92d294d81e462173" +checksum = "bbcf33c2a618cbe41ee43ae6e9f2e48368cd9f9db2896f10167d8d762679f639" dependencies = [ - "nix 0.25.0", - "winapi", + "nix 0.26.2", + "windows-sys 0.45.0", ] [[package]] @@ -1172,22 +1421,23 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "4.0.0-pre.1" +version = "4.0.0-rc.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4033478fbf70d6acf2655ac70da91ee65852d69daf7a67bf7a2f518fb47aafcf" +checksum = "8da00a7a9a4eb92a0a0f8e75660926d48f0d0f3c537e455c457bcdaa1e16b1ac" dependencies = [ - "byteorder", - "digest 0.9.0", - "rand_core 0.6.4", + "cfg-if", + "fiat-crypto", + "packed_simd_2", + "platforms 3.0.2", "subtle", "zeroize", ] [[package]] name = "cxx" -version = "1.0.80" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b7d4e43b25d3c994662706a1d4fcfc32aaa6afd287502c111b237093bb23f3a" +checksum = "bc831ee6a32dd495436e317595e639a587aa9907bef96fe6e6abc290ab6204e9" dependencies = [ "cc", "cxxbridge-flags", @@ -1197,9 +1447,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.80" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84f8829ddc213e2c1368e51a2564c552b65a8cb6a28f31e576270ac81d5e5827" +checksum = "94331d54f1b1a8895cd81049f7eaaaef9d05a7dcb4d1fd08bf3ff0806246789d" dependencies = [ "cc", "codespan-reporting", @@ -1212,15 +1462,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.80" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e72537424b474af1460806647c41d4b6d35d09ef7fe031c5c2fa5766047cc56a" +checksum = "48dcd35ba14ca9b40d6e4b4b39961f23d835dbb8eed74565ded361d93e1feb8a" [[package]] name = "cxxbridge-macro" -version = "1.0.80" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "309e4fb93eed90e1e14bea0da16b209f81813ba9fc7830c20ed151dd7bc0a4d7" +checksum = "81bbeb29798b407ccd82a3324ade1a7286e0d29851475990b612670f6f5124d2" dependencies = [ "proc-macro2", "quote", @@ -1233,8 +1483,18 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.13.4", + "darling_macro 0.13.4", +] + +[[package]] +name = "darling" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0808e1bd8671fb44a113a14e13497557533369847788fa2ae912b6ebfce9fa8" +dependencies = [ + "darling_core 0.14.3", + "darling_macro 0.14.3", ] [[package]] @@ -1251,13 +1511,38 @@ dependencies = [ "syn", ] +[[package]] +name = "darling_core" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "001d80444f28e193f30c2f293455da62dcf9a6b29918a4253152ae2b1de592cb" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.10.0", + "syn", +] + [[package]] name = "darling_macro" version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" dependencies = [ - "darling_core", + "darling_core 0.13.4", + "quote", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b36230598a2d5de7ec1c6f51f72d8a99a9208daff41de2084d06e3fd3ea56685" +dependencies = [ + "darling_core 0.14.3", "quote", "syn", ] @@ -1284,9 +1569,29 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.3.2" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" +checksum = "23d8666cb01533c39dde32bcbab8e227b4ed6679b2c925eba05feabea39508fb" + +[[package]] +name = "data-encoding-macro" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86927b7cd2fe88fa698b87404b287ab98d1a0063a34071d92e575b72d3029aca" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5bbed42daaa95e780b60a50546aa345b8413a1e46f9a40a12907d3598f038db" +dependencies = [ + "data-encoding", + "syn", +] [[package]] name = "database_manager" @@ -1338,14 +1643,43 @@ dependencies = [ [[package]] name = "der" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13dd2ae565c0a381dde7fade45fce95984c568bdcb4700a4fdbe3175e0380b2f" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" dependencies = [ "const-oid", + "pem-rfc7468", "zeroize", ] +[[package]] +name = "der-parser" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe398ac75057914d7d07307bf67dc7f3f574a26783b4fc7805a20ffa9f506e82" +dependencies = [ + "asn1-rs 0.3.1", + "displaydoc", + "nom 7.1.3", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "der-parser" +version = "8.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d4bc9b0db0a0df9ae64634ac5bdefb7afcb534e182275ca0beadbe486701c1" +dependencies = [ + "asn1-rs 0.5.1", + "displaydoc", + "nom 7.1.3", + "num-bigint", + "num-traits", + "rusticata-macros", +] + [[package]] name = "derivative" version = "2.2.0" @@ -1359,15 +1693,46 @@ dependencies = [ [[package]] name = "derive_arbitrary" -version = "1.2.0" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4903dff04948f22033ca30232ab8eca2c3fc4c913a8b6a34ee5199699814817f" +checksum = "8beee4701e2e229e8098bbdecdca12449bc3e322f137d269182fa1291e20bd00" dependencies = [ "proc-macro2", "quote", "syn", ] +[[package]] +name = "derive_builder" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d07adf7be193b71cc36b193d0f5fe60b918a3a9db4dad0449f57bcfd519704a3" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f91d4cfa921f1c05904dc3c57b4a32c38aed3340cce209f3a6fd1478babafc4" +dependencies = [ + "darling 0.14.3", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_builder_macro" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f0314b72bed045f3a68671b3c86328386762c93f82d98c65c3cb5e5f573dd68" +dependencies = [ + "derive_builder_core", + "syn", +] + [[package]] name = "derive_more" version = "0.99.17" @@ -1392,9 +1757,9 @@ dependencies = [ [[package]] name = "digest" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c" +checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" dependencies = [ "block-buffer 0.10.3", "crypto-common", @@ -1457,8 +1822,8 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d767c0e59b3e8d65222d95df723cc2ea1da92bb0f27c563607e6f0bde064f255" dependencies = [ - "aes", - "aes-gcm", + "aes 0.7.5", + "aes-gcm 0.9.4", "arrayvec", "delay_map", "enr", @@ -1468,8 +1833,8 @@ dependencies = [ "hex", "hkdf", "lazy_static", - "libp2p-core", - "lru", + "libp2p-core 0.36.0", + "lru 0.7.8", "more-asserts", "parking_lot 0.11.2", "rand 0.8.5", @@ -1486,10 +1851,21 @@ dependencies = [ ] [[package]] -name = "dtoa" -version = "1.0.4" +name = "displaydoc" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8a6eee2d5d0d113f015688310da018bd1d864d86bd567c8fca9c266889e1bfa" +checksum = "3bf95dc3f046b9da4f2d51833c0d3547d8564ef6910f5c1ed130306a75b92886" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dtoa" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c00704156a7de8df8da0911424e30c2049957b0a714542a44e05fe693dd85313" [[package]] name = "ecdsa" @@ -1505,9 +1881,9 @@ dependencies = [ [[package]] name = "ed25519" -version = "1.5.2" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9c280362032ea4203659fc489832d0204ef09f247a0506f170dafcac08c369" +checksum = "91cff35c70bba8a626e3185d8cd48cc11b5437e1a5bcd15b9b5fa3c64b6dfee7" dependencies = [ "signature", ] @@ -1538,7 +1914,7 @@ dependencies = [ "derivative", "eth2_ssz", "eth2_ssz_derive", - "ethereum-types 0.12.1", + "ethereum-types 0.14.1", "execution_layer", "fork_choice", "fs2", @@ -1559,9 +1935,9 @@ dependencies = [ [[package]] name = "either" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] name = "elliptic-curve" @@ -1572,10 +1948,12 @@ dependencies = [ "base16ct", "crypto-bigint", "der", - "digest 0.10.5", + "digest 0.10.6", "ff", "generic-array", "group", + "hkdf", + "pem-rfc7468", "pkcs8", "rand_core 0.6.4", "sec1", @@ -1585,9 +1963,9 @@ dependencies = [ [[package]] name = "encoding_rs" -version = "0.8.31" +version = "0.8.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" +checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" dependencies = [ "cfg-if", ] @@ -1598,7 +1976,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26fa0a0be8915790626d5759eb51fe47435a8eac92c2f212bd2da9aa7f30ea56" dependencies = [ - "base64", + "base64 0.13.1", "bs58", "bytes", "ed25519-dalek", @@ -1614,9 +1992,9 @@ dependencies = [ [[package]] name = "enum-as-inner" -version = "0.4.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21cdad81446a7f7dc43f6a77409efeb9733d2fa65553efef6018ef257c959b73" +checksum = "c9720bba047d567ffc8a3cba48bf19126600e249ab7f128e9233e6376976a116" dependencies = [ "heck", "proc-macro2", @@ -1636,9 +2014,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c90bf5f19754d10198ccb95b70664fc925bd1fc090a0fd9a6ebc54acc8cd6272" +checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" dependencies = [ "atty", "humantime", @@ -1775,7 +2153,7 @@ dependencies = [ name = "eth2_interop_keypairs" version = "0.2.0" dependencies = [ - "base64", + "base64 0.13.1", "bls", "eth2_hashing", "hex", @@ -1802,7 +2180,7 @@ dependencies = [ name = "eth2_keystore" version = "0.1.0" dependencies = [ - "aes", + "aes 0.7.5", "bls", "eth2_key_derivation", "hex", @@ -1816,7 +2194,7 @@ dependencies = [ "sha2 0.9.9", "tempfile", "unicode-normalization", - "uuid", + "uuid 0.8.2", "zeroize", ] @@ -1837,7 +2215,7 @@ dependencies = [ name = "eth2_serde_utils" version = "0.1.1" dependencies = [ - "ethereum-types 0.12.1", + "ethereum-types 0.14.1", "hex", "serde", "serde_derive", @@ -1849,7 +2227,7 @@ name = "eth2_ssz" version = "0.4.1" dependencies = [ "eth2_ssz_derive", - "ethereum-types 0.12.1", + "ethereum-types 0.14.1", "itertools", "smallvec", ] @@ -1858,7 +2236,7 @@ dependencies = [ name = "eth2_ssz_derive" version = "0.3.0" dependencies = [ - "darling", + "darling 0.13.4", "proc-macro2", "quote", "syn", @@ -1894,7 +2272,7 @@ dependencies = [ "serde_repr", "tempfile", "tiny-bip39", - "uuid", + "uuid 0.8.2", ] [[package]] @@ -1923,11 +2301,11 @@ dependencies = [ [[package]] name = "ethabi" -version = "17.2.0" +version = "18.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4966fba78396ff92db3b817ee71143eccd98acf0f876b8d600e585a670c5d1b" +checksum = "7413c5f74cc903ea37386a8965a936cbeb334bd270862fdece542c1b2dcbc898" dependencies = [ - "ethereum-types 0.13.1", + "ethereum-types 0.14.1", "hex", "once_cell", "regex", @@ -1945,22 +2323,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfb684ac8fa8f6c5759f788862bb22ec6fe3cb392f6bfd08e3c64b603661e3f8" dependencies = [ "crunchy", - "fixed-hash", + "fixed-hash 0.7.0", "impl-rlp", - "impl-serde", + "impl-serde 0.3.2", "tiny-keccak", ] [[package]] name = "ethbloom" -version = "0.12.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11da94e443c60508eb62cf256243a64da87304c2802ac2528847f79d750007ef" +checksum = "c22d4b5885b6aa2fe5e8b9329fb8d232bf739e434e6b87347c63bdd00c120f60" dependencies = [ "crunchy", - "fixed-hash", + "fixed-hash 0.8.0", + "impl-codec 0.6.0", "impl-rlp", - "impl-serde", + "impl-serde 0.4.0", + "scale-info", "tiny-keccak", ] @@ -1975,7 +2355,7 @@ dependencies = [ "enr", "hex", "integer-sqrt", - "multiaddr", + "multiaddr 0.14.0", "rand 0.8.5", "serde", "serde_json", @@ -1993,46 +2373,47 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05136f7057fe789f06e6d41d07b34e6f70d8c86e5693b60f97aaa6553553bdaf" dependencies = [ "ethbloom 0.11.1", - "fixed-hash", + "fixed-hash 0.7.0", "impl-rlp", - "impl-serde", + "impl-serde 0.3.2", "primitive-types 0.10.1", "uint", ] [[package]] name = "ethereum-types" -version = "0.13.1" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2827b94c556145446fcce834ca86b7abf0c39a805883fe20e72c5bfdb5a0dc6" +checksum = "02d215cbf040552efcbe99a38372fe80ab9d00268e20012b79fcd0f073edd8ee" dependencies = [ - "ethbloom 0.12.1", - "fixed-hash", + "ethbloom 0.13.0", + "fixed-hash 0.8.0", + "impl-codec 0.6.0", "impl-rlp", - "impl-serde", - "primitive-types 0.11.1", + "impl-serde 0.4.0", + "primitive-types 0.12.1", + "scale-info", "uint", ] [[package]] name = "ethers-core" -version = "0.17.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ebdd63c828f58aa067f40f9adcbea5e114fb1f90144b3a1e2858e0c9b1ff4e8" +checksum = "ade3e9c97727343984e1ceada4fdab11142d2ee3472d2c67027d56b1251d4f15" dependencies = [ "arrayvec", "bytes", "chrono", "elliptic-curve", - "ethabi 17.2.0", - "fastrlp", + "ethabi 18.0.0", "generic-array", "hex", "k256", + "open-fastrlp", "rand 0.8.5", "rlp", "rlp-derive", - "rust_decimal", "serde", "serde_json", "strum", @@ -2043,13 +2424,13 @@ dependencies = [ [[package]] name = "ethers-providers" -version = "0.17.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46482e4d1e79b20c338fd9db9e166184eb387f0a4e7c05c5b5c0aa2e8c8900c" +checksum = "a1a9e0597aa6b2fdc810ff58bc95e4eeaa2c219b3e615ed025106ecb027407d8" dependencies = [ "async-trait", "auto_impl", - "base64", + "base64 0.13.1", "ethers-core", "futures-channel", "futures-core", @@ -2078,6 +2459,12 @@ dependencies = [ "ws_stream_wasm", ] +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + [[package]] name = "execution_engine_integration" version = "0.1.0" @@ -2118,11 +2505,14 @@ dependencies = [ "exit-future", "fork_choice", "futures", + "hash-db", + "hash256-std-hasher", "hex", "jsonwebtoken", + "keccak-hash", "lazy_static", "lighthouse_metrics", - "lru", + "lru 0.7.8", "mev-build-rs", "parking_lot 0.12.1", "rand 0.8.5", @@ -2141,6 +2531,7 @@ dependencies = [ "tokio-stream", "tree_hash", "tree_hash_derive", + "triehash", "types", "warp", "zeroize", @@ -2176,31 +2567,6 @@ dependencies = [ "instant", ] -[[package]] -name = "fastrlp" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "089263294bb1c38ac73649a6ad563dd9a5142c8dc0482be15b8b9acb22a1611e" -dependencies = [ - "arrayvec", - "auto_impl", - "bytes", - "ethereum-types 0.13.1", - "fastrlp-derive", -] - -[[package]] -name = "fastrlp-derive" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9e9158c1d8f0a7a716c9191562eaabba70268ba64972ef4871ce8d66fd08872" -dependencies = [ - "bytes", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "ff" version = "0.12.1" @@ -2217,13 +2583,19 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec54ac60a7f2ee9a97cad9946f9bf629a3bc6a7ae59e68983dc9318f5a54b81a" +[[package]] +name = "fiat-crypto" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a214f5bb88731d436478f3ae1f8a277b62124089ba9fb67f4f93fb100ef73c90" + [[package]] name = "field-offset" version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e1c54951450cbd39f3dbcf1005ac413b49487dabf18a720ad2383eccfeffb92" dependencies = [ - "memoffset", + "memoffset 0.6.5", "rustc_version 0.3.3", ] @@ -2238,7 +2610,20 @@ dependencies = [ [[package]] name = "fixed-hash" version = "0.7.0" -source = "git+https://github.com/paritytech/parity-common?rev=df638ab0885293d21d656dc300d39236b69ce57d#df638ab0885293d21d656dc300d39236b69ce57d" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcf0ed7fe52a17a03854ec54a9f76d6d84508d1c0e66bc1793301c73fc8493c" +dependencies = [ + "byteorder", + "rand 0.8.5", + "rustc-hex", + "static_assertions", +] + +[[package]] +name = "fixed-hash" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "835c052cb0c08c1acf6ffd71c022172e18723949c8282f2b9f27efbc51e64534" dependencies = [ "arbitrary", "byteorder", @@ -2255,9 +2640,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" +checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" dependencies = [ "crc32fast", "libz-sys", @@ -2333,9 +2718,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0" +checksum = "13e2792b0ff0340399d58445b88fd9770e3489eff258a4cbc1523418f12abf84" dependencies = [ "futures-channel", "futures-core", @@ -2348,9 +2733,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" +checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5" dependencies = [ "futures-core", "futures-sink", @@ -2358,15 +2743,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" +checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608" [[package]] name = "futures-executor" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7acc85df6714c176ab5edf386123fafe217be88c0840ec11f199441134a074e2" +checksum = "e8de0a35a6ab97ec8869e32a2473f4b1324459e14c29275d14b10cb1fd19b50e" dependencies = [ "futures-core", "futures-task", @@ -2376,15 +2761,30 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" +checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531" + +[[package]] +name = "futures-lite" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite 0.2.9", + "waker-fn", +] [[package]] name = "futures-macro" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d" +checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70" dependencies = [ "proc-macro2", "quote", @@ -2398,21 +2798,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2411eed028cdf8c8034eaf21f9915f956b6c3abec4d4c7949ee67f0721127bd" dependencies = [ "futures-io", - "rustls 0.20.7", + "rustls 0.20.8", "webpki 0.22.0", ] [[package]] name = "futures-sink" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" +checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364" [[package]] name = "futures-task" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" +checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366" [[package]] name = "futures-timer" @@ -2422,9 +2822,9 @@ checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" [[package]] name = "futures-util" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" +checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1" dependencies = [ "futures-channel", "futures-core", @@ -2504,6 +2904,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "ghash" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97304e4cd182c3846f7575ced3890c53012ce534ad9114046b0a9e00bb30a375" +dependencies = [ + "opaque-debug", + "polyval 0.4.5", +] + [[package]] name = "ghash" version = "0.4.4" @@ -2511,14 +2921,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1583cc1656d7839fd3732b80cf4f38850336cdb9b8ded1cd399ca62958de3c99" dependencies = [ "opaque-debug", - "polyval", + "polyval 0.5.3", ] [[package]] name = "gimli" -version = "0.26.2" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" +checksum = "221996f774192f0f718773def8201c4ae31f02616a54ccfc2d358bb0e5cefdec" [[package]] name = "git-version" @@ -2544,9 +2954,9 @@ dependencies = [ [[package]] name = "glob" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "group" @@ -2584,6 +2994,21 @@ version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" +[[package]] +name = "hash-db" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d23bd4e7b5eda0d0f3a307e8b381fdc8ba9000f26fbe912250c0a4cc3956364a" + +[[package]] +name = "hash256-std-hasher" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92c171d55b98633f4ed3860808f004099b36c1cc29c42cfc53aa8591b21efcf2" +dependencies = [ + "crunchy", +] + [[package]] name = "hashbrown" version = "0.11.2" @@ -2626,7 +3051,7 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3e372db8e5c0d213e0cd0b9be18be2aca3d44cf2fe30a9d46a65581cd454584" dependencies = [ - "base64", + "base64 0.13.1", "bitflags", "bytes", "headers-core", @@ -2647,9 +3072,9 @@ dependencies = [ [[package]] name = "heck" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" @@ -2660,6 +3085,15 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + [[package]] name = "hex" version = "0.4.3" @@ -2691,6 +3125,16 @@ dependencies = [ "digest 0.9.0", ] +[[package]] +name = "hmac" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" +dependencies = [ + "crypto-mac 0.10.1", + "digest 0.9.0", +] + [[package]] name = "hmac" version = "0.11.0" @@ -2707,7 +3151,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest 0.10.5", + "digest 0.10.6", ] [[package]] @@ -2740,7 +3184,7 @@ checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" dependencies = [ "bytes", "fnv", - "itoa 1.0.4", + "itoa 1.0.5", ] [[package]] @@ -2779,7 +3223,7 @@ dependencies = [ "lighthouse_network", "lighthouse_version", "logging", - "lru", + "lru 0.7.8", "network", "parking_lot 0.12.1", "proto_array", @@ -2844,9 +3288,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.22" +version = "0.14.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abfba89e19b959ca163c7752ba59d737c1ceea53a5d31a149c805446fc958064" +checksum = "5e011372fa0b68db8350aa7a248930ecc7839bf46d8485577d69f117a75f164c" dependencies = [ "bytes", "futures-channel", @@ -2857,7 +3301,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa 1.0.4", + "itoa 1.0.5", "pin-project-lite 0.2.9", "socket2", "tokio", @@ -2868,13 +3312,13 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.23.0" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d87c48c02e0dc5e3b849a2041db3029fd066650f8f717c07bf8ed78ccb895cac" +checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" dependencies = [ "http", "hyper", - "rustls 0.20.7", + "rustls 0.20.8", "tokio", "tokio-rustls 0.23.4", ] @@ -2974,6 +3418,25 @@ dependencies = [ "libc", ] +[[package]] +name = "if-watch" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba7abdbb86e485125dad06c2691e1e393bf3b08c7b743b43aa162a00fd39062e" +dependencies = [ + "async-io", + "core-foundation", + "fnv", + "futures", + "if-addrs 0.7.0", + "ipnet", + "log", + "rtnetlink", + "system-configuration", + "tokio", + "windows", +] + [[package]] name = "igd" version = "0.11.1" @@ -3002,7 +3465,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba6a270039626615617f3f36d15fc827041df3b78c439da2cadfa47455a77f2f" dependencies = [ - "parity-scale-codec 3.2.1", + "parity-scale-codec 3.3.0", ] [[package]] @@ -3023,6 +3486,15 @@ dependencies = [ "serde", ] +[[package]] +name = "impl-serde" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc88fc67028ae3db0c853baa36269d398d5f45b6982f95549ff5def78c935cd" +dependencies = [ + "serde", +] + [[package]] name = "impl-trait-for-tuples" version = "0.2.2" @@ -3036,9 +3508,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" +checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" dependencies = [ "autocfg 1.1.0", "hashbrown 0.12.3", @@ -3075,22 +3547,41 @@ dependencies = [ ] [[package]] -name = "ipconfig" -version = "0.3.0" +name = "interceptor" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "723519edce41262b05d4143ceb95050e4c614f483e78e9fd9e39a8275a84ad98" +checksum = "1e8a11ae2da61704edada656798b61c94b35ecac2c58eb955156987d5e6be90b" +dependencies = [ + "async-trait", + "bytes", + "log", + "rand 0.8.5", + "rtcp", + "rtp", + "thiserror", + "tokio", + "waitgroup", + "webrtc-srtp", + "webrtc-util", +] + +[[package]] +name = "ipconfig" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd302af1b90f2463a98fa5ad469fc212c8e3175a41c3068601bfa2727591c5be" dependencies = [ "socket2", "widestring 0.5.1", "winapi", - "winreg 0.7.0", + "winreg", ] [[package]] name = "ipnet" -version = "2.5.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" +checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" [[package]] name = "itertools" @@ -3109,15 +3600,46 @@ checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" [[package]] name = "itoa" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" +checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" + +[[package]] +name = "jemalloc-ctl" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1891c671f3db85d8ea8525dd43ab147f9977041911d24a03e5a36187a7bfde9" +dependencies = [ + "jemalloc-sys", + "libc", + "paste", +] + +[[package]] +name = "jemalloc-sys" +version = "0.5.3+5.3.0-patched" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9bd5d616ea7ed58b571b2e209a65759664d7fb021a0819d7a790afc67e47ca1" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "jemallocator" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16c2514137880c52b0b4822b563fadd38257c1f380858addb74a400889696ea6" +dependencies = [ + "jemalloc-sys", + "libc", +] [[package]] name = "js-sys" -version = "0.3.60" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" dependencies = [ "wasm-bindgen", ] @@ -3139,11 +3661,11 @@ dependencies = [ [[package]] name = "jsonwebtoken" -version = "8.1.1" +version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aa4b4af834c6cfd35d8763d359661b90f2e45d8f750a0849156c7f4671af09c" +checksum = "09f4f04699947111ec1733e71778d763555737579e44b85844cae8e1940a1828" dependencies = [ - "base64", + "base64 0.13.1", "pem", "ring", "serde", @@ -3166,9 +3688,22 @@ dependencies = [ [[package]] name = "keccak" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9b7d56ba4a8344d6be9729995e6b06f928af29998cdf79fe390cbf6b1fee838" +checksum = "3afef3b6eff9ce9d8ff9b3601125eec7f0c8cbac7abd14f355d053fa56c98768" +dependencies = [ + "cpufeatures", +] + +[[package]] +name = "keccak-hash" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b286e6b663fb926e1eeb68528e69cb70ed46c6d65871a21b2215ae8154c6d3c" +dependencies = [ + "primitive-types 0.12.1", + "tiny-keccak", +] [[package]] name = "lazy_static" @@ -3187,7 +3722,7 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "lcli" -version = "3.3.0" +version = "3.4.0" dependencies = [ "account_utils", "beacon_chain", @@ -3196,7 +3731,7 @@ dependencies = [ "clap_utils", "deposit_contract", "directory", - "env_logger 0.9.1", + "env_logger 0.9.3", "environment", "eth1_test_rig", "eth2", @@ -3208,6 +3743,7 @@ dependencies = [ "lighthouse_network", "lighthouse_version", "log", + "malloc_utils", "sensitive_url", "serde", "serde_json", @@ -3246,9 +3782,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.137" +version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" +checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" [[package]] name = "libflate" @@ -3272,9 +3808,9 @@ dependencies = [ [[package]] name = "libloading" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efbc0f03f9a775e9f6aed295c6a1ba2253c5757a9e03d55c6caa46a681abcddd" +checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" dependencies = [ "cfg-if", "winapi", @@ -3282,9 +3818,15 @@ dependencies = [ [[package]] name = "libm" -version = "0.2.5" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "292a948cd991e376cf75541fe5b97a1081d713c618b4f1b9500f8844e49eb565" +checksum = "7fc7aa29613bd6a620df431842069224d8bc9011086b1db4c0e0cd47fa03ec9a" + +[[package]] +name = "libm" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" [[package]] name = "libmdbx" @@ -3303,33 +3845,33 @@ dependencies = [ [[package]] name = "libp2p" -version = "0.48.0" +version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94c996fe5bfdba47f5a5af71d48ecbe8cec900b7b97391cc1d3ba1afb0e2d3b6" +checksum = "2e0a0d2f693675f49ded13c5d510c48b78069e23cbd9108d7ccd59f6dc568819" dependencies = [ "bytes", "futures", "futures-timer", "getrandom 0.2.8", "instant", - "lazy_static", - "libp2p-core", + "libp2p-core 0.38.0", "libp2p-dns", "libp2p-gossipsub", "libp2p-identify", + "libp2p-mdns", "libp2p-metrics", "libp2p-mplex", "libp2p-noise", "libp2p-plaintext", + "libp2p-quic", "libp2p-swarm", - "libp2p-swarm-derive", "libp2p-tcp", + "libp2p-webrtc", "libp2p-websocket", "libp2p-yamux", - "multiaddr", + "multiaddr 0.16.0", "parking_lot 0.12.1", "pin-project", - "rand 0.7.3", "smallvec", ] @@ -3350,9 +3892,9 @@ dependencies = [ "lazy_static", "libsecp256k1", "log", - "multiaddr", + "multiaddr 0.14.0", "multihash", - "multistream-select", + "multistream-select 0.11.0", "p256", "parking_lot 0.12.1", "pin-project", @@ -3369,13 +3911,49 @@ dependencies = [ ] [[package]] -name = "libp2p-dns" -version = "0.36.0" +name = "libp2p-core" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb3c16e3bb2f76c751ae12f0f26e788c89d353babdded40411e7923f01fc978" +checksum = "b6a8fcd392ff67af6cc3f03b1426c41f7f26b6b9aff2dc632c1c56dd649e571f" +dependencies = [ + "asn1_der", + "bs58", + "ed25519-dalek", + "either", + "fnv", + "futures", + "futures-timer", + "instant", + "libsecp256k1", + "log", + "multiaddr 0.16.0", + "multihash", + "multistream-select 0.12.1", + "once_cell", + "p256", + "parking_lot 0.12.1", + "pin-project", + "prost", + "prost-build", + "rand 0.8.5", + "rw-stream-sink", + "sec1", + "sha2 0.10.6", + "smallvec", + "thiserror", + "unsigned-varint 0.7.1", + "void", + "zeroize", +] + +[[package]] +name = "libp2p-dns" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e42a271c1b49f789b92f7fc87749fa79ce5c7bdc88cbdfacb818a4bca47fec5" dependencies = [ "futures", - "libp2p-core", + "libp2p-core 0.38.0", "log", "parking_lot 0.12.1", "smallvec", @@ -3384,45 +3962,47 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" -version = "0.41.0" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2185aac44b162c95180ae4ddd1f4dfb705217ea1cb8e16bdfc70d31496fd80fa" +checksum = "a173171c71c29bb156f98886c7c4824596de3903dadf01e2e79d2ccdcf38cd9f" dependencies = [ "asynchronous-codec", - "base64", + "base64 0.13.1", "byteorder", "bytes", "fnv", "futures", "hex_fmt", "instant", - "libp2p-core", + "libp2p-core 0.38.0", "libp2p-swarm", "log", "prometheus-client", "prost", "prost-build", - "rand 0.7.3", + "prost-codec", + "rand 0.8.5", "regex", "sha2 0.10.6", "smallvec", + "thiserror", "unsigned-varint 0.7.1", "wasm-timer", ] [[package]] name = "libp2p-identify" -version = "0.39.0" +version = "0.41.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f19440c84b509d69b13f0c9c28caa9bd3a059d25478527e937e86761f25c821e" +checksum = "c052d0026f4817b44869bfb6810f4e1112f43aec8553f2cb38881c524b563abf" dependencies = [ "asynchronous-codec", "futures", "futures-timer", - "libp2p-core", + "libp2p-core 0.38.0", "libp2p-swarm", "log", - "lru", + "lru 0.8.1", "prost", "prost-build", "prost-codec", @@ -3432,12 +4012,32 @@ dependencies = [ ] [[package]] -name = "libp2p-metrics" -version = "0.9.0" +name = "libp2p-mdns" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a74ab339e8b5d989e8c1000a78adb5c064a6319245bb22d1e70b415ec18c39b8" +checksum = "04f378264aade9872d6ccd315c0accc18be3a35d15fc1b9c36e5b6f983b62b5b" dependencies = [ - "libp2p-core", + "data-encoding", + "futures", + "if-watch", + "libp2p-core 0.38.0", + "libp2p-swarm", + "log", + "rand 0.8.5", + "smallvec", + "socket2", + "tokio", + "trust-dns-proto", + "void", +] + +[[package]] +name = "libp2p-metrics" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ad8a64f29da86005c86a4d2728b8a0719e9b192f4092b609fd8790acb9dec55" +dependencies = [ + "libp2p-core 0.38.0", "libp2p-gossipsub", "libp2p-identify", "libp2p-swarm", @@ -3446,54 +4046,55 @@ dependencies = [ [[package]] name = "libp2p-mplex" -version = "0.36.0" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce53169351226ee0eb18ee7bef8d38f308fa8ad7244f986ae776390c0ae8a44d" +checksum = "03805b44107aa013e7cbbfa5627b31c36cbedfdfb00603c0311998882bc4bace" dependencies = [ "asynchronous-codec", "bytes", "futures", - "libp2p-core", + "libp2p-core 0.38.0", "log", "nohash-hasher", "parking_lot 0.12.1", - "rand 0.7.3", + "rand 0.8.5", "smallvec", "unsigned-varint 0.7.1", ] [[package]] name = "libp2p-noise" -version = "0.39.0" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cb0f939a444b06779ce551b3d78ebf13970ac27906ada452fd70abd160b09b8" +checksum = "a978cb57efe82e892ec6f348a536bfbd9fee677adbe5689d7a93ad3a9bffbf2e" dependencies = [ "bytes", "curve25519-dalek 3.2.0", "futures", - "lazy_static", - "libp2p-core", + "libp2p-core 0.38.0", "log", + "once_cell", "prost", "prost-build", "rand 0.8.5", "sha2 0.10.6", "snow", "static_assertions", - "x25519-dalek", + "thiserror", + "x25519-dalek 1.1.1", "zeroize", ] [[package]] name = "libp2p-plaintext" -version = "0.36.0" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "328e8c654a55ac7f093eb96dfd0386244dd337f2bd2822dc019522b743ea8add" +checksum = "4c43ab37fb4102682ae9a248dc2e6a8e7b941ec75cf24aed103060a788e0fd15" dependencies = [ "asynchronous-codec", "bytes", "futures", - "libp2p-core", + "libp2p-core 0.38.0", "log", "prost", "prost-build", @@ -3502,30 +4103,53 @@ dependencies = [ ] [[package]] -name = "libp2p-swarm" -version = "0.39.0" +name = "libp2p-quic" +version = "0.7.0-alpha" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ad2db60c06603606b54b58e4247e32efec87a93cb4387be24bf32926c600f2" +checksum = "01e7c867e95c8130667b24409d236d37598270e6da69b3baf54213ba31ffca59" +dependencies = [ + "bytes", + "futures", + "futures-timer", + "if-watch", + "libp2p-core 0.38.0", + "libp2p-tls", + "log", + "parking_lot 0.12.1", + "quinn-proto", + "rand 0.8.5", + "rustls 0.20.8", + "thiserror", + "tokio", +] + +[[package]] +name = "libp2p-swarm" +version = "0.41.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a35472fe3276b3855c00f1c032ea8413615e030256429ad5349cdf67c6e1a0" dependencies = [ "either", "fnv", "futures", "futures-timer", "instant", - "libp2p-core", + "libp2p-core 0.38.0", + "libp2p-swarm-derive", "log", "pin-project", - "rand 0.7.3", + "rand 0.8.5", "smallvec", "thiserror", + "tokio", "void", ] [[package]] name = "libp2p-swarm-derive" -version = "0.30.1" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0eddc4497a8b5a506013c40e8189864f9c3a00db2b25671f428ae9007f3ba32" +checksum = "9d527d5827582abd44a6d80c07ff8b50b4ee238a8979e05998474179e79dc400" dependencies = [ "heck", "quote", @@ -3534,31 +4158,79 @@ dependencies = [ [[package]] name = "libp2p-tcp" -version = "0.36.0" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9675432b4c94b3960f3d2c7e57427b81aea92aab67fd0eebef09e2ae0ff54895" +checksum = "b4b257baf6df8f2df39678b86c578961d48cc8b68642a12f0f763f56c8e5858d" dependencies = [ "futures", "futures-timer", - "if-addrs 0.7.0", - "ipnet", + "if-watch", "libc", - "libp2p-core", + "libp2p-core 0.38.0", "log", "socket2", "tokio", ] [[package]] -name = "libp2p-websocket" -version = "0.38.0" +name = "libp2p-tls" +version = "0.1.0-alpha" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de8a9e825cc03f2fc194d2e1622113d7fe18e1c7f4458a582b83140c9b9aea27" +checksum = "f7905ce0d040576634e8a3229a7587cc8beab83f79db6023800f1792895defa8" +dependencies = [ + "futures", + "futures-rustls", + "libp2p-core 0.38.0", + "rcgen 0.10.0", + "ring", + "rustls 0.20.8", + "thiserror", + "webpki 0.22.0", + "x509-parser 0.14.0", + "yasna", +] + +[[package]] +name = "libp2p-webrtc" +version = "0.4.0-alpha" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb6cd86dd68cba72308ea05de1cebf3ba0ae6e187c40548167955d4e3970f6a" +dependencies = [ + "async-trait", + "asynchronous-codec", + "bytes", + "futures", + "futures-timer", + "hex", + "if-watch", + "libp2p-core 0.38.0", + "libp2p-noise", + "log", + "multihash", + "prost", + "prost-build", + "prost-codec", + "rand 0.8.5", + "rcgen 0.9.3", + "serde", + "stun", + "thiserror", + "tinytemplate", + "tokio", + "tokio-util 0.7.4", + "webrtc", +] + +[[package]] +name = "libp2p-websocket" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d705506030d5c0aaf2882437c70dab437605f21c5f9811978f694e6917a3b54" dependencies = [ "either", "futures", "futures-rustls", - "libp2p-core", + "libp2p-core 0.38.0", "log", "parking_lot 0.12.1", "quicksink", @@ -3570,12 +4242,13 @@ dependencies = [ [[package]] name = "libp2p-yamux" -version = "0.40.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b74ec8dc042b583f0b2b93d52917f3b374c1e4b1cfa79ee74c7672c41257694c" +checksum = "4f63594a0aa818642d9d4915c791945053877253f08a3626f13416b5cd928a29" dependencies = [ "futures", - "libp2p-core", + "libp2p-core 0.38.0", + "log", "parking_lot 0.12.1", "thiserror", "yamux", @@ -3588,7 +4261,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95b09eff1b35ed3b33b877ced3a691fc7a481919c7e29c53c906226fcf55e2a1" dependencies = [ "arrayref", - "base64", + "base64 0.13.1", "digest 0.9.0", "hmac-drbg", "libsecp256k1-core", @@ -3653,7 +4326,7 @@ dependencies = [ [[package]] name = "lighthouse" -version = "3.3.0" +version = "3.4.0" dependencies = [ "account_manager", "account_utils", @@ -3664,7 +4337,7 @@ dependencies = [ "clap_utils", "database_manager", "directory", - "env_logger 0.9.1", + "env_logger 0.9.3", "environment", "eth1", "eth2", @@ -3721,7 +4394,7 @@ dependencies = [ "libp2p", "lighthouse_metrics", "lighthouse_version", - "lru", + "lru 0.7.8", "parking_lot 0.12.1", "prometheus-client", "quickcheck", @@ -3761,9 +4434,9 @@ dependencies = [ [[package]] name = "link-cplusplus" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9272ab7b96c9046fbc5bc56c06c117cb639fe2d509df0c421cad82d2915cf369" +checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" dependencies = [ "cc", ] @@ -3842,6 +4515,15 @@ dependencies = [ "hashbrown 0.12.3", ] +[[package]] +name = "lru" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6e8aaa3f231bb4bd57b84b2d5dc3ae7f350265df8aa96492e0bc394a1571909" +dependencies = [ + "hashbrown 0.12.3", +] + [[package]] name = "lru-cache" version = "0.1.2" @@ -3871,6 +4553,8 @@ dependencies = [ name = "malloc_utils" version = "0.1.0" dependencies = [ + "jemalloc-ctl", + "jemallocator", "lazy_static", "libc", "lighthouse_metrics", @@ -3900,9 +4584,9 @@ dependencies = [ [[package]] name = "matches" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" +checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" [[package]] name = "matchit" @@ -3910,6 +4594,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb" +[[package]] +name = "md-5" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +dependencies = [ + "digest 0.10.6", +] + [[package]] name = "mdbx-sys" version = "0.11.6-4" @@ -3936,18 +4629,50 @@ dependencies = [ "autocfg 1.1.0", ] +[[package]] +name = "memoffset" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +dependencies = [ + "autocfg 1.1.0", +] + [[package]] name = "merkle_proof" version = "0.2.0" dependencies = [ "eth2_hashing", - "ethereum-types 0.12.1", + "ethereum-types 0.14.1", "lazy_static", "quickcheck", "quickcheck_macros", "safe_arith", ] +[[package]] +name = "metastruct" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "734788dec2091fe9afa39530ca2ea7994f4a2c9aff3dbfebb63f2c1945c6f10b" +dependencies = [ + "metastruct_macro", +] + +[[package]] +name = "metastruct_macro" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ded15e7570c2a507a23e6c3a1c8d74507b779476e43afe93ddfc261d44173d" +dependencies = [ + "darling 0.13.4", + "itertools", + "proc-macro2", + "quote", + "smallvec", + "syn", +] + [[package]] name = "mev-build-rs" version = "0.2.1" @@ -4000,9 +4725,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.5.4" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34" +checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" dependencies = [ "adler", ] @@ -4063,6 +4788,35 @@ dependencies = [ "url", ] +[[package]] +name = "multiaddr" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aebdb21e90f81d13ed01dc84123320838e53963c2ca94b60b305d3fa64f31e" +dependencies = [ + "arrayref", + "byteorder", + "data-encoding", + "multibase", + "multihash", + "percent-encoding", + "serde", + "static_assertions", + "unsigned-varint 0.7.1", + "url", +] + +[[package]] +name = "multibase" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b3539ec3c1f04ac9748a260728e855f261b4977f5c3406612c884564f329404" +dependencies = [ + "base-x", + "data-encoding", + "data-encoding-macro", +] + [[package]] name = "multihash" version = "0.16.3" @@ -4070,7 +4824,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c346cf9999c631f002d8f977c4eaeaa0e6386f16007202308d0b3757522c2cc" dependencies = [ "core2", - "digest 0.10.5", + "digest 0.10.6", "multihash-derive", "sha2 0.10.6", "unsigned-varint 0.7.1", @@ -4078,9 +4832,9 @@ dependencies = [ [[package]] name = "multihash-derive" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc076939022111618a5026d3be019fd8b366e76314538ff9a1b59ffbcbf98bcd" +checksum = "1d6d4752e6230d8ef7adf7bd5d8c4b1f6561c1014c5ba9a37445ccefe18aa1db" dependencies = [ "proc-macro-crate", "proc-macro-error", @@ -4129,10 +4883,24 @@ dependencies = [ ] [[package]] -name = "native-tls" -version = "0.2.10" +name = "multistream-select" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" +checksum = "c8552ab875c1313b97b8d20cb857b9fd63e2d1d6a0a1b53ce9821e575405f27a" +dependencies = [ + "bytes", + "futures", + "log", + "pin-project", + "smallvec", + "unsigned-varint 0.7.1", +] + +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" dependencies = [ "lazy_static", "libc", @@ -4146,6 +4914,72 @@ dependencies = [ "tempfile", ] +[[package]] +name = "netlink-packet-core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "345b8ab5bd4e71a2986663e88c56856699d060e78e152e6e9d7966fcd5491297" +dependencies = [ + "anyhow", + "byteorder", + "libc", + "netlink-packet-utils", +] + +[[package]] +name = "netlink-packet-route" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9ea4302b9759a7a88242299225ea3688e63c85ea136371bb6cf94fd674efaab" +dependencies = [ + "anyhow", + "bitflags", + "byteorder", + "libc", + "netlink-packet-core", + "netlink-packet-utils", +] + +[[package]] +name = "netlink-packet-utils" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ede8a08c71ad5a95cdd0e4e52facd37190977039a4704eb82a283f713747d34" +dependencies = [ + "anyhow", + "byteorder", + "paste", + "thiserror", +] + +[[package]] +name = "netlink-proto" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65b4b14489ab424703c092062176d52ba55485a89c076b4f9db05092b7223aa6" +dependencies = [ + "bytes", + "futures", + "log", + "netlink-packet-core", + "netlink-sys", + "thiserror", + "tokio", +] + +[[package]] +name = "netlink-sys" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "260e21fbb6f3d253a14df90eb0000a6066780a15dd901a7519ce02d77a94985b" +dependencies = [ + "bytes", + "futures", + "libc", + "log", + "tokio", +] + [[package]] name = "network" version = "0.2.0" @@ -4157,7 +4991,8 @@ dependencies = [ "error-chain", "eth2_ssz", "eth2_ssz_types", - "ethereum-types 0.12.1", + "ethereum-types 0.14.1", + "execution_layer", "exit-future", "fnv", "futures", @@ -4192,27 +5027,39 @@ dependencies = [ [[package]] name = "nix" -version = "0.23.1" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f866317acbd3a240710c63f065ffb1e4fd466259045ccb504130b7f668f35c6" +checksum = "8f3790c00a0150112de0f4cd161e3d7fc4b2d8a5542ffc35f099a2562aecb35c" dependencies = [ "bitflags", "cc", "cfg-if", "libc", - "memoffset", + "memoffset 0.6.5", ] [[package]] name = "nix" -version = "0.25.0" +version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e322c04a9e3440c327fca7b6c8a63e6890a32fa2ad689db972425f07e0d22abb" +checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" dependencies = [ - "autocfg 1.1.0", "bitflags", "cfg-if", "libc", + "memoffset 0.6.5", +] + +[[package]] +name = "nix" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" +dependencies = [ + "bitflags", + "cfg-if", + "libc", + "static_assertions", ] [[package]] @@ -4244,9 +5091,9 @@ checksum = "cf51a729ecf40266a2368ad335a5fdde43471f545a967109cd62146ecf8b66ff" [[package]] name = "nom" -version = "7.1.1" +version = "7.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" dependencies = [ "memchr", "minimal-lexical", @@ -4291,7 +5138,7 @@ dependencies = [ "autocfg 0.1.8", "byteorder", "lazy_static", - "libm", + "libm 0.2.6", "num-integer", "num-iter", "num-traits", @@ -4333,11 +5180,11 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.13.1" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ - "hermit-abi", + "hermit-abi 0.2.6", "libc", ] @@ -4352,18 +5199,36 @@ dependencies = [ [[package]] name = "object" -version = "0.29.0" +version = "0.30.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53" +checksum = "ea86265d3d3dcb6a27fc51bd29a4bf387fae9d2986b823079d4986af253eb439" dependencies = [ "memchr", ] [[package]] -name = "once_cell" -version = "1.16.0" +name = "oid-registry" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +checksum = "38e20717fa0541f39bd146692035c37bedfa532b3e5071b35761082407546b2a" +dependencies = [ + "asn1-rs 0.3.1", +] + +[[package]] +name = "oid-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bedf36ffb6ba96c2eb7144ef6270557b52e54b20c0a8e1eb2ff99a6c6959bff" +dependencies = [ + "asn1-rs 0.5.1", +] + +[[package]] +name = "once_cell" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" [[package]] name = "oneshot_broadcast" @@ -4385,10 +5250,35 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] -name = "openssl" -version = "0.10.42" +name = "open-fastrlp" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12fc0523e3bd51a692c8850d075d74dc062ccf251c0110668cbd921917118a13" +checksum = "786393f80485445794f6043fd3138854dd109cc6c4bd1a6383db304c9ce9b9ce" +dependencies = [ + "arrayvec", + "auto_impl", + "bytes", + "ethereum-types 0.14.1", + "open-fastrlp-derive", +] + +[[package]] +name = "open-fastrlp-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "003b2be5c6c53c1cfeb0a238b8a1c3915cd410feb684457a36c10038f764bb1c" +dependencies = [ + "bytes", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl" +version = "0.10.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1" dependencies = [ "bitflags", "cfg-if", @@ -4418,18 +5308,18 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "111.22.0+1.1.1q" +version = "111.25.0+1.1.1t" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f31f0d509d1c1ae9cada2f9539ff8f37933831fd5098879e482aa687d659853" +checksum = "3173cd3626c43e3854b1b727422a276e568d9ec5fe8cec197822cf52cfb743d6" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.77" +version = "0.9.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b03b84c3b2d099b81f0953422b4d4ad58761589d0229b5506356afca05a3670a" +checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7" dependencies = [ "autocfg 1.1.0", "cc", @@ -4479,6 +5369,27 @@ dependencies = [ "sha2 0.10.6", ] +[[package]] +name = "p384" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc8c5bf642dde52bb9e87c0ecd8ca5a76faac2eeed98dedb7c717997e1080aa" +dependencies = [ + "ecdsa", + "elliptic-curve", + "sha2 0.10.6", +] + +[[package]] +name = "packed_simd_2" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1914cd452d8fccd6f9db48147b29fd4ae05bea9dc5d9ad578509f72415de282" +dependencies = [ + "cfg-if", + "libm 0.1.4", +] + [[package]] name = "parity-scale-codec" version = "2.3.1" @@ -4495,15 +5406,15 @@ dependencies = [ [[package]] name = "parity-scale-codec" -version = "3.2.1" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "366e44391a8af4cfd6002ef6ba072bae071a96aafca98d7d448a34c5dca38b6a" +checksum = "c3840933452adf7b3b9145e27086a5a3376c619dca1a21b1e5a5af0d54979bed" dependencies = [ "arrayvec", "bitvec 1.0.1", "byte-slice-cast", "impl-trait-for-tuples", - "parity-scale-codec-derive 3.1.3", + "parity-scale-codec-derive 3.1.4", "serde", ] @@ -4521,9 +5432,9 @@ dependencies = [ [[package]] name = "parity-scale-codec-derive" -version = "3.1.3" +version = "3.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9299338969a3d2f491d65f140b00ddec470858402f888af98e8642fb5e8965cd" +checksum = "86b26a931f824dd4eca30b3e43bb4f31cd5f0d3a403c5f5ff27106b805bfde7b" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -4531,6 +5442,12 @@ dependencies = [ "syn", ] +[[package]] +name = "parking" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" + [[package]] name = "parking_lot" version = "0.11.2" @@ -4539,7 +5456,7 @@ checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" dependencies = [ "instant", "lock_api", - "parking_lot_core 0.8.5", + "parking_lot_core 0.8.6", ] [[package]] @@ -4549,14 +5466,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.4", + "parking_lot_core 0.9.7", ] [[package]] name = "parking_lot_core" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" dependencies = [ "cfg-if", "instant", @@ -4568,22 +5485,22 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.4" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dc9e0dc2adc1c69d09143aff38d3d30c5c3f0df0dad82e6d25547af174ebec0" +checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] name = "paste" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1de2e551fb905ac83f73f7aedf2f0cb4a0da7e35efa24a202a936269f1f18e1" +checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba" [[package]] name = "pbkdf2" @@ -4611,11 +5528,20 @@ checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" [[package]] name = "pem" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03c64931a1a212348ec4f3b4362585eca7159d0d09cbdf4a7f74f02173596fd4" +checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" dependencies = [ - "base64", + "base64 0.13.1", +] + +[[package]] +name = "pem-rfc7468" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d159833a9105500e0398934e205e0773f0b27529557134ecfc51c27646adac" +dependencies = [ + "base64ct", ] [[package]] @@ -4626,9 +5552,9 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "pest" -version = "2.4.0" +version = "2.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc7bc69c062e492337d74d59b120c274fd3d261b6bf6d3207d499b4b379c41a" +checksum = "028accff104c4e513bad663bbcd2ad7cfd5304144404c31ed0a77ac103d00660" dependencies = [ "thiserror", "ucd-trie", @@ -4636,9 +5562,9 @@ dependencies = [ [[package]] name = "petgraph" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143" +checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" dependencies = [ "fixedbitset", "indexmap", @@ -4714,6 +5640,12 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8d0eef3571242013a0d5dc84861c3ae4a652e56e12adf8bdc26ff5f8cb34c94" +[[package]] +name = "platforms" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d7ddaed09e0eb771a79ab0fd64609ba0afb0a8366421957936ad14cbd13630" + [[package]] name = "plotters" version = "0.3.4" @@ -4742,6 +5674,20 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "polling" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22122d5ec4f9fe1b3916419b76be1e80bcb93f618d071d2edf841b137b2a2bd6" +dependencies = [ + "autocfg 1.1.0", + "cfg-if", + "libc", + "log", + "wepoll-ffi", + "windows-sys 0.42.0", +] + [[package]] name = "poly1305" version = "0.7.2" @@ -4753,6 +5699,17 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "polyval" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eebcc4aa140b9abd2bc40d9c3f7ccec842679cd79045ac3a7ac698c1a064b7cd" +dependencies = [ + "cpuid-bool", + "opaque-debug", + "universal-hash", +] + [[package]] name = "polyval" version = "0.5.3" @@ -4767,9 +5724,19 @@ dependencies = [ [[package]] name = "ppv-lite86" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "prettyplease" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e97e3215779627f01ee256d2fad52f3d95e8e1c11e9fc6fd08f7cd455d5d5c78" +dependencies = [ + "proc-macro2", + "syn", +] [[package]] name = "primitive-types" @@ -4777,33 +5744,33 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05e4722c697a58a99d5d06a08c30821d7c082a4632198de1eaa5a6c22ef42373" dependencies = [ - "fixed-hash", + "fixed-hash 0.7.0", "impl-codec 0.5.1", "impl-rlp", - "impl-serde", + "impl-serde 0.3.2", "uint", ] [[package]] name = "primitive-types" -version = "0.11.1" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28720988bff275df1f51b171e1b2a18c30d194c4d2b61defdacecd625a5d94a" +checksum = "9f3486ccba82358b11a77516035647c34ba167dfa53312630de83b12bd4f3d66" dependencies = [ - "fixed-hash", + "fixed-hash 0.8.0", "impl-codec 0.6.0", "impl-rlp", - "impl-serde", + "impl-serde 0.4.0", + "scale-info", "uint", ] [[package]] name = "proc-macro-crate" -version = "1.2.1" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eda0fc3b0fb7c975631757e14d9049da17374063edb6ebbcbc54d880d4fe94e9" +checksum = "e17d47ce914bf4de440332250b0edd23ce48c005f59fab39d3335866b114f11a" dependencies = [ - "once_cell", "thiserror", "toml", ] @@ -4834,15 +5801,15 @@ dependencies = [ [[package]] name = "proc-macro-hack" -version = "0.5.19" +version = "0.5.20+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" +checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.47" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" dependencies = [ "unicode-ident", ] @@ -4881,7 +5848,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83cd1b99916654a69008fd66b4f9397fbe08e6e51dfe23d4417acf5d3b8cb87c" dependencies = [ "dtoa", - "itoa 1.0.4", + "itoa 1.0.5", "parking_lot 0.12.1", "prometheus-client-derive-text-encode", ] @@ -4899,9 +5866,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.11.0" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "399c3c31cdec40583bb68f0b18403400d01ec4289c383aa047560439952c4dd7" +checksum = "21dc42e00223fc37204bd4aa177e69420c604ca4a183209a8f9de30c6d934698" dependencies = [ "bytes", "prost-derive", @@ -4909,9 +5876,9 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.11.1" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f835c582e6bd972ba8347313300219fed5bfa52caf175298d860b61ff6069bb" +checksum = "a3f8ad728fb08fe212df3c05169e940fbb6d9d16a877ddde14644a983ba2012e" dependencies = [ "bytes", "heck", @@ -4920,18 +5887,20 @@ dependencies = [ "log", "multimap", "petgraph", + "prettyplease", "prost", "prost-types", "regex", + "syn", "tempfile", "which", ] [[package]] name = "prost-codec" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "011ae9ff8359df7915f97302d591cdd9e0e27fbd5a4ddc5bd13b71079bb20987" +checksum = "0dc34979ff898b6e141106178981ce2596c387ea6e62533facfc61a37fc879c0" dependencies = [ "asynchronous-codec", "bytes", @@ -4942,9 +5911,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.11.0" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7345d5f0e08c0536d7ac7229952590239e77abf0a0100a1b1d890add6ea96364" +checksum = "8bda8c0881ea9f722eb9629376db3d0b903b462477c1aafcb0566610ac28ac5d" dependencies = [ "anyhow", "itertools", @@ -4955,9 +5924,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.11.1" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dfaa718ad76a44b3415e6c4d53b17c8f99160dcb3a99b10470fce8ad43f6e3e" +checksum = "a5e0526209433e96d83d750dd81a99118edbc55739e7e61a46764fd2ad537788" dependencies = [ "bytes", "prost", @@ -4993,10 +5962,10 @@ dependencies = [ "derive_more", "glob", "mach", - "nix 0.23.1", + "nix 0.23.2", "num_cpus", "once_cell", - "platforms", + "platforms 2.0.0", "thiserror", "unescape", ] @@ -5042,10 +6011,28 @@ dependencies = [ ] [[package]] -name = "quote" -version = "1.0.21" +name = "quinn-proto" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +checksum = "72ef4ced82a24bb281af338b9e8f94429b6eca01b4e66d899f40031f074e74c9" +dependencies = [ + "bytes", + "rand 0.8.5", + "ring", + "rustc-hash", + "rustls 0.20.8", + "slab", + "thiserror", + "tinyvec", + "tracing", + "webpki 0.22.0", +] + +[[package]] +name = "quote" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" dependencies = [ "proc-macro2", ] @@ -5165,21 +6152,19 @@ dependencies = [ [[package]] name = "rayon" -version = "1.5.3" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d" +checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" dependencies = [ - "autocfg 1.1.0", - "crossbeam-deque", "either", "rayon-core", ] [[package]] name = "rayon-core" -version = "1.9.3" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f" +checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b" dependencies = [ "crossbeam-channel", "crossbeam-deque", @@ -5187,6 +6172,31 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "rcgen" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6413f3de1edee53342e6138e75b56d32e7bc6e332b3bd62d497b1929d4cfbcdd" +dependencies = [ + "pem", + "ring", + "time 0.3.17", + "x509-parser 0.13.2", + "yasna", +] + +[[package]] +name = "rcgen" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbe84efe2f38dea12e9bfc1f65377fdf03e53a18cb3b995faedf7934c7e785b" +dependencies = [ + "pem", + "ring", + "time 0.3.17", + "yasna", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -5209,9 +6219,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.6.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" +checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" dependencies = [ "aho-corasick", "memchr", @@ -5229,9 +6239,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.27" +version = "0.6.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" +checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" [[package]] name = "remove_dir_all" @@ -5244,11 +6254,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.12" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "431949c384f4e2ae07605ccaa56d1d9d2ecdb5cadd4f9577ccfab29f2e5149fc" +checksum = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9" dependencies = [ - "base64", + "base64 0.21.0", "bytes", "encoding_rs", "futures-core", @@ -5267,7 +6277,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite 0.2.9", - "rustls 0.20.7", + "rustls 0.20.8", "rustls-pemfile", "serde", "serde_json", @@ -5280,9 +6290,10 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", "webpki-roots", - "winreg 0.10.1", + "winreg", ] [[package]] @@ -5297,9 +6308,9 @@ dependencies = [ [[package]] name = "rfc6979" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88c86280f057430a52f4861551b092a01b419b8eacefc7c995eacb9dc132fe32" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" dependencies = [ "crypto-bigint", "hmac 0.12.1", @@ -5358,6 +6369,46 @@ dependencies = [ "winapi", ] +[[package]] +name = "rtcp" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1919efd6d4a6a85d13388f9487549bb8e359f17198cc03ffd72f79b553873691" +dependencies = [ + "bytes", + "thiserror", + "webrtc-util", +] + +[[package]] +name = "rtnetlink" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "322c53fd76a18698f1c27381d58091de3a043d356aa5bd0d510608b565f469a0" +dependencies = [ + "futures", + "log", + "netlink-packet-route", + "netlink-proto", + "nix 0.24.3", + "thiserror", + "tokio", +] + +[[package]] +name = "rtp" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2a095411ff00eed7b12e4c6a118ba984d113e1079582570d56a5ee723f11f80" +dependencies = [ + "async-trait", + "bytes", + "rand 0.8.5", + "serde", + "thiserror", + "webrtc-util", +] + [[package]] name = "rusqlite" version = "0.25.4" @@ -5373,17 +6424,6 @@ dependencies = [ "smallvec", ] -[[package]] -name = "rust_decimal" -version = "1.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee9164faf726e4f3ece4978b25ca877ddc6802fa77f38cdccb32c7f805ecd70c" -dependencies = [ - "arrayvec", - "num-traits", - "serde", -] - [[package]] name = "rustc-demangle" version = "0.1.21" @@ -5426,7 +6466,16 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.14", + "semver 1.0.16", +] + +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom 7.1.3", ] [[package]] @@ -5435,7 +6484,7 @@ version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" dependencies = [ - "base64", + "base64 0.13.1", "log", "ring", "sct 0.6.1", @@ -5444,9 +6493,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.20.7" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "539a2bfe908f471bfa933876bd1eb6a19cf2176d375f82ef7f99530a40e48c2c" +checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" dependencies = [ "log", "ring", @@ -5456,18 +6505,18 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0864aeff53f8c05aa08d86e5ef839d3dfcf07aeba2db32f12db0ef716e87bd55" +checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" dependencies = [ - "base64", + "base64 0.21.0", ] [[package]] name = "rustversion" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8" +checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70" [[package]] name = "rw-stream-sink" @@ -5482,9 +6531,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" +checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" [[package]] name = "safe_arith" @@ -5502,7 +6551,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ecbd2eb639fd7cab5804a0837fe373cc2172d15437e804c054a9fb885cb923b0" dependencies = [ - "cipher", + "cipher 0.3.0", ] [[package]] @@ -5515,13 +6564,36 @@ dependencies = [ ] [[package]] -name = "schannel" -version = "0.1.20" +name = "scale-info" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" +checksum = "001cf62ece89779fd16105b5f515ad0e5cedcd5440d3dd806bb067978e7c3608" dependencies = [ - "lazy_static", - "windows-sys 0.36.1", + "cfg-if", + "derive_more", + "parity-scale-codec 3.3.0", + "scale-info-derive", +] + +[[package]] +name = "scale-info-derive" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "303959cf613a6f6efd19ed4b4ad5bf79966a13352716299ad532cfb115f4205c" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "schannel" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" +dependencies = [ + "windows-sys 0.42.0", ] [[package]] @@ -5547,9 +6619,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "scratch" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" +checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" [[package]] name = "scrypt" @@ -5583,6 +6655,18 @@ dependencies = [ "untrusted", ] +[[package]] +name = "sdp" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d22a5ef407871893fd72b4562ee15e4742269b173959db4b8df6f538c414e13" +dependencies = [ + "rand 0.8.5", + "substring", + "thiserror", + "url", +] + [[package]] name = "sec1" version = "0.3.0" @@ -5617,9 +6701,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.7.0" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c" +checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" dependencies = [ "bitflags", "core-foundation", @@ -5630,9 +6714,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.6.1" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" +checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" dependencies = [ "core-foundation-sys", "libc", @@ -5658,9 +6742,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.14" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4" +checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a" [[package]] name = "semver-parser" @@ -5679,9 +6763,9 @@ dependencies = [ [[package]] name = "send_wrapper" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "930c0acf610d3fdb5e2ab6213019aaa04e227ebe9547b0649ba599b16d788bd7" +checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" [[package]] name = "sensitive_url" @@ -5693,9 +6777,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.147" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965" +checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" dependencies = [ "serde_derive", ] @@ -5722,9 +6806,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.147" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f1d362ca8fc9c3e3a7484440752472d68a6caa98f1ab81d99b5dfe517cec852" +checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ "proc-macro2", "quote", @@ -5733,20 +6817,20 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.87" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce777b7b150d76b9cf60d28b55f5847135a003f7d7350c6be7a773508ce7d45" +checksum = "7434af0dc1cbd59268aa98b4c22c131c0584d2232f6fb166efb993e2832e896a" dependencies = [ - "itoa 1.0.4", + "itoa 1.0.5", "ryu", "serde", ] [[package]] name = "serde_repr" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fe39d9fbb0ebf5eb2c7cb7e2a47e4f462fad1379f1166b8ae49ad9eae89a7ca" +checksum = "9a5ec9fa74a20ebbe5d9ac23dac1fc96ba0ecfe9f50f2843b52e537b10fbcb4e" dependencies = [ "proc-macro2", "quote", @@ -5760,7 +6844,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa 1.0.4", + "itoa 1.0.5", "ryu", "serde", ] @@ -5781,7 +6865,7 @@ version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" dependencies = [ - "darling", + "darling 0.13.4", "proc-macro2", "quote", "syn", @@ -5814,13 +6898,13 @@ dependencies = [ [[package]] name = "sha-1" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" +checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.10.5", + "digest 0.10.6", ] [[package]] @@ -5831,7 +6915,7 @@ checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.10.5", + "digest 0.10.6", ] [[package]] @@ -5855,7 +6939,7 @@ checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.10.5", + "digest 0.10.6", ] [[package]] @@ -5876,7 +6960,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bdf0c33fae925bdc080598b84bc15c55e7b9a4a43b3c704da051f977469691c9" dependencies = [ - "digest 0.10.5", + "digest 0.10.6", "keccak", ] @@ -5910,7 +6994,7 @@ version = "1.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" dependencies = [ - "digest 0.10.5", + "digest 0.10.6", "rand_core 0.6.4", ] @@ -5923,7 +7007,7 @@ dependencies = [ "num-bigint", "num-traits", "thiserror", - "time 0.3.16", + "time 0.3.17", ] [[package]] @@ -5931,7 +7015,7 @@ name = "simulator" version = "0.2.0" dependencies = [ "clap", - "env_logger 0.9.1", + "env_logger 0.9.3", "eth1", "eth1_test_rig", "execution_layer", @@ -5969,7 +7053,7 @@ dependencies = [ "lmdb-rkv", "lmdb-rkv-sys", "logging", - "lru", + "lru 0.7.8", "maplit", "parking_lot 0.12.1", "rand 0.8.5", @@ -6049,7 +7133,7 @@ dependencies = [ "serde", "serde_json", "slog", - "time 0.3.16", + "time 0.3.17", ] [[package]] @@ -6094,7 +7178,7 @@ dependencies = [ "slog", "term", "thread_local", - "time 0.3.16", + "time 0.3.17", ] [[package]] @@ -6139,20 +7223,20 @@ checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "snap" -version = "1.0.5" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45456094d1983e2ee2a18fdfebce3189fa451699d0502cb8e3b49dba5ba41451" +checksum = "5e9f0ab6ef7eb7353d9119c170a436d1bf248eea575ac42d19d12f4e34130831" [[package]] name = "snow" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "774d05a3edae07ce6d68ea6984f3c05e9bba8927e3dd591e3b479e5b03213d0d" +checksum = "12ba5f4d4ff12bdb6a169ed51b7c48c0e0ac4b0b4b31012b2571e97d78d3201d" dependencies = [ - "aes-gcm", + "aes-gcm 0.9.4", "blake2", "chacha20poly1305", - "curve25519-dalek 4.0.0-pre.1", + "curve25519-dalek 4.0.0-rc.0", "rand_core 0.6.4", "ring", "rustc_version 0.4.0", @@ -6176,7 +7260,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41d1c5305e39e09653383c2c7244f2f78b3bcae37cf50c64cb4789c9f5096ec2" dependencies = [ - "base64", + "base64 0.13.1", "bytes", "flate2", "futures", @@ -6235,7 +7319,7 @@ dependencies = [ "beacon_chain", "bls", "derivative", - "env_logger 0.9.1", + "env_logger 0.9.3", "eth2_hashing", "eth2_ssz", "eth2_ssz_derive", @@ -6285,7 +7369,7 @@ dependencies = [ "lazy_static", "leveldb", "lighthouse_metrics", - "lru", + "lru 0.7.8", "parking_lot 0.12.1", "serde", "serde_derive", @@ -6331,6 +7415,34 @@ dependencies = [ "syn", ] +[[package]] +name = "stun" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7e94b1ec00bad60e6410e058b52f1c66de3dc5fe4d62d09b3e52bb7d3b73e25" +dependencies = [ + "base64 0.13.1", + "crc", + "lazy_static", + "md-5", + "rand 0.8.5", + "ring", + "subtle", + "thiserror", + "tokio", + "url", + "webrtc-util", +] + +[[package]] +name = "substring" +version = "1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ee6433ecef213b2e72f587ef64a2f5943e7cd16fbd82dbe8bc07486c534c86" +dependencies = [ + "autocfg 1.1.0", +] + [[package]] name = "subtle" version = "2.4.1" @@ -6343,7 +7455,7 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95a99807a055ff4ff5d249bb84c80d9eabb55ca3c452187daae43fd5b51ef695" dependencies = [ - "darling", + "darling 0.13.4", "itertools", "proc-macro2", "quote", @@ -6357,14 +7469,14 @@ version = "0.2.0" dependencies = [ "criterion", "eth2_hashing", - "ethereum-types 0.12.1", + "ethereum-types 0.14.1", ] [[package]] name = "syn" -version = "1.0.103" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d" +checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" dependencies = [ "proc-macro2", "quote", @@ -6373,9 +7485,9 @@ dependencies = [ [[package]] name = "sync_wrapper" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] name = "synstructure" @@ -6391,9 +7503,9 @@ dependencies = [ [[package]] name = "sysinfo" -version = "0.26.7" +version = "0.26.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c375d5fd899e32847b8566e10598d6e9f1d9b55ec6de3cdf9e7da4bdc51371bc" +checksum = "5c18a6156d1f27a9592ee18c1a846ca8dd5c258b7179fc193ae87c74ebb666f5" dependencies = [ "cfg-if", "core-foundation-sys", @@ -6404,6 +7516,27 @@ dependencies = [ "winapi", ] +[[package]] +name = "system-configuration" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d75182f12f490e953596550b65ee31bda7c8e043d9386174b353bda50838c3fd" +dependencies = [ + "bitflags", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "system_health" version = "0.1.0" @@ -6482,9 +7615,9 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.1.3" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" dependencies = [ "winapi-util", ] @@ -6516,18 +7649,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" +checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" +checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" dependencies = [ "proc-macro2", "quote", @@ -6554,9 +7687,9 @@ dependencies = [ [[package]] name = "time" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" dependencies = [ "libc", "wasi 0.10.0+wasi-snapshot-preview1", @@ -6565,11 +7698,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fab5c8b9980850e06d92ddbe3ab839c062c801f3927c0fb8abd6fc8e918fbca" +checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" dependencies = [ - "itoa 1.0.4", + "itoa 1.0.5", "libc", "num_threads", "serde", @@ -6585,9 +7718,9 @@ checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" [[package]] name = "time-macros" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bb801831d812c562ae7d2bfb531f26e66e4e1f6b17307ba4149c5064710e5b" +checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2" dependencies = [ "time-core", ] @@ -6652,15 +7785,15 @@ dependencies = [ [[package]] name = "tinyvec_macros" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.21.2" +version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099" +checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" dependencies = [ "autocfg 1.1.0", "bytes", @@ -6673,7 +7806,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "winapi", + "windows-sys 0.42.0", ] [[package]] @@ -6688,9 +7821,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "1.8.0" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" +checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" dependencies = [ "proc-macro2", "quote", @@ -6699,9 +7832,9 @@ dependencies = [ [[package]] name = "tokio-native-tls" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", "tokio", @@ -6724,7 +7857,7 @@ version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ - "rustls 0.20.7", + "rustls 0.20.8", "tokio", "webpki 0.22.0", ] @@ -6762,7 +7895,7 @@ checksum = "f714dd15bead90401d77e04243611caec13726c2408afd5b31901dfcdcb3b181" dependencies = [ "futures-util", "log", - "rustls 0.20.7", + "rustls 0.20.8", "tokio", "tokio-rustls 0.23.4", "tungstenite 0.17.3", @@ -6794,6 +7927,7 @@ checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" dependencies = [ "bytes", "futures-core", + "futures-io", "futures-sink", "pin-project-lite 0.2.9", "tokio", @@ -6802,9 +7936,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.9" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" dependencies = [ "serde", ] @@ -6827,9 +7961,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c530c8675c1dbf98facee631536fa116b5fb6382d7dd6dc1b118d970eafe3ba" +checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" dependencies = [ "bitflags", "bytes", @@ -6956,7 +8090,7 @@ dependencies = [ "eth2_hashing", "eth2_ssz", "eth2_ssz_derive", - "ethereum-types 0.12.1", + "ethereum-types 0.14.1", "rand 0.8.5", "smallvec", "tree_hash_derive", @@ -6967,16 +8101,26 @@ dependencies = [ name = "tree_hash_derive" version = "0.4.0" dependencies = [ - "darling", + "darling 0.13.4", "quote", "syn", ] [[package]] -name = "trust-dns-proto" -version = "0.21.2" +name = "triehash" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c31f240f59877c3d4bb3b3ea0ec5a6a0cff07323580ff8c7a605cd7d08b255d" +checksum = "a1631b201eb031b563d2e85ca18ec8092508e262a3196ce9bd10a67ec87b9f5c" +dependencies = [ + "hash-db", + "rlp", +] + +[[package]] +name = "trust-dns-proto" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f7f83d1e4a0e4358ac54c5c3681e5d7da5efc5a7a632c90bb6d6669ddd9bc26" dependencies = [ "async-trait", "cfg-if", @@ -6988,40 +8132,41 @@ dependencies = [ "idna 0.2.3", "ipnet", "lazy_static", - "log", "rand 0.8.5", "smallvec", + "socket2", "thiserror", "tinyvec", "tokio", + "tracing", "url", ] [[package]] name = "trust-dns-resolver" -version = "0.21.2" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4ba72c2ea84515690c9fcef4c6c660bb9df3036ed1051686de84605b74fd558" +checksum = "aff21aa4dcefb0a1afbfac26deb0adc93888c7d295fb63ab273ef276ba2b7cfe" dependencies = [ "cfg-if", "futures-util", "ipconfig", "lazy_static", - "log", "lru-cache", "parking_lot 0.12.1", "resolv-conf", "smallvec", "thiserror", "tokio", + "tracing", "trust-dns-proto", ] [[package]] name = "try-lock" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "tungstenite" @@ -7029,7 +8174,7 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0b2d8558abd2e276b0a8df5c05a2ec762609344191e5fd23e292c910e9165b5" dependencies = [ - "base64", + "base64 0.13.1", "byteorder", "bytes", "http", @@ -7048,21 +8193,40 @@ version = "0.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e27992fd6a8c29ee7eef28fc78349aa244134e10ad447ce3b9f0ac0ed0fa4ce0" dependencies = [ - "base64", + "base64 0.13.1", "byteorder", "bytes", "http", "httparse", "log", "rand 0.8.5", - "rustls 0.20.7", - "sha-1 0.10.0", + "rustls 0.20.8", + "sha-1 0.10.1", "thiserror", "url", "utf-8", "webpki 0.22.0", ] +[[package]] +name = "turn" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4712ee30d123ec7ae26d1e1b218395a16c87cdbaf4b3925d170d684af62ea5e8" +dependencies = [ + "async-trait", + "base64 0.13.1", + "futures", + "log", + "md-5", + "rand 0.8.5", + "ring", + "stun", + "thiserror", + "tokio", + "webrtc-util", +] + [[package]] name = "twoway" version = "0.1.8" @@ -7074,9 +8238,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "types" @@ -7096,7 +8260,7 @@ dependencies = [ "eth2_ssz", "eth2_ssz_derive", "eth2_ssz_types", - "ethereum-types 0.12.1", + "ethereum-types 0.14.1", "hex", "int_to_bytes", "itertools", @@ -7104,6 +8268,7 @@ dependencies = [ "log", "maplit", "merkle_proof", + "metastruct", "parking_lot 0.12.1", "rand 0.8.5", "rand_xorshift", @@ -7136,9 +8301,9 @@ checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81" [[package]] name = "uint" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a45526d29728d135c2900b0d30573fe3ee79fceb12ef534c7bb30e810a91b601" +checksum = "76f64bba2c53b04fcab63c01a7d7427eadc821e3bc48c34dc9ba29c501164b52" dependencies = [ "arbitrary", "byteorder", @@ -7164,15 +8329,15 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.8" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" +checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" [[package]] name = "unicode-ident" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" +checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" [[package]] name = "unicode-normalization" @@ -7262,6 +8427,15 @@ dependencies = [ "serde", ] +[[package]] +name = "uuid" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1674845326ee10d37ca60470760d4288a6f80f304007d92e5c53bab78c9cfd79" +dependencies = [ + "getrandom 0.2.8", +] + [[package]] name = "validator_client" version = "0.3.5" @@ -7390,6 +8564,21 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +[[package]] +name = "waitgroup" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1f50000a783467e6c0200f9d10642f4bc424e39efc1b770203e88b488f79292" +dependencies = [ + "atomic-waker", +] + +[[package]] +name = "waker-fn" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" + [[package]] name = "walkdir" version = "2.3.2" @@ -7479,9 +8668,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -7489,9 +8678,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" dependencies = [ "bumpalo", "log", @@ -7504,9 +8693,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" +checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" dependencies = [ "cfg-if", "js-sys", @@ -7516,9 +8705,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -7526,9 +8715,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2", "quote", @@ -7539,15 +8728,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" [[package]] name = "wasm-bindgen-test" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d2fff962180c3fadf677438054b1db62bee4aa32af26a45388af07d1287e1d" +checksum = "6db36fc0f9fb209e88fb3642590ae0205bb5a56216dabd963ba15879fe53a30b" dependencies = [ "console_error_panic_hook", "js-sys", @@ -7559,14 +8748,27 @@ dependencies = [ [[package]] name = "wasm-bindgen-test-macro" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4683da3dfc016f704c9f82cf401520c4f1cb3ee440f7f52b3d6ac29506a49ca7" +checksum = "0734759ae6b3b1717d661fe4f016efcfb9828f5edb4520c18eaee05af3b43be9" dependencies = [ "proc-macro2", "quote", ] +[[package]] +name = "wasm-streams" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasm-timer" version = "0.2.5" @@ -7584,9 +8786,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.60" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" dependencies = [ "js-sys", "wasm-bindgen", @@ -7599,7 +8801,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44f258e254752d210b84fe117b31f1e3cc9cbf04c0d747eb7f8cf7cf5e370f6d" dependencies = [ "arrayvec", - "base64", + "base64 0.13.1", "bytes", "derive_more", "ethabi 16.0.0", @@ -7686,18 +8888,239 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.22.5" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368bfe657969fb01238bb756d351dcade285e0f6fcbd36dcb23359a5169975be" +checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" dependencies = [ "webpki 0.22.0", ] [[package]] -name = "which" -version = "4.3.0" +name = "webrtc" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c831fbbee9e129a8cf93e7747a82da9d95ba8e16621cae60ec2cdc849bacb7b" +checksum = "2d3bc9049bdb2cea52f5fd4f6f728184225bdb867ed0dc2410eab6df5bdd67bb" +dependencies = [ + "arc-swap", + "async-trait", + "bytes", + "hex", + "interceptor", + "lazy_static", + "log", + "rand 0.8.5", + "rcgen 0.9.3", + "regex", + "ring", + "rtcp", + "rtp", + "rustls 0.19.1", + "sdp", + "serde", + "serde_json", + "sha2 0.10.6", + "stun", + "thiserror", + "time 0.3.17", + "tokio", + "turn", + "url", + "waitgroup", + "webrtc-data", + "webrtc-dtls", + "webrtc-ice", + "webrtc-mdns", + "webrtc-media", + "webrtc-sctp", + "webrtc-srtp", + "webrtc-util", +] + +[[package]] +name = "webrtc-data" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef36a4d12baa6e842582fe9ec16a57184ba35e1a09308307b67d43ec8883100" +dependencies = [ + "bytes", + "derive_builder", + "log", + "thiserror", + "tokio", + "webrtc-sctp", + "webrtc-util", +] + +[[package]] +name = "webrtc-dtls" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7021987ae0a2ed6c8cd33f68e98e49bb6e74ffe9543310267b48a1bbe3900e5f" +dependencies = [ + "aes 0.6.0", + "aes-gcm 0.8.0", + "async-trait", + "bincode", + "block-modes", + "byteorder", + "ccm", + "curve25519-dalek 3.2.0", + "der-parser 8.1.0", + "elliptic-curve", + "hkdf", + "hmac 0.10.1", + "log", + "oid-registry 0.6.1", + "p256", + "p384", + "rand 0.8.5", + "rand_core 0.6.4", + "rcgen 0.9.3", + "ring", + "rustls 0.19.1", + "sec1", + "serde", + "sha-1 0.9.8", + "sha2 0.9.9", + "signature", + "subtle", + "thiserror", + "tokio", + "webpki 0.21.4", + "webrtc-util", + "x25519-dalek 2.0.0-pre.1", + "x509-parser 0.13.2", +] + +[[package]] +name = "webrtc-ice" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494483fbb2f5492620871fdc78b084aed8807377f6e3fe88b2e49f0a9c9c41d7" +dependencies = [ + "arc-swap", + "async-trait", + "crc", + "log", + "rand 0.8.5", + "serde", + "serde_json", + "stun", + "thiserror", + "tokio", + "turn", + "url", + "uuid 1.3.0", + "waitgroup", + "webrtc-mdns", + "webrtc-util", +] + +[[package]] +name = "webrtc-mdns" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f08dfd7a6e3987e255c4dbe710dde5d94d0f0574f8a21afa95d171376c143106" +dependencies = [ + "log", + "socket2", + "thiserror", + "tokio", + "webrtc-util", +] + +[[package]] +name = "webrtc-media" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2a3c157a040324e5049bcbd644ffc9079e6738fa2cfab2bcff64e5cc4c00d7" +dependencies = [ + "byteorder", + "bytes", + "derive_builder", + "displaydoc", + "rand 0.8.5", + "rtp", + "thiserror", + "webrtc-util", +] + +[[package]] +name = "webrtc-sctp" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d47adcd9427eb3ede33d5a7f3424038f63c965491beafcc20bc650a2f6679c0" +dependencies = [ + "arc-swap", + "async-trait", + "bytes", + "crc", + "log", + "rand 0.8.5", + "thiserror", + "tokio", + "webrtc-util", +] + +[[package]] +name = "webrtc-srtp" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6183edc4c1c6c0175f8812eefdce84dfa0aea9c3ece71c2bf6ddd3c964de3da5" +dependencies = [ + "aead 0.4.3", + "aes 0.7.5", + "aes-gcm 0.9.4", + "async-trait", + "byteorder", + "bytes", + "ctr 0.8.0", + "hmac 0.11.0", + "log", + "rtcp", + "rtp", + "sha-1 0.9.8", + "subtle", + "thiserror", + "tokio", + "webrtc-util", +] + +[[package]] +name = "webrtc-util" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93f1db1727772c05cf7a2cfece52c3aca8045ca1e176cd517d323489aa3c6d87" +dependencies = [ + "async-trait", + "bitflags", + "bytes", + "cc", + "ipnet", + "lazy_static", + "libc", + "log", + "nix 0.24.3", + "rand 0.8.5", + "thiserror", + "tokio", + "winapi", +] + +[[package]] +name = "wepoll-ffi" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d743fdedc5c64377b5fc2bc036b01c7fd642205a0d96356034ae3404d49eb7fb" +dependencies = [ + "cc", +] + +[[package]] +name = "which" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" dependencies = [ "either", "libc", @@ -7747,6 +9170,19 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45296b64204227616fdbf2614cefa4c236b98ee64dfaaaa435207ed99fe7829f" +dependencies = [ + "windows_aarch64_msvc 0.34.0", + "windows_i686_gnu 0.34.0", + "windows_i686_msvc 0.34.0", + "windows_x86_64_gnu 0.34.0", + "windows_x86_64_msvc 0.34.0", +] + [[package]] name = "windows-acl" version = "0.3.0" @@ -7759,19 +9195,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "windows-sys" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" -dependencies = [ - "windows_aarch64_msvc 0.36.1", - "windows_i686_gnu 0.36.1", - "windows_i686_msvc 0.36.1", - "windows_x86_64_gnu 0.36.1", - "windows_x86_64_msvc 0.36.1", -] - [[package]] name = "windows-sys" version = "0.42.0" @@ -7779,94 +9202,109 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ "windows_aarch64_gnullvm", - "windows_aarch64_msvc 0.42.0", - "windows_i686_gnu 0.42.0", - "windows_i686_msvc 0.42.0", - "windows_x86_64_gnu 0.42.0", + "windows_aarch64_msvc 0.42.1", + "windows_i686_gnu 0.42.1", + "windows_i686_msvc 0.42.1", + "windows_x86_64_gnu 0.42.1", "windows_x86_64_gnullvm", - "windows_x86_64_msvc 0.42.0", + "windows_x86_64_msvc 0.42.1", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc 0.42.1", + "windows_i686_gnu 0.42.1", + "windows_i686_msvc 0.42.1", + "windows_x86_64_gnu 0.42.1", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc 0.42.1", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" +checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" [[package]] name = "windows_aarch64_msvc" -version = "0.36.1" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" +checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d" [[package]] name = "windows_aarch64_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" +checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" [[package]] name = "windows_i686_gnu" -version = "0.36.1" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" +checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed" [[package]] name = "windows_i686_gnu" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" +checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" [[package]] name = "windows_i686_msvc" -version = "0.36.1" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" +checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956" [[package]] name = "windows_i686_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" +checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" [[package]] name = "windows_x86_64_gnu" -version = "0.36.1" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" +checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4" [[package]] name = "windows_x86_64_gnu" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" +checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" +checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" [[package]] name = "windows_x86_64_msvc" -version = "0.36.1" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" +checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9" [[package]] name = "windows_x86_64_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" - -[[package]] -name = "winreg" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" -dependencies = [ - "winapi", -] +checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" [[package]] name = "winreg" @@ -7879,13 +9317,14 @@ dependencies = [ [[package]] name = "ws_stream_wasm" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47ca1ab42f5afed7fc332b22b6e932ca5414b209465412c8cdf0ad23bc0de645" +checksum = "7999f5f4217fe3818726b66257a4475f71e74ffd190776ad053fa159e50737f5" dependencies = [ "async_io_stream", "futures", "js-sys", + "log", "pharos", "rustc_version 0.4.0", "send_wrapper", @@ -7903,9 +9342,9 @@ checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214" [[package]] name = "wyz" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b31594f29d27036c383b53b59ed3476874d518f0efb151b27a4c275141390e" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" dependencies = [ "tap", ] @@ -7921,6 +9360,54 @@ dependencies = [ "zeroize", ] +[[package]] +name = "x25519-dalek" +version = "2.0.0-pre.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5da623d8af10a62342bcbbb230e33e58a63255a58012f8653c578e54bab48df" +dependencies = [ + "curve25519-dalek 3.2.0", + "rand_core 0.6.4", + "zeroize", +] + +[[package]] +name = "x509-parser" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9bace5b5589ffead1afb76e43e34cff39cd0f3ce7e170ae0c29e53b88eb1c" +dependencies = [ + "asn1-rs 0.3.1", + "base64 0.13.1", + "data-encoding", + "der-parser 7.0.0", + "lazy_static", + "nom 7.1.3", + "oid-registry 0.4.0", + "ring", + "rusticata-macros", + "thiserror", + "time 0.3.17", +] + +[[package]] +name = "x509-parser" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0ecbeb7b67ce215e40e3cc7f2ff902f94a223acf44995934763467e7b1febc8" +dependencies = [ + "asn1-rs 0.5.1", + "base64 0.13.1", + "data-encoding", + "der-parser 8.1.0", + "lazy_static", + "nom 7.1.3", + "oid-registry 0.6.1", + "rusticata-macros", + "thiserror", + "time 0.3.17", +] + [[package]] name = "xml-rs" version = "0.8.4" @@ -7959,6 +9446,15 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "yasna" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aed2e7a52e3744ab4d0c05c20aa065258e84c49fd4226f5191b2ed29712710b4" +dependencies = [ + "time 0.3.17", +] + [[package]] name = "zeroize" version = "1.5.7" @@ -7970,9 +9466,9 @@ dependencies = [ [[package]] name = "zeroize_derive" -version = "1.3.2" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f8f187641dad4f680d25c4bfc4225b418165984179f26ca76ec4fb6441d3a17" +checksum = "44bf07cb3e50ea2003396695d58bf46bc9887a1f362260446fad6bc4e79bd36c" dependencies = [ "proc-macro2", "quote", @@ -7991,5 +9487,5 @@ dependencies = [ "crc32fast", "flate2", "thiserror", - "time 0.1.44", + "time 0.1.45", ] diff --git a/Cargo.toml b/Cargo.toml index c9fad65a1d..ee1491b460 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -90,10 +90,10 @@ members = [ "validator_manager", ] +resolver = "2" [patch] [patch.crates-io] -fixed-hash = { git = "https://github.com/paritytech/parity-common", rev="df638ab0885293d21d656dc300d39236b69ce57d" } warp = { git = "https://github.com/macladson/warp", rev="7e75acc368229a46a236a8c991bf251fe7fe50ef" } eth2_ssz = { path = "consensus/ssz" } eth2_ssz_derive = { path = "consensus/ssz_derive" } diff --git a/Makefile b/Makefile index 33077a6c93..85872f016d 100644 --- a/Makefile +++ b/Makefile @@ -14,8 +14,16 @@ BUILD_PATH_AARCH64 = "target/$(AARCH64_TAG)/release" PINNED_NIGHTLY ?= nightly CLIPPY_PINNED_NIGHTLY=nightly-2022-05-19 +# List of features to use when building natively. Can be overriden via the environment. +# No jemalloc on Windows +ifeq ($(OS),Windows_NT) + FEATURES?= +else + FEATURES?=jemalloc +endif + # List of features to use when cross-compiling. Can be overridden via the environment. -CROSS_FEATURES ?= gnosis,slasher-lmdb,slasher-mdbx +CROSS_FEATURES ?= gnosis,slasher-lmdb,slasher-mdbx,jemalloc # Cargo profile for Cross builds. Default is for local builds, CI uses an override. CROSS_PROFILE ?= release @@ -101,10 +109,6 @@ cargo-fmt: check-benches: cargo check --workspace --benches -# Typechecks consensus code *without* allowing deprecated legacy arithmetic or metrics. -check-consensus: - cargo check -p state_processing --no-default-features - # Runs only the ef-test vectors. run-ef-tests: rm -rf $(EF_TESTS)/.accessed_file_log.txt @@ -160,7 +164,8 @@ lint: -A clippy::from-over-into \ -A clippy::upper-case-acronyms \ -A clippy::vec-init-then-push \ - -A clippy::question-mark + -A clippy::question-mark \ + -A clippy::uninlined-format-args nightly-lint: cp .github/custom/clippy.toml . @@ -185,7 +190,7 @@ arbitrary-fuzz: # Runs cargo audit (Audit Cargo.lock files for crates with security vulnerabilities reported to the RustSec Advisory Database) audit: cargo install --force cargo-audit - cargo audit --ignore RUSTSEC-2020-0071 --ignore RUSTSEC-2020-0159 + cargo audit --ignore RUSTSEC-2020-0071 # Runs `cargo vendor` to make sure dependencies can be vendored for packaging, reproducibility and archival purpose. vendor: diff --git a/README.md b/README.md index 859d5c4c63..3565882d6e 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ of the Lighthouse book. The best place for discussion is the [Lighthouse Discord server](https://discord.gg/cyAszAh). -Sign up to the [Lighthouse Development Updates](https://eepurl.com/dh9Lvb/) mailing list for email +Sign up to the [Lighthouse Development Updates](https://eepurl.com/dh9Lvb) mailing list for email notifications about releases, network status and other important information. Encrypt sensitive messages using our [PGP diff --git a/account_manager/src/wallet/list.rs b/account_manager/src/wallet/list.rs index 5b671b1dce..9190de3915 100644 --- a/account_manager/src/wallet/list.rs +++ b/account_manager/src/wallet/list.rs @@ -10,7 +10,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { } pub fn cli_run(wallet_base_dir: PathBuf) -> Result<(), String> { - let mgr = WalletManager::open(&wallet_base_dir) + let mgr = WalletManager::open(wallet_base_dir) .map_err(|e| format!("Unable to open --{}: {:?}", WALLETS_DIR_FLAG, e))?; for (name, _uuid) in mgr diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index d47f77da93..cca8cc969e 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "beacon_node" -version = "3.3.0" +version = "3.4.0" authors = ["Paul Hauner ", "Age Manning BeaconChain { + pub fn compute_attestation_rewards( + &self, + epoch: Epoch, + validators: Vec, + log: Logger, + ) -> Result { + debug!(log, "computing attestation rewards"; "epoch" => epoch, "validator_count" => validators.len()); + + // Get state + let spec = &self.spec; + + let state_slot = (epoch + 1).end_slot(T::EthSpec::slots_per_epoch()); + + let state_root = self + .state_root_at_slot(state_slot)? + .ok_or(BeaconChainError::NoStateForSlot(state_slot))?; + + let mut state = self + .get_state(&state_root, Some(state_slot))? + .ok_or(BeaconChainError::MissingBeaconState(state_root))?; + + // Calculate ideal_rewards + let participation_cache = ParticipationCache::new(&state, spec)?; + + let previous_epoch = state.previous_epoch(); + + let mut ideal_rewards_hashmap = HashMap::new(); + + for flag_index in 0..PARTICIPATION_FLAG_WEIGHTS.len() { + let weight = get_flag_weight(flag_index) + .map_err(|_| BeaconChainError::AttestationRewardsError)?; + + let unslashed_participating_indices = participation_cache + .get_unslashed_participating_indices(flag_index, previous_epoch)?; + + let unslashed_participating_balance = + unslashed_participating_indices + .total_balance() + .map_err(|_| BeaconChainError::AttestationRewardsError)?; + + let unslashed_participating_increments = + unslashed_participating_balance.safe_div(spec.effective_balance_increment)?; + + let total_active_balance = participation_cache.current_epoch_total_active_balance(); + + let active_increments = + total_active_balance.safe_div(spec.effective_balance_increment)?; + + let base_reward_per_increment = + BaseRewardPerIncrement::new(total_active_balance, spec)?; + + for effective_balance_eth in 0..=32 { + let effective_balance = + effective_balance_eth.safe_mul(spec.effective_balance_increment)?; + let base_reward = + effective_balance_eth.safe_mul(base_reward_per_increment.as_u64())?; + + let penalty = -(base_reward.safe_mul(weight)?.safe_div(WEIGHT_DENOMINATOR)? as i64); + + let reward_numerator = base_reward + .safe_mul(weight)? + .safe_mul(unslashed_participating_increments)?; + + let ideal_reward = reward_numerator + .safe_div(active_increments)? + .safe_div(WEIGHT_DENOMINATOR)?; + if !state.is_in_inactivity_leak(previous_epoch, spec) { + ideal_rewards_hashmap + .insert((flag_index, effective_balance), (ideal_reward, penalty)); + } else { + ideal_rewards_hashmap.insert((flag_index, effective_balance), (0, penalty)); + } + } + } + + // Calculate total_rewards + let mut total_rewards: Vec = Vec::new(); + + let validators = if validators.is_empty() { + participation_cache.eligible_validator_indices().to_vec() + } else { + validators + .into_iter() + .map(|validator| match validator { + ValidatorId::Index(i) => Ok(i as usize), + ValidatorId::PublicKey(pubkey) => state + .get_validator_index(&pubkey)? + .ok_or(BeaconChainError::ValidatorPubkeyUnknown(pubkey)), + }) + .collect::, _>>()? + }; + + for validator_index in &validators { + let eligible = state.is_eligible_validator(previous_epoch, *validator_index)?; + let mut head_reward = 0u64; + let mut target_reward = 0i64; + let mut source_reward = 0i64; + + if eligible { + let effective_balance = state.get_effective_balance(*validator_index)?; + + for flag_index in 0..PARTICIPATION_FLAG_WEIGHTS.len() { + let (ideal_reward, penalty) = ideal_rewards_hashmap + .get(&(flag_index, effective_balance)) + .ok_or(BeaconChainError::AttestationRewardsError)?; + let voted_correctly = participation_cache + .get_unslashed_participating_indices(flag_index, previous_epoch) + .map_err(|_| BeaconChainError::AttestationRewardsError)? + .contains(*validator_index) + .map_err(|_| BeaconChainError::AttestationRewardsError)?; + if voted_correctly { + if flag_index == TIMELY_HEAD_FLAG_INDEX { + head_reward += ideal_reward; + } else if flag_index == TIMELY_TARGET_FLAG_INDEX { + target_reward += *ideal_reward as i64; + } else if flag_index == TIMELY_SOURCE_FLAG_INDEX { + source_reward += *ideal_reward as i64; + } + } else if flag_index == TIMELY_HEAD_FLAG_INDEX { + head_reward = 0; + } else if flag_index == TIMELY_TARGET_FLAG_INDEX { + target_reward = *penalty; + } else if flag_index == TIMELY_SOURCE_FLAG_INDEX { + source_reward = *penalty; + } + } + } + total_rewards.push(TotalAttestationRewards { + validator_index: *validator_index as u64, + head: head_reward, + target: target_reward, + source: source_reward, + }); + } + + // Convert hashmap to vector + let mut ideal_rewards: Vec = ideal_rewards_hashmap + .iter() + .map( + |((flag_index, effective_balance), (ideal_reward, _penalty))| { + (flag_index, effective_balance, ideal_reward) + }, + ) + .fold( + HashMap::new(), + |mut acc, (flag_index, &effective_balance, ideal_reward)| { + let entry = acc + .entry(effective_balance) + .or_insert(IdealAttestationRewards { + effective_balance, + head: 0, + target: 0, + source: 0, + }); + match *flag_index { + TIMELY_SOURCE_FLAG_INDEX => entry.source += ideal_reward, + TIMELY_TARGET_FLAG_INDEX => entry.target += ideal_reward, + TIMELY_HEAD_FLAG_INDEX => entry.head += ideal_reward, + _ => {} + } + acc + }, + ) + .into_values() + .collect::>(); + ideal_rewards.sort_by(|a, b| a.effective_balance.cmp(&b.effective_balance)); + + Ok(StandardAttestationRewards { + ideal_rewards, + total_rewards, + }) + } +} diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index b60ce7efe5..04f601fad9 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -27,6 +27,11 @@ //! â–¼ //! impl VerifiedAttestation //! ``` + +// Ignore this lint for `AttestationSlashInfo` which is of comparable size to the non-error types it +// is returned alongside. +#![allow(clippy::result_large_err)] + mod batch; use crate::{ diff --git a/beacon_node/beacon_chain/src/beacon_block_reward.rs b/beacon_node/beacon_chain/src/beacon_block_reward.rs new file mode 100644 index 0000000000..3f186c37c1 --- /dev/null +++ b/beacon_node/beacon_chain/src/beacon_block_reward.rs @@ -0,0 +1,237 @@ +use crate::{BeaconChain, BeaconChainError, BeaconChainTypes}; +use eth2::lighthouse::StandardBlockReward; +use operation_pool::RewardCache; +use safe_arith::SafeArith; +use slog::error; +use state_processing::{ + common::{ + altair, get_attestation_participation_flag_indices, get_attesting_indices_from_state, + }, + per_block_processing::{ + altair::sync_committee::compute_sync_aggregate_rewards, get_slashable_indices, + }, +}; +use store::{ + consts::altair::{PARTICIPATION_FLAG_WEIGHTS, PROPOSER_WEIGHT, WEIGHT_DENOMINATOR}, + RelativeEpoch, +}; +use types::{BeaconBlockRef, BeaconState, BeaconStateError, ExecPayload, Hash256}; + +type BeaconBlockSubRewardValue = u64; + +impl BeaconChain { + pub fn compute_beacon_block_reward>( + &self, + block: BeaconBlockRef<'_, T::EthSpec, Payload>, + block_root: Hash256, + state: &mut BeaconState, + ) -> Result { + if block.slot() != state.slot() { + return Err(BeaconChainError::BlockRewardSlotError); + } + + state.build_committee_cache(RelativeEpoch::Previous, &self.spec)?; + state.build_committee_cache(RelativeEpoch::Current, &self.spec)?; + + let proposer_index = block.proposer_index(); + + let sync_aggregate_reward = + self.compute_beacon_block_sync_aggregate_reward(block, state)?; + + let proposer_slashing_reward = self + .compute_beacon_block_proposer_slashing_reward(block, state) + .map_err(|e| { + error!( + self.log, + "Error calculating proposer slashing reward"; + "error" => ?e + ); + BeaconChainError::BlockRewardError + })?; + + let attester_slashing_reward = self + .compute_beacon_block_attester_slashing_reward(block, state) + .map_err(|e| { + error!( + self.log, + "Error calculating attester slashing reward"; + "error" => ?e + ); + BeaconChainError::BlockRewardError + })?; + + let block_attestation_reward = if let BeaconState::Base(_) = state { + self.compute_beacon_block_attestation_reward_base(block, block_root, state) + .map_err(|e| { + error!( + self.log, + "Error calculating base block attestation reward"; + "error" => ?e + ); + BeaconChainError::BlockRewardAttestationError + })? + } else { + self.compute_beacon_block_attestation_reward_altair(block, state) + .map_err(|e| { + error!( + self.log, + "Error calculating altair block attestation reward"; + "error" => ?e + ); + BeaconChainError::BlockRewardAttestationError + })? + }; + + let total_reward = sync_aggregate_reward + .safe_add(proposer_slashing_reward)? + .safe_add(attester_slashing_reward)? + .safe_add(block_attestation_reward)?; + + Ok(StandardBlockReward { + proposer_index, + total: total_reward, + attestations: block_attestation_reward, + sync_aggregate: sync_aggregate_reward, + proposer_slashings: proposer_slashing_reward, + attester_slashings: attester_slashing_reward, + }) + } + + fn compute_beacon_block_sync_aggregate_reward>( + &self, + block: BeaconBlockRef<'_, T::EthSpec, Payload>, + state: &BeaconState, + ) -> Result { + if let Ok(sync_aggregate) = block.body().sync_aggregate() { + let (_, proposer_reward_per_bit) = compute_sync_aggregate_rewards(state, &self.spec) + .map_err(|_| BeaconChainError::BlockRewardSyncError)?; + Ok(sync_aggregate.sync_committee_bits.num_set_bits() as u64 * proposer_reward_per_bit) + } else { + Ok(0) + } + } + + fn compute_beacon_block_proposer_slashing_reward>( + &self, + block: BeaconBlockRef<'_, T::EthSpec, Payload>, + state: &BeaconState, + ) -> Result { + let mut proposer_slashing_reward = 0; + + let proposer_slashings = block.body().proposer_slashings(); + + for proposer_slashing in proposer_slashings { + proposer_slashing_reward.safe_add_assign( + state + .get_validator(proposer_slashing.proposer_index() as usize)? + .effective_balance + .safe_div(self.spec.whistleblower_reward_quotient)?, + )?; + } + + Ok(proposer_slashing_reward) + } + + fn compute_beacon_block_attester_slashing_reward>( + &self, + block: BeaconBlockRef<'_, T::EthSpec, Payload>, + state: &BeaconState, + ) -> Result { + let mut attester_slashing_reward = 0; + + let attester_slashings = block.body().attester_slashings(); + + for attester_slashing in attester_slashings { + for attester_index in get_slashable_indices(state, attester_slashing)? { + attester_slashing_reward.safe_add_assign( + state + .get_validator(attester_index as usize)? + .effective_balance + .safe_div(self.spec.whistleblower_reward_quotient)?, + )?; + } + } + + Ok(attester_slashing_reward) + } + + fn compute_beacon_block_attestation_reward_base>( + &self, + block: BeaconBlockRef<'_, T::EthSpec, Payload>, + block_root: Hash256, + state: &BeaconState, + ) -> Result { + // Call compute_block_reward in the base case + // Since base does not have sync aggregate, we only grab attesation portion of the returned + // value + let mut reward_cache = RewardCache::default(); + let block_attestation_reward = self + .compute_block_reward(block, block_root, state, &mut reward_cache, true)? + .attestation_rewards + .total; + + Ok(block_attestation_reward) + } + + fn compute_beacon_block_attestation_reward_altair>( + &self, + block: BeaconBlockRef<'_, T::EthSpec, Payload>, + state: &mut BeaconState, + ) -> Result { + let total_active_balance = state.get_total_active_balance()?; + let base_reward_per_increment = + altair::BaseRewardPerIncrement::new(total_active_balance, &self.spec)?; + + let mut total_proposer_reward = 0; + + let proposer_reward_denominator = WEIGHT_DENOMINATOR + .safe_sub(PROPOSER_WEIGHT)? + .safe_mul(WEIGHT_DENOMINATOR)? + .safe_div(PROPOSER_WEIGHT)?; + + for attestation in block.body().attestations() { + let data = &attestation.data; + let inclusion_delay = state.slot().safe_sub(data.slot)?.as_u64(); + let participation_flag_indices = get_attestation_participation_flag_indices( + state, + data, + inclusion_delay, + &self.spec, + )?; + + let attesting_indices = get_attesting_indices_from_state(state, attestation)?; + + let mut proposer_reward_numerator = 0; + for index in attesting_indices { + let index = index as usize; + for (flag_index, &weight) in PARTICIPATION_FLAG_WEIGHTS.iter().enumerate() { + let epoch_participation = + state.get_epoch_participation_mut(data.target.epoch)?; + let validator_participation = epoch_participation + .get_mut(index) + .ok_or(BeaconStateError::ParticipationOutOfBounds(index))?; + + if participation_flag_indices.contains(&flag_index) + && !validator_participation.has_flag(flag_index)? + { + validator_participation.add_flag(flag_index)?; + proposer_reward_numerator.safe_add_assign( + altair::get_base_reward( + state, + index, + base_reward_per_increment, + &self.spec, + )? + .safe_mul(weight)?, + )?; + } + } + } + total_proposer_reward.safe_add_assign( + proposer_reward_numerator.safe_div(proposer_reward_denominator)?, + )?; + } + + Ok(total_proposer_reward) + } +} diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 55d6ae29ef..4ec13f8f53 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -8,7 +8,7 @@ use crate::beacon_proposer_cache::compute_proposer_duties_from_head; use crate::beacon_proposer_cache::BeaconProposerCache; use crate::block_times_cache::BlockTimesCache; use crate::block_verification::{ - check_block_is_finalized_descendant, check_block_relevancy, get_block_root, + check_block_is_finalized_checkpoint_or_descendant, check_block_relevancy, get_block_root, signature_verify_chain_segment, BlockError, ExecutionPendingBlock, GossipVerifiedBlock, IntoExecutionPendingBlock, PayloadVerificationOutcome, POS_PANDA_BANNER, }; @@ -975,7 +975,9 @@ impl BeaconChain { .ok_or(Error::ExecutionLayerMissing)? .get_payload_by_block_hash(exec_block_hash) .await - .map_err(|e| Error::ExecutionLayerErrorPayloadReconstruction(exec_block_hash, e))? + .map_err(|e| { + Error::ExecutionLayerErrorPayloadReconstruction(exec_block_hash, Box::new(e)) + })? .ok_or(Error::BlockHashMissingFromExecutionLayer(exec_block_hash))?; // Verify payload integrity. @@ -992,8 +994,6 @@ impl BeaconChain { return Err(Error::InconsistentPayloadReconstructed { slot: blinded_block.slot(), exec_block_hash, - canonical_payload_root: execution_payload_header.tree_hash_root(), - reconstructed_payload_root: header_from_payload.tree_hash_root(), canonical_transactions_root: execution_payload_header.transactions_root, reconstructed_transactions_root: header_from_payload.transactions_root, }); @@ -2714,7 +2714,7 @@ impl BeaconChain { // is so we don't have to think about lock ordering with respect to the fork choice lock. // There are a bunch of places where we lock both fork choice and the pubkey cache and it // would be difficult to check that they all lock fork choice first. - let mut kv_store_ops = self + let mut ops = self .validator_pubkey_cache .try_write_for(VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT) .ok_or(Error::ValidatorPubkeyCacheLockTimeout)? @@ -2736,7 +2736,7 @@ impl BeaconChain { let mut fork_choice = self.canonical_head.fork_choice_write_lock(); // Do not import a block that doesn't descend from the finalized root. - check_block_is_finalized_descendant(self, &fork_choice, &signed_block)?; + check_block_is_finalized_checkpoint_or_descendant(self, &fork_choice, &signed_block)?; // Register the new block with the fork choice service. { @@ -2816,9 +2816,14 @@ impl BeaconChain { // ---------------------------- BLOCK PROBABLY ATTESTABLE ---------------------------------- // Most blocks are now capable of being attested to thanks to the `early_attester_cache` // cache above. Resume non-essential processing. + // + // It is important NOT to return errors here before the database commit, because the block + // has already been added to fork choice and the database would be left in an inconsistent + // state if we returned early without committing. In other words, an error here would + // corrupt the node's database permanently. // ----------------------------------------------------------------------------------------- - self.import_block_update_shuffling_cache(block_root, &mut state)?; + self.import_block_update_shuffling_cache(block_root, &mut state); self.import_block_observe_attestations( block, &state, @@ -2841,17 +2846,16 @@ impl BeaconChain { // If the write fails, revert fork choice to the version from disk, else we can // end up with blocks in fork choice that are missing from disk. // See https://github.com/sigp/lighthouse/issues/2028 - let mut ops: Vec<_> = confirmed_state_roots - .into_iter() - .map(StoreOp::DeleteStateTemporaryFlag) - .collect(); + ops.extend( + confirmed_state_roots + .into_iter() + .map(StoreOp::DeleteStateTemporaryFlag), + ); ops.push(StoreOp::PutBlock(block_root, signed_block.clone())); ops.push(StoreOp::PutState(block.state_root(), &state)); let txn_lock = self.store.hot_db.begin_rw_transaction(); - kv_store_ops.extend(self.store.convert_to_kv_batch(ops)?); - - if let Err(e) = self.store.hot_db.do_atomically(kv_store_ops) { + if let Err(e) = self.store.do_atomically(ops) { error!( self.log, "Database write failed!"; @@ -3280,13 +3284,27 @@ impl BeaconChain { } } + // For the current and next epoch of this state, ensure we have the shuffling from this + // block in our cache. fn import_block_update_shuffling_cache( &self, block_root: Hash256, state: &mut BeaconState, + ) { + if let Err(e) = self.import_block_update_shuffling_cache_fallible(block_root, state) { + warn!( + self.log, + "Failed to prime shuffling cache"; + "error" => ?e + ); + } + } + + fn import_block_update_shuffling_cache_fallible( + &self, + block_root: Hash256, + state: &mut BeaconState, ) -> Result<(), BlockError> { - // For the current and next epoch of this state, ensure we have the shuffling from this - // block in our cache. for relative_epoch in [RelativeEpoch::Current, RelativeEpoch::Next] { let shuffling_id = AttestationShufflingId::new(block_root, state, relative_epoch)?; diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index ab317e96b9..4f65a05c56 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -42,6 +42,11 @@ //! END //! //! ``` + +// Ignore this lint for `BlockSlashInfo` which is of comparable size to the non-error types it is +// returned alongside. +#![allow(clippy::result_large_err)] + use crate::eth1_finalization_cache::Eth1FinalizationData; use crate::execution_payload::{ is_optimistic_candidate_block, validate_execution_payload_for_gossip, validate_merge_block, @@ -739,7 +744,7 @@ impl GossipVerifiedBlock { // Do not process a block that doesn't descend from the finalized root. // // We check this *before* we load the parent so that we can return a more detailed error. - check_block_is_finalized_descendant( + check_block_is_finalized_checkpoint_or_descendant( chain, &chain.canonical_head.fork_choice_write_lock(), &block, @@ -1559,12 +1564,12 @@ fn check_block_against_finalized_slot( /// ## Warning /// /// Taking a lock on the `chain.canonical_head.fork_choice` might cause a deadlock here. -pub fn check_block_is_finalized_descendant( +pub fn check_block_is_finalized_checkpoint_or_descendant( chain: &BeaconChain, fork_choice: &BeaconForkChoice, block: &Arc>, ) -> Result<(), BlockError> { - if fork_choice.is_descendant_of_finalized(block.parent_root()) { + if fork_choice.is_finalized_checkpoint_or_descendant(block.parent_root()) { Ok(()) } else { // If fork choice does *not* consider the parent to be a descendant of the finalized block, diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index eff50701d7..48419d46ed 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -579,11 +579,13 @@ where mut self, auto_register: bool, validators: Vec, + individual_metrics_threshold: usize, log: Logger, ) -> Self { self.validator_monitor = Some(ValidatorMonitor::new( validators, auto_register, + individual_metrics_threshold, log.clone(), )); self @@ -989,6 +991,7 @@ fn descriptive_db_error(item: &str, error: &StoreError) -> String { #[cfg(test)] mod test { use super::*; + use crate::validator_monitor::DEFAULT_INDIVIDUAL_TRACKING_THRESHOLD; use eth2_hashing::hash; use genesis::{ generate_deterministic_keypairs, interop_genesis_state, DEFAULT_ETH1_BLOCK_HASH, @@ -1045,7 +1048,12 @@ mod test { .testing_slot_clock(Duration::from_secs(1)) .expect("should configure testing slot clock") .shutdown_sender(shutdown_tx) - .monitor_validators(true, vec![], log.clone()) + .monitor_validators( + true, + vec![], + DEFAULT_INDIVIDUAL_TRACKING_THRESHOLD, + log.clone(), + ) .build() .expect("should build"); diff --git a/beacon_node/beacon_chain/src/chain_config.rs b/beacon_node/beacon_chain/src/chain_config.rs index c4c6966732..cce2fbb971 100644 --- a/beacon_node/beacon_chain/src/chain_config.rs +++ b/beacon_node/beacon_chain/src/chain_config.rs @@ -65,6 +65,8 @@ pub struct ChainConfig { /// Low values are useful for execution engines which don't improve their payload after the /// first call, and high values are useful for ensuring the EL is given ample notice. pub prepare_payload_lookahead: Duration, + /// Use EL-free optimistic sync for the finalized part of the chain. + pub optimistic_finalized_sync: bool, } impl Default for ChainConfig { @@ -89,6 +91,7 @@ impl Default for ChainConfig { count_unrealized_full: CountUnrealizedFull::default(), checkpoint_sync_url_timeout: 60, prepare_payload_lookahead: Duration::from_secs(4), + optimistic_finalized_sync: true, } } } diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index 17f58b223f..e6f44f6654 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -50,7 +50,6 @@ pub enum BeaconChainError { }, SlotClockDidNotStart, NoStateForSlot(Slot), - UnableToFindTargetRoot(Slot), BeaconStateError(BeaconStateError), DBInconsistent(String), DBError(store::Error), @@ -141,13 +140,11 @@ pub enum BeaconChainError { BuilderMissing, ExecutionLayerMissing, BlockVariantLacksExecutionPayload(Hash256), - ExecutionLayerErrorPayloadReconstruction(ExecutionBlockHash, execution_layer::Error), + ExecutionLayerErrorPayloadReconstruction(ExecutionBlockHash, Box), BlockHashMissingFromExecutionLayer(ExecutionBlockHash), InconsistentPayloadReconstructed { slot: Slot, exec_block_hash: ExecutionBlockHash, - canonical_payload_root: Hash256, - reconstructed_payload_root: Hash256, canonical_transactions_root: Hash256, reconstructed_transactions_root: Hash256, }, @@ -157,9 +154,12 @@ pub enum BeaconChainError { ExecutionForkChoiceUpdateInvalid { status: PayloadStatus, }, + BlockRewardError, BlockRewardSlotError, BlockRewardAttestationError, BlockRewardSyncError, + SyncCommitteeRewardsSyncError, + AttestationRewardsError, HeadMissingFromForkChoice(Hash256), FinalizedBlockMissingFromForkChoice(Hash256), HeadBlockMissingFromForkChoice(Hash256), diff --git a/beacon_node/beacon_chain/src/execution_payload.rs b/beacon_node/beacon_chain/src/execution_payload.rs index 85f7629bb7..7435c3a8cc 100644 --- a/beacon_node/beacon_chain/src/execution_payload.rs +++ b/beacon_node/beacon_chain/src/execution_payload.rs @@ -15,7 +15,7 @@ use crate::{ use execution_layer::{BuilderParams, PayloadStatus}; use fork_choice::{InvalidationOperation, PayloadVerificationStatus}; use proto_array::{Block as ProtoBlock, ExecutionStatus}; -use slog::debug; +use slog::{debug, warn}; use slot_clock::SlotClock; use state_processing::per_block_processing::{ compute_timestamp_at_slot, is_execution_enabled, is_merge_transition_complete, @@ -59,26 +59,46 @@ impl PayloadNotifier { state: &BeaconState, notify_execution_layer: NotifyExecutionLayer, ) -> Result> { - let payload_verification_status = match notify_execution_layer { - NotifyExecutionLayer::No => Some(PayloadVerificationStatus::Optimistic), - NotifyExecutionLayer::Yes => { - if is_execution_enabled(state, block.message().body()) { - // Perform the initial stages of payload verification. - // - // We will duplicate these checks again during `per_block_processing`, however these checks - // are cheap and doing them here ensures we protect the execution engine from junk. - partially_verify_execution_payload( - state, - block.slot(), - block.message().execution_payload()?, - &chain.spec, - ) - .map_err(BlockError::PerBlockProcessingError)?; - None - } else { - Some(PayloadVerificationStatus::Irrelevant) + let payload_verification_status = if is_execution_enabled(state, block.message().body()) { + // Perform the initial stages of payload verification. + // + // We will duplicate these checks again during `per_block_processing`, however these + // checks are cheap and doing them here ensures we have verified them before marking + // the block as optimistically imported. This is particularly relevant in the case + // where we do not send the block to the EL at all. + let block_message = block.message(); + let payload = block_message.execution_payload()?; + partially_verify_execution_payload(state, block.slot(), payload, &chain.spec) + .map_err(BlockError::PerBlockProcessingError)?; + + match notify_execution_layer { + NotifyExecutionLayer::No if chain.config.optimistic_finalized_sync => { + // Verify the block hash here in Lighthouse and immediately mark the block as + // optimistically imported. This saves a lot of roundtrips to the EL. + let execution_layer = chain + .execution_layer + .as_ref() + .ok_or(ExecutionPayloadError::NoExecutionConnection)?; + + if let Err(e) = + execution_layer.verify_payload_block_hash(&payload.execution_payload) + { + warn!( + chain.log, + "Falling back to slow block hash verification"; + "block_number" => payload.block_number(), + "info" => "you can silence this warning with --disable-optimistic-finalized-sync", + "error" => ?e, + ); + None + } else { + Some(PayloadVerificationStatus::Optimistic) + } } + _ => None, } + } else { + Some(PayloadVerificationStatus::Irrelevant) }; Ok(Self { diff --git a/beacon_node/beacon_chain/src/fork_choice_signal.rs b/beacon_node/beacon_chain/src/fork_choice_signal.rs index fd92de661d..f5424d417e 100644 --- a/beacon_node/beacon_chain/src/fork_choice_signal.rs +++ b/beacon_node/beacon_chain/src/fork_choice_signal.rs @@ -43,7 +43,7 @@ impl ForkChoiceSignalTx { /// /// Return an error if the provided `slot` is strictly less than any previously provided slot. pub fn notify_fork_choice_complete(&self, slot: Slot) -> Result<(), BeaconChainError> { - let &(ref lock, ref condvar) = &*self.pair; + let (lock, condvar) = &*self.pair; let mut current_slot = lock.lock(); @@ -72,7 +72,7 @@ impl Default for ForkChoiceSignalTx { impl ForkChoiceSignalRx { pub fn wait_for_fork_choice(&self, slot: Slot, timeout: Duration) -> ForkChoiceWaitResult { - let &(ref lock, ref condvar) = &*self.pair; + let (lock, condvar) = &*self.pair; let mut current_slot = lock.lock(); diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index ae1c5e4b76..5e75c2a632 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -1,6 +1,8 @@ #![recursion_limit = "128"] // For lazy-static +pub mod attestation_rewards; pub mod attestation_verification; mod attester_cache; +pub mod beacon_block_reward; mod beacon_chain; mod beacon_fork_choice_store; pub mod beacon_proposer_cache; @@ -40,6 +42,7 @@ pub mod schema_change; mod shuffling_cache; mod snapshot_cache; pub mod state_advance_timer; +pub mod sync_committee_rewards; pub mod sync_committee_verification; pub mod test_utils; mod timeout_rw_lock; diff --git a/beacon_node/beacon_chain/src/light_client_optimistic_update_verification.rs b/beacon_node/beacon_chain/src/light_client_optimistic_update_verification.rs index ec9c90e735..20d7181808 100644 --- a/beacon_node/beacon_chain/src/light_client_optimistic_update_verification.rs +++ b/beacon_node/beacon_chain/src/light_client_optimistic_update_verification.rs @@ -2,6 +2,7 @@ use crate::{ beacon_chain::MAXIMUM_GOSSIP_CLOCK_DISPARITY, BeaconChain, BeaconChainError, BeaconChainTypes, }; use derivative::Derivative; +use eth2::types::Hash256; use slot_clock::SlotClock; use std::time::Duration; use strum::AsRefStr; @@ -36,6 +37,8 @@ pub enum Error { SigSlotStartIsNone, /// Failed to construct a LightClientOptimisticUpdate from state. FailedConstructingUpdate, + /// Unknown block with parent root. + UnknownBlockParentRoot(Hash256), /// Beacon chain error occured. BeaconChainError(BeaconChainError), LightClientUpdateError(LightClientUpdateError), @@ -58,6 +61,7 @@ impl From for Error { #[derivative(Clone(bound = "T: BeaconChainTypes"))] pub struct VerifiedLightClientOptimisticUpdate { light_client_optimistic_update: LightClientOptimisticUpdate, + pub parent_root: Hash256, seen_timestamp: Duration, } @@ -107,6 +111,16 @@ impl VerifiedLightClientOptimisticUpdate { None => return Err(Error::SigSlotStartIsNone), } + // check if we can process the optimistic update immediately + // otherwise queue + let canonical_root = light_client_optimistic_update + .attested_header + .canonical_root(); + + if canonical_root != head_block.message().parent_root() { + return Err(Error::UnknownBlockParentRoot(canonical_root)); + } + let optimistic_update = LightClientOptimisticUpdate::new(&chain.spec, head_block, &attested_state)?; @@ -119,6 +133,7 @@ impl VerifiedLightClientOptimisticUpdate { Ok(Self { light_client_optimistic_update, + parent_root: canonical_root, seen_timestamp, }) } diff --git a/beacon_node/beacon_chain/src/naive_aggregation_pool.rs b/beacon_node/beacon_chain/src/naive_aggregation_pool.rs index 252e9915d8..7eeb9bb56f 100644 --- a/beacon_node/beacon_chain/src/naive_aggregation_pool.rs +++ b/beacon_node/beacon_chain/src/naive_aggregation_pool.rs @@ -402,7 +402,7 @@ impl NaiveAggregationPool { /// Returns the total number of items stored in `self`. pub fn num_items(&self) -> usize { - self.maps.iter().map(|(_, map)| map.len()).sum() + self.maps.values().map(T::len).sum() } /// Returns an aggregated `T::Value` with the given `T::Data`, if any. @@ -448,11 +448,7 @@ impl NaiveAggregationPool { // If we have too many maps, remove the lowest amount to ensure we only have // `SLOTS_RETAINED` left. if self.maps.len() > SLOTS_RETAINED { - let mut slots = self - .maps - .iter() - .map(|(slot, _map)| *slot) - .collect::>(); + let mut slots = self.maps.keys().copied().collect::>(); // Sort is generally pretty slow, however `SLOTS_RETAINED` is quite low so it should be // negligible. slots.sort_unstable(); diff --git a/beacon_node/beacon_chain/src/sync_committee_rewards.rs b/beacon_node/beacon_chain/src/sync_committee_rewards.rs new file mode 100644 index 0000000000..561fed1a86 --- /dev/null +++ b/beacon_node/beacon_chain/src/sync_committee_rewards.rs @@ -0,0 +1,87 @@ +use crate::{BeaconChain, BeaconChainError, BeaconChainTypes}; + +use eth2::lighthouse::SyncCommitteeReward; +use safe_arith::SafeArith; +use slog::error; +use state_processing::per_block_processing::altair::sync_committee::compute_sync_aggregate_rewards; +use std::collections::HashMap; +use store::RelativeEpoch; +use types::{BeaconBlockRef, BeaconState, ExecPayload}; + +impl BeaconChain { + pub fn compute_sync_committee_rewards>( + &self, + block: BeaconBlockRef<'_, T::EthSpec, Payload>, + state: &mut BeaconState, + ) -> Result, BeaconChainError> { + if block.slot() != state.slot() { + return Err(BeaconChainError::BlockRewardSlotError); + } + + let spec = &self.spec; + + state.build_committee_cache(RelativeEpoch::Current, spec)?; + + let sync_aggregate = block.body().sync_aggregate()?; + + let sync_committee = state.current_sync_committee()?.clone(); + + let sync_committee_indices = state.get_sync_committee_indices(&sync_committee)?; + + let (participant_reward_value, proposer_reward_per_bit) = + compute_sync_aggregate_rewards(state, spec).map_err(|e| { + error!( + self.log, "Error calculating sync aggregate rewards"; + "error" => ?e + ); + BeaconChainError::SyncCommitteeRewardsSyncError + })?; + + let mut balances = HashMap::::new(); + + let mut total_proposer_rewards = 0; + let proposer_index = state.get_beacon_proposer_index(block.slot(), spec)?; + + // Apply rewards to participant balances. Keep track of proposer rewards + for (validator_index, participant_bit) in sync_committee_indices + .iter() + .zip(sync_aggregate.sync_committee_bits.iter()) + { + let participant_balance = balances + .entry(*validator_index) + .or_insert_with(|| state.balances()[*validator_index]); + + if participant_bit { + participant_balance.safe_add_assign(participant_reward_value)?; + + balances + .entry(proposer_index) + .or_insert_with(|| state.balances()[proposer_index]) + .safe_add_assign(proposer_reward_per_bit)?; + + total_proposer_rewards.safe_add_assign(proposer_reward_per_bit)?; + } else { + *participant_balance = participant_balance.saturating_sub(participant_reward_value); + } + } + + Ok(balances + .iter() + .filter_map(|(i, new_balance)| { + let reward = if *i != proposer_index { + *new_balance as i64 - state.balances()[*i] as i64 + } else if sync_committee_indices.contains(i) { + *new_balance as i64 + - state.balances()[*i] as i64 + - total_proposer_rewards as i64 + } else { + return None; + }; + Some(SyncCommitteeReward { + validator_index: *i as u64, + reward, + }) + }) + .collect()) + } +} diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index d6e8787f4e..749487dc5a 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -2,6 +2,8 @@ pub use crate::persisted_beacon_chain::PersistedBeaconChain; pub use crate::{ beacon_chain::{BEACON_CHAIN_DB_KEY, ETH1_CACHE_DB_KEY, FORK_CHOICE_DB_KEY, OP_POOL_DB_KEY}, migrate::MigratorConfig, + sync_committee_verification::Error as SyncCommitteeError, + validator_monitor::DEFAULT_INDIVIDUAL_TRACKING_THRESHOLD, BeaconChainError, NotifyExecutionLayer, ProduceBlockVerification, }; use crate::{ @@ -472,7 +474,7 @@ where log.clone(), 5, ))) - .monitor_validators(true, vec![], log); + .monitor_validators(true, vec![], DEFAULT_INDIVIDUAL_TRACKING_THRESHOLD, log); builder = if let Some(mutator) = self.initial_mutator { mutator(builder) @@ -1459,7 +1461,7 @@ where let proposer_index = state.get_beacon_proposer_index(slot, &self.spec).unwrap(); let signed_block = block.sign( - &self.validator_keypairs[proposer_index as usize].sk, + &self.validator_keypairs[proposer_index].sk, &state.fork(), state.genesis_validators_root(), &self.spec, @@ -1979,6 +1981,30 @@ where (honest_head, faulty_head) } + + pub fn process_sync_contributions( + &self, + sync_contributions: HarnessSyncContributions, + ) -> Result<(), SyncCommitteeError> { + let mut verified_contributions = Vec::with_capacity(sync_contributions.len()); + + for (_, contribution_and_proof) in sync_contributions { + let signed_contribution_and_proof = contribution_and_proof.unwrap(); + + let verified_contribution = self + .chain + .verify_sync_contribution_for_gossip(signed_contribution_and_proof)?; + + verified_contributions.push(verified_contribution); + } + + for verified_contribution in verified_contributions { + self.chain + .add_contribution_to_block_inclusion_pool(verified_contribution)?; + } + + Ok(()) + } } // Junk `Debug` impl to satistfy certain trait bounds during testing. diff --git a/beacon_node/beacon_chain/src/validator_monitor.rs b/beacon_node/beacon_chain/src/validator_monitor.rs index 2d093ff886..dad5e1517a 100644 --- a/beacon_node/beacon_chain/src/validator_monitor.rs +++ b/beacon_node/beacon_chain/src/validator_monitor.rs @@ -21,10 +21,21 @@ use types::{ SignedContributionAndProof, Slot, SyncCommitteeMessage, VoluntaryExit, }; +/// Used for Prometheus labels. +/// +/// We've used `total` for this value to align with Nimbus, as per: +/// https://github.com/sigp/lighthouse/pull/3728#issuecomment-1375173063 +const TOTAL_LABEL: &str = "total"; + /// The validator monitor collects per-epoch data about each monitored validator. Historical data /// will be kept around for `HISTORIC_EPOCHS` before it is pruned. pub const HISTORIC_EPOCHS: usize = 4; +/// Once the validator monitor reaches this number of validators it will stop +/// tracking their metrics/logging individually in an effort to reduce +/// Prometheus cardinality and log volume. +pub const DEFAULT_INDIVIDUAL_TRACKING_THRESHOLD: usize = 64; + #[derive(Debug)] pub enum Error { InvalidPubkey(String), @@ -258,16 +269,27 @@ pub struct ValidatorMonitor { indices: HashMap, /// If true, allow the automatic registration of validators. auto_register: bool, + /// Once the number of monitored validators goes above this threshold, we + /// will stop tracking metrics/logs on a per-validator basis. This prevents + /// large validator counts causing infeasibly high cardinailty for + /// Prometheus and high log volumes. + individual_tracking_threshold: usize, log: Logger, _phantom: PhantomData, } impl ValidatorMonitor { - pub fn new(pubkeys: Vec, auto_register: bool, log: Logger) -> Self { + pub fn new( + pubkeys: Vec, + auto_register: bool, + individual_tracking_threshold: usize, + log: Logger, + ) -> Self { let mut s = Self { validators: <_>::default(), indices: <_>::default(), auto_register, + individual_tracking_threshold, log, _phantom: PhantomData, }; @@ -277,6 +299,13 @@ impl ValidatorMonitor { s } + /// Returns `true` when the validator count is sufficiently low enough to + /// emit metrics and logs on a per-validator basis (rather than just an + /// aggregated basis). + fn individual_tracking(&self) -> bool { + self.validators.len() <= self.individual_tracking_threshold + } + /// Add some validators to `self` for additional monitoring. fn add_validator_pubkey(&mut self, pubkey: PublicKeyBytes) { let index_opt = self @@ -317,6 +346,12 @@ impl ValidatorMonitor { for monitored_validator in self.validators.values() { if let Some(i) = monitored_validator.index { monitored_validator.touch_epoch_summary(current_epoch); + + // Only log the per-validator metrics if it's enabled. + if !self.individual_tracking() { + continue; + } + let i = i as usize; let id = &monitored_validator.id; @@ -379,6 +414,24 @@ impl ValidatorMonitor { } } + /// Run `func` with the `TOTAL_LABEL` and optionally the + /// `individual_id`. + /// + /// This function is used for registering metrics that can be applied to + /// both all validators and an indivdual validator. For example, the count + /// of missed head votes can be aggregated across all validators in a single + /// metric and also tracked on a per-validator basis. + /// + /// We allow disabling tracking metrics on an individual validator basis + /// since it can result in untenable cardinality with high validator counts. + fn aggregatable_metric(&self, individual_id: &str, func: F) { + func(TOTAL_LABEL); + + if self.individual_tracking() { + func(individual_id); + } + } + pub fn process_validator_statuses( &self, epoch: Epoch, @@ -431,72 +484,92 @@ impl ValidatorMonitor { // For Base states, this will be *any* attestation whatsoever. For Altair states, // this will be any attestation that matched a "timely" flag. if previous_epoch_matched_any { - metrics::inc_counter_vec( - &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_ATTESTER_HIT, - &[id], - ); + self.aggregatable_metric(id, |label| { + metrics::inc_counter_vec( + &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_ATTESTER_HIT, + &[label], + ) + }); attestation_success.push(id); - debug!( - self.log, - "Previous epoch attestation success"; - "matched_source" => previous_epoch_matched_source, - "matched_target" => previous_epoch_matched_target, - "matched_head" => previous_epoch_matched_head, - "epoch" => prev_epoch, - "validator" => id, - ) + if self.individual_tracking() { + debug!( + self.log, + "Previous epoch attestation success"; + "matched_source" => previous_epoch_matched_source, + "matched_target" => previous_epoch_matched_target, + "matched_head" => previous_epoch_matched_head, + "epoch" => prev_epoch, + "validator" => id, + ) + } } else { - metrics::inc_counter_vec( - &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_ATTESTER_MISS, - &[id], - ); + self.aggregatable_metric(id, |label| { + metrics::inc_counter_vec( + &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_ATTESTER_MISS, + &[label], + ); + }); attestation_miss.push(id); - debug!( - self.log, - "Previous epoch attestation missing"; - "epoch" => prev_epoch, - "validator" => id, - ) + if self.individual_tracking() { + debug!( + self.log, + "Previous epoch attestation missing"; + "epoch" => prev_epoch, + "validator" => id, + ) + } } // Indicates if any on-chain attestation hit the head. if previous_epoch_matched_head { - metrics::inc_counter_vec( - &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_HEAD_ATTESTER_HIT, - &[id], - ); + self.aggregatable_metric(id, |label| { + metrics::inc_counter_vec( + &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_HEAD_ATTESTER_HIT, + &[label], + ); + }); } else { - metrics::inc_counter_vec( - &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_HEAD_ATTESTER_MISS, - &[id], - ); + self.aggregatable_metric(id, |label| { + metrics::inc_counter_vec( + &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_HEAD_ATTESTER_MISS, + &[label], + ); + }); head_miss.push(id); - debug!( - self.log, - "Attestation failed to match head"; - "epoch" => prev_epoch, - "validator" => id, - ); + if self.individual_tracking() { + debug!( + self.log, + "Attestation failed to match head"; + "epoch" => prev_epoch, + "validator" => id, + ); + } } // Indicates if any on-chain attestation hit the target. if previous_epoch_matched_target { - metrics::inc_counter_vec( - &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_TARGET_ATTESTER_HIT, - &[id], - ); + self.aggregatable_metric(id, |label| { + metrics::inc_counter_vec( + &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_TARGET_ATTESTER_HIT, + &[label], + ); + }); } else { - metrics::inc_counter_vec( - &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_TARGET_ATTESTER_MISS, - &[id], - ); + self.aggregatable_metric(id, |label| { + metrics::inc_counter_vec( + &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_TARGET_ATTESTER_MISS, + &[label], + ); + }); target_miss.push(id); - debug!( - self.log, - "Attestation failed to match target"; - "epoch" => prev_epoch, - "validator" => id, - ); + if self.individual_tracking() { + debug!( + self.log, + "Attestation failed to match target"; + "epoch" => prev_epoch, + "validator" => id, + ); + } } // Get the minimum value among the validator monitor observed inclusion distance @@ -511,21 +584,25 @@ impl ValidatorMonitor { if let Some(inclusion_delay) = min_inclusion_distance { if inclusion_delay > spec.min_attestation_inclusion_delay { suboptimal_inclusion.push(id); - debug!( - self.log, - "Potential sub-optimal inclusion delay"; - "optimal" => spec.min_attestation_inclusion_delay, - "delay" => inclusion_delay, - "epoch" => prev_epoch, - "validator" => id, - ); + if self.individual_tracking() { + debug!( + self.log, + "Potential sub-optimal inclusion delay"; + "optimal" => spec.min_attestation_inclusion_delay, + "delay" => inclusion_delay, + "epoch" => prev_epoch, + "validator" => id, + ); + } } - metrics::set_int_gauge( - &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_INCLUSION_DISTANCE, - &[id], - inclusion_delay as i64, - ); + if self.individual_tracking() { + metrics::set_int_gauge( + &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_INCLUSION_DISTANCE, + &[id], + inclusion_delay as i64, + ); + } } // Indicates the number of sync committee signatures that made it into @@ -536,13 +613,19 @@ impl ValidatorMonitor { let current_epoch = epoch - 1; if let Some(sync_committee) = summary.sync_committee() { if sync_committee.contains(pubkey) { - metrics::set_int_gauge( - &metrics::VALIDATOR_MONITOR_VALIDATOR_IN_CURRENT_SYNC_COMMITTEE, - &[id], - 1, - ); + if self.individual_tracking() { + metrics::set_int_gauge( + &metrics::VALIDATOR_MONITOR_VALIDATOR_IN_CURRENT_SYNC_COMMITTEE, + &[id], + 1, + ); + } let epoch_summary = monitored_validator.summaries.read(); if let Some(summary) = epoch_summary.get(¤t_epoch) { + // This log is not gated by + // `self.individual_tracking()` since the number of + // logs that can be generated is capped by the size + // of the sync committee. info!( self.log, "Current epoch sync signatures"; @@ -552,7 +635,7 @@ impl ValidatorMonitor { "validator" => id, ); } - } else { + } else if self.individual_tracking() { metrics::set_int_gauge( &metrics::VALIDATOR_MONITOR_VALIDATOR_IN_CURRENT_SYNC_COMMITTEE, &[id], @@ -631,10 +714,7 @@ impl ValidatorMonitor { // Return the `id`'s of all monitored validators. pub fn get_all_monitored_validators(&self) -> Vec { - self.validators - .iter() - .map(|(_, val)| val.id.clone()) - .collect() + self.validators.values().map(|val| val.id.clone()).collect() } /// If `self.auto_register == true`, add the `validator_index` to `self.monitored_validators`. @@ -696,12 +776,17 @@ impl ValidatorMonitor { let id = &validator.id; let delay = get_block_delay_ms(seen_timestamp, block, slot_clock); - metrics::inc_counter_vec(&metrics::VALIDATOR_MONITOR_BEACON_BLOCK_TOTAL, &[src, id]); - metrics::observe_timer_vec( - &metrics::VALIDATOR_MONITOR_BEACON_BLOCK_DELAY_SECONDS, - &[src, id], - delay, - ); + self.aggregatable_metric(id, |label| { + metrics::inc_counter_vec( + &metrics::VALIDATOR_MONITOR_BEACON_BLOCK_TOTAL, + &[src, label], + ); + metrics::observe_timer_vec( + &metrics::VALIDATOR_MONITOR_BEACON_BLOCK_DELAY_SECONDS, + &[src, label], + delay, + ); + }); info!( self.log, @@ -767,27 +852,31 @@ impl ValidatorMonitor { if let Some(validator) = self.get_validator(*i) { let id = &validator.id; - metrics::inc_counter_vec( - &metrics::VALIDATOR_MONITOR_UNAGGREGATED_ATTESTATION_TOTAL, - &[src, id], - ); - metrics::observe_timer_vec( - &metrics::VALIDATOR_MONITOR_UNAGGREGATED_ATTESTATION_DELAY_SECONDS, - &[src, id], - delay, - ); + self.aggregatable_metric(id, |label| { + metrics::inc_counter_vec( + &metrics::VALIDATOR_MONITOR_UNAGGREGATED_ATTESTATION_TOTAL, + &[src, label], + ); + metrics::observe_timer_vec( + &metrics::VALIDATOR_MONITOR_UNAGGREGATED_ATTESTATION_DELAY_SECONDS, + &[src, label], + delay, + ); + }); - info!( - self.log, - "Unaggregated attestation"; - "head" => ?data.beacon_block_root, - "index" => %data.index, - "delay_ms" => %delay.as_millis(), - "epoch" => %epoch, - "slot" => %data.slot, - "src" => src, - "validator" => %id, - ); + if self.individual_tracking() { + info!( + self.log, + "Unaggregated attestation"; + "head" => ?data.beacon_block_root, + "index" => %data.index, + "delay_ms" => %delay.as_millis(), + "epoch" => %epoch, + "slot" => %data.slot, + "src" => src, + "validator" => %id, + ); + } validator.with_epoch_summary(epoch, |summary| { summary.register_unaggregated_attestation(delay) @@ -851,27 +940,31 @@ impl ValidatorMonitor { if let Some(validator) = self.get_validator(aggregator_index) { let id = &validator.id; - metrics::inc_counter_vec( - &metrics::VALIDATOR_MONITOR_AGGREGATED_ATTESTATION_TOTAL, - &[src, id], - ); - metrics::observe_timer_vec( - &metrics::VALIDATOR_MONITOR_AGGREGATED_ATTESTATION_DELAY_SECONDS, - &[src, id], - delay, - ); + self.aggregatable_metric(id, |label| { + metrics::inc_counter_vec( + &metrics::VALIDATOR_MONITOR_AGGREGATED_ATTESTATION_TOTAL, + &[src, label], + ); + metrics::observe_timer_vec( + &metrics::VALIDATOR_MONITOR_AGGREGATED_ATTESTATION_DELAY_SECONDS, + &[src, label], + delay, + ); + }); - info!( - self.log, - "Aggregated attestation"; - "head" => ?data.beacon_block_root, - "index" => %data.index, - "delay_ms" => %delay.as_millis(), - "epoch" => %epoch, - "slot" => %data.slot, - "src" => src, - "validator" => %id, - ); + if self.individual_tracking() { + info!( + self.log, + "Aggregated attestation"; + "head" => ?data.beacon_block_root, + "index" => %data.index, + "delay_ms" => %delay.as_millis(), + "epoch" => %epoch, + "slot" => %data.slot, + "src" => src, + "validator" => %id, + ); + } validator.with_epoch_summary(epoch, |summary| { summary.register_aggregated_attestation(delay) @@ -882,27 +975,31 @@ impl ValidatorMonitor { if let Some(validator) = self.get_validator(*i) { let id = &validator.id; - metrics::inc_counter_vec( - &metrics::VALIDATOR_MONITOR_ATTESTATION_IN_AGGREGATE_TOTAL, - &[src, id], - ); - metrics::observe_timer_vec( - &metrics::VALIDATOR_MONITOR_ATTESTATION_IN_AGGREGATE_DELAY_SECONDS, - &[src, id], - delay, - ); + self.aggregatable_metric(id, |label| { + metrics::inc_counter_vec( + &metrics::VALIDATOR_MONITOR_ATTESTATION_IN_AGGREGATE_TOTAL, + &[src, label], + ); + metrics::observe_timer_vec( + &metrics::VALIDATOR_MONITOR_ATTESTATION_IN_AGGREGATE_DELAY_SECONDS, + &[src, label], + delay, + ); + }); - info!( - self.log, - "Attestation included in aggregate"; - "head" => ?data.beacon_block_root, - "index" => %data.index, - "delay_ms" => %delay.as_millis(), - "epoch" => %epoch, - "slot" => %data.slot, - "src" => src, - "validator" => %id, - ); + if self.individual_tracking() { + info!( + self.log, + "Attestation included in aggregate"; + "head" => ?data.beacon_block_root, + "index" => %data.index, + "delay_ms" => %delay.as_millis(), + "epoch" => %epoch, + "slot" => %data.slot, + "src" => src, + "validator" => %id, + ); + } validator.with_epoch_summary(epoch, |summary| { summary.register_aggregate_attestation_inclusion() @@ -936,26 +1033,31 @@ impl ValidatorMonitor { if let Some(validator) = self.get_validator(*i) { let id = &validator.id; - metrics::inc_counter_vec( - &metrics::VALIDATOR_MONITOR_ATTESTATION_IN_BLOCK_TOTAL, - &["block", id], - ); - metrics::set_int_gauge( - &metrics::VALIDATOR_MONITOR_ATTESTATION_IN_BLOCK_DELAY_SLOTS, - &["block", id], - delay.as_u64() as i64, - ); + self.aggregatable_metric(id, |label| { + metrics::inc_counter_vec( + &metrics::VALIDATOR_MONITOR_ATTESTATION_IN_BLOCK_TOTAL, + &["block", label], + ); + }); - info!( - self.log, - "Attestation included in block"; - "head" => ?data.beacon_block_root, - "index" => %data.index, - "inclusion_lag" => format!("{} slot(s)", delay), - "epoch" => %epoch, - "slot" => %data.slot, - "validator" => %id, - ); + if self.individual_tracking() { + metrics::set_int_gauge( + &metrics::VALIDATOR_MONITOR_ATTESTATION_IN_BLOCK_DELAY_SLOTS, + &["block", id], + delay.as_u64() as i64, + ); + + info!( + self.log, + "Attestation included in block"; + "head" => ?data.beacon_block_root, + "index" => %data.index, + "inclusion_lag" => format!("{} slot(s)", delay), + "epoch" => %epoch, + "slot" => %data.slot, + "validator" => %id, + ); + } validator.with_epoch_summary(epoch, |summary| { summary.register_attestation_block_inclusion(inclusion_distance) @@ -1013,26 +1115,30 @@ impl ValidatorMonitor { slot_clock, ); - metrics::inc_counter_vec( - &metrics::VALIDATOR_MONITOR_SYNC_COMMITTEE_MESSAGES_TOTAL, - &[src, id], - ); - metrics::observe_timer_vec( - &metrics::VALIDATOR_MONITOR_SYNC_COMMITTEE_MESSAGES_DELAY_SECONDS, - &[src, id], - delay, - ); + self.aggregatable_metric(id, |label| { + metrics::inc_counter_vec( + &metrics::VALIDATOR_MONITOR_SYNC_COMMITTEE_MESSAGES_TOTAL, + &[src, label], + ); + metrics::observe_timer_vec( + &metrics::VALIDATOR_MONITOR_SYNC_COMMITTEE_MESSAGES_DELAY_SECONDS, + &[src, label], + delay, + ); + }); - info!( - self.log, - "Sync committee message"; - "head" => %sync_committee_message.beacon_block_root, - "delay_ms" => %delay.as_millis(), - "epoch" => %epoch, - "slot" => %sync_committee_message.slot, - "src" => src, - "validator" => %id, - ); + if self.individual_tracking() { + info!( + self.log, + "Sync committee message"; + "head" => %sync_committee_message.beacon_block_root, + "delay_ms" => %delay.as_millis(), + "epoch" => %epoch, + "slot" => %sync_committee_message.slot, + "src" => src, + "validator" => %id, + ); + } validator.with_epoch_summary(epoch, |summary| { summary.register_sync_committee_message(delay) @@ -1097,26 +1203,30 @@ impl ValidatorMonitor { if let Some(validator) = self.get_validator(aggregator_index) { let id = &validator.id; - metrics::inc_counter_vec( - &metrics::VALIDATOR_MONITOR_SYNC_CONTRIBUTIONS_TOTAL, - &[src, id], - ); - metrics::observe_timer_vec( - &metrics::VALIDATOR_MONITOR_SYNC_CONTRIBUTIONS_DELAY_SECONDS, - &[src, id], - delay, - ); + self.aggregatable_metric(id, |label| { + metrics::inc_counter_vec( + &metrics::VALIDATOR_MONITOR_SYNC_CONTRIBUTIONS_TOTAL, + &[src, label], + ); + metrics::observe_timer_vec( + &metrics::VALIDATOR_MONITOR_SYNC_CONTRIBUTIONS_DELAY_SECONDS, + &[src, label], + delay, + ); + }); - info!( - self.log, - "Sync contribution"; - "head" => %beacon_block_root, - "delay_ms" => %delay.as_millis(), - "epoch" => %epoch, - "slot" => %slot, - "src" => src, - "validator" => %id, - ); + if self.individual_tracking() { + info!( + self.log, + "Sync contribution"; + "head" => %beacon_block_root, + "delay_ms" => %delay.as_millis(), + "epoch" => %epoch, + "slot" => %slot, + "src" => src, + "validator" => %id, + ); + } validator.with_epoch_summary(epoch, |summary| { summary.register_sync_committee_contribution(delay) @@ -1127,21 +1237,25 @@ impl ValidatorMonitor { if let Some(validator) = self.validators.get(validator_pubkey) { let id = &validator.id; - metrics::inc_counter_vec( - &metrics::VALIDATOR_MONITOR_SYNC_COMMITTEE_MESSAGE_IN_CONTRIBUTION_TOTAL, - &[src, id], - ); + self.aggregatable_metric(id, |label| { + metrics::inc_counter_vec( + &metrics::VALIDATOR_MONITOR_SYNC_COMMITTEE_MESSAGE_IN_CONTRIBUTION_TOTAL, + &[src, label], + ); + }); - info!( - self.log, - "Sync signature included in contribution"; - "head" => %beacon_block_root, - "delay_ms" => %delay.as_millis(), - "epoch" => %epoch, - "slot" => %slot, - "src" => src, - "validator" => %id, - ); + if self.individual_tracking() { + info!( + self.log, + "Sync signature included in contribution"; + "head" => %beacon_block_root, + "delay_ms" => %delay.as_millis(), + "epoch" => %epoch, + "slot" => %slot, + "src" => src, + "validator" => %id, + ); + } validator.with_epoch_summary(epoch, |summary| { summary.register_sync_signature_contribution_inclusion() @@ -1163,19 +1277,23 @@ impl ValidatorMonitor { if let Some(validator) = self.validators.get(validator_pubkey) { let id = &validator.id; - metrics::inc_counter_vec( - &metrics::VALIDATOR_MONITOR_SYNC_COMMITTEE_MESSAGE_IN_BLOCK_TOTAL, - &["block", id], - ); + self.aggregatable_metric(id, |label| { + metrics::inc_counter_vec( + &metrics::VALIDATOR_MONITOR_SYNC_COMMITTEE_MESSAGE_IN_BLOCK_TOTAL, + &["block", label], + ); + }); - info!( - self.log, - "Sync signature included in block"; - "head" => %beacon_block_root, - "epoch" => %epoch, - "slot" => %slot, - "validator" => %id, - ); + if self.individual_tracking() { + info!( + self.log, + "Sync signature included in block"; + "head" => %beacon_block_root, + "epoch" => %epoch, + "slot" => %slot, + "validator" => %id, + ); + } validator.with_epoch_summary(epoch, |summary| { summary.register_sync_signature_block_inclusions(); @@ -1204,8 +1322,12 @@ impl ValidatorMonitor { let id = &validator.id; let epoch = exit.epoch; - metrics::inc_counter_vec(&metrics::VALIDATOR_MONITOR_EXIT_TOTAL, &[src, id]); + self.aggregatable_metric(id, |label| { + metrics::inc_counter_vec(&metrics::VALIDATOR_MONITOR_EXIT_TOTAL, &[src, label]); + }); + // Not gated behind `self.individual_tracking()` since it's an + // infrequent and interesting message. info!( self.log, "Voluntary exit"; @@ -1243,11 +1365,15 @@ impl ValidatorMonitor { if let Some(validator) = self.get_validator(proposer) { let id = &validator.id; - metrics::inc_counter_vec( - &metrics::VALIDATOR_MONITOR_PROPOSER_SLASHING_TOTAL, - &[src, id], - ); + self.aggregatable_metric(id, |label| { + metrics::inc_counter_vec( + &metrics::VALIDATOR_MONITOR_PROPOSER_SLASHING_TOTAL, + &[src, label], + ); + }); + // Not gated behind `self.individual_tracking()` since it's an + // infrequent and interesting message. crit!( self.log, "Proposer slashing"; @@ -1296,11 +1422,15 @@ impl ValidatorMonitor { let id = &validator.id; let epoch = data.slot.epoch(T::slots_per_epoch()); - metrics::inc_counter_vec( - &metrics::VALIDATOR_MONITOR_ATTESTER_SLASHING_TOTAL, - &[src, id], - ); + self.aggregatable_metric(id, |label| { + metrics::inc_counter_vec( + &metrics::VALIDATOR_MONITOR_ATTESTER_SLASHING_TOTAL, + &[src, label], + ); + }); + // Not gated behind `self.individual_tracking()` since it's an + // infrequent and interesting message. crit!( self.log, "Attester slashing"; @@ -1350,69 +1480,80 @@ impl ValidatorMonitor { /* * Attestations */ - metrics::set_gauge_vec( - &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATIONS_TOTAL, - &[id], - summary.attestations as i64, - ); if let Some(delay) = summary.attestation_min_delay { - metrics::observe_timer_vec( - &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATIONS_MIN_DELAY_SECONDS, - &[id], - delay, - ); + self.aggregatable_metric(id, |tag| { + metrics::observe_timer_vec( + &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATIONS_MIN_DELAY_SECONDS, + &[tag], + delay, + ); + }); } - metrics::set_gauge_vec( - &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATION_AGGREGATE_INCLUSIONS, - &[id], - summary.attestation_aggregate_inclusions as i64, - ); - metrics::set_gauge_vec( - &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATION_BLOCK_INCLUSIONS, - &[id], - summary.attestation_block_inclusions as i64, - ); - if let Some(distance) = summary.attestation_min_block_inclusion_distance { + if self.individual_tracking() { metrics::set_gauge_vec( - &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATION_BLOCK_MIN_INCLUSION_DISTANCE, + &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATIONS_TOTAL, &[id], - distance.as_u64() as i64, + summary.attestations as i64, ); + metrics::set_gauge_vec( + &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATION_AGGREGATE_INCLUSIONS, + &[id], + summary.attestation_aggregate_inclusions as i64, + ); + metrics::set_gauge_vec( + &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATION_BLOCK_INCLUSIONS, + &[id], + summary.attestation_block_inclusions as i64, + ); + + if let Some(distance) = summary.attestation_min_block_inclusion_distance { + metrics::set_gauge_vec( + &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTATION_BLOCK_MIN_INCLUSION_DISTANCE, + &[id], + distance.as_u64() as i64, + ); + } } /* * Sync committee messages */ - metrics::set_gauge_vec( - &metrics::VALIDATOR_MONITOR_PREV_EPOCH_SYNC_COMMITTEE_MESSAGES_TOTAL, - &[id], - summary.sync_committee_messages as i64, - ); if let Some(delay) = summary.sync_committee_message_min_delay { - metrics::observe_timer_vec( - &metrics::VALIDATOR_MONITOR_PREV_EPOCH_SYNC_COMMITTEE_MESSAGES_MIN_DELAY_SECONDS, + self.aggregatable_metric(id, |tag| { + metrics::observe_timer_vec( + &metrics::VALIDATOR_MONITOR_PREV_EPOCH_SYNC_COMMITTEE_MESSAGES_MIN_DELAY_SECONDS, + &[tag], + delay, + ); + }); + } + if self.individual_tracking() { + metrics::set_gauge_vec( + &metrics::VALIDATOR_MONITOR_PREV_EPOCH_SYNC_COMMITTEE_MESSAGES_TOTAL, &[id], - delay, + summary.sync_committee_messages as i64, + ); + metrics::set_gauge_vec( + &metrics::VALIDATOR_MONITOR_PREV_EPOCH_SYNC_CONTRIBUTION_INCLUSIONS, + &[id], + summary.sync_signature_contribution_inclusions as i64, + ); + metrics::set_gauge_vec( + &metrics::VALIDATOR_MONITOR_PREV_EPOCH_SYNC_SIGNATURE_BLOCK_INCLUSIONS, + &[id], + summary.sync_signature_block_inclusions as i64, ); } - metrics::set_gauge_vec( - &metrics::VALIDATOR_MONITOR_PREV_EPOCH_SYNC_CONTRIBUTION_INCLUSIONS, - &[id], - summary.sync_signature_contribution_inclusions as i64, - ); - metrics::set_gauge_vec( - &metrics::VALIDATOR_MONITOR_PREV_EPOCH_SYNC_SIGNATURE_BLOCK_INCLUSIONS, - &[id], - summary.sync_signature_block_inclusions as i64, - ); /* * Sync contributions */ - metrics::set_gauge_vec( - &metrics::VALIDATOR_MONITOR_PREV_EPOCH_SYNC_CONTRIBUTIONS_TOTAL, - &[id], - summary.sync_contributions as i64, - ); + if self.individual_tracking() { + metrics::set_gauge_vec( + &metrics::VALIDATOR_MONITOR_PREV_EPOCH_SYNC_CONTRIBUTIONS_TOTAL, + &[id], + summary.sync_contributions as i64, + ); + } if let Some(delay) = summary.sync_contribution_min_delay { metrics::observe_timer_vec( &metrics::VALIDATOR_MONITOR_PREV_EPOCH_SYNC_CONTRIBUTION_MIN_DELAY_SECONDS, @@ -1424,51 +1565,61 @@ impl ValidatorMonitor { /* * Blocks */ - metrics::set_gauge_vec( - &metrics::VALIDATOR_MONITOR_PREV_EPOCH_BEACON_BLOCKS_TOTAL, - &[id], - summary.blocks as i64, - ); - if let Some(delay) = summary.block_min_delay { - metrics::observe_timer_vec( - &metrics::VALIDATOR_MONITOR_PREV_EPOCH_BEACON_BLOCKS_MIN_DELAY_SECONDS, + if self.individual_tracking() { + metrics::set_gauge_vec( + &metrics::VALIDATOR_MONITOR_PREV_EPOCH_BEACON_BLOCKS_TOTAL, &[id], - delay, + summary.blocks as i64, ); } + if let Some(delay) = summary.block_min_delay { + self.aggregatable_metric(id, |tag| { + metrics::observe_timer_vec( + &metrics::VALIDATOR_MONITOR_PREV_EPOCH_BEACON_BLOCKS_MIN_DELAY_SECONDS, + &[tag], + delay, + ); + }); + } /* * Aggregates */ - metrics::set_gauge_vec( - &metrics::VALIDATOR_MONITOR_PREV_EPOCH_AGGREGATES_TOTAL, - &[id], - summary.aggregates as i64, - ); - if let Some(delay) = summary.aggregate_min_delay { - metrics::observe_timer_vec( - &metrics::VALIDATOR_MONITOR_PREV_EPOCH_AGGREGATES_MIN_DELAY_SECONDS, + if self.individual_tracking() { + metrics::set_gauge_vec( + &metrics::VALIDATOR_MONITOR_PREV_EPOCH_AGGREGATES_TOTAL, &[id], - delay, + summary.aggregates as i64, ); } + if let Some(delay) = summary.aggregate_min_delay { + self.aggregatable_metric(id, |tag| { + metrics::observe_timer_vec( + &metrics::VALIDATOR_MONITOR_PREV_EPOCH_AGGREGATES_MIN_DELAY_SECONDS, + &[tag], + delay, + ); + }); + } /* * Other */ - metrics::set_gauge_vec( - &metrics::VALIDATOR_MONITOR_PREV_EPOCH_EXITS_TOTAL, - &[id], - summary.exits as i64, - ); - metrics::set_gauge_vec( - &metrics::VALIDATOR_MONITOR_PREV_EPOCH_PROPOSER_SLASHINGS_TOTAL, - &[id], - summary.proposer_slashings as i64, - ); - metrics::set_gauge_vec( - &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTER_SLASHINGS_TOTAL, - &[id], - summary.attester_slashings as i64, - ); + if self.individual_tracking() { + metrics::set_gauge_vec( + &metrics::VALIDATOR_MONITOR_PREV_EPOCH_EXITS_TOTAL, + &[id], + summary.exits as i64, + ); + metrics::set_gauge_vec( + &metrics::VALIDATOR_MONITOR_PREV_EPOCH_PROPOSER_SLASHINGS_TOTAL, + &[id], + summary.proposer_slashings as i64, + ); + metrics::set_gauge_vec( + &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ATTESTER_SLASHINGS_TOTAL, + &[id], + summary.attester_slashings as i64, + ); + } } } } diff --git a/beacon_node/beacon_chain/src/validator_pubkey_cache.rs b/beacon_node/beacon_chain/src/validator_pubkey_cache.rs index 26aea2d272..79910df292 100644 --- a/beacon_node/beacon_chain/src/validator_pubkey_cache.rs +++ b/beacon_node/beacon_chain/src/validator_pubkey_cache.rs @@ -4,7 +4,7 @@ use ssz::{Decode, Encode}; use std::collections::HashMap; use std::convert::TryInto; use std::marker::PhantomData; -use store::{DBColumn, Error as StoreError, KeyValueStore, KeyValueStoreOp, StoreItem}; +use store::{DBColumn, Error as StoreError, StoreItem, StoreOp}; use types::{BeaconState, Hash256, PublicKey, PublicKeyBytes}; /// Provides a mapping of `validator_index -> validator_publickey`. @@ -38,7 +38,7 @@ impl ValidatorPubkeyCache { }; let store_ops = cache.import_new_pubkeys(state)?; - store.hot_db.do_atomically(store_ops)?; + store.do_atomically(store_ops)?; Ok(cache) } @@ -79,7 +79,7 @@ impl ValidatorPubkeyCache { pub fn import_new_pubkeys( &mut self, state: &BeaconState, - ) -> Result, BeaconChainError> { + ) -> Result>, BeaconChainError> { if state.validators().len() > self.pubkeys.len() { self.import( state.validators()[self.pubkeys.len()..] @@ -92,7 +92,10 @@ impl ValidatorPubkeyCache { } /// Adds zero or more validators to `self`. - fn import(&mut self, validator_keys: I) -> Result, BeaconChainError> + fn import( + &mut self, + validator_keys: I, + ) -> Result>, BeaconChainError> where I: Iterator + ExactSizeIterator, { @@ -112,7 +115,9 @@ impl ValidatorPubkeyCache { // It will be committed atomically when the block that introduced it is written to disk. // Notably it is NOT written while the write lock on the cache is held. // See: https://github.com/sigp/lighthouse/issues/2327 - store_ops.push(DatabasePubkey(pubkey).as_kv_store_op(DatabasePubkey::key_for_index(i))); + store_ops.push(StoreOp::KeyValueOp( + DatabasePubkey(pubkey).as_kv_store_op(DatabasePubkey::key_for_index(i)), + )); self.pubkeys.push( (&pubkey) @@ -294,7 +299,7 @@ mod test { let ops = cache .import_new_pubkeys(&state) .expect("should import pubkeys"); - store.hot_db.do_atomically(ops).unwrap(); + store.do_atomically(ops).unwrap(); check_cache_get(&cache, &keypairs[..]); drop(cache); diff --git a/beacon_node/beacon_chain/tests/main.rs b/beacon_node/beacon_chain/tests/main.rs index 1c61e9927f..eceb4f2e85 100644 --- a/beacon_node/beacon_chain/tests/main.rs +++ b/beacon_node/beacon_chain/tests/main.rs @@ -4,6 +4,7 @@ mod block_verification; mod merge; mod op_verification; mod payload_invalidation; +mod rewards; mod store_tests; mod sync_committee_verification; mod tests; diff --git a/beacon_node/beacon_chain/tests/rewards.rs b/beacon_node/beacon_chain/tests/rewards.rs new file mode 100644 index 0000000000..b61bea1242 --- /dev/null +++ b/beacon_node/beacon_chain/tests/rewards.rs @@ -0,0 +1,121 @@ +#![cfg(test)] + +use std::collections::HashMap; + +use beacon_chain::test_utils::{ + generate_deterministic_keypairs, BeaconChainHarness, EphemeralHarnessType, +}; +use beacon_chain::{ + test_utils::{AttestationStrategy, BlockStrategy, RelativeSyncCommittee}, + types::{Epoch, EthSpec, Keypair, MinimalEthSpec}, +}; +use lazy_static::lazy_static; + +pub const VALIDATOR_COUNT: usize = 64; + +lazy_static! { + static ref KEYPAIRS: Vec = generate_deterministic_keypairs(VALIDATOR_COUNT); +} + +fn get_harness() -> BeaconChainHarness> { + let mut spec = E::default_spec(); + + spec.altair_fork_epoch = Some(Epoch::new(0)); // We use altair for all tests + + let harness = BeaconChainHarness::builder(E::default()) + .spec(spec) + .keypairs(KEYPAIRS.to_vec()) + .fresh_ephemeral_store() + .build(); + + harness.advance_slot(); + + harness +} + +#[tokio::test] +async fn test_sync_committee_rewards() { + let num_block_produced = MinimalEthSpec::slots_per_epoch(); + let harness = get_harness::(); + + let latest_block_root = harness + .extend_chain( + num_block_produced as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + + // Create and add sync committee message to op_pool + let sync_contributions = harness.make_sync_contributions( + &harness.get_current_state(), + latest_block_root, + harness.get_current_slot(), + RelativeSyncCommittee::Current, + ); + + harness + .process_sync_contributions(sync_contributions) + .unwrap(); + + // Add block + let chain = &harness.chain; + let (head_state, head_state_root) = harness.get_current_state_and_root(); + let target_slot = harness.get_current_slot() + 1; + + let (block_root, mut state) = harness + .add_attested_block_at_slot(target_slot, head_state, head_state_root, &[]) + .await + .unwrap(); + + let block = harness.get_block(block_root).unwrap(); + let parent_block = chain + .get_blinded_block(&block.parent_root()) + .unwrap() + .unwrap(); + let parent_state = chain + .get_state(&parent_block.state_root(), Some(parent_block.slot())) + .unwrap() + .unwrap(); + + let reward_payload = chain + .compute_sync_committee_rewards(block.message(), &mut state) + .unwrap(); + + let rewards = reward_payload + .iter() + .map(|reward| (reward.validator_index, reward.reward)) + .collect::>(); + + let proposer_index = state + .get_beacon_proposer_index(target_slot, &MinimalEthSpec::default_spec()) + .unwrap(); + + let mut mismatches = vec![]; + + for validator in state.validators() { + let validator_index = state + .clone() + .get_validator_index(&validator.pubkey) + .unwrap() + .unwrap(); + let pre_state_balance = parent_state.balances()[validator_index]; + let post_state_balance = state.balances()[validator_index]; + let sync_committee_reward = rewards.get(&(validator_index as u64)).unwrap_or(&0); + + if validator_index == proposer_index { + continue; // Ignore proposer + } + + if pre_state_balance as i64 + *sync_committee_reward != post_state_balance as i64 { + mismatches.push(validator_index.to_string()); + } + } + + assert_eq!( + mismatches.len(), + 0, + "Expect 0 mismatches, but these validators have mismatches on balance: {} ", + mismatches.join(",") + ); +} diff --git a/beacon_node/beacon_chain/tests/store_tests.rs b/beacon_node/beacon_chain/tests/store_tests.rs index b2fc7a6402..8a6ea9cfe1 100644 --- a/beacon_node/beacon_chain/tests/store_tests.rs +++ b/beacon_node/beacon_chain/tests/store_tests.rs @@ -5,6 +5,7 @@ use beacon_chain::builder::BeaconChainBuilder; use beacon_chain::test_utils::{ test_spec, AttestationStrategy, BeaconChainHarness, BlockStrategy, DiskHarnessType, }; +use beacon_chain::validator_monitor::DEFAULT_INDIVIDUAL_TRACKING_THRESHOLD; use beacon_chain::{ historical_blocks::HistoricalBlockError, migrate::MigratorConfig, BeaconChain, BeaconChainError, BeaconChainTypes, BeaconSnapshot, ChainConfig, NotifyExecutionLayer, @@ -2121,7 +2122,7 @@ async fn weak_subjectivity_sync() { log.clone(), 1, ))) - .monitor_validators(true, vec![], log) + .monitor_validators(true, vec![], DEFAULT_INDIVIDUAL_TRACKING_THRESHOLD, log) .build() .expect("should build"), ); diff --git a/beacon_node/builder_client/Cargo.toml b/beacon_node/builder_client/Cargo.toml index c4d21c59ab..48ac0300c9 100644 --- a/beacon_node/builder_client/Cargo.toml +++ b/beacon_node/builder_client/Cargo.toml @@ -9,4 +9,4 @@ reqwest = { version = "0.11.0", features = ["json","stream"] } sensitive_url = { path = "../../common/sensitive_url" } eth2 = { path = "../../common/eth2" } serde = { version = "1.0.116", features = ["derive"] } -serde_json = "1.0.58" \ No newline at end of file +serde_json = "1.0.58" diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index f3e937b2e5..3b016ebda9 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -173,6 +173,7 @@ where .monitor_validators( config.validator_monitor_auto, config.validator_monitor_pubkeys.clone(), + config.validator_monitor_individual_tracking_threshold, runtime_context .service_context("val_mon".to_string()) .log() diff --git a/beacon_node/client/src/config.rs b/beacon_node/client/src/config.rs index 0a2997762a..22b868256a 100644 --- a/beacon_node/client/src/config.rs +++ b/beacon_node/client/src/config.rs @@ -1,3 +1,4 @@ +use beacon_chain::validator_monitor::DEFAULT_INDIVIDUAL_TRACKING_THRESHOLD; use directory::DEFAULT_ROOT_DIR; use environment::LoggerConfig; use network::NetworkConfig; @@ -59,6 +60,11 @@ pub struct Config { pub validator_monitor_auto: bool, /// A list of validator pubkeys to monitor. pub validator_monitor_pubkeys: Vec, + /// Once the number of monitored validators goes above this threshold, we + /// will stop tracking metrics on a per-validator basis. This prevents large + /// validator counts causing infeasibly high cardinailty for Prometheus and + /// high log volumes. + pub validator_monitor_individual_tracking_threshold: usize, #[serde(skip)] /// The `genesis` field is not serialized or deserialized by `serde` to ensure it is defined /// via the CLI at runtime, instead of from a configuration file saved to disk. @@ -97,6 +103,7 @@ impl Default for Config { slasher: None, validator_monitor_auto: false, validator_monitor_pubkeys: vec![], + validator_monitor_individual_tracking_threshold: DEFAULT_INDIVIDUAL_TRACKING_THRESHOLD, logger_config: LoggerConfig::default(), } } diff --git a/beacon_node/eth1/src/deposit_cache.rs b/beacon_node/eth1/src/deposit_cache.rs index ab07b380d1..75391e58a0 100644 --- a/beacon_node/eth1/src/deposit_cache.rs +++ b/beacon_node/eth1/src/deposit_cache.rs @@ -675,7 +675,7 @@ pub mod tests { #[test] fn test_finalization_boundaries() { let n = 8; - let half = (n / 2) as usize; + let half = n / 2; let mut deposit_cache = get_cache_with_deposits(n as u64); @@ -828,9 +828,9 @@ pub mod tests { // get_log(half+quarter) should return log with index `half+quarter` assert_eq!( q3_log_before_finalization.index, - (half + quarter) as u64, + half + quarter, "log index should be {}", - (half + quarter), + half + quarter, ); // get lower quarter of deposits with max deposit count diff --git a/beacon_node/execution_layer/Cargo.toml b/beacon_node/execution_layer/Cargo.toml index bfc748d5b6..76788b102e 100644 --- a/beacon_node/execution_layer/Cargo.toml +++ b/beacon_node/execution_layer/Cargo.toml @@ -37,7 +37,7 @@ rand = "0.8.5" zeroize = { version = "1.4.2", features = ["zeroize_derive"] } lighthouse_metrics = { path = "../../common/lighthouse_metrics" } lazy_static = "1.4.0" -ethers-core = "0.17.0" +ethers-core = "1.0.2" builder_client = { path = "../builder_client" } fork_choice = { path = "../../consensus/fork_choice" } mev-build-rs = { git = "https://github.com/ralexstokes/mev-rs", rev = "6c99b0fbdc0427b1625469d2e575303ce08de5b8" } @@ -45,3 +45,7 @@ ethereum-consensus = { git = "https://github.com/ralexstokes/ethereum-consensus" ssz-rs = { git = "https://github.com/ralexstokes/ssz-rs", rev = "cb08f1" } tokio-stream = { version = "0.1.9", features = [ "sync" ] } strum = "0.24.0" +keccak-hash = "0.10.0" +hash256-std-hasher = "0.15.2" +triehash = "0.8.4" +hash-db = "0.15.2" diff --git a/beacon_node/execution_layer/src/block_hash.rs b/beacon_node/execution_layer/src/block_hash.rs new file mode 100644 index 0000000000..f023c038ae --- /dev/null +++ b/beacon_node/execution_layer/src/block_hash.rs @@ -0,0 +1,163 @@ +use crate::{ + keccak::{keccak256, KeccakHasher}, + metrics, Error, ExecutionLayer, +}; +use ethers_core::utils::rlp::RlpStream; +use keccak_hash::KECCAK_EMPTY_LIST_RLP; +use triehash::ordered_trie_root; +use types::{ + map_execution_block_header_fields, Address, EthSpec, ExecutionBlockHash, ExecutionBlockHeader, + ExecutionPayload, Hash256, Hash64, Uint256, +}; + +impl ExecutionLayer { + /// Verify `payload.block_hash` locally within Lighthouse. + /// + /// No remote calls to the execution client will be made, so this is quite a cheap check. + pub fn verify_payload_block_hash(&self, payload: &ExecutionPayload) -> Result<(), Error> { + let _timer = metrics::start_timer(&metrics::EXECUTION_LAYER_VERIFY_BLOCK_HASH); + + // Calculate the transactions root. + // We're currently using a deprecated Parity library for this. We should move to a + // better alternative when one appears, possibly following Reth. + let rlp_transactions_root = ordered_trie_root::( + payload.transactions.iter().map(|txn_bytes| &**txn_bytes), + ); + + // Construct the block header. + let exec_block_header = ExecutionBlockHeader::from_payload( + payload, + KECCAK_EMPTY_LIST_RLP.as_fixed_bytes().into(), + rlp_transactions_root, + ); + + // Hash the RLP encoding of the block header. + let rlp_block_header = rlp_encode_block_header(&exec_block_header); + let header_hash = ExecutionBlockHash::from_root(keccak256(&rlp_block_header)); + + if header_hash != payload.block_hash { + return Err(Error::BlockHashMismatch { + computed: header_hash, + payload: payload.block_hash, + transactions_root: rlp_transactions_root, + }); + } + + Ok(()) + } +} + +/// RLP encode an execution block header. +pub fn rlp_encode_block_header(header: &ExecutionBlockHeader) -> Vec { + let mut rlp_header_stream = RlpStream::new(); + rlp_header_stream.begin_unbounded_list(); + map_execution_block_header_fields!(&header, |_, field| { + rlp_header_stream.append(field); + }); + rlp_header_stream.finalize_unbounded_list(); + rlp_header_stream.out().into() +} + +#[cfg(test)] +mod test { + use super::*; + use hex::FromHex; + use std::str::FromStr; + + fn test_rlp_encoding( + header: &ExecutionBlockHeader, + expected_rlp: Option<&str>, + expected_hash: Hash256, + ) { + let rlp_encoding = rlp_encode_block_header(header); + + if let Some(expected_rlp) = expected_rlp { + let computed_rlp = hex::encode(&rlp_encoding); + assert_eq!(expected_rlp, computed_rlp); + } + + let computed_hash = keccak256(&rlp_encoding); + assert_eq!(expected_hash, computed_hash); + } + + #[test] + fn test_rlp_encode_eip1559_block() { + let header = ExecutionBlockHeader { + parent_hash: Hash256::from_str("e0a94a7a3c9617401586b1a27025d2d9671332d22d540e0af72b069170380f2a").unwrap(), + ommers_hash: Hash256::from_str("1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347").unwrap(), + beneficiary: Address::from_str("ba5e000000000000000000000000000000000000").unwrap(), + state_root: Hash256::from_str("ec3c94b18b8a1cff7d60f8d258ec723312932928626b4c9355eb4ab3568ec7f7").unwrap(), + transactions_root: Hash256::from_str("50f738580ed699f0469702c7ccc63ed2e51bc034be9479b7bff4e68dee84accf").unwrap(), + receipts_root: Hash256::from_str("29b0562f7140574dd0d50dee8a271b22e1a0a7b78fca58f7c60370d8317ba2a9").unwrap(), + logs_bloom: <[u8; 256]>::from_hex("00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000").unwrap().into(), + difficulty: 0x020000.into(), + number: 0x01_u64.into(), + gas_limit: 0x016345785d8a0000_u64.into(), + gas_used: 0x015534_u64.into(), + timestamp: 0x079e, + extra_data: vec![0x42], + mix_hash: Hash256::from_str("0000000000000000000000000000000000000000000000000000000000000000").unwrap(), + nonce: Hash64::zero(), + base_fee_per_gas: 0x036b_u64.into(), + }; + let expected_rlp = "f90200a0e0a94a7a3c9617401586b1a27025d2d9671332d22d540e0af72b069170380f2aa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d4934794ba5e000000000000000000000000000000000000a0ec3c94b18b8a1cff7d60f8d258ec723312932928626b4c9355eb4ab3568ec7f7a050f738580ed699f0469702c7ccc63ed2e51bc034be9479b7bff4e68dee84accfa029b0562f7140574dd0d50dee8a271b22e1a0a7b78fca58f7c60370d8317ba2a9b9010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000830200000188016345785d8a00008301553482079e42a0000000000000000000000000000000000000000000000000000000000000000088000000000000000082036b"; + let expected_hash = + Hash256::from_str("6a251c7c3c5dca7b42407a3752ff48f3bbca1fab7f9868371d9918daf1988d1f") + .unwrap(); + test_rlp_encoding(&header, Some(expected_rlp), expected_hash); + } + + #[test] + fn test_rlp_encode_merge_block() { + let header = ExecutionBlockHeader { + parent_hash: Hash256::from_str("927ca537f06c783a3a2635b8805eef1c8c2124f7444ad4a3389898dd832f2dbe").unwrap(), + ommers_hash: Hash256::from_str("1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347").unwrap(), + beneficiary: Address::from_str("ba5e000000000000000000000000000000000000").unwrap(), + state_root: Hash256::from_str("0xe97859b065bd8dbbb4519c7cb935024de2484c2b7f881181b4360492f0b06b82").unwrap(), + transactions_root: Hash256::from_str("50f738580ed699f0469702c7ccc63ed2e51bc034be9479b7bff4e68dee84accf").unwrap(), + receipts_root: Hash256::from_str("29b0562f7140574dd0d50dee8a271b22e1a0a7b78fca58f7c60370d8317ba2a9").unwrap(), + logs_bloom: <[u8; 256]>::from_hex("00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000").unwrap().into(), + difficulty: 0x00.into(), + number: 0x01_u64.into(), + gas_limit: 0x016345785d8a0000_u64.into(), + gas_used: 0x015534_u64.into(), + timestamp: 0x079e, + extra_data: vec![0x42], + mix_hash: Hash256::from_str("0000000000000000000000000000000000000000000000000000000000020000").unwrap(), + nonce: Hash64::zero(), + base_fee_per_gas: 0x036b_u64.into(), + }; + let expected_rlp = "f901fda0927ca537f06c783a3a2635b8805eef1c8c2124f7444ad4a3389898dd832f2dbea01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d4934794ba5e000000000000000000000000000000000000a0e97859b065bd8dbbb4519c7cb935024de2484c2b7f881181b4360492f0b06b82a050f738580ed699f0469702c7ccc63ed2e51bc034be9479b7bff4e68dee84accfa029b0562f7140574dd0d50dee8a271b22e1a0a7b78fca58f7c60370d8317ba2a9b9010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800188016345785d8a00008301553482079e42a0000000000000000000000000000000000000000000000000000000000002000088000000000000000082036b"; + let expected_hash = + Hash256::from_str("0x5b1f0f2efdaa19e996b4aea59eeb67620259f09732732a339a10dac311333684") + .unwrap(); + test_rlp_encoding(&header, Some(expected_rlp), expected_hash); + } + + // Test a real payload from mainnet. + #[test] + fn test_rlp_encode_block_16182891() { + let header = ExecutionBlockHeader { + parent_hash: Hash256::from_str("3e9c7b3f403947f110f68c4564a004b73dd8ebf73b143e46cc637926eec01a6d").unwrap(), + ommers_hash: Hash256::from_str("1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347").unwrap(), + beneficiary: Address::from_str("dafea492d9c6733ae3d56b7ed1adb60692c98bc5").unwrap(), + state_root: Hash256::from_str("5a8183d230818a167477420ce3a393ca3ef8706a7d596694ab6059894ed6fda9").unwrap(), + transactions_root: Hash256::from_str("0223f0cb35f184d2ac409e89dc0768ad738f777bd1c85d3302ca50f307180c94").unwrap(), + receipts_root: Hash256::from_str("371c76821b1cc21232574604eac5349d51647eb530e2a45d4f6fe2c501351aa5").unwrap(), + logs_bloom: <[u8; 256]>::from_hex("1a2c559955848d2662a0634cb40c7a6192a1524f11061203689bcbcdec901b054084d4f4d688009d24c10918e0089b48e72fe2d7abafb903889d10c3827c6901096612d259801b1b7ba1663a4201f5f88f416a9997c55bcc2c54785280143b057a008764c606182e324216822a2d5913e797a05c16cc1468d001acf3783b18e00e0203033e43106178db554029e83ca46402dc49d929d7882a04a0e7215041bdabf7430bd10ef4bb658a40f064c63c4816660241c2480862f26742fdf9ca41637731350301c344e439428182a03e384484e6d65d0c8a10117c6739ca201b60974519a1ae6b0c3966c0f650b449d10eae065dab2c83ab4edbab5efdea50bbc801").unwrap().into(), + difficulty: 0.into(), + number: 16182891.into(), + gas_limit: 0x1c9c380.into(), + gas_used: 0xe9b752.into(), + timestamp: 0x6399bf63, + extra_data: hex::decode("496c6c756d696e61746520446d6f63726174697a6520447374726962757465").unwrap(), + mix_hash: Hash256::from_str("bf5289894b2ceab3549f92f063febbac896b280ddb18129a57cff13113c11b13").unwrap(), + nonce: Hash64::zero(), + base_fee_per_gas: 0x34187b238_u64.into(), + }; + let expected_hash = + Hash256::from_str("6da69709cd5a34079b6604d29cd78fc01dacd7c6268980057ad92a2bede87351") + .unwrap(); + test_rlp_encoding(&header, None, expected_hash); + } +} diff --git a/beacon_node/execution_layer/src/engine_api/auth.rs b/beacon_node/execution_layer/src/engine_api/auth.rs index 8fcdb2543d..2f4c0cd1e8 100644 --- a/beacon_node/execution_layer/src/engine_api/auth.rs +++ b/beacon_node/execution_layer/src/engine_api/auth.rs @@ -27,7 +27,7 @@ impl From for Error { /// Provides wrapper around `[u8; JWT_SECRET_LENGTH]` that implements `Zeroize`. #[derive(Zeroize, Clone)] #[zeroize(drop)] -pub struct JwtKey([u8; JWT_SECRET_LENGTH as usize]); +pub struct JwtKey([u8; JWT_SECRET_LENGTH]); impl JwtKey { /// Wrap given slice in `Self`. Returns an error if slice.len() != `JWT_SECRET_LENGTH`. diff --git a/beacon_node/execution_layer/src/engines.rs b/beacon_node/execution_layer/src/engines.rs index 339006c1ba..eaaa271c51 100644 --- a/beacon_node/execution_layer/src/engines.rs +++ b/beacon_node/execution_layer/src/engines.rs @@ -5,7 +5,7 @@ use crate::engine_api::{ }; use crate::HttpJsonRpc; use lru::LruCache; -use slog::{debug, error, info, Logger}; +use slog::{debug, error, info, warn, Logger}; use std::future::Future; use std::sync::Arc; use task_executor::TaskExecutor; @@ -325,7 +325,7 @@ impl Engine { Ok(result) } Err(error) => { - error!( + warn!( self.log, "Execution engine call failed"; "error" => ?error, diff --git a/beacon_node/execution_layer/src/keccak.rs b/beacon_node/execution_layer/src/keccak.rs new file mode 100644 index 0000000000..c4c9689272 --- /dev/null +++ b/beacon_node/execution_layer/src/keccak.rs @@ -0,0 +1,35 @@ +// Copyright 2017, 2018 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +use hash256_std_hasher::Hash256StdHasher; +use hash_db::Hasher; +use types::Hash256; + +pub fn keccak256(bytes: &[u8]) -> Hash256 { + Hash256::from(ethers_core::utils::keccak256(bytes)) +} + +/// Keccak hasher. +#[derive(Default, Debug, Clone, PartialEq)] +pub struct KeccakHasher; + +impl Hasher for KeccakHasher { + type Out = Hash256; + type StdHasher = Hash256StdHasher; + + const LENGTH: usize = 32; + + fn hash(x: &[u8]) -> Self::Out { + keccak256(x) + } +} diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index ec1415f80b..5b0fecbf20 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -40,8 +40,10 @@ use types::{ ProposerPreparationData, PublicKeyBytes, Signature, SignedBeaconBlock, Slot, Uint256, }; +mod block_hash; mod engine_api; mod engines; +mod keccak; mod metrics; pub mod payload_cache; mod payload_status; @@ -90,6 +92,11 @@ pub enum Error { ShuttingDown, FeeRecipientUnspecified, MissingLatestValidHash, + BlockHashMismatch { + computed: ExecutionBlockHash, + payload: ExecutionBlockHash, + transactions_root: Hash256, + }, InvalidJWTSecret(String), } @@ -1568,10 +1575,10 @@ impl ExecutionLayer { &metrics::EXECUTION_LAYER_BUILDER_REVEAL_PAYLOAD_OUTCOME, &[metrics::FAILURE], ); - error!( + warn!( self.log(), "Builder failed to reveal payload"; - "info" => "this relay failure may cause a missed proposal", + "info" => "this is common behaviour for some builders and may not indicate an issue", "error" => ?e, "relay_response_ms" => duration.as_millis(), "block_root" => ?block_root, diff --git a/beacon_node/execution_layer/src/metrics.rs b/beacon_node/execution_layer/src/metrics.rs index bb5a1088d1..287050f66b 100644 --- a/beacon_node/execution_layer/src/metrics.rs +++ b/beacon_node/execution_layer/src/metrics.rs @@ -45,6 +45,11 @@ lazy_static::lazy_static! { "execution_layer_get_payload_by_block_hash_time", "Time to reconstruct a payload from the EE using eth_getBlockByHash" ); + pub static ref EXECUTION_LAYER_VERIFY_BLOCK_HASH: Result = try_create_histogram_with_buckets( + "execution_layer_verify_block_hash_time", + "Time to verify the execution block hash in Lighthouse, without the EL", + Ok(vec![10e-6, 50e-6, 100e-6, 500e-6, 1e-3, 5e-3, 10e-3, 50e-3, 100e-3, 500e-3]), + ); pub static ref EXECUTION_LAYER_PAYLOAD_STATUS: Result = try_create_int_counter_vec( "execution_layer_payload_status", "Indicates the payload status returned for a particular method", diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index b018f9c737..60e5d2adf4 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -15,7 +15,9 @@ mod database; mod metrics; mod proposer_duties; mod publish_blocks; +mod standard_block_rewards; mod state_id; +mod sync_committee_rewards; mod sync_committees; mod ui; mod validator_inclusion; @@ -1699,6 +1701,114 @@ pub fn serve( }, ); + let beacon_rewards_path = eth_v1 + .and(warp::path("beacon")) + .and(warp::path("rewards")) + .and(chain_filter.clone()); + + // GET beacon/rewards/blocks/{block_id} + let get_beacon_rewards_blocks = beacon_rewards_path + .clone() + .and(warp::path("blocks")) + .and(block_id_or_err) + .and(warp::path::end()) + .and_then(|chain: Arc>, block_id: BlockId| { + blocking_json_task(move || { + let (rewards, execution_optimistic) = + standard_block_rewards::compute_beacon_block_rewards(chain, block_id)?; + Ok(rewards) + .map(api_types::GenericResponse::from) + .map(|resp| resp.add_execution_optimistic(execution_optimistic)) + }) + }); + + /* + * beacon/rewards + */ + + let beacon_rewards_path = eth_v1 + .and(warp::path("beacon")) + .and(warp::path("rewards")) + .and(chain_filter.clone()); + + // POST beacon/rewards/attestations/{epoch} + let post_beacon_rewards_attestations = beacon_rewards_path + .clone() + .and(warp::path("attestations")) + .and(warp::path::param::()) + .and(warp::path::end()) + .and(warp::body::json()) + .and(log_filter.clone()) + .and_then( + |chain: Arc>, + epoch: Epoch, + validators: Vec, + log: Logger| { + blocking_json_task(move || { + let attestation_rewards = chain + .compute_attestation_rewards(epoch, validators, log) + .map_err(|e| match e { + BeaconChainError::MissingBeaconState(root) => { + warp_utils::reject::custom_not_found(format!( + "missing state {root:?}", + )) + } + BeaconChainError::NoStateForSlot(slot) => { + warp_utils::reject::custom_not_found(format!( + "missing state at slot {slot}" + )) + } + BeaconChainError::BeaconStateError( + BeaconStateError::UnknownValidator(validator_index), + ) => warp_utils::reject::custom_bad_request(format!( + "validator is unknown: {validator_index}" + )), + BeaconChainError::ValidatorPubkeyUnknown(pubkey) => { + warp_utils::reject::custom_bad_request(format!( + "validator pubkey is unknown: {pubkey:?}" + )) + } + e => warp_utils::reject::custom_server_error(format!( + "unexpected error: {:?}", + e + )), + })?; + let execution_optimistic = + chain.is_optimistic_or_invalid_head().unwrap_or_default(); + + Ok(attestation_rewards) + .map(api_types::GenericResponse::from) + .map(|resp| resp.add_execution_optimistic(execution_optimistic)) + }) + }, + ); + + // POST beacon/rewards/sync_committee/{block_id} + let post_beacon_rewards_sync_committee = beacon_rewards_path + .clone() + .and(warp::path("sync_committee")) + .and(block_id_or_err) + .and(warp::path::end()) + .and(warp::body::json()) + .and(log_filter.clone()) + .and_then( + |chain: Arc>, + block_id: BlockId, + validators: Vec, + log: Logger| { + blocking_json_task(move || { + let (rewards, execution_optimistic) = + sync_committee_rewards::compute_sync_committee_rewards( + chain, block_id, validators, log, + )?; + + Ok(rewards) + .map(api_types::GenericResponse::from) + .map(|resp| resp.add_execution_optimistic(execution_optimistic)) + }) + }, + ); + /* * config */ @@ -2745,7 +2855,7 @@ pub fn serve( .await .map(|resp| warp::reply::json(&resp)) .map_err(|e| { - error!( + warn!( log, "Relay error when registering validator(s)"; "num_registrations" => filtered_registration_data.len(), @@ -2840,7 +2950,7 @@ pub fn serve( let is_live = chain.validator_seen_at_epoch(index as usize, request_data.epoch); api_types::LivenessResponseData { - index: index as u64, + index, epoch: request_data.epoch, is_live, } @@ -2876,7 +2986,7 @@ pub fn serve( .and_then( |sysinfo, app_start: std::time::Instant, data_dir, network_globals| { blocking_json_task(move || { - let app_uptime = app_start.elapsed().as_secs() as u64; + let app_uptime = app_start.elapsed().as_secs(); Ok(api_types::GenericResponse::from(observe_system_health_bn( sysinfo, data_dir, @@ -3345,6 +3455,7 @@ pub fn serve( .or(get_beacon_pool_proposer_slashings.boxed()) .or(get_beacon_pool_voluntary_exits.boxed()) .or(get_beacon_deposit_snapshot.boxed()) + .or(get_beacon_rewards_blocks.boxed()) .or(get_config_fork_schedule.boxed()) .or(get_config_spec.boxed()) .or(get_config_deposit_contract.boxed()) @@ -3383,7 +3494,8 @@ pub fn serve( .or(get_lighthouse_attestation_performance.boxed()) .or(get_lighthouse_block_packing_efficiency.boxed()) .or(get_lighthouse_merge_readiness.boxed()) - .or(get_events.boxed()), + .or(get_events.boxed()) + .recover(warp_utils::reject::handle_rejection), ) .boxed() .or(warp::post().and( @@ -3395,6 +3507,8 @@ pub fn serve( .or(post_beacon_pool_proposer_slashings.boxed()) .or(post_beacon_pool_voluntary_exits.boxed()) .or(post_beacon_pool_sync_committees.boxed()) + .or(post_beacon_rewards_attestations.boxed()) + .or(post_beacon_rewards_sync_committee.boxed()) .or(post_validator_duties_attester.boxed()) .or(post_validator_duties_sync.boxed()) .or(post_validator_aggregate_and_proofs.boxed()) @@ -3407,7 +3521,8 @@ pub fn serve( .or(post_lighthouse_database_reconstruct.boxed()) .or(post_lighthouse_database_historical_blocks.boxed()) .or(post_lighthouse_block_rewards.boxed()) - .or(post_lighthouse_ui_validator_metrics.boxed()), + .or(post_lighthouse_ui_validator_metrics.boxed()) + .recover(warp_utils::reject::handle_rejection), )) .recover(warp_utils::reject::handle_rejection) .with(slog_logging(log.clone())) diff --git a/beacon_node/http_api/src/standard_block_rewards.rs b/beacon_node/http_api/src/standard_block_rewards.rs new file mode 100644 index 0000000000..b3c90d08a4 --- /dev/null +++ b/beacon_node/http_api/src/standard_block_rewards.rs @@ -0,0 +1,27 @@ +use crate::sync_committee_rewards::get_state_before_applying_block; +use crate::BlockId; +use crate::ExecutionOptimistic; +use beacon_chain::{BeaconChain, BeaconChainTypes}; +use eth2::lighthouse::StandardBlockReward; +use std::sync::Arc; +use warp_utils::reject::beacon_chain_error; +//// The difference between block_rewards and beacon_block_rewards is the later returns block +//// reward format that satisfies beacon-api specs +pub fn compute_beacon_block_rewards( + chain: Arc>, + block_id: BlockId, +) -> Result<(StandardBlockReward, ExecutionOptimistic), warp::Rejection> { + let (block, execution_optimistic) = block_id.blinded_block(&chain)?; + + let block_ref = block.message(); + + let block_root = block.canonical_root(); + + let mut state = get_state_before_applying_block(chain.clone(), &block)?; + + let rewards = chain + .compute_beacon_block_reward(block_ref, block_root, &mut state) + .map_err(beacon_chain_error)?; + + Ok((rewards, execution_optimistic)) +} diff --git a/beacon_node/http_api/src/sync_committee_rewards.rs b/beacon_node/http_api/src/sync_committee_rewards.rs new file mode 100644 index 0000000000..cefa98db41 --- /dev/null +++ b/beacon_node/http_api/src/sync_committee_rewards.rs @@ -0,0 +1,77 @@ +use crate::{BlockId, ExecutionOptimistic}; +use beacon_chain::{BeaconChain, BeaconChainError, BeaconChainTypes}; +use eth2::lighthouse::SyncCommitteeReward; +use eth2::types::ValidatorId; +use slog::{debug, Logger}; +use state_processing::BlockReplayer; +use std::sync::Arc; +use types::{BeaconState, SignedBlindedBeaconBlock}; +use warp_utils::reject::{beacon_chain_error, custom_not_found}; + +pub fn compute_sync_committee_rewards( + chain: Arc>, + block_id: BlockId, + validators: Vec, + log: Logger, +) -> Result<(Option>, ExecutionOptimistic), warp::Rejection> { + let (block, execution_optimistic) = block_id.blinded_block(&chain)?; + + let mut state = get_state_before_applying_block(chain.clone(), &block)?; + + let reward_payload = chain + .compute_sync_committee_rewards(block.message(), &mut state) + .map_err(beacon_chain_error)?; + + let data = if reward_payload.is_empty() { + debug!(log, "compute_sync_committee_rewards returned empty"); + None + } else if validators.is_empty() { + Some(reward_payload) + } else { + Some( + reward_payload + .into_iter() + .filter(|reward| { + validators.iter().any(|validator| match validator { + ValidatorId::Index(i) => reward.validator_index == *i, + ValidatorId::PublicKey(pubkey) => match state.get_validator_index(pubkey) { + Ok(Some(i)) => reward.validator_index == i as u64, + _ => false, + }, + }) + }) + .collect::>(), + ) + }; + + Ok((data, execution_optimistic)) +} + +pub fn get_state_before_applying_block( + chain: Arc>, + block: &SignedBlindedBeaconBlock, +) -> Result, warp::reject::Rejection> { + let parent_block: SignedBlindedBeaconBlock = chain + .get_blinded_block(&block.parent_root()) + .and_then(|maybe_block| { + maybe_block.ok_or_else(|| BeaconChainError::MissingBeaconBlock(block.parent_root())) + }) + .map_err(|e| custom_not_found(format!("Parent block is not available! {:?}", e)))?; + + let parent_state = chain + .get_state(&parent_block.state_root(), Some(parent_block.slot())) + .and_then(|maybe_state| { + maybe_state + .ok_or_else(|| BeaconChainError::MissingBeaconState(parent_block.state_root())) + }) + .map_err(|e| custom_not_found(format!("Parent state is not available! {:?}", e)))?; + + let replayer = BlockReplayer::new(parent_state, &chain.spec) + .no_signature_verification() + .state_root_iter([Ok((parent_block.state_root(), parent_block.slot()))].into_iter()) + .minimal_block_root_verification() + .apply_blocks(vec![], Some(block.slot())) + .map_err(beacon_chain_error)?; + + Ok(replayer.into_state()) +} diff --git a/beacon_node/http_api/tests/common.rs b/beacon_node/http_api/tests/common.rs index 9d6ad4050b..7c228d9803 100644 --- a/beacon_node/http_api/tests/common.rs +++ b/beacon_node/http_api/tests/common.rs @@ -7,7 +7,13 @@ use eth2::{BeaconNodeHttpClient, Timeouts}; use http_api::{Config, Context}; use lighthouse_network::{ discv5::enr::{CombinedKey, EnrBuilder}, - libp2p::{core::connection::ConnectionId, swarm::NetworkBehaviour}, + libp2p::{ + core::connection::ConnectionId, + swarm::{ + behaviour::{ConnectionEstablished, FromSwarm}, + NetworkBehaviour, + }, + }, rpc::methods::{MetaData, MetaDataV2}, types::{EnrAttestationBitfield, EnrSyncCommitteeBitfield, SyncState}, ConnectedPoint, Enr, NetworkGlobals, PeerId, PeerManager, @@ -143,12 +149,18 @@ pub async fn create_api_server_on_port( // add a peer let peer_id = PeerId::random(); - let connected_point = ConnectedPoint::Listener { + let endpoint = &ConnectedPoint::Listener { local_addr: EXTERNAL_ADDR.parse().unwrap(), send_back_addr: EXTERNAL_ADDR.parse().unwrap(), }; - let con_id = ConnectionId::new(1); - pm.inject_connection_established(&peer_id, &con_id, &connected_point, None, 0); + let connection_id = ConnectionId::new(1); + pm.on_swarm_event(FromSwarm::ConnectionEstablished(ConnectionEstablished { + peer_id, + connection_id, + endpoint, + failed_addresses: &[], + other_established: 0, + })); *network_globals.sync_state.write() = SyncState::Synced; let eth1_service = diff --git a/beacon_node/lighthouse_network/Cargo.toml b/beacon_node/lighthouse_network/Cargo.toml index 977f0a1088..474ebebb50 100644 --- a/beacon_node/lighthouse_network/Cargo.toml +++ b/beacon_node/lighthouse_network/Cargo.toml @@ -40,11 +40,12 @@ superstruct = "0.5.0" prometheus-client = "0.18.0" unused_port = { path = "../../common/unused_port" } delay_map = "0.1.1" +void = "1" [dependencies.libp2p] -version = "0.48.0" +version = "0.50.0" default-features = false -features = ["websocket", "identify", "mplex", "yamux", "noise", "gossipsub", "dns-tokio", "tcp-tokio", "plaintext", "secp256k1"] +features = ["websocket", "identify", "mplex", "yamux", "noise", "gossipsub", "dns", "tcp", "tokio", "plaintext", "secp256k1", "macros", "ecdsa"] [dev-dependencies] slog-term = "2.6.0" diff --git a/beacon_node/lighthouse_network/src/config.rs b/beacon_node/lighthouse_network/src/config.rs index 0ae3d9a23b..009aab8e3c 100644 --- a/beacon_node/lighthouse_network/src/config.rs +++ b/beacon_node/lighthouse_network/src/config.rs @@ -1,3 +1,4 @@ +use crate::rpc::config::OutboundRateLimiterConfig; use crate::types::GossipKind; use crate::{Enr, PeerIdSerialized}; use directory::{ @@ -133,6 +134,9 @@ pub struct Config { /// Whether light client protocols should be enabled. pub enable_light_client_server: bool, + + /// Configuration for the outbound rate limiter (requests made by this node). + pub outbound_rate_limiter_config: Option, } impl Default for Config { @@ -211,6 +215,7 @@ impl Default for Config { topics: Vec::new(), metrics_enabled: false, enable_light_client_server: false, + outbound_rate_limiter_config: None, } } } diff --git a/beacon_node/lighthouse_network/src/discovery/mod.rs b/beacon_node/lighthouse_network/src/discovery/mod.rs index 8e528f09d2..c41844c2c5 100644 --- a/beacon_node/lighthouse_network/src/discovery/mod.rs +++ b/beacon_node/lighthouse_network/src/discovery/mod.rs @@ -22,12 +22,13 @@ use enr::{ATTESTATION_BITFIELD_ENR_KEY, ETH2_ENR_KEY, SYNC_COMMITTEE_BITFIELD_EN use futures::prelude::*; use futures::stream::FuturesUnordered; use libp2p::multiaddr::Protocol; +use libp2p::swarm::behaviour::{DialFailure, FromSwarm}; use libp2p::swarm::AddressScore; pub use libp2p::{ core::{connection::ConnectionId, ConnectedPoint, Multiaddr, PeerId}, swarm::{ - handler::ConnectionHandler, DialError, NetworkBehaviour, - NetworkBehaviourAction as NBAction, NotifyHandler, PollParameters, SubstreamProtocol, + dummy::ConnectionHandler, DialError, NetworkBehaviour, NetworkBehaviourAction as NBAction, + NotifyHandler, PollParameters, SubstreamProtocol, }, }; use lru::LruCache; @@ -927,11 +928,11 @@ impl Discovery { impl NetworkBehaviour for Discovery { // Discovery is not a real NetworkBehaviour... - type ConnectionHandler = libp2p::swarm::handler::DummyConnectionHandler; + type ConnectionHandler = ConnectionHandler; type OutEvent = DiscoveredPeers; fn new_handler(&mut self) -> Self::ConnectionHandler { - libp2p::swarm::handler::DummyConnectionHandler::default() + ConnectionHandler } // Handles the libp2p request to obtain multiaddrs for peer_id's in order to dial them. @@ -947,40 +948,6 @@ impl NetworkBehaviour for Discovery { } } - fn inject_event( - &mut self, - _: PeerId, - _: ConnectionId, - _: ::OutEvent, - ) { - } - - fn inject_dial_failure( - &mut self, - peer_id: Option, - _handler: Self::ConnectionHandler, - error: &DialError, - ) { - if let Some(peer_id) = peer_id { - match error { - DialError::Banned - | DialError::LocalPeerId - | DialError::InvalidPeerId(_) - | DialError::ConnectionIo(_) - | DialError::NoAddresses - | DialError::Transport(_) - | DialError::WrongPeerId { .. } => { - // set peer as disconnected in discovery DHT - debug!(self.log, "Marking peer disconnected in DHT"; "peer_id" => %peer_id); - self.disconnect_peer(&peer_id); - } - DialError::ConnectionLimit(_) - | DialError::DialPeerConditionFalse(_) - | DialError::Aborted => {} - } - } - } - // Main execution loop to drive the behaviour fn poll( &mut self, @@ -1067,6 +1034,50 @@ impl NetworkBehaviour for Discovery { } Poll::Pending } + + fn on_swarm_event(&mut self, event: FromSwarm) { + match event { + FromSwarm::DialFailure(DialFailure { peer_id, error, .. }) => { + self.on_dial_failure(peer_id, error) + } + FromSwarm::ConnectionEstablished(_) + | FromSwarm::ConnectionClosed(_) + | FromSwarm::AddressChange(_) + | FromSwarm::ListenFailure(_) + | FromSwarm::NewListener(_) + | FromSwarm::NewListenAddr(_) + | FromSwarm::ExpiredListenAddr(_) + | FromSwarm::ListenerError(_) + | FromSwarm::ListenerClosed(_) + | FromSwarm::NewExternalAddr(_) + | FromSwarm::ExpiredExternalAddr(_) => { + // Ignore events not relevant to discovery + } + } + } +} + +impl Discovery { + fn on_dial_failure(&mut self, peer_id: Option, error: &DialError) { + if let Some(peer_id) = peer_id { + match error { + DialError::Banned + | DialError::LocalPeerId + | DialError::InvalidPeerId(_) + | DialError::ConnectionIo(_) + | DialError::NoAddresses + | DialError::Transport(_) + | DialError::WrongPeerId { .. } => { + // set peer as disconnected in discovery DHT + debug!(self.log, "Marking peer disconnected in DHT"; "peer_id" => %peer_id); + self.disconnect_peer(&peer_id); + } + DialError::ConnectionLimit(_) + | DialError::DialPeerConditionFalse(_) + | DialError::Aborted => {} + } + } + } } #[cfg(test)] diff --git a/beacon_node/lighthouse_network/src/peer_manager/mod.rs b/beacon_node/lighthouse_network/src/peer_manager/mod.rs index a468239a9e..89670a2eb3 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/mod.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/mod.rs @@ -7,7 +7,7 @@ use crate::{NetworkGlobals, PeerId}; use crate::{Subnet, SubnetDiscovery}; use delay_map::HashSetDelay; use discv5::Enr; -use libp2p::identify::IdentifyInfo; +use libp2p::identify::Info as IdentifyInfo; use peerdb::{client::ClientKind, BanOperation, BanResult, ScoreUpdateResult}; use rand::seq::SliceRandom; use slog::{debug, error, trace, warn}; diff --git a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs index 175dfaf018..42eb270c40 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs @@ -1,14 +1,12 @@ use std::task::{Context, Poll}; use futures::StreamExt; -use libp2p::core::connection::ConnectionId; use libp2p::core::ConnectedPoint; +use libp2p::swarm::behaviour::{ConnectionClosed, ConnectionEstablished, DialFailure, FromSwarm}; use libp2p::swarm::dial_opts::{DialOpts, PeerCondition}; -use libp2p::swarm::handler::DummyConnectionHandler; -use libp2p::swarm::{ - ConnectionHandler, DialError, NetworkBehaviour, NetworkBehaviourAction, PollParameters, -}; -use libp2p::{Multiaddr, PeerId}; +use libp2p::swarm::dummy::ConnectionHandler; +use libp2p::swarm::{NetworkBehaviour, NetworkBehaviourAction, PollParameters}; +use libp2p::PeerId; use slog::{debug, error}; use types::EthSpec; @@ -20,23 +18,14 @@ use super::peerdb::BanResult; use super::{ConnectingType, PeerManager, PeerManagerEvent, ReportSource}; impl NetworkBehaviour for PeerManager { - type ConnectionHandler = DummyConnectionHandler; + type ConnectionHandler = ConnectionHandler; type OutEvent = PeerManagerEvent; /* Required trait members */ fn new_handler(&mut self) -> Self::ConnectionHandler { - DummyConnectionHandler::default() - } - - fn inject_event( - &mut self, - _: PeerId, - _: ConnectionId, - _: ::OutEvent, - ) { - unreachable!("Dummy handler does not emit events") + ConnectionHandler } fn poll( @@ -114,19 +103,46 @@ impl NetworkBehaviour for PeerManager { Poll::Pending } - /* Overwritten trait members */ + fn on_swarm_event(&mut self, event: FromSwarm) { + match event { + FromSwarm::ConnectionEstablished(ConnectionEstablished { + peer_id, + endpoint, + other_established, + .. + }) => self.on_connection_established(peer_id, endpoint, other_established), + FromSwarm::ConnectionClosed(ConnectionClosed { + peer_id, + remaining_established, + .. + }) => self.on_connection_closed(peer_id, remaining_established), + FromSwarm::DialFailure(DialFailure { peer_id, .. }) => self.on_dial_failure(peer_id), + FromSwarm::AddressChange(_) + | FromSwarm::ListenFailure(_) + | FromSwarm::NewListener(_) + | FromSwarm::NewListenAddr(_) + | FromSwarm::ExpiredListenAddr(_) + | FromSwarm::ListenerError(_) + | FromSwarm::ListenerClosed(_) + | FromSwarm::NewExternalAddr(_) + | FromSwarm::ExpiredExternalAddr(_) => { + // The rest of the events we ignore since they are handled in their associated + // `SwarmEvent` + } + } + } +} - fn inject_connection_established( +impl PeerManager { + fn on_connection_established( &mut self, - peer_id: &PeerId, - _connection_id: &ConnectionId, + peer_id: PeerId, endpoint: &ConnectedPoint, - _failed_addresses: Option<&Vec>, other_established: usize, ) { debug!(self.log, "Connection established"; "peer_id" => %peer_id, "connection" => ?endpoint.to_endpoint()); if other_established == 0 { - self.events.push(PeerManagerEvent::MetaData(*peer_id)); + self.events.push(PeerManagerEvent::MetaData(peer_id)); } // Check NAT if metrics are enabled @@ -135,20 +151,20 @@ impl NetworkBehaviour for PeerManager { } // Check to make sure the peer is not supposed to be banned - match self.ban_status(peer_id) { + match self.ban_status(&peer_id) { // TODO: directly emit the ban event? BanResult::BadScore => { // This is a faulty state error!(self.log, "Connected to a banned peer. Re-banning"; "peer_id" => %peer_id); // Reban the peer - self.goodbye_peer(peer_id, GoodbyeReason::Banned, ReportSource::PeerManager); + self.goodbye_peer(&peer_id, GoodbyeReason::Banned, ReportSource::PeerManager); return; } BanResult::BannedIp(ip_addr) => { // A good peer has connected to us via a banned IP address. We ban the peer and // prevent future connections. debug!(self.log, "Peer connected via banned IP. Banning"; "peer_id" => %peer_id, "banned_ip" => %ip_addr); - self.goodbye_peer(peer_id, GoodbyeReason::BannedIP, ReportSource::PeerManager); + self.goodbye_peer(&peer_id, GoodbyeReason::BannedIP, ReportSource::PeerManager); return; } BanResult::NotBanned => {} @@ -162,11 +178,11 @@ impl NetworkBehaviour for PeerManager { .network_globals .peers .read() - .peer_info(peer_id) + .peer_info(&peer_id) .map_or(true, |peer| !peer.has_future_duty()) { // Gracefully disconnect the peer. - self.disconnect_peer(*peer_id, GoodbyeReason::TooManyPeers); + self.disconnect_peer(peer_id, GoodbyeReason::TooManyPeers); return; } @@ -174,14 +190,14 @@ impl NetworkBehaviour for PeerManager { // does not need to know about these peers. match endpoint { ConnectedPoint::Listener { send_back_addr, .. } => { - self.inject_connect_ingoing(peer_id, send_back_addr.clone(), None); + self.inject_connect_ingoing(&peer_id, send_back_addr.clone(), None); self.events - .push(PeerManagerEvent::PeerConnectedIncoming(*peer_id)); + .push(PeerManagerEvent::PeerConnectedIncoming(peer_id)); } ConnectedPoint::Dialer { address, .. } => { - self.inject_connect_outgoing(peer_id, address.clone(), None); + self.inject_connect_outgoing(&peer_id, address.clone(), None); self.events - .push(PeerManagerEvent::PeerConnectedOutgoing(*peer_id)); + .push(PeerManagerEvent::PeerConnectedOutgoing(peer_id)); } } @@ -189,14 +205,8 @@ impl NetworkBehaviour for PeerManager { self.update_connected_peer_metrics(); metrics::inc_counter(&metrics::PEER_CONNECT_EVENT_COUNT); } - fn inject_connection_closed( - &mut self, - peer_id: &PeerId, - _: &ConnectionId, - _: &ConnectedPoint, - _: DummyConnectionHandler, - remaining_established: usize, - ) { + + fn on_connection_closed(&mut self, peer_id: PeerId, remaining_established: usize) { if remaining_established > 0 { return; } @@ -206,62 +216,33 @@ impl NetworkBehaviour for PeerManager { .network_globals .peers .read() - .is_connected_or_disconnecting(peer_id) + .is_connected_or_disconnecting(&peer_id) { // We are disconnecting the peer or the peer has already been connected. // Both these cases, the peer has been previously registered by the peer manager and // potentially the application layer. // Inform the application. self.events - .push(PeerManagerEvent::PeerDisconnected(*peer_id)); + .push(PeerManagerEvent::PeerDisconnected(peer_id)); debug!(self.log, "Peer disconnected"; "peer_id" => %peer_id); } // NOTE: It may be the case that a rejected node, due to too many peers is disconnected // here and the peer manager has no knowledge of its connection. We insert it here for // reference so that peer manager can track this peer. - self.inject_disconnect(peer_id); + self.inject_disconnect(&peer_id); // Update the prometheus metrics self.update_connected_peer_metrics(); metrics::inc_counter(&metrics::PEER_DISCONNECT_EVENT_COUNT); } - fn inject_address_change( - &mut self, - _peer_id: &PeerId, - _connection_id: &ConnectionId, - old: &ConnectedPoint, - new: &ConnectedPoint, - ) { - debug_assert!( - matches!( - (old, new), - ( - // inbound remains inbound - ConnectedPoint::Listener { .. }, - ConnectedPoint::Listener { .. } - ) | ( - // outbound remains outbound - ConnectedPoint::Dialer { .. }, - ConnectedPoint::Dialer { .. } - ) - ), - "A peer has changed between inbound and outbound" - ) - } - /// A dial attempt has failed. /// /// NOTE: It can be the case that we are dialing a peer and during the dialing process the peer /// connects and the dial attempt later fails. To handle this, we only update the peer_db if /// the peer is not already connected. - fn inject_dial_failure( - &mut self, - peer_id: Option, - _handler: DummyConnectionHandler, - _error: &DialError, - ) { + fn on_dial_failure(&mut self, peer_id: Option) { if let Some(peer_id) = peer_id { if !self.network_globals.peers.read().is_connected(&peer_id) { self.inject_disconnect(&peer_id); diff --git a/beacon_node/lighthouse_network/src/peer_manager/peerdb/client.rs b/beacon_node/lighthouse_network/src/peer_manager/peerdb/client.rs index dcc121b7f4..1178dbcb9c 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/peerdb/client.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/peerdb/client.rs @@ -2,7 +2,7 @@ //! //! Currently using identify to fingerprint. -use libp2p::identify::IdentifyInfo; +use libp2p::identify::Info as IdentifyInfo; use serde::Serialize; use strum::{AsRefStr, EnumIter, IntoStaticStr}; diff --git a/beacon_node/lighthouse_network/src/peer_manager/peerdb/score.rs b/beacon_node/lighthouse_network/src/peer_manager/peerdb/score.rs index fca665db98..bafa355d68 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/peerdb/score.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/peerdb/score.rs @@ -186,14 +186,7 @@ impl RealScore { /// Add an f64 to the score abiding by the limits. fn add(&mut self, score: f64) { - let mut new_score = self.lighthouse_score + score; - if new_score > MAX_SCORE { - new_score = MAX_SCORE; - } - if new_score < MIN_SCORE { - new_score = MIN_SCORE; - } - + let new_score = (self.lighthouse_score + score).clamp(MIN_SCORE, MAX_SCORE); self.set_lighthouse_score(new_score); } diff --git a/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs b/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs index a4dd602b3f..eccbf0dd62 100644 --- a/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs +++ b/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs @@ -443,7 +443,7 @@ fn handle_length( // Note: length-prefix of > 10 bytes(uint64) would be a decoding error match uvi_codec.decode(bytes).map_err(RPCError::from)? { Some(length) => { - *len = Some(length as usize); + *len = Some(length); Ok(Some(length)) } None => Ok(None), // need more bytes to decode length diff --git a/beacon_node/lighthouse_network/src/rpc/config.rs b/beacon_node/lighthouse_network/src/rpc/config.rs new file mode 100644 index 0000000000..bea0929fb0 --- /dev/null +++ b/beacon_node/lighthouse_network/src/rpc/config.rs @@ -0,0 +1,173 @@ +use std::{ + fmt::{Debug, Display}, + str::FromStr, + time::Duration, +}; + +use super::{methods, rate_limiter::Quota, Protocol}; + +use serde_derive::{Deserialize, Serialize}; + +/// Auxiliary struct to aid on configuration parsing. +/// +/// A protocol's quota is specified as `protocol_name:tokens/time_in_seconds`. +#[derive(Debug, PartialEq, Eq)] +struct ProtocolQuota { + protocol: Protocol, + quota: Quota, +} + +impl Display for ProtocolQuota { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}:{}/{}", + self.protocol.as_ref(), + self.quota.max_tokens, + self.quota.replenish_all_every.as_secs() + ) + } +} + +impl FromStr for ProtocolQuota { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + let (protocol_str, quota_str) = s + .split_once(':') + .ok_or("Missing ':' from quota definition.")?; + let protocol = protocol_str + .parse() + .map_err(|_parse_err| "Wrong protocol representation in quota")?; + let (tokens_str, time_str) = quota_str + .split_once('/') + .ok_or("Quota should be defined as \"n/t\" (t in seconds). Missing '/' from quota.")?; + let tokens = tokens_str + .parse() + .map_err(|_| "Failed to parse tokens from quota.")?; + let seconds = time_str + .parse::() + .map_err(|_| "Failed to parse time in seconds from quota.")?; + Ok(ProtocolQuota { + protocol, + quota: Quota { + replenish_all_every: Duration::from_secs(seconds), + max_tokens: tokens, + }, + }) + } +} + +/// Configurations for the rate limiter applied to outbound requests (made by the node itself). +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct OutboundRateLimiterConfig { + pub(super) ping_quota: Quota, + pub(super) meta_data_quota: Quota, + pub(super) status_quota: Quota, + pub(super) goodbye_quota: Quota, + pub(super) blocks_by_range_quota: Quota, + pub(super) blocks_by_root_quota: Quota, +} + +impl OutboundRateLimiterConfig { + pub const DEFAULT_PING_QUOTA: Quota = Quota::n_every(2, 10); + pub const DEFAULT_META_DATA_QUOTA: Quota = Quota::n_every(2, 5); + pub const DEFAULT_STATUS_QUOTA: Quota = Quota::n_every(5, 15); + pub const DEFAULT_GOODBYE_QUOTA: Quota = Quota::one_every(10); + pub const DEFAULT_BLOCKS_BY_RANGE_QUOTA: Quota = + Quota::n_every(methods::MAX_REQUEST_BLOCKS, 10); + pub const DEFAULT_BLOCKS_BY_ROOT_QUOTA: Quota = Quota::n_every(128, 10); +} + +impl Default for OutboundRateLimiterConfig { + fn default() -> Self { + OutboundRateLimiterConfig { + ping_quota: Self::DEFAULT_PING_QUOTA, + meta_data_quota: Self::DEFAULT_META_DATA_QUOTA, + status_quota: Self::DEFAULT_STATUS_QUOTA, + goodbye_quota: Self::DEFAULT_GOODBYE_QUOTA, + blocks_by_range_quota: Self::DEFAULT_BLOCKS_BY_RANGE_QUOTA, + blocks_by_root_quota: Self::DEFAULT_BLOCKS_BY_ROOT_QUOTA, + } + } +} + +impl Debug for OutboundRateLimiterConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + macro_rules! fmt_q { + ($quota:expr) => { + &format_args!( + "{}/{}s", + $quota.max_tokens, + $quota.replenish_all_every.as_secs() + ) + }; + } + + f.debug_struct("OutboundRateLimiterConfig") + .field("ping", fmt_q!(&self.ping_quota)) + .field("metadata", fmt_q!(&self.meta_data_quota)) + .field("status", fmt_q!(&self.status_quota)) + .field("goodbye", fmt_q!(&self.goodbye_quota)) + .field("blocks_by_range", fmt_q!(&self.blocks_by_range_quota)) + .field("blocks_by_root", fmt_q!(&self.blocks_by_root_quota)) + .finish() + } +} + +/// Parse configurations for the outbound rate limiter. Protocols that are not specified use +/// the default values. Protocol specified more than once use only the first given Quota. +/// +/// The expected format is a ';' separated list of [`ProtocolQuota`]. +impl FromStr for OutboundRateLimiterConfig { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + let mut ping_quota = None; + let mut meta_data_quota = None; + let mut status_quota = None; + let mut goodbye_quota = None; + let mut blocks_by_range_quota = None; + let mut blocks_by_root_quota = None; + for proto_def in s.split(';') { + let ProtocolQuota { protocol, quota } = proto_def.parse()?; + let quota = Some(quota); + match protocol { + Protocol::Status => status_quota = status_quota.or(quota), + Protocol::Goodbye => goodbye_quota = goodbye_quota.or(quota), + Protocol::BlocksByRange => blocks_by_range_quota = blocks_by_range_quota.or(quota), + Protocol::BlocksByRoot => blocks_by_root_quota = blocks_by_root_quota.or(quota), + Protocol::Ping => ping_quota = ping_quota.or(quota), + Protocol::MetaData => meta_data_quota = meta_data_quota.or(quota), + Protocol::LightClientBootstrap => return Err("Lighthouse does not send LightClientBootstrap requests. Quota should not be set."), + } + } + Ok(OutboundRateLimiterConfig { + ping_quota: ping_quota.unwrap_or(Self::DEFAULT_PING_QUOTA), + meta_data_quota: meta_data_quota.unwrap_or(Self::DEFAULT_META_DATA_QUOTA), + status_quota: status_quota.unwrap_or(Self::DEFAULT_STATUS_QUOTA), + goodbye_quota: goodbye_quota.unwrap_or(Self::DEFAULT_GOODBYE_QUOTA), + blocks_by_range_quota: blocks_by_range_quota + .unwrap_or(Self::DEFAULT_BLOCKS_BY_RANGE_QUOTA), + blocks_by_root_quota: blocks_by_root_quota + .unwrap_or(Self::DEFAULT_BLOCKS_BY_ROOT_QUOTA), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_quota_inverse() { + let quota = ProtocolQuota { + protocol: Protocol::Goodbye, + quota: Quota { + replenish_all_every: Duration::from_secs(10), + max_tokens: 8, + }, + }; + assert_eq!(quota.to_string().parse(), Ok(quota)) + } +} diff --git a/beacon_node/lighthouse_network/src/rpc/handler.rs b/beacon_node/lighthouse_network/src/rpc/handler.rs index 9d6229eb38..a1743c15fb 100644 --- a/beacon_node/lighthouse_network/src/rpc/handler.rs +++ b/beacon_node/lighthouse_network/src/rpc/handler.rs @@ -327,61 +327,6 @@ where self.listen_protocol.clone() } - fn inject_fully_negotiated_inbound( - &mut self, - substream: >::Output, - _info: Self::InboundOpenInfo, - ) { - // only accept new peer requests when active - if !matches!(self.state, HandlerState::Active) { - return; - } - - let (req, substream) = substream; - let expected_responses = req.expected_responses(); - - // store requests that expect responses - if expected_responses > 0 { - if self.inbound_substreams.len() < MAX_INBOUND_SUBSTREAMS { - // Store the stream and tag the output. - let delay_key = self.inbound_substreams_delay.insert( - self.current_inbound_substream_id, - Duration::from_secs(RESPONSE_TIMEOUT), - ); - let awaiting_stream = InboundState::Idle(substream); - self.inbound_substreams.insert( - self.current_inbound_substream_id, - InboundInfo { - state: awaiting_stream, - pending_items: VecDeque::with_capacity(expected_responses as usize), - delay_key: Some(delay_key), - protocol: req.protocol(), - request_start_time: Instant::now(), - remaining_chunks: expected_responses, - }, - ); - } else { - self.events_out.push(Err(HandlerErr::Inbound { - id: self.current_inbound_substream_id, - proto: req.protocol(), - error: RPCError::HandlerRejected, - })); - return self.shutdown(None); - } - } - - // If we received a goodbye, shutdown the connection. - if let InboundRequest::Goodbye(_) = req { - self.shutdown(None); - } - - self.events_out.push(Ok(RPCReceived::Request( - self.current_inbound_substream_id, - req, - ))); - self.current_inbound_substream_id.0 += 1; - } - fn inject_fully_negotiated_outbound( &mut self, out: >::Output, @@ -438,6 +383,64 @@ where } } + fn inject_fully_negotiated_inbound( + &mut self, + substream: >::Output, + _info: Self::InboundOpenInfo, + ) { + // only accept new peer requests when active + if !matches!(self.state, HandlerState::Active) { + return; + } + + let (req, substream) = substream; + let expected_responses = req.expected_responses(); + + // store requests that expect responses + if expected_responses > 0 { + if self.inbound_substreams.len() < MAX_INBOUND_SUBSTREAMS { + // Store the stream and tag the output. + let delay_key = self.inbound_substreams_delay.insert( + self.current_inbound_substream_id, + Duration::from_secs(RESPONSE_TIMEOUT), + ); + let awaiting_stream = InboundState::Idle(substream); + self.inbound_substreams.insert( + self.current_inbound_substream_id, + InboundInfo { + state: awaiting_stream, + pending_items: VecDeque::with_capacity(std::cmp::min( + expected_responses, + 128, + ) as usize), + delay_key: Some(delay_key), + protocol: req.protocol(), + request_start_time: Instant::now(), + remaining_chunks: expected_responses, + }, + ); + } else { + self.events_out.push(Err(HandlerErr::Inbound { + id: self.current_inbound_substream_id, + proto: req.protocol(), + error: RPCError::HandlerRejected, + })); + return self.shutdown(None); + } + } + + // If we received a goodbye, shutdown the connection. + if let InboundRequest::Goodbye(_) = req { + self.shutdown(None); + } + + self.events_out.push(Ok(RPCReceived::Request( + self.current_inbound_substream_id, + req, + ))); + self.current_inbound_substream_id.0 += 1; + } + fn inject_event(&mut self, rpc_event: Self::InEvent) { match rpc_event { RPCSend::Request(id, req) => self.send_request(id, req), diff --git a/beacon_node/lighthouse_network/src/rpc/mod.rs b/beacon_node/lighthouse_network/src/rpc/mod.rs index 203a642a8b..31569b820b 100644 --- a/beacon_node/lighthouse_network/src/rpc/mod.rs +++ b/beacon_node/lighthouse_network/src/rpc/mod.rs @@ -12,7 +12,7 @@ use libp2p::swarm::{ PollParameters, SubstreamProtocol, }; use libp2p::PeerId; -use rate_limiter::{RPCRateLimiter as RateLimiter, RPCRateLimiterBuilder, RateLimitedErr}; +use rate_limiter::{RPCRateLimiter as RateLimiter, RateLimitedErr}; use slog::{crit, debug, o}; use std::marker::PhantomData; use std::sync::Arc; @@ -32,12 +32,17 @@ pub use methods::{ pub(crate) use outbound::OutboundRequest; pub use protocol::{max_rpc_size, Protocol, RPCError}; +use self::config::OutboundRateLimiterConfig; +use self::self_limiter::SelfRateLimiter; + pub(crate) mod codec; +pub mod config; mod handler; pub mod methods; mod outbound; mod protocol; mod rate_limiter; +mod self_limiter; /// Composite trait for a request id. pub trait ReqId: Send + 'static + std::fmt::Debug + Copy + Clone {} @@ -100,13 +105,18 @@ pub struct RPCMessage { pub event: HandlerEvent, } +type BehaviourAction = + NetworkBehaviourAction, RPCHandler>; + /// Implements the libp2p `NetworkBehaviour` trait and therefore manages network-level /// logic. pub struct RPC { /// Rate limiter limiter: RateLimiter, + /// Rate limiter for our own requests. + self_limiter: Option>, /// Queue of events to be processed. - events: Vec, RPCHandler>>, + events: Vec>, fork_context: Arc, enable_light_client_server: bool, /// Slog logger for RPC behaviour. @@ -117,10 +127,12 @@ impl RPC { pub fn new( fork_context: Arc, enable_light_client_server: bool, + outbound_rate_limiter_config: Option, log: slog::Logger, ) -> Self { let log = log.new(o!("service" => "libp2p_rpc")); - let limiter = RPCRateLimiterBuilder::new() + + let limiter = RateLimiter::builder() .n_every(Protocol::MetaData, 2, Duration::from_secs(5)) .n_every(Protocol::Ping, 2, Duration::from_secs(10)) .n_every(Protocol::Status, 5, Duration::from_secs(15)) @@ -134,8 +146,14 @@ impl RPC { .n_every(Protocol::BlocksByRoot, 128, Duration::from_secs(10)) .build() .expect("Configuration parameters are valid"); + + let self_limiter = outbound_rate_limiter_config.map(|config| { + SelfRateLimiter::new(config, log.clone()).expect("Configuration parameters are valid") + }); + RPC { limiter, + self_limiter, events: Vec::new(), fork_context, enable_light_client_server, @@ -162,12 +180,24 @@ impl RPC { /// Submits an RPC request. /// /// The peer must be connected for this to succeed. - pub fn send_request(&mut self, peer_id: PeerId, request_id: Id, event: OutboundRequest) { - self.events.push(NetworkBehaviourAction::NotifyHandler { - peer_id, - handler: NotifyHandler::Any, - event: RPCSend::Request(request_id, event), - }); + pub fn send_request(&mut self, peer_id: PeerId, request_id: Id, req: OutboundRequest) { + let event = if let Some(self_limiter) = self.self_limiter.as_mut() { + match self_limiter.allows(peer_id, request_id, req) { + Ok(event) => event, + Err(_e) => { + // Request is logged and queued internally in the self rate limiter. + return; + } + } + } else { + NetworkBehaviourAction::NotifyHandler { + peer_id, + handler: NotifyHandler::Any, + event: RPCSend::Request(request_id, req), + } + }; + + self.events.push(event); } /// Lighthouse wishes to disconnect from this peer by sending a Goodbye message. This @@ -272,11 +302,19 @@ where cx: &mut Context, _: &mut impl PollParameters, ) -> Poll> { - // let the rate limiter prune + // let the rate limiter prune. let _ = self.limiter.poll_unpin(cx); + + if let Some(self_limiter) = self.self_limiter.as_mut() { + if let Poll::Ready(event) = self_limiter.poll_ready(cx) { + self.events.push(event) + } + } + if !self.events.is_empty() { return Poll::Ready(self.events.remove(0)); } + Poll::Pending } } diff --git a/beacon_node/lighthouse_network/src/rpc/protocol.rs b/beacon_node/lighthouse_network/src/rpc/protocol.rs index 1f40f81971..e5d784d800 100644 --- a/beacon_node/lighthouse_network/src/rpc/protocol.rs +++ b/beacon_node/lighthouse_network/src/rpc/protocol.rs @@ -14,7 +14,7 @@ use std::io; use std::marker::PhantomData; use std::sync::Arc; use std::time::Duration; -use strum::IntoStaticStr; +use strum::{AsRefStr, Display, EnumString, IntoStaticStr}; use tokio_io_timeout::TimeoutStream; use tokio_util::{ codec::Framed, @@ -139,21 +139,26 @@ pub fn rpc_block_limits_by_fork(current_fork: ForkName) -> RpcLimits { } /// Protocol names to be used. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EnumString, AsRefStr, Display)] +#[strum(serialize_all = "snake_case")] pub enum Protocol { /// The Status protocol name. Status, /// The Goodbye protocol name. Goodbye, /// The `BlocksByRange` protocol name. + #[strum(serialize = "beacon_blocks_by_range")] BlocksByRange, /// The `BlocksByRoot` protocol name. + #[strum(serialize = "beacon_blocks_by_root")] BlocksByRoot, /// The `Ping` protocol name. Ping, /// The `MetaData` protocol name. + #[strum(serialize = "metadata")] MetaData, /// The `LightClientBootstrap` protocol name. + #[strum(serialize = "light_client_bootstrap")] LightClientBootstrap, } @@ -172,21 +177,6 @@ pub enum Encoding { SSZSnappy, } -impl std::fmt::Display for Protocol { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let repr = match self { - Protocol::Status => "status", - Protocol::Goodbye => "goodbye", - Protocol::BlocksByRange => "beacon_blocks_by_range", - Protocol::BlocksByRoot => "beacon_blocks_by_root", - Protocol::Ping => "ping", - Protocol::MetaData => "metadata", - Protocol::LightClientBootstrap => "light_client_bootstrap", - }; - f.write_str(repr) - } -} - impl std::fmt::Display for Encoding { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let repr = match self { diff --git a/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs b/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs index 6ba9f6e941..a1f7b89a2f 100644 --- a/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs +++ b/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs @@ -1,6 +1,7 @@ -use crate::rpc::{InboundRequest, Protocol}; +use crate::rpc::Protocol; use fnv::FnvHashMap; use libp2p::PeerId; +use serde_derive::{Deserialize, Serialize}; use std::convert::TryInto; use std::future::Future; use std::hash::Hash; @@ -47,12 +48,31 @@ type Nanosecs = u64; /// n*`replenish_all_every`/`max_tokens` units of time since their last request. /// /// To produce hard limits, set `max_tokens` to 1. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct Quota { /// How often are `max_tokens` fully replenished. - replenish_all_every: Duration, + pub(super) replenish_all_every: Duration, /// Token limit. This translates on how large can an instantaneous batch of /// tokens be. - max_tokens: u64, + pub(super) max_tokens: u64, +} + +impl Quota { + /// A hard limit of one token every `seconds`. + pub const fn one_every(seconds: u64) -> Self { + Quota { + replenish_all_every: Duration::from_secs(seconds), + max_tokens: 1, + } + } + + /// Allow `n` tokens to be use used every `seconds`. + pub const fn n_every(n: u64, seconds: u64) -> Self { + Quota { + replenish_all_every: Duration::from_secs(seconds), + max_tokens: n, + } + } } /// Manages rate limiting of requests per peer, with differentiated rates per protocol. @@ -78,6 +98,7 @@ pub struct RPCRateLimiter { } /// Error type for non conformant requests +#[derive(Debug)] pub enum RateLimitedErr { /// Required tokens for this request exceed the maximum TooLarge, @@ -86,7 +107,7 @@ pub enum RateLimitedErr { } /// User-friendly builder of a `RPCRateLimiter` -#[derive(Default)] +#[derive(Default, Clone)] pub struct RPCRateLimiterBuilder { /// Quota for the Goodbye protocol. goodbye_quota: Option, @@ -105,13 +126,8 @@ pub struct RPCRateLimiterBuilder { } impl RPCRateLimiterBuilder { - /// Get an empty `RPCRateLimiterBuilder`. - pub fn new() -> Self { - Default::default() - } - /// Set a quota for a protocol. - fn set_quota(mut self, protocol: Protocol, quota: Quota) -> Self { + pub fn set_quota(mut self, protocol: Protocol, quota: Quota) -> Self { let q = Some(quota); match protocol { Protocol::Ping => self.ping_quota = q, @@ -191,11 +207,40 @@ impl RPCRateLimiterBuilder { } } +pub trait RateLimiterItem { + fn protocol(&self) -> Protocol; + fn expected_responses(&self) -> u64; +} + +impl RateLimiterItem for super::InboundRequest { + fn protocol(&self) -> Protocol { + self.protocol() + } + + fn expected_responses(&self) -> u64 { + self.expected_responses() + } +} + +impl RateLimiterItem for super::OutboundRequest { + fn protocol(&self) -> Protocol { + self.protocol() + } + + fn expected_responses(&self) -> u64 { + self.expected_responses() + } +} impl RPCRateLimiter { - pub fn allows( + /// Get a builder instance. + pub fn builder() -> RPCRateLimiterBuilder { + RPCRateLimiterBuilder::default() + } + + pub fn allows( &mut self, peer_id: &PeerId, - request: &InboundRequest, + request: &Item, ) -> Result<(), RateLimitedErr> { let time_since_start = self.init_time.elapsed(); let tokens = request.expected_responses().max(1); diff --git a/beacon_node/lighthouse_network/src/rpc/self_limiter.rs b/beacon_node/lighthouse_network/src/rpc/self_limiter.rs new file mode 100644 index 0000000000..451c6206f3 --- /dev/null +++ b/beacon_node/lighthouse_network/src/rpc/self_limiter.rs @@ -0,0 +1,202 @@ +use std::{ + collections::{hash_map::Entry, HashMap, VecDeque}, + task::{Context, Poll}, + time::Duration, +}; + +use futures::FutureExt; +use libp2p::{swarm::NotifyHandler, PeerId}; +use slog::{crit, debug, Logger}; +use smallvec::SmallVec; +use tokio_util::time::DelayQueue; +use types::EthSpec; + +use super::{ + config::OutboundRateLimiterConfig, + rate_limiter::{RPCRateLimiter as RateLimiter, RateLimitedErr}, + BehaviourAction, OutboundRequest, Protocol, RPCSend, ReqId, +}; + +/// A request that was rate limited or waiting on rate limited requests for the same peer and +/// protocol. +struct QueuedRequest { + req: OutboundRequest, + request_id: Id, +} + +pub(crate) struct SelfRateLimiter { + /// Requests queued for sending per peer. This requests are stored when the self rate + /// limiter rejects them. Rate limiting is based on a Peer and Protocol basis, therefore + /// are stored in the same way. + delayed_requests: HashMap<(PeerId, Protocol), VecDeque>>, + /// The delay required to allow a peer's outbound request per protocol. + next_peer_request: DelayQueue<(PeerId, Protocol)>, + /// Rate limiter for our own requests. + limiter: RateLimiter, + /// Requests that are ready to be sent. + ready_requests: SmallVec<[BehaviourAction; 3]>, + /// Slog logger. + log: Logger, +} + +/// Error returned when the rate limiter does not accept a request. +// NOTE: this is currently not used, but might be useful for debugging. +pub enum Error { + /// There are queued requests for this same peer and protocol. + PendingRequests, + /// Request was tried but rate limited. + RateLimited, +} + +impl SelfRateLimiter { + /// Creates a new [`SelfRateLimiter`] based on configration values. + pub fn new(config: OutboundRateLimiterConfig, log: Logger) -> Result { + debug!(log, "Using self rate limiting params"; "config" => ?config); + // Destructure to make sure every configuration value is used. + let OutboundRateLimiterConfig { + ping_quota, + meta_data_quota, + status_quota, + goodbye_quota, + blocks_by_range_quota, + blocks_by_root_quota, + } = config; + + let limiter = RateLimiter::builder() + .set_quota(Protocol::Ping, ping_quota) + .set_quota(Protocol::MetaData, meta_data_quota) + .set_quota(Protocol::Status, status_quota) + .set_quota(Protocol::Goodbye, goodbye_quota) + .set_quota(Protocol::BlocksByRange, blocks_by_range_quota) + .set_quota(Protocol::BlocksByRoot, blocks_by_root_quota) + // Manually set the LightClientBootstrap quota, since we use the same rate limiter for + // inbound and outbound requests, and the LightClientBootstrap is an only inbound + // protocol. + .one_every(Protocol::LightClientBootstrap, Duration::from_secs(10)) + .build()?; + + Ok(SelfRateLimiter { + delayed_requests: Default::default(), + next_peer_request: Default::default(), + limiter, + ready_requests: Default::default(), + log, + }) + } + + /// Checks if the rate limiter allows the request. If it's allowed, returns the + /// [`NetworkBehaviourAction`] that should be emitted. When not allowed, the request is delayed + /// until it can be sent. + pub fn allows( + &mut self, + peer_id: PeerId, + request_id: Id, + req: OutboundRequest, + ) -> Result, Error> { + let protocol = req.protocol(); + // First check that there are not already other requests waiting to be sent. + if let Some(queued_requests) = self.delayed_requests.get_mut(&(peer_id, protocol)) { + queued_requests.push_back(QueuedRequest { req, request_id }); + + return Err(Error::PendingRequests); + } + match Self::try_send_request(&mut self.limiter, peer_id, request_id, req, &self.log) { + Err((rate_limited_req, wait_time)) => { + let key = (peer_id, protocol); + self.next_peer_request.insert(key, wait_time); + self.delayed_requests + .entry(key) + .or_default() + .push_back(rate_limited_req); + + Err(Error::RateLimited) + } + Ok(event) => Ok(event), + } + } + + /// Auxiliary function to deal with self rate limiting outcomes. If the rate limiter allows the + /// request, the [`NetworkBehaviourAction`] that should be emitted is returned. If the request + /// should be delayed, it's returned with the duration to wait. + fn try_send_request( + limiter: &mut RateLimiter, + peer_id: PeerId, + request_id: Id, + req: OutboundRequest, + log: &Logger, + ) -> Result, (QueuedRequest, Duration)> { + match limiter.allows(&peer_id, &req) { + Ok(()) => Ok(BehaviourAction::NotifyHandler { + peer_id, + handler: NotifyHandler::Any, + event: RPCSend::Request(request_id, req), + }), + Err(e) => { + let protocol = req.protocol(); + match e { + RateLimitedErr::TooLarge => { + // this should never happen with default parameters. Let's just send the request. + // Log a crit since this is a config issue. + crit!( + log, + "Self rate limiting error for a batch that will never fit. Sending request anyway. Check configuration parameters."; + "protocol" => %req.protocol() + ); + Ok(BehaviourAction::NotifyHandler { + peer_id, + handler: NotifyHandler::Any, + event: RPCSend::Request(request_id, req), + }) + } + RateLimitedErr::TooSoon(wait_time) => { + debug!(log, "Self rate limiting"; "protocol" => %protocol, "wait_time_ms" => wait_time.as_millis(), "peer_id" => %peer_id); + Err((QueuedRequest { req, request_id }, wait_time)) + } + } + } + } + } + + /// When a peer and protocol are allowed to send a next request, this function checks the + /// queued requests and attempts marking as ready as many as the limiter allows. + fn next_peer_request_ready(&mut self, peer_id: PeerId, protocol: Protocol) { + if let Entry::Occupied(mut entry) = self.delayed_requests.entry((peer_id, protocol)) { + let queued_requests = entry.get_mut(); + while let Some(QueuedRequest { req, request_id }) = queued_requests.pop_front() { + match Self::try_send_request(&mut self.limiter, peer_id, request_id, req, &self.log) + { + Err((rate_limited_req, wait_time)) => { + let key = (peer_id, protocol); + self.next_peer_request.insert(key, wait_time); + queued_requests.push_back(rate_limited_req); + // If one fails just wait for the next window that allows sending requests. + return; + } + Ok(event) => self.ready_requests.push(event), + } + } + if queued_requests.is_empty() { + entry.remove(); + } + } + } + + pub fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + // First check the requests that were self rate limited, since those might add events to + // the queue. Also do this this before rate limiter prunning to avoid removing and + // immediately adding rate limiting keys. + if let Poll::Ready(Some(Ok(expired))) = self.next_peer_request.poll_expired(cx) { + let (peer_id, protocol) = expired.into_inner(); + self.next_peer_request_ready(peer_id, protocol); + } + // Prune the rate limiter. + let _ = self.limiter.poll_unpin(cx); + + // Finally return any queued events. + if !self.ready_requests.is_empty() { + return Poll::Ready(self.ready_requests.remove(0)); + } + + Poll::Pending + } +} diff --git a/beacon_node/lighthouse_network/src/service/behaviour.rs b/beacon_node/lighthouse_network/src/service/behaviour.rs index 3adc940a6a..7d20b87ad1 100644 --- a/beacon_node/lighthouse_network/src/service/behaviour.rs +++ b/beacon_node/lighthouse_network/src/service/behaviour.rs @@ -7,8 +7,8 @@ use libp2p::gossipsub::subscription_filter::{ MaxCountSubscriptionFilter, WhitelistSubscriptionFilter, }; use libp2p::gossipsub::Gossipsub as BaseGossipsub; -use libp2p::identify::Identify; -use libp2p::NetworkBehaviour; +use libp2p::identify::Behaviour as Identify; +use libp2p::swarm::NetworkBehaviour; use types::EthSpec; use super::api_types::RequestId; diff --git a/beacon_node/lighthouse_network/src/service/gossipsub_scoring_parameters.rs b/beacon_node/lighthouse_network/src/service/gossipsub_scoring_parameters.rs index 71a3953ece..88becd686e 100644 --- a/beacon_node/lighthouse_network/src/service/gossipsub_scoring_parameters.rs +++ b/beacon_node/lighthouse_network/src/service/gossipsub_scoring_parameters.rs @@ -270,11 +270,11 @@ impl PeerScoreSettings { let modulo_smaller = max( 1, - smaller_committee_size / self.target_aggregators_per_committee as usize, + smaller_committee_size / self.target_aggregators_per_committee, ); let modulo_larger = max( 1, - (smaller_committee_size + 1) / self.target_aggregators_per_committee as usize, + (smaller_committee_size + 1) / self.target_aggregators_per_committee, ); Ok(( diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index a6f1ce20ad..832f025c43 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -26,7 +26,7 @@ use libp2p::gossipsub::subscription_filter::MaxCountSubscriptionFilter; use libp2p::gossipsub::{ GossipsubEvent, IdentTopic as Topic, MessageAcceptance, MessageAuthenticity, MessageId, }; -use libp2p::identify::{Identify, IdentifyConfig, IdentifyEvent}; +use libp2p::identify::{Behaviour as Identify, Config as IdentifyConfig, Event as IdentifyEvent}; use libp2p::multiaddr::{Multiaddr, Protocol as MProtocol}; use libp2p::swarm::{ConnectionLimits, Swarm, SwarmBuilder, SwarmEvent}; use libp2p::PeerId; @@ -262,6 +262,7 @@ impl Network { let eth2_rpc = RPC::new( ctx.fork_context.clone(), config.enable_light_client_server, + config.outbound_rate_limiter_config.clone(), log.clone(), ); @@ -316,7 +317,7 @@ impl Network { // use the executor for libp2p struct Executor(task_executor::TaskExecutor); - impl libp2p::core::Executor for Executor { + impl libp2p::swarm::Executor for Executor { fn exec(&self, f: Pin + Send>>) { self.0.spawn(f, "libp2p"); } @@ -341,12 +342,16 @@ impl Network { .with_max_established_per_peer(Some(MAX_CONNECTIONS_PER_PEER)); ( - SwarmBuilder::new(transport, behaviour, local_peer_id) - .notify_handler_buffer_size(std::num::NonZeroUsize::new(7).expect("Not zero")) - .connection_event_buffer_size(64) - .connection_limits(limits) - .executor(Box::new(Executor(executor))) - .build(), + SwarmBuilder::with_executor( + transport, + behaviour, + local_peer_id, + Executor(executor), + ) + .notify_handler_buffer_size(std::num::NonZeroUsize::new(7).expect("Not zero")) + .connection_event_buffer_size(64) + .connection_limits(limits) + .build(), bandwidth, ) }; diff --git a/beacon_node/lighthouse_network/src/service/utils.rs b/beacon_node/lighthouse_network/src/service/utils.rs index 09a8d1a863..addaaf5b5e 100644 --- a/beacon_node/lighthouse_network/src/service/utils.rs +++ b/beacon_node/lighthouse_network/src/service/utils.rs @@ -44,8 +44,7 @@ type BoxedTransport = Boxed<(PeerId, StreamMuxerBox)>; pub fn build_transport( local_private_key: Keypair, ) -> std::io::Result<(BoxedTransport, Arc)> { - let tcp = - libp2p::tcp::TokioTcpTransport::new(libp2p::tcp::GenTcpConfig::default().nodelay(true)); + let tcp = libp2p::tcp::tokio::Transport::new(libp2p::tcp::Config::default().nodelay(true)); let transport = libp2p::dns::TokioDnsConfig::system(tcp)?; #[cfg(feature = "libp2p-websocket")] let transport = { @@ -88,7 +87,7 @@ fn keypair_from_hex(hex_bytes: &str) -> error::Result { hex_bytes.to_string() }; - hex::decode(&hex_bytes) + hex::decode(hex_bytes) .map_err(|e| format!("Failed to parse p2p secret key bytes: {:?}", e).into()) .and_then(keypair_from_bytes) } diff --git a/beacon_node/lighthouse_network/tests/common/mod.rs b/beacon_node/lighthouse_network/tests/common.rs similarity index 98% rename from beacon_node/lighthouse_network/tests/common/mod.rs rename to beacon_node/lighthouse_network/tests/common.rs index a3c32d0fb1..b67b412cfc 100644 --- a/beacon_node/lighthouse_network/tests/common/mod.rs +++ b/beacon_node/lighthouse_network/tests/common.rs @@ -15,13 +15,6 @@ use types::{ }; use unused_port::unused_tcp_port; -#[allow(clippy::type_complexity)] -#[allow(unused)] -pub mod behaviour; -#[allow(clippy::type_complexity)] -#[allow(unused)] -pub mod swarm; - type E = MinimalEthSpec; type ReqId = usize; diff --git a/beacon_node/lighthouse_network/tests/common/behaviour.rs b/beacon_node/lighthouse_network/tests/common/behaviour.rs deleted file mode 100644 index 50fe6941db..0000000000 --- a/beacon_node/lighthouse_network/tests/common/behaviour.rs +++ /dev/null @@ -1,395 +0,0 @@ -// NOTE: Taken from libp2p's swarm's testing utils. -// -// Copyright 2020 Parity Technologies (UK) Ltd. -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -use std::collections::HashMap; -use std::task::{Context, Poll}; - -use libp2p::core::connection::{ConnectedPoint, ConnectionId}; -use libp2p::core::transport::ListenerId; -use libp2p::swarm::handler::{ConnectionHandler, DummyConnectionHandler, IntoConnectionHandler}; -use libp2p::swarm::{DialError, NetworkBehaviour, NetworkBehaviourAction, PollParameters}; -use libp2p::{Multiaddr, PeerId}; - -/// A `MockBehaviour` is a `NetworkBehaviour` that allows for -/// the instrumentation of return values, without keeping -/// any further state. -pub struct MockBehaviour< - THandler = DummyConnectionHandler, - TOutEvent = ::OutEvent, -> where - THandler: ConnectionHandler, -{ - /// The prototype protocols handler that is cloned for every - /// invocation of `new_handler`. - pub handler_proto: THandler, - /// The addresses to return from `addresses_of_peer`. - pub addresses: HashMap>, - /// The next action to return from `poll`. - /// - /// An action is only returned once. - pub next_action: Option>, -} - -impl MockBehaviour -where - THandler: ConnectionHandler, -{ - pub fn new(handler_proto: THandler) -> Self { - MockBehaviour { - handler_proto, - addresses: HashMap::new(), - next_action: None, - } - } -} - -impl NetworkBehaviour for MockBehaviour -where - THandler: ConnectionHandler + Clone, - THandler::OutEvent: Clone, - TOutEvent: Send + 'static, -{ - type ConnectionHandler = THandler; - type OutEvent = TOutEvent; - - fn new_handler(&mut self) -> Self::ConnectionHandler { - self.handler_proto.clone() - } - - fn addresses_of_peer(&mut self, p: &PeerId) -> Vec { - self.addresses.get(p).map_or(Vec::new(), |v| v.clone()) - } - - fn inject_event(&mut self, _: PeerId, _: ConnectionId, _: THandler::OutEvent) {} - - fn poll( - &mut self, - _: &mut Context, - _: &mut impl PollParameters, - ) -> Poll> { - Option::take(&mut self.next_action).map_or(Poll::Pending, Poll::Ready) - } -} - -/// A `CallTraceBehaviour` is a `NetworkBehaviour` that tracks invocations of callback methods and -/// their arguments, wrapping around an inner behaviour. It ensures certain invariants are met. -pub struct CallTraceBehaviour -where - TInner: NetworkBehaviour, -{ - inner: TInner, - - pub addresses_of_peer: Vec, - pub inject_connection_established: Vec<(PeerId, ConnectionId, ConnectedPoint, usize)>, - pub inject_connection_closed: Vec<(PeerId, ConnectionId, ConnectedPoint, usize)>, - pub inject_event: Vec<( - PeerId, - ConnectionId, - <::Handler as ConnectionHandler>::OutEvent, - )>, - pub inject_dial_failure: Vec>, - pub inject_new_listener: Vec, - pub inject_new_listen_addr: Vec<(ListenerId, Multiaddr)>, - pub inject_new_external_addr: Vec, - pub inject_expired_listen_addr: Vec<(ListenerId, Multiaddr)>, - pub inject_expired_external_addr: Vec, - pub inject_listener_error: Vec, - pub inject_listener_closed: Vec<(ListenerId, bool)>, - pub poll: usize, -} - -impl CallTraceBehaviour -where - TInner: NetworkBehaviour, -{ - pub fn new(inner: TInner) -> Self { - Self { - inner, - addresses_of_peer: Vec::new(), - inject_connection_established: Vec::new(), - inject_connection_closed: Vec::new(), - inject_event: Vec::new(), - inject_dial_failure: Vec::new(), - inject_new_listener: Vec::new(), - inject_new_listen_addr: Vec::new(), - inject_new_external_addr: Vec::new(), - inject_expired_listen_addr: Vec::new(), - inject_expired_external_addr: Vec::new(), - inject_listener_error: Vec::new(), - inject_listener_closed: Vec::new(), - poll: 0, - } - } - - #[allow(dead_code)] - pub fn reset(&mut self) { - self.addresses_of_peer = Vec::new(); - self.inject_connection_established = Vec::new(); - self.inject_connection_closed = Vec::new(); - self.inject_event = Vec::new(); - self.inject_dial_failure = Vec::new(); - self.inject_new_listen_addr = Vec::new(); - self.inject_new_external_addr = Vec::new(); - self.inject_expired_listen_addr = Vec::new(); - self.inject_listener_error = Vec::new(); - self.inject_listener_closed = Vec::new(); - self.poll = 0; - } - - pub fn inner(&mut self) -> &mut TInner { - &mut self.inner - } - - /// Checks that when the expected number of closed connection notifications are received, a - /// given number of expected disconnections have been received as well. - /// - /// Returns if the first condition is met. - pub fn assert_disconnected( - &self, - expected_closed_connections: usize, - expected_disconnections: usize, - ) -> bool { - if self.inject_connection_closed.len() == expected_closed_connections { - assert_eq!( - self.inject_connection_closed - .iter() - .filter(|(.., remaining_established)| { *remaining_established == 0 }) - .count(), - expected_disconnections - ); - return true; - } - - false - } - - /// Checks that when the expected number of established connection notifications are received, - /// a given number of expected connections have been received as well. - /// - /// Returns if the first condition is met. - pub fn assert_connected( - &self, - expected_established_connections: usize, - expected_connections: usize, - ) -> bool { - if self.inject_connection_established.len() == expected_established_connections { - assert_eq!( - self.inject_connection_established - .iter() - .filter(|(.., reported_aditional_connections)| { - *reported_aditional_connections == 0 - }) - .count(), - expected_connections - ); - return true; - } - - false - } -} - -impl NetworkBehaviour for CallTraceBehaviour -where - TInner: NetworkBehaviour, - <::Handler as ConnectionHandler>::OutEvent: - Clone, -{ - type ConnectionHandler = TInner::ConnectionHandler; - type OutEvent = TInner::OutEvent; - - fn new_handler(&mut self) -> Self::ConnectionHandler { - self.inner.new_handler() - } - - fn addresses_of_peer(&mut self, p: &PeerId) -> Vec { - self.addresses_of_peer.push(*p); - self.inner.addresses_of_peer(p) - } - - fn inject_connection_established( - &mut self, - p: &PeerId, - c: &ConnectionId, - e: &ConnectedPoint, - errors: Option<&Vec>, - other_established: usize, - ) { - let mut other_peer_connections = self - .inject_connection_established - .iter() - .rev() // take last to first - .filter_map(|(peer, .., other_established)| { - if p == peer { - Some(other_established) - } else { - None - } - }) - .take(other_established); - - // We are informed that there are `other_established` additional connections. Ensure that the - // number of previous connections is consistent with this - if let Some(&prev) = other_peer_connections.next() { - if prev < other_established { - assert_eq!( - prev, - other_established - 1, - "Inconsistent connection reporting" - ) - } - assert_eq!(other_peer_connections.count(), other_established - 1); - } else { - assert_eq!(other_established, 0) - } - self.inject_connection_established - .push((*p, *c, e.clone(), other_established)); - self.inner - .inject_connection_established(p, c, e, errors, other_established); - } - - fn inject_connection_closed( - &mut self, - p: &PeerId, - c: &ConnectionId, - e: &ConnectedPoint, - handler: ::Handler, - remaining_established: usize, - ) { - let mut other_closed_connections = self - .inject_connection_established - .iter() - .rev() // take last to first - .filter_map(|(peer, .., remaining_established)| { - if p == peer { - Some(remaining_established) - } else { - None - } - }) - .take(remaining_established); - - // We are informed that there are `other_established` additional connections. Ensure that the - // number of previous connections is consistent with this - if let Some(&prev) = other_closed_connections.next() { - if prev < remaining_established { - assert_eq!( - prev, - remaining_established - 1, - "Inconsistent closed connection reporting" - ) - } - assert_eq!(other_closed_connections.count(), remaining_established - 1); - } else { - assert_eq!(remaining_established, 0) - } - assert!( - self.inject_connection_established - .iter() - .any(|(peer, conn_id, endpoint, _)| (peer, conn_id, endpoint) == (p, c, e)), - "`inject_connection_closed` is called only for connections for \ - which `inject_connection_established` was called first." - ); - self.inject_connection_closed - .push((*p, *c, e.clone(), remaining_established)); - self.inner - .inject_connection_closed(p, c, e, handler, remaining_established); - } - - fn inject_event( - &mut self, - p: PeerId, - c: ConnectionId, - e: <::Handler as ConnectionHandler>::OutEvent, - ) { - assert!( - self.inject_connection_established - .iter() - .any(|(peer_id, conn_id, ..)| *peer_id == p && c == *conn_id), - "`inject_event` is called for reported connections." - ); - assert!( - !self - .inject_connection_closed - .iter() - .any(|(peer_id, conn_id, ..)| *peer_id == p && c == *conn_id), - "`inject_event` is never called for closed connections." - ); - - self.inject_event.push((p, c, e.clone())); - self.inner.inject_event(p, c, e); - } - - fn inject_dial_failure( - &mut self, - p: Option, - handler: Self::ConnectionHandler, - error: &DialError, - ) { - self.inject_dial_failure.push(p); - self.inner.inject_dial_failure(p, handler, error); - } - - fn inject_new_listener(&mut self, id: ListenerId) { - self.inject_new_listener.push(id); - self.inner.inject_new_listener(id); - } - - fn inject_new_listen_addr(&mut self, id: ListenerId, a: &Multiaddr) { - self.inject_new_listen_addr.push((id, a.clone())); - self.inner.inject_new_listen_addr(id, a); - } - - fn inject_expired_listen_addr(&mut self, id: ListenerId, a: &Multiaddr) { - self.inject_expired_listen_addr.push((id, a.clone())); - self.inner.inject_expired_listen_addr(id, a); - } - - fn inject_new_external_addr(&mut self, a: &Multiaddr) { - self.inject_new_external_addr.push(a.clone()); - self.inner.inject_new_external_addr(a); - } - - fn inject_expired_external_addr(&mut self, a: &Multiaddr) { - self.inject_expired_external_addr.push(a.clone()); - self.inner.inject_expired_external_addr(a); - } - - fn inject_listener_error(&mut self, l: ListenerId, e: &(dyn std::error::Error + 'static)) { - self.inject_listener_error.push(l); - self.inner.inject_listener_error(l, e); - } - - fn inject_listener_closed(&mut self, l: ListenerId, r: Result<(), &std::io::Error>) { - self.inject_listener_closed.push((l, r.is_ok())); - self.inner.inject_listener_closed(l, r); - } - - fn poll( - &mut self, - cx: &mut Context, - args: &mut impl PollParameters, - ) -> Poll> { - self.poll += 1; - self.inner.poll(cx, args) - } -} diff --git a/beacon_node/lighthouse_network/tests/common/swarm.rs b/beacon_node/lighthouse_network/tests/common/swarm.rs deleted file mode 100644 index aa41a5c066..0000000000 --- a/beacon_node/lighthouse_network/tests/common/swarm.rs +++ /dev/null @@ -1,99 +0,0 @@ -use std::collections::HashMap; -use std::pin::Pin; - -use super::behaviour::{CallTraceBehaviour, MockBehaviour}; - -use futures::stream::Stream; -use futures::task::{Context, Poll}; -use libp2p::swarm::handler::ConnectionHandler; -use libp2p::swarm::{IntoConnectionHandler, NetworkBehaviour, Swarm, SwarmBuilder, SwarmEvent}; -use libp2p::{PeerId, Transport}; - -use futures::StreamExt; - -pub fn new_test_swarm(behaviour: B) -> Swarm -where - B: NetworkBehaviour, -{ - let id_keys = libp2p::identity::Keypair::generate_ed25519(); - let local_public_key = id_keys.public(); - let transport = libp2p::core::transport::MemoryTransport::default() - .upgrade(libp2p::core::upgrade::Version::V1) - .authenticate(libp2p::plaintext::PlainText2Config { - local_public_key: local_public_key.clone(), - }) - .multiplex(libp2p::yamux::YamuxConfig::default()) - .boxed(); - SwarmBuilder::new(transport, behaviour, local_public_key.into()).build() -} - -pub fn random_multiaddr() -> libp2p::multiaddr::Multiaddr { - libp2p::multiaddr::Protocol::Memory(rand::random::()).into() -} - -/// Bind a memory multiaddr to a compatible swarm. -pub async fn bind_listener( - swarm: &mut Swarm, -) -> libp2p::multiaddr::Multiaddr { - swarm.listen_on(random_multiaddr()).unwrap(); - match swarm.select_next_some().await { - SwarmEvent::NewListenAddr { - listener_id: _, - address, - } => address, - _ => panic!("Testing swarm's first event should be a new listener"), - } -} - -#[derive(Default)] -pub struct SwarmPool { - swarms: HashMap>, -} - -impl SwarmPool { - pub fn with_capacity(capacity: usize) -> Self { - Self { - swarms: HashMap::with_capacity(capacity), - } - } - pub fn insert(&mut self, swarm: Swarm) -> PeerId { - let peer_id = *swarm.local_peer_id(); - self.swarms.insert(peer_id, swarm); - peer_id - } - - pub fn remove(&mut self, peer_id: &PeerId) { - self.swarms.remove(peer_id); - } - - pub fn get_mut(&mut self, peer_id: &PeerId) -> Option<&mut Swarm> { - self.swarms.get_mut(peer_id) - } - - pub fn swarms(&self) -> &HashMap> { - &self.swarms - } - - pub fn swarms_mut(&mut self) -> &mut HashMap> { - &mut self.swarms - } -} - -impl Stream for SwarmPool -where - B: NetworkBehaviour, - ::ConnectionHandler: ConnectionHandler, -{ - type Item = (PeerId, - SwarmEvent<::OutEvent, <<::ConnectionHandler as IntoConnectionHandler>::Handler as ConnectionHandler>::Error>); - - fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let mut polls = self - .get_mut() - .swarms - .iter_mut() - .map(|(&peer_id, swarm)| swarm.map(move |ev| (peer_id, ev))) - .collect::>(); - polls.poll_next_unpin(cx) - } -} diff --git a/beacon_node/lighthouse_network/tests/pm_tests.rs b/beacon_node/lighthouse_network/tests/pm_tests.rs deleted file mode 100644 index 17a044ced0..0000000000 --- a/beacon_node/lighthouse_network/tests/pm_tests.rs +++ /dev/null @@ -1,203 +0,0 @@ -#![cfg(not(debug_assertions))] - -mod common; -use std::{ - collections::{HashMap, HashSet}, - sync::Arc, -}; - -use common::{ - behaviour::{CallTraceBehaviour, MockBehaviour}, - swarm, -}; -use lighthouse_network::{ - peer_manager::{config::Config, PeerManagerEvent}, - NetworkGlobals, PeerAction, PeerInfo, PeerManager, ReportSource, -}; -use types::MinimalEthSpec as E; - -use futures::StreamExt; -use libp2p::{ - core::either::EitherError, - swarm::SwarmEvent, - swarm::{handler::DummyConnectionHandler, DummyBehaviour, KeepAlive, Swarm}, - NetworkBehaviour, -}; - -use slog::debug; - -/// Struct that mimics the lighthouse_network::Service with respect to handling peer manager -/// events. -// TODO: make this a real struct for more accurate testing. -struct Service { - swarm: Swarm, -} - -impl Service { - async fn select_next_some(&mut self) -> SwarmEvent> { - let ev = self.swarm.select_next_some().await; - match &ev { - SwarmEvent::Behaviour(Ev(PeerManagerEvent::Banned(peer_id, _addr_vec))) => { - self.swarm.ban_peer_id(*peer_id); - } - SwarmEvent::Behaviour(Ev(PeerManagerEvent::UnBanned(peer_id, _addr_vec))) => { - self.swarm.unban_peer_id(*peer_id); - } - SwarmEvent::Behaviour(Ev(PeerManagerEvent::DisconnectPeer(peer_id, _reason))) => { - // directly disconnect here. - let _ = self.swarm.disconnect_peer_id(*peer_id); - } - _ => {} - } - ev - } -} - -#[derive(Debug)] -struct Ev(PeerManagerEvent); -impl From for Ev { - fn from(_: void::Void) -> Self { - unreachable!("No events are emmited") - } -} -impl From for Ev { - fn from(ev: PeerManagerEvent) -> Self { - Ev(ev) - } -} - -#[derive(NetworkBehaviour)] -#[behaviour(out_event = "Ev")] -struct Behaviour { - pm_call_trace: CallTraceBehaviour>, - sibling: MockBehaviour, -} - -impl Behaviour { - fn new(pm: PeerManager) -> Self { - Behaviour { - pm_call_trace: CallTraceBehaviour::new(pm), - sibling: MockBehaviour::new(DummyConnectionHandler { - // The peer manager votes No, so we make sure the combined handler stays alive this - // way. - keep_alive: KeepAlive::Yes, - }), - } - } -} - -#[tokio::test] -async fn banned_peers_consistency() { - let log = common::build_log(slog::Level::Debug, false); - let pm_log = log.new(slog::o!("who" => "[PM]")); - let globals: Arc> = Arc::new(NetworkGlobals::new_test_globals(&log)); - - // Build the peer manager. - let (mut pm_service, pm_addr) = { - let pm_config = Config { - discovery_enabled: false, - ..Default::default() - }; - let pm = PeerManager::new(pm_config, globals.clone(), &pm_log).unwrap(); - let mut pm_swarm = swarm::new_test_swarm(Behaviour::new(pm)); - let pm_addr = swarm::bind_listener(&mut pm_swarm).await; - let service = Service { swarm: pm_swarm }; - (service, pm_addr) - }; - - let excess_banned_peers = 15; - let peers_to_ban = - lighthouse_network::peer_manager::peerdb::MAX_BANNED_PEERS + excess_banned_peers; - - // Build all the dummy peers needed. - let (mut swarm_pool, peers) = { - let mut pool = swarm::SwarmPool::with_capacity(peers_to_ban); - let mut peers = HashSet::with_capacity(peers_to_ban); - for _ in 0..peers_to_ban { - let mut peer_swarm = - swarm::new_test_swarm(DummyBehaviour::with_keep_alive(KeepAlive::Yes)); - let _peer_addr = swarm::bind_listener(&mut peer_swarm).await; - // It is ok to dial all at the same time since the swarm handles an event at a time. - peer_swarm.dial(pm_addr.clone()).unwrap(); - let peer_id = pool.insert(peer_swarm); - peers.insert(peer_id); - } - (pool, peers) - }; - - // we track banned peers at the swarm level here since there is no access to that info. - let mut swarm_banned_peers = HashMap::with_capacity(peers_to_ban); - let mut peers_unbanned = 0; - let timeout = tokio::time::sleep(tokio::time::Duration::from_secs(30)); - futures::pin_mut!(timeout); - - loop { - // poll the pm and dummy swarms. - tokio::select! { - pm_event = pm_service.select_next_some() => { - debug!(log, "[PM] {:?}", pm_event); - match pm_event { - SwarmEvent::Behaviour(Ev(ev)) => match ev { - PeerManagerEvent::Banned(peer_id, _) => { - let has_been_unbanned = false; - swarm_banned_peers.insert(peer_id, has_been_unbanned); - } - PeerManagerEvent::UnBanned(peer_id, _) => { - *swarm_banned_peers.get_mut(&peer_id).expect("Unbanned peer must be banned first") = true; - peers_unbanned += 1; - } - _ => {} - } - SwarmEvent::ConnectionEstablished { - peer_id, - endpoint: _, - num_established: _, - concurrent_dial_errors: _, - } => { - assert!(peers.contains(&peer_id)); - // now we report the peer as banned. - pm_service - .swarm - .behaviour_mut() - .pm_call_trace - .inner() - .report_peer( - &peer_id, - PeerAction::Fatal, - ReportSource::Processor, - None, - "" - ); - }, - _ => {} - } - } - Some((_peer_id, _peer_ev)) = swarm_pool.next() => { - // we need to poll the swarms to keep the peers going - } - _ = timeout.as_mut() => { - panic!("Test timeout.") - } - } - - if peers_unbanned == excess_banned_peers { - let pdb = globals.peers.read(); - let inconsistencies = swarm_banned_peers - .into_iter() - .map(|(peer_id, was_unbanned)| { - was_unbanned - != pdb.peer_info(&peer_id).map_or( - false, /* We forgot about a banned peer */ - PeerInfo::is_banned, - ) - }); - assert_eq!( - inconsistencies - .filter(|is_consistent| *is_consistent) - .count(), - peers_to_ban - ); - return; - } - } -} diff --git a/beacon_node/network/Cargo.toml b/beacon_node/network/Cargo.toml index 2e7b2227b2..1b036b32c6 100644 --- a/beacon_node/network/Cargo.toml +++ b/beacon_node/network/Cargo.toml @@ -44,7 +44,8 @@ strum = "0.24.0" tokio-util = { version = "0.6.3", features = ["time"] } derivative = "2.2.0" delay_map = "0.1.1" -ethereum-types = { version = "0.12.1", optional = true } +ethereum-types = { version = "0.14.1", optional = true } +execution_layer = { path = "../execution_layer" } [features] deterministic_long_lived_attnets = [ "ethereum-types" ] diff --git a/beacon_node/network/src/beacon_processor/mod.rs b/beacon_node/network/src/beacon_processor/mod.rs index 743a97a29c..8118443a65 100644 --- a/beacon_node/network/src/beacon_processor/mod.rs +++ b/beacon_node/network/src/beacon_processor/mod.rs @@ -67,7 +67,8 @@ use types::{ SignedVoluntaryExit, SubnetId, SyncCommitteeMessage, SyncSubnetId, }; use work_reprocessing_queue::{ - spawn_reprocess_scheduler, QueuedAggregate, QueuedRpcBlock, QueuedUnaggregate, ReadyWork, + spawn_reprocess_scheduler, QueuedAggregate, QueuedLightClientUpdate, QueuedRpcBlock, + QueuedUnaggregate, ReadyWork, }; use worker::{Toolbox, Worker}; @@ -137,6 +138,10 @@ const MAX_GOSSIP_FINALITY_UPDATE_QUEUE_LEN: usize = 1_024; /// before we start dropping them. const MAX_GOSSIP_OPTIMISTIC_UPDATE_QUEUE_LEN: usize = 1_024; +/// The maximum number of queued `LightClientOptimisticUpdate` objects received on gossip that will be stored +/// for reprocessing before we start dropping them. +const MAX_GOSSIP_OPTIMISTIC_UPDATE_REPROCESS_QUEUE_LEN: usize = 128; + /// The maximum number of queued `SyncCommitteeMessage` objects that will be stored before we start dropping /// them. const MAX_SYNC_MESSAGE_QUEUE_LEN: usize = 2048; @@ -213,6 +218,7 @@ pub const BLOCKS_BY_ROOTS_REQUEST: &str = "blocks_by_roots_request"; pub const LIGHT_CLIENT_BOOTSTRAP_REQUEST: &str = "light_client_bootstrap"; pub const UNKNOWN_BLOCK_ATTESTATION: &str = "unknown_block_attestation"; pub const UNKNOWN_BLOCK_AGGREGATE: &str = "unknown_block_aggregate"; +pub const UNKNOWN_LIGHT_CLIENT_UPDATE: &str = "unknown_light_client_update"; /// A simple first-in-first-out queue with a maximum length. struct FifoQueue { @@ -694,6 +700,21 @@ impl std::convert::From> for WorkEvent { seen_timestamp, }, }, + ReadyWork::LightClientUpdate(QueuedLightClientUpdate { + peer_id, + message_id, + light_client_optimistic_update, + seen_timestamp, + .. + }) => Self { + drop_during_sync: true, + work: Work::UnknownLightClientOptimisticUpdate { + message_id, + peer_id, + light_client_optimistic_update, + seen_timestamp, + }, + }, } } } @@ -733,6 +754,12 @@ pub enum Work { aggregate: Box>, seen_timestamp: Duration, }, + UnknownLightClientOptimisticUpdate { + message_id: MessageId, + peer_id: PeerId, + light_client_optimistic_update: Box>, + seen_timestamp: Duration, + }, GossipAggregateBatch { packages: Vec>, }, @@ -845,6 +872,7 @@ impl Work { Work::LightClientBootstrapRequest { .. } => LIGHT_CLIENT_BOOTSTRAP_REQUEST, Work::UnknownBlockAttestation { .. } => UNKNOWN_BLOCK_ATTESTATION, Work::UnknownBlockAggregate { .. } => UNKNOWN_BLOCK_AGGREGATE, + Work::UnknownLightClientOptimisticUpdate { .. } => UNKNOWN_LIGHT_CLIENT_UPDATE, } } } @@ -979,6 +1007,8 @@ impl BeaconProcessor { // Using a FIFO queue for light client updates to maintain sequence order. let mut finality_update_queue = FifoQueue::new(MAX_GOSSIP_FINALITY_UPDATE_QUEUE_LEN); let mut optimistic_update_queue = FifoQueue::new(MAX_GOSSIP_OPTIMISTIC_UPDATE_QUEUE_LEN); + let mut unknown_light_client_update_queue = + FifoQueue::new(MAX_GOSSIP_OPTIMISTIC_UPDATE_REPROCESS_QUEUE_LEN); // Using a FIFO queue since blocks need to be imported sequentially. let mut rpc_block_queue = FifoQueue::new(MAX_RPC_BLOCK_QUEUE_LEN); @@ -1346,6 +1376,9 @@ impl BeaconProcessor { Work::UnknownBlockAggregate { .. } => { unknown_block_aggregate_queue.push(work) } + Work::UnknownLightClientOptimisticUpdate { .. } => { + unknown_light_client_update_queue.push(work, work_id, &self.log) + } } } } @@ -1665,6 +1698,7 @@ impl BeaconProcessor { message_id, peer_id, *light_client_optimistic_update, + Some(work_reprocessing_tx), seen_timestamp, ) }), @@ -1787,6 +1821,20 @@ impl BeaconProcessor { seen_timestamp, ) }), + Work::UnknownLightClientOptimisticUpdate { + message_id, + peer_id, + light_client_optimistic_update, + seen_timestamp, + } => task_spawner.spawn_blocking(move || { + worker.process_gossip_optimistic_update( + message_id, + peer_id, + *light_client_optimistic_update, + None, + seen_timestamp, + ) + }), }; } } diff --git a/beacon_node/network/src/beacon_processor/work_reprocessing_queue.rs b/beacon_node/network/src/beacon_processor/work_reprocessing_queue.rs index 2aeec11c32..8c568a7eef 100644 --- a/beacon_node/network/src/beacon_processor/work_reprocessing_queue.rs +++ b/beacon_node/network/src/beacon_processor/work_reprocessing_queue.rs @@ -19,7 +19,7 @@ use futures::task::Poll; use futures::{Stream, StreamExt}; use lighthouse_network::{MessageId, PeerId}; use logging::TimeLatch; -use slog::{crit, debug, error, warn, Logger}; +use slog::{crit, debug, error, trace, warn, Logger}; use slot_clock::SlotClock; use std::collections::{HashMap, HashSet}; use std::pin::Pin; @@ -30,12 +30,16 @@ use task_executor::TaskExecutor; use tokio::sync::mpsc::{self, Receiver, Sender}; use tokio::time::error::Error as TimeError; use tokio_util::time::delay_queue::{DelayQueue, Key as DelayKey}; -use types::{Attestation, EthSpec, Hash256, SignedAggregateAndProof, SignedBeaconBlock, SubnetId}; +use types::{ + Attestation, EthSpec, Hash256, LightClientOptimisticUpdate, SignedAggregateAndProof, + SignedBeaconBlock, SubnetId, +}; const TASK_NAME: &str = "beacon_processor_reprocess_queue"; const GOSSIP_BLOCKS: &str = "gossip_blocks"; const RPC_BLOCKS: &str = "rpc_blocks"; const ATTESTATIONS: &str = "attestations"; +const LIGHT_CLIENT_UPDATES: &str = "lc_updates"; /// Queue blocks for re-processing with an `ADDITIONAL_QUEUED_BLOCK_DELAY` after the slot starts. /// This is to account for any slight drift in the system clock. @@ -44,6 +48,9 @@ const ADDITIONAL_QUEUED_BLOCK_DELAY: Duration = Duration::from_millis(5); /// For how long to queue aggregated and unaggregated attestations for re-processing. pub const QUEUED_ATTESTATION_DELAY: Duration = Duration::from_secs(12); +/// For how long to queue light client updates for re-processing. +pub const QUEUED_LIGHT_CLIENT_UPDATE_DELAY: Duration = Duration::from_secs(12); + /// For how long to queue rpc blocks before sending them back for reprocessing. pub const QUEUED_RPC_BLOCK_DELAY: Duration = Duration::from_secs(3); @@ -55,6 +62,9 @@ const MAXIMUM_QUEUED_BLOCKS: usize = 16; /// How many attestations we keep before new ones get dropped. const MAXIMUM_QUEUED_ATTESTATIONS: usize = 16_384; +/// How many light client updates we keep before new ones get dropped. +const MAXIMUM_QUEUED_LIGHT_CLIENT_UPDATES: usize = 128; + /// Messages that the scheduler can receive. pub enum ReprocessQueueMessage { /// A block that has been received early and we should queue for later processing. @@ -62,13 +72,18 @@ pub enum ReprocessQueueMessage { /// A gossip block for hash `X` is being imported, we should queue the rpc block for the same /// hash until the gossip block is imported. RpcBlock(QueuedRpcBlock), - /// A block that was successfully processed. We use this to handle attestations for unknown - /// blocks. - BlockImported(Hash256), + /// A block that was successfully processed. We use this to handle attestations and light client updates + /// for unknown blocks. + BlockImported { + block_root: Hash256, + parent_root: Hash256, + }, /// An unaggregated attestation that references an unknown block. UnknownBlockUnaggregate(QueuedUnaggregate), /// An aggregated attestation that references an unknown block. UnknownBlockAggregate(QueuedAggregate), + /// A light client optimistic update that references a parent root that has not been seen as a parent. + UnknownLightClientOptimisticUpdate(QueuedLightClientUpdate), } /// Events sent by the scheduler once they are ready for re-processing. @@ -77,6 +92,7 @@ pub enum ReadyWork { RpcBlock(QueuedRpcBlock), Unaggregate(QueuedUnaggregate), Aggregate(QueuedAggregate), + LightClientUpdate(QueuedLightClientUpdate), } /// An Attestation for which the corresponding block was not seen while processing, queued for @@ -99,6 +115,16 @@ pub struct QueuedAggregate { pub seen_timestamp: Duration, } +/// A light client update for which the corresponding parent block was not seen while processing, +/// queued for later. +pub struct QueuedLightClientUpdate { + pub peer_id: PeerId, + pub message_id: MessageId, + pub light_client_optimistic_update: Box>, + pub parent_root: Hash256, + pub seen_timestamp: Duration, +} + /// A block that arrived early and has been queued for later import. pub struct QueuedGossipBlock { pub peer_id: PeerId, @@ -127,6 +153,8 @@ enum InboundEvent { ReadyRpcBlock(QueuedRpcBlock), /// An aggregated or unaggregated attestation is ready for re-processing. ReadyAttestation(QueuedAttestationId), + /// A light client update that is ready for re-processing. + ReadyLightClientUpdate(QueuedLightClientUpdateId), /// A `DelayQueue` returned an error. DelayQueueError(TimeError, &'static str), /// A message sent to the `ReprocessQueue` @@ -147,6 +175,8 @@ struct ReprocessQueue { rpc_block_delay_queue: DelayQueue>, /// Queue to manage scheduled attestations. attestations_delay_queue: DelayQueue, + /// Queue to manage scheduled light client updates. + lc_updates_delay_queue: DelayQueue, /* Queued items */ /// Queued blocks. @@ -157,15 +187,23 @@ struct ReprocessQueue { queued_unaggregates: FnvHashMap, DelayKey)>, /// Attestations (aggregated and unaggregated) per root. awaiting_attestations_per_root: HashMap>, + /// Queued Light Client Updates. + queued_lc_updates: FnvHashMap, DelayKey)>, + /// Light Client Updates per parent_root. + awaiting_lc_updates_per_parent_root: HashMap>, /* Aux */ /// Next attestation id, used for both aggregated and unaggregated attestations next_attestation: usize, + next_lc_update: usize, early_block_debounce: TimeLatch, rpc_block_debounce: TimeLatch, attestation_delay_debounce: TimeLatch, + lc_update_delay_debounce: TimeLatch, } +pub type QueuedLightClientUpdateId = usize; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum QueuedAttestationId { Aggregate(usize), @@ -235,6 +273,20 @@ impl Stream for ReprocessQueue { Poll::Ready(None) | Poll::Pending => (), } + match self.lc_updates_delay_queue.poll_expired(cx) { + Poll::Ready(Some(Ok(lc_id))) => { + return Poll::Ready(Some(InboundEvent::ReadyLightClientUpdate( + lc_id.into_inner(), + ))); + } + Poll::Ready(Some(Err(e))) => { + return Poll::Ready(Some(InboundEvent::DelayQueueError(e, "lc_updates_queue"))); + } + // `Poll::Ready(None)` means that there are no more entries in the delay queue and we + // will continue to get this result until something else is added into the queue. + Poll::Ready(None) | Poll::Pending => (), + } + // Last empty the messages channel. match self.work_reprocessing_rx.poll_recv(cx) { Poll::Ready(Some(message)) => return Poll::Ready(Some(InboundEvent::Msg(message))), @@ -264,14 +316,19 @@ pub fn spawn_reprocess_scheduler( gossip_block_delay_queue: DelayQueue::new(), rpc_block_delay_queue: DelayQueue::new(), attestations_delay_queue: DelayQueue::new(), + lc_updates_delay_queue: DelayQueue::new(), queued_gossip_block_roots: HashSet::new(), + queued_lc_updates: FnvHashMap::default(), queued_aggregates: FnvHashMap::default(), queued_unaggregates: FnvHashMap::default(), awaiting_attestations_per_root: HashMap::new(), + awaiting_lc_updates_per_parent_root: HashMap::new(), next_attestation: 0, + next_lc_update: 0, early_block_debounce: TimeLatch::default(), rpc_block_debounce: TimeLatch::default(), attestation_delay_debounce: TimeLatch::default(), + lc_update_delay_debounce: TimeLatch::default(), }; executor.spawn( @@ -473,9 +530,49 @@ impl ReprocessQueue { self.next_attestation += 1; } - InboundEvent::Msg(BlockImported(root)) => { + InboundEvent::Msg(UnknownLightClientOptimisticUpdate( + queued_light_client_optimistic_update, + )) => { + if self.lc_updates_delay_queue.len() >= MAXIMUM_QUEUED_LIGHT_CLIENT_UPDATES { + if self.lc_update_delay_debounce.elapsed() { + error!( + log, + "Light client updates delay queue is full"; + "queue_size" => MAXIMUM_QUEUED_LIGHT_CLIENT_UPDATES, + "msg" => "check system clock" + ); + } + // Drop the light client update. + return; + } + + let lc_id: QueuedLightClientUpdateId = self.next_lc_update; + + // Register the delay. + let delay_key = self + .lc_updates_delay_queue + .insert(lc_id, QUEUED_LIGHT_CLIENT_UPDATE_DELAY); + + // Register the light client update for the corresponding root. + self.awaiting_lc_updates_per_parent_root + .entry(queued_light_client_optimistic_update.parent_root) + .or_default() + .push(lc_id); + + // Store the light client update and its info. + self.queued_lc_updates.insert( + self.next_lc_update, + (queued_light_client_optimistic_update, delay_key), + ); + + self.next_lc_update += 1; + } + InboundEvent::Msg(BlockImported { + block_root, + parent_root, + }) => { // Unqueue the attestations we have for this root, if any. - if let Some(queued_ids) = self.awaiting_attestations_per_root.remove(&root) { + if let Some(queued_ids) = self.awaiting_attestations_per_root.remove(&block_root) { for id in queued_ids { metrics::inc_counter( &metrics::BEACON_PROCESSOR_REPROCESSING_QUEUE_MATCHED_ATTESTATIONS, @@ -511,12 +608,62 @@ impl ReprocessQueue { error!( log, "Unknown queued attestation for block root"; - "block_root" => ?root, + "block_root" => ?block_root, "att_id" => ?id, ); } } } + // Unqueue the light client optimistic updates we have for this root, if any. + if let Some(queued_lc_id) = self + .awaiting_lc_updates_per_parent_root + .remove(&parent_root) + { + debug!( + log, + "Dequeuing light client optimistic updates"; + "parent_root" => %parent_root, + "count" => queued_lc_id.len(), + ); + + for lc_id in queued_lc_id { + metrics::inc_counter( + &metrics::BEACON_PROCESSOR_REPROCESSING_QUEUE_MATCHED_OPTIMISTIC_UPDATES, + ); + if let Some((work, delay_key)) = self.queued_lc_updates.remove(&lc_id).map( + |(light_client_optimistic_update, delay_key)| { + ( + ReadyWork::LightClientUpdate(light_client_optimistic_update), + delay_key, + ) + }, + ) { + // Remove the delay + self.lc_updates_delay_queue.remove(&delay_key); + + // Send the work + match self.ready_work_tx.try_send(work) { + Ok(_) => trace!( + log, + "reprocessing light client update sent"; + ), + Err(_) => error!( + log, + "Failed to send scheduled light client update"; + ), + } + } else { + // There is a mismatch between the light client update ids registered for this + // root and the queued light client updates. This should never happen. + error!( + log, + "Unknown queued light client update for parent root"; + "parent_root" => ?parent_root, + "lc_id" => ?lc_id, + ); + } + } + } } // A block that was queued for later processing is now ready to be processed. InboundEvent::ReadyGossipBlock(ready_block) => { @@ -591,6 +738,38 @@ impl ReprocessQueue { } } } + InboundEvent::ReadyLightClientUpdate(queued_id) => { + metrics::inc_counter( + &metrics::BEACON_PROCESSOR_REPROCESSING_QUEUE_EXPIRED_OPTIMISTIC_UPDATES, + ); + + if let Some((parent_root, work)) = self.queued_lc_updates.remove(&queued_id).map( + |(queued_lc_update, _delay_key)| { + ( + queued_lc_update.parent_root, + ReadyWork::LightClientUpdate(queued_lc_update), + ) + }, + ) { + if self.ready_work_tx.try_send(work).is_err() { + error!( + log, + "Failed to send scheduled light client optimistic update"; + ); + } + + if let Some(queued_lc_updates) = self + .awaiting_lc_updates_per_parent_root + .get_mut(&parent_root) + { + if let Some(index) = + queued_lc_updates.iter().position(|&id| id == queued_id) + { + queued_lc_updates.swap_remove(index); + } + } + } + } } metrics::set_gauge_vec( @@ -608,5 +787,10 @@ impl ReprocessQueue { &[ATTESTATIONS], self.attestations_delay_queue.len() as i64, ); + metrics::set_gauge_vec( + &metrics::BEACON_PROCESSOR_REPROCESSING_QUEUE_TOTAL, + &[LIGHT_CLIENT_UPDATES], + self.lc_updates_delay_queue.len() as i64, + ); } } diff --git a/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs b/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs index ef23f6761f..3601ccb195 100644 --- a/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs @@ -28,7 +28,8 @@ use types::{ use super::{ super::work_reprocessing_queue::{ - QueuedAggregate, QueuedGossipBlock, QueuedUnaggregate, ReprocessQueueMessage, + QueuedAggregate, QueuedGossipBlock, QueuedLightClientUpdate, QueuedUnaggregate, + ReprocessQueueMessage, }, Worker, }; @@ -715,6 +716,10 @@ impl Worker { &metrics::BEACON_BLOCK_GOSSIP_SLOT_START_DELAY_TIME, block_delay, ); + metrics::set_gauge( + &metrics::BEACON_BLOCK_LAST_DELAY, + block_delay.as_millis() as i64, + ); let verification_result = self .chain @@ -949,7 +954,10 @@ impl Worker { metrics::inc_counter(&metrics::BEACON_PROCESSOR_GOSSIP_BLOCK_IMPORTED_TOTAL); if reprocess_tx - .try_send(ReprocessQueueMessage::BlockImported(block_root)) + .try_send(ReprocessQueueMessage::BlockImported { + block_root, + parent_root: block.message().parent_root(), + }) .is_err() { error!( @@ -1326,7 +1334,7 @@ impl Worker { LightClientFinalityUpdateError::InvalidLightClientFinalityUpdate => { debug!( self.log, - "LC invalid finality update"; + "Light client invalid finality update"; "peer" => %peer_id, "error" => ?e, ); @@ -1340,7 +1348,7 @@ impl Worker { LightClientFinalityUpdateError::TooEarly => { debug!( self.log, - "LC finality update too early"; + "Light client finality update too early"; "peer" => %peer_id, "error" => ?e, ); @@ -1353,7 +1361,7 @@ impl Worker { } LightClientFinalityUpdateError::FinalityUpdateAlreadySeen => debug!( self.log, - "LC finality update already seen"; + "Light client finality update already seen"; "peer" => %peer_id, "error" => ?e, ), @@ -1362,7 +1370,7 @@ impl Worker { | LightClientFinalityUpdateError::SigSlotStartIsNone | LightClientFinalityUpdateError::FailedConstructingUpdate => debug!( self.log, - "LC error constructing finality update"; + "Light client error constructing finality update"; "peer" => %peer_id, "error" => ?e, ), @@ -1377,22 +1385,77 @@ impl Worker { message_id: MessageId, peer_id: PeerId, light_client_optimistic_update: LightClientOptimisticUpdate, + reprocess_tx: Option>>, seen_timestamp: Duration, ) { - match self - .chain - .verify_optimistic_update_for_gossip(light_client_optimistic_update, seen_timestamp) - { - Ok(_verified_light_client_optimistic_update) => { + match self.chain.verify_optimistic_update_for_gossip( + light_client_optimistic_update.clone(), + seen_timestamp, + ) { + Ok(verified_light_client_optimistic_update) => { + debug!( + self.log, + "Light client successful optimistic update"; + "peer" => %peer_id, + "parent_root" => %verified_light_client_optimistic_update.parent_root, + ); + self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Accept); } Err(e) => { - metrics::register_optimistic_update_error(&e); match e { - LightClientOptimisticUpdateError::InvalidLightClientOptimisticUpdate => { + LightClientOptimisticUpdateError::UnknownBlockParentRoot(parent_root) => { + metrics::inc_counter( + &metrics::BEACON_PROCESSOR_REPROCESSING_QUEUE_SENT_OPTIMISTIC_UPDATES, + ); debug!( self.log, - "LC invalid optimistic update"; + "Optimistic update for unknown block"; + "peer_id" => %peer_id, + "parent_root" => ?parent_root + ); + + if let Some(sender) = reprocess_tx { + let msg = ReprocessQueueMessage::UnknownLightClientOptimisticUpdate( + QueuedLightClientUpdate { + peer_id, + message_id, + light_client_optimistic_update: Box::new( + light_client_optimistic_update, + ), + parent_root, + seen_timestamp, + }, + ); + + if sender.try_send(msg).is_err() { + error!( + self.log, + "Failed to send optimistic update for re-processing"; + ) + } + } else { + debug!( + self.log, + "Not sending light client update because it had been reprocessed"; + "peer_id" => %peer_id, + "parent_root" => ?parent_root + ); + + self.propagate_validation_result( + message_id, + peer_id, + MessageAcceptance::Ignore, + ); + } + return; + } + LightClientOptimisticUpdateError::InvalidLightClientOptimisticUpdate => { + metrics::register_optimistic_update_error(&e); + + debug!( + self.log, + "Light client invalid optimistic update"; "peer" => %peer_id, "error" => ?e, ); @@ -1404,9 +1467,10 @@ impl Worker { ) } LightClientOptimisticUpdateError::TooEarly => { + metrics::register_optimistic_update_error(&e); debug!( self.log, - "LC optimistic update too early"; + "Light client optimistic update too early"; "peer" => %peer_id, "error" => ?e, ); @@ -1417,21 +1481,29 @@ impl Worker { "light_client_gossip_error", ); } - LightClientOptimisticUpdateError::OptimisticUpdateAlreadySeen => debug!( - self.log, - "LC optimistic update already seen"; - "peer" => %peer_id, - "error" => ?e, - ), + LightClientOptimisticUpdateError::OptimisticUpdateAlreadySeen => { + metrics::register_optimistic_update_error(&e); + + debug!( + self.log, + "Light client optimistic update already seen"; + "peer" => %peer_id, + "error" => ?e, + ) + } LightClientOptimisticUpdateError::BeaconChainError(_) | LightClientOptimisticUpdateError::LightClientUpdateError(_) | LightClientOptimisticUpdateError::SigSlotStartIsNone - | LightClientOptimisticUpdateError::FailedConstructingUpdate => debug!( - self.log, - "LC error constructing optimistic update"; - "peer" => %peer_id, - "error" => ?e, - ), + | LightClientOptimisticUpdateError::FailedConstructingUpdate => { + metrics::register_optimistic_update_error(&e); + + debug!( + self.log, + "Light client error constructing optimistic update"; + "peer" => %peer_id, + "error" => ?e, + ) + } } self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Ignore); } diff --git a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs index 3e354a70d2..afcc15280d 100644 --- a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs @@ -7,7 +7,7 @@ use itertools::process_results; use lighthouse_network::rpc::StatusMessage; use lighthouse_network::rpc::*; use lighthouse_network::{PeerId, PeerRequestId, ReportSource, Response, SyncInfo}; -use slog::{debug, error}; +use slog::{debug, error, warn}; use slot_clock::SlotClock; use std::sync::Arc; use task_executor::TaskExecutor; @@ -392,12 +392,35 @@ impl Worker { break; } Err(e) => { - error!( - self.log, - "Error fetching block for peer"; - "block_root" => ?root, - "error" => ?e + if matches!( + e, + BeaconChainError::ExecutionLayerErrorPayloadReconstruction(_block_hash, ref boxed_error) + if matches!(**boxed_error, execution_layer::Error::EngineError(_)) + ) { + warn!( + self.log, + "Error rebuilding payload for peer"; + "info" => "this may occur occasionally when the EE is busy", + "block_root" => ?root, + "error" => ?e, + ); + } else { + error!( + self.log, + "Error fetching block for peer"; + "block_root" => ?root, + "error" => ?e + ); + } + + // send the stream terminator + self.send_error_response( + peer_id, + RPCResponseErrorCode::ServerError, + "Failed fetching blocks".into(), + request_id, ); + send_response = false; break; } } diff --git a/beacon_node/network/src/beacon_processor/worker/sync_methods.rs b/beacon_node/network/src/beacon_processor/worker/sync_methods.rs index 1ec045e97e..6e6e681550 100644 --- a/beacon_node/network/src/beacon_processor/worker/sync_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/sync_methods.rs @@ -84,6 +84,7 @@ impl Worker { } }; let slot = block.slot(); + let parent_root = block.message().parent_root(); let result = self .chain .process_block( @@ -101,7 +102,10 @@ impl Worker { info!(self.log, "New RPC block received"; "slot" => slot, "hash" => %hash); // Trigger processing for work referencing this block. - let reprocess_msg = ReprocessQueueMessage::BlockImported(hash); + let reprocess_msg = ReprocessQueueMessage::BlockImported { + block_root: hash, + parent_root, + }; if reprocess_tx.try_send(reprocess_msg).is_err() { error!(self.log, "Failed to inform block import"; "source" => "rpc", "block_root" => %hash) }; diff --git a/beacon_node/network/src/metrics.rs b/beacon_node/network/src/metrics.rs index b4f3f29f93..8dc76877a1 100644 --- a/beacon_node/network/src/metrics.rs +++ b/beacon_node/network/src/metrics.rs @@ -335,10 +335,18 @@ lazy_static! { pub static ref BEACON_BLOCK_GOSSIP_SLOT_START_DELAY_TIME: Result = try_create_histogram_with_buckets( "beacon_block_gossip_slot_start_delay_time", "Duration between when the block is received and the start of the slot it belongs to.", + // Create a custom bucket list for greater granularity in block delay + Ok(vec![0.1, 0.2, 0.3,0.4,0.5,0.75,1.0,1.25,1.5,1.75,2.0,2.5,3.0,3.5,4.0,5.0,6.0,7.0,8.0,9.0,10.0,15.0,20.0]) + // NOTE: Previous values, which we may want to switch back to. // [0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50] - decimal_buckets(-1,2) + //decimal_buckets(-1,2) ); + pub static ref BEACON_BLOCK_LAST_DELAY: Result = try_create_int_gauge( + "beacon_block_last_delay", + "Keeps track of the last block's delay from the start of the slot" + ); + pub static ref BEACON_BLOCK_GOSSIP_ARRIVED_LATE_TOTAL: Result = try_create_int_counter( "beacon_block_gossip_arrived_late_total", "Count of times when a gossip block arrived from the network later than the attestation deadline.", @@ -362,6 +370,21 @@ lazy_static! { "Number of queued attestations where as matching block has been imported." ); + /* + * Light client update reprocessing queue metrics. + */ + pub static ref BEACON_PROCESSOR_REPROCESSING_QUEUE_EXPIRED_OPTIMISTIC_UPDATES: Result = try_create_int_counter( + "beacon_processor_reprocessing_queue_expired_optimistic_updates", + "Number of queued light client optimistic updates which have expired before a matching block has been found." + ); + pub static ref BEACON_PROCESSOR_REPROCESSING_QUEUE_MATCHED_OPTIMISTIC_UPDATES: Result = try_create_int_counter( + "beacon_processor_reprocessing_queue_matched_optimistic_updates", + "Number of queued light client optimistic updates where as matching block has been imported." + ); + pub static ref BEACON_PROCESSOR_REPROCESSING_QUEUE_SENT_OPTIMISTIC_UPDATES: Result = try_create_int_counter( + "beacon_processor_reprocessing_queue_sent_optimistic_updates", + "Number of queued light client optimistic updates where as matching block has been imported." + ); } pub fn update_bandwidth_metrics(bandwidth: Arc) { diff --git a/beacon_node/network/src/subnet_service/tests/mod.rs b/beacon_node/network/src/subnet_service/tests/mod.rs index 30f030eba7..9e1c9f51bc 100644 --- a/beacon_node/network/src/subnet_service/tests/mod.rs +++ b/beacon_node/network/src/subnet_service/tests/mod.rs @@ -2,6 +2,7 @@ use super::*; use beacon_chain::{ builder::{BeaconChainBuilder, Witness}, eth1_chain::CachingEth1Backend, + validator_monitor::DEFAULT_INDIVIDUAL_TRACKING_THRESHOLD, BeaconChain, }; use futures::prelude::*; @@ -75,7 +76,7 @@ impl TestBeaconChain { Duration::from_millis(SLOT_DURATION_MILLIS), )) .shutdown_sender(shutdown_tx) - .monitor_validators(true, vec![], log) + .monitor_validators(true, vec![], DEFAULT_INDIVIDUAL_TRACKING_THRESHOLD, log) .build() .expect("should build"), ); diff --git a/beacon_node/operation_pool/src/attestation.rs b/beacon_node/operation_pool/src/attestation.rs index 4af4edc0e4..fbbd5d7ddc 100644 --- a/beacon_node/operation_pool/src/attestation.rs +++ b/beacon_node/operation_pool/src/attestation.rs @@ -49,7 +49,7 @@ impl<'a, T: EthSpec> AttMaxCover<'a, T> { let indices = get_attesting_indices::(committee.committee, &fresh_validators).ok()?; let fresh_validators_rewards: HashMap = indices .iter() - .map(|i| *i as u64) + .copied() .flat_map(|validator_index| { let reward = base::get_base_reward( state, diff --git a/beacon_node/operation_pool/src/lib.rs b/beacon_node/operation_pool/src/lib.rs index 4fe5a72545..bb370ed5b2 100644 --- a/beacon_node/operation_pool/src/lib.rs +++ b/beacon_node/operation_pool/src/lib.rs @@ -8,7 +8,7 @@ mod persistence; mod reward_cache; mod sync_aggregate_id; -pub use attestation::AttMaxCover; +pub use attestation::{earliest_attestation_validators, AttMaxCover}; pub use attestation_storage::{AttestationRef, SplitAttestation}; pub use max_cover::MaxCover; pub use persistence::{ diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index c5f4cc8adf..b4da83315c 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -194,6 +194,21 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .help("Lighthouse by default does not discover private IP addresses. Set this flag to enable connection attempts to local addresses.") .takes_value(false), ) + .arg( + Arg::with_name("self-limiter") + .long("self-limiter") + .help( + "Enables the outbound rate limiter (requests made by this node).\ + \ + Rate limit quotas per protocol can be set in the form of \ + :/. To set quotas for multiple protocols, \ + separate them by ';'. If the self rate limiter is enabled and a protocol is not \ + present in the configuration, the quotas used for the inbound rate limiter will be \ + used." + ) + .min_values(0) + .hidden(true) + ) /* REST API related arguments */ .arg( Arg::with_name("http") @@ -753,6 +768,17 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .value_name("PATH") .takes_value(true) ) + .arg( + Arg::with_name("validator-monitor-individual-tracking-threshold") + .long("validator-monitor-individual-tracking-threshold") + .help("Once the validator monitor reaches this number of local validators \ + it will stop collecting per-validator Prometheus metrics and issuing \ + per-validator logs. Instead, it will provide aggregate metrics and logs. \ + This avoids infeasibly high cardinality in the Prometheus database and \ + high log volume when using many validators. Defaults to 64.") + .value_name("INTEGER") + .takes_value(true) + ) .arg( Arg::with_name("disable-lock-timeouts") .long("disable-lock-timeouts") @@ -900,6 +926,13 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { Useful if you intend to run a non-validating beacon node.") .takes_value(false) ) + .arg( + Arg::with_name("disable-optimistic-finalized-sync") + .long("disable-optimistic-finalized-sync") + .help("Force Lighthouse to verify every execution block hash with the execution \ + client during finalized sync. By default block hashes will be checked in \ + Lighthouse and only passed to the EL if initial verification fails.") + ) .arg( Arg::with_name("light-client-server") .long("light-client-server") diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index b3bfa696ed..726f8368ea 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -675,6 +675,12 @@ pub fn get_config( .extend_from_slice(&pubkeys); } + if let Some(count) = + clap_utils::parse_optional(cli_args, "validator-monitor-individual-tracking-threshold")? + { + client_config.validator_monitor_individual_tracking_threshold = count; + } + if cli_args.is_present("disable-lock-timeouts") { client_config.chain.enable_lock_timeouts = false; } @@ -741,6 +747,10 @@ pub fn get_config( client_config.validator_monitor_auto = true; } + // Optimistic finalized sync. + client_config.chain.optimistic_finalized_sync = + !cli_args.is_present("disable-optimistic-finalized-sync"); + Ok(client_config) } @@ -957,6 +967,13 @@ pub fn set_network_config( // Light client server config. config.enable_light_client_server = cli_args.is_present("light-client-server"); + // This flag can be used both with or without a value. Try to parse it first with a value, if + // no value is defined but the flag is present, use the default params. + config.outbound_rate_limiter_config = clap_utils::parse_optional(cli_args, "self-limiter")?; + if cli_args.is_present("self-limiter") && config.outbound_rate_limiter_config.is_none() { + config.outbound_rate_limiter_config = Some(Default::default()); + } + Ok(()) } diff --git a/beacon_node/store/src/chunked_vector.rs b/beacon_node/store/src/chunked_vector.rs index 25169b4790..8c64d4bcc0 100644 --- a/beacon_node/store/src/chunked_vector.rs +++ b/beacon_node/store/src/chunked_vector.rs @@ -801,7 +801,7 @@ mod test { fn needs_genesis_value_test_randao>(_: F) { let spec = &TestSpec::default_spec(); - let max = TestSpec::slots_per_epoch() as u64 * (F::Length::to_u64() - 1); + let max = TestSpec::slots_per_epoch() * (F::Length::to_u64() - 1); for i in 0..max { assert!( F::slot_needs_genesis_value(Slot::new(i), spec), diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index 4f63f4e7f9..6028d0ddcf 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -727,6 +727,10 @@ impl, Cold: ItemStore> HotColdDB let key = get_key_for_col(DBColumn::ExecPayload.into(), block_root.as_bytes()); key_value_batch.push(KeyValueStoreOp::DeleteKey(key)); } + + StoreOp::KeyValueOp(kv_op) => { + key_value_batch.push(kv_op); + } } } Ok(key_value_batch) @@ -758,6 +762,8 @@ impl, Cold: ItemStore> HotColdDB StoreOp::DeleteState(_, _) => (), StoreOp::DeleteExecutionPayload(_) => (), + + StoreOp::KeyValueOp(_) => (), } } diff --git a/beacon_node/store/src/lib.rs b/beacon_node/store/src/lib.rs index 75aeca058b..9d15dd4043 100644 --- a/beacon_node/store/src/lib.rs +++ b/beacon_node/store/src/lib.rs @@ -161,6 +161,7 @@ pub enum StoreOp<'a, E: EthSpec> { DeleteBlock(Hash256), DeleteState(Hash256, Option), DeleteExecutionPayload(Hash256), + KeyValueOp(KeyValueStoreOp), } /// A unique column identifier. diff --git a/book/src/checkpoint-sync.md b/book/src/checkpoint-sync.md index 736aa08f1c..47dc03b20c 100644 --- a/book/src/checkpoint-sync.md +++ b/book/src/checkpoint-sync.md @@ -48,17 +48,6 @@ The Ethereum community provides various [public endpoints](https://eth-clients.g lighthouse bn --checkpoint-sync-url https://example.com/ ... ``` -### Use Infura as a remote beacon node provider - -You can use Infura as the remote beacon node provider to load the initial checkpoint state. - -1. Sign up for the free Infura ETH2 API using the `Create new project tab` on the [Infura dashboard](https://infura.io/dashboard). -2. Copy the HTTPS endpoint for the required network (Mainnet/Prater). -3. Use it as the url for the `--checkpoint-sync-url` flag. e.g. -``` -lighthouse bn --checkpoint-sync-url https://:@eth2-beacon-mainnet.infura.io ... -``` - ## Backfilling Blocks Once forwards sync completes, Lighthouse will commence a "backfill sync" to download the blocks @@ -108,7 +97,7 @@ You can opt-in to reconstructing all of the historic states by providing the The database keeps track of three markers to determine the availability of historic blocks and states: -* `oldest_block_slot`: All blocks with slots less than or equal to this value are available in the +* `oldest_block_slot`: All blocks with slots greater than or equal to this value are available in the database. Additionally, the genesis block is always available. * `state_lower_limit`: All states with slots _less than or equal to_ this value are available in the database. The minimum value is 0, indicating that the genesis state is always available. diff --git a/book/src/docker.md b/book/src/docker.md index f22b8a2008..7484f9f525 100644 --- a/book/src/docker.md +++ b/book/src/docker.md @@ -57,7 +57,7 @@ $ docker pull sigp/lighthouse:latest-modern Image tags follow this format: ``` -${version}${arch}${stability}${modernity} +${version}${arch}${stability}${modernity}${features} ``` The `version` is: @@ -81,6 +81,12 @@ The `modernity` is: * `-modern` for optimized builds * empty for a `portable` unoptimized build +The `features` is: + +* `-dev` for a development build with `minimal-spec` preset enabled. +* empty for a standard build with no custom feature enabled. + + Examples: * `latest-unstable-modern`: most recent `unstable` build for all modern CPUs (x86_64 or ARM) diff --git a/book/src/installation-source.md b/book/src/installation-source.md index b3d83ef9f9..8e515a41bd 100644 --- a/book/src/installation-source.md +++ b/book/src/installation-source.md @@ -64,6 +64,7 @@ choco install protoc These dependencies are for compiling Lighthouse natively on Windows. Lighthouse can also run successfully under the [Windows Subsystem for Linux (WSL)][WSL]. If using Ubuntu under WSL, you should follow the instructions for Ubuntu listed in the [Dependencies (Ubuntu)](#ubuntu) section. + [WSL]: https://docs.microsoft.com/en-us/windows/wsl/about ## Build Lighthouse @@ -128,8 +129,12 @@ Commonly used features include: * `gnosis`: support for the Gnosis Beacon Chain. * `portable`: support for legacy hardware. * `modern`: support for exclusively modern hardware. -* `slasher-mdbx`: support for the MDBX slasher backend (enabled by default). +* `slasher-mdbx`: support for the MDBX slasher backend. Enabled by default. * `slasher-lmdb`: support for the LMDB slasher backend. +* `jemalloc`: use [`jemalloc`][jemalloc] to allocate memory. Enabled by default on Linux and macOS. + Not supported on Windows. + +[jemalloc]: https://jemalloc.net/ ## Compilation Profiles diff --git a/book/src/merge-migration.md b/book/src/merge-migration.md index 08f1b51e42..ec9aeaaee8 100644 --- a/book/src/merge-migration.md +++ b/book/src/merge-migration.md @@ -58,7 +58,7 @@ supported. Each execution engine has its own flags for configuring the engine API and JWT. Please consult the relevant page for your execution engine for the required flags: -- [Geth: Connecting to Consensus Clients](https://geth.ethereum.org/docs/interface/consensus-clients) +- [Geth: Connecting to Consensus Clients](https://geth.ethereum.org/docs/getting-started/consensus-clients) - [Nethermind: Running Nethermind Post Merge](https://docs.nethermind.io/nethermind/first-steps-with-nethermind/running-nethermind-post-merge) - [Besu: Prepare For The Merge](https://besu.hyperledger.org/en/stable/HowTo/Upgrade/Prepare-for-The-Merge/) - [Erigon: Beacon Chain (Consensus Layer)](https://github.com/ledgerwatch/erigon#beacon-chain-consensus-layer) @@ -203,5 +203,5 @@ guidance for specific setups. - [Ethereum.org: The Merge](https://ethereum.org/en/upgrades/merge/) - [Ethereum Staking Launchpad: Merge Readiness](https://launchpad.ethereum.org/en/merge-readiness). - [CoinCashew: Ethereum Merge Upgrade Checklist](https://www.coincashew.com/coins/overview-eth/ethereum-merge-upgrade-checklist-for-home-stakers-and-validators) -- [EthDocker: Merge Preparation](https://eth-docker.net/docs/About/MergePrep/) +- [EthDocker: Merge Preparation](https://eth-docker.net/About/MergePrep/) - [Remy Roy: How to join the Goerli/Prater merge testnet](https://github.com/remyroy/ethstaker/blob/main/merge-goerli-prater.md) diff --git a/book/src/run_a_node.md b/book/src/run_a_node.md index 5ce42aa630..fb112c3675 100644 --- a/book/src/run_a_node.md +++ b/book/src/run_a_node.md @@ -26,7 +26,7 @@ has authority to control the execution engine. Each execution engine has its own flags for configuring the engine API and JWT. Please consult the relevant page of your execution engine for the required flags: -- [Geth: Connecting to Consensus Clients](https://geth.ethereum.org/docs/interface/consensus-clients) +- [Geth: Connecting to Consensus Clients](https://geth.ethereum.org/docs/getting-started/consensus-clients) - [Nethermind: Running Nethermind & CL](https://docs.nethermind.io/nethermind/first-steps-with-nethermind/running-nethermind-post-merge) - [Besu: Connect to Mainnet](https://besu.hyperledger.org/en/stable/public-networks/get-started/connect/mainnet/) - [Erigon: Beacon Chain (Consensus Layer)](https://github.com/ledgerwatch/erigon#beacon-chain-consensus-layer) diff --git a/book/src/validator-inclusion.md b/book/src/validator-inclusion.md index e6fbc0f16f..0793af20db 100644 --- a/book/src/validator-inclusion.md +++ b/book/src/validator-inclusion.md @@ -59,14 +59,7 @@ The following fields are returned: - `previous_epoch_head_attesting_gwei`: the total staked gwei that attested to a head beacon block that is in the canonical chain. -From this data you can calculate some interesting figures: - -#### Participation Rate - -`previous_epoch_attesting_gwei / previous_epoch_active_gwei` - -Expresses the ratio of validators that managed to have an attestation -voting upon the previous epoch included in a block. +From this data you can calculate: #### Justification/Finalization Rate diff --git a/book/src/voluntary-exit.md b/book/src/voluntary-exit.md index 69c2d7598b..5056040e4c 100644 --- a/book/src/voluntary-exit.md +++ b/book/src/voluntary-exit.md @@ -12,10 +12,10 @@ This number can be much higher depending on how many other validators are queued ## Withdrawal of exited funds -Even though users can perform a voluntary exit in phase 0, they **cannot withdraw their exited funds at this point in time**. -This implies that the staked funds are effectively **frozen** until withdrawals are enabled in future phases. +Even though users can currently perform a voluntary exit, they **cannot withdraw their exited funds at this point in time**. +This implies that the staked funds are effectively **frozen** until withdrawals are enabled in a future hard fork (Capella). -To understand the phased rollout strategy for Ethereum upgrades, please visit . +To understand the rollout strategy for Ethereum upgrades, please visit . diff --git a/boot_node/Cargo.toml b/boot_node/Cargo.toml index 1dedabe4a4..a03ba36caa 100644 --- a/boot_node/Cargo.toml +++ b/boot_node/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "boot_node" -version = "3.3.0" +version = "3.4.0" authors = ["Sigma Prime "] edition = "2021" diff --git a/bors.toml b/bors.toml index dbe92c68f4..9e633d63f5 100644 --- a/bors.toml +++ b/bors.toml @@ -10,7 +10,6 @@ status = [ "merge-transition-ubuntu", "no-eth1-simulator-ubuntu", "check-benchmarks", - "check-consensus", "clippy", "arbitrary-check", "cargo-audit", @@ -23,7 +22,6 @@ status = [ "check-msrv", "slasher-tests", "syncing-simulator-ubuntu", - "disallowed-from-async-lint", "compile-with-beta-compiler" ] use_squash_merge = true diff --git a/common/account_utils/src/validator_definitions.rs b/common/account_utils/src/validator_definitions.rs index c0addb0568..1b01bedb74 100644 --- a/common/account_utils/src/validator_definitions.rs +++ b/common/account_utils/src/validator_definitions.rs @@ -227,7 +227,7 @@ impl ValidatorDefinitions { .write(true) .read(true) .create_new(false) - .open(&config_path) + .open(config_path) .map_err(Error::UnableToOpenFile)?; serde_yaml::from_reader(file).map_err(Error::UnableToParseFile) } diff --git a/common/clap_utils/Cargo.toml b/common/clap_utils/Cargo.toml index b370eb0825..62eb8aa3d5 100644 --- a/common/clap_utils/Cargo.toml +++ b/common/clap_utils/Cargo.toml @@ -12,7 +12,7 @@ hex = "0.4.2" dirs = "3.0.1" eth2_network_config = { path = "../eth2_network_config" } eth2_ssz = "0.4.1" -ethereum-types = "0.12.1" +ethereum-types = "0.14.1" serde = "1.0.116" serde_json = "1.0.59" serde_yaml = "0.8.13" diff --git a/common/compare_fields/src/lib.rs b/common/compare_fields/src/lib.rs index a0166eb500..bc2f5446ad 100644 --- a/common/compare_fields/src/lib.rs +++ b/common/compare_fields/src/lib.rs @@ -115,11 +115,7 @@ impl Comparison { let mut children = vec![]; for i in 0..std::cmp::max(a.len(), b.len()) { - children.push(FieldComparison::new( - format!("{:}", i), - &a.get(i), - &b.get(i), - )); + children.push(FieldComparison::new(format!("{i}"), &a.get(i), &b.get(i))); } Self::parent(field_name, a == b, children) @@ -164,8 +160,8 @@ impl FieldComparison { Self { field_name, equal: a == b, - a: format!("{:?}", a), - b: format!("{:?}", b), + a: format!("{a:?}"), + b: format!("{b:?}"), } } diff --git a/common/compare_fields_derive/src/lib.rs b/common/compare_fields_derive/src/lib.rs index beabc6ca9b..752c09ee05 100644 --- a/common/compare_fields_derive/src/lib.rs +++ b/common/compare_fields_derive/src/lib.rs @@ -32,7 +32,7 @@ pub fn compare_fields_derive(input: TokenStream) -> TokenStream { _ => panic!("compare_fields_derive only supports named struct fields."), }; - let field_name = format!("{:}", ident_a); + let field_name = ident_a.to_string(); let ident_b = ident_a.clone(); let quote = if is_slice(field) { diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 58b4c88b3c..653c6c0bcc 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -1026,6 +1026,58 @@ impl BeaconNodeHttpClient { .transpose() } + /// `POST beacon/rewards/sync_committee` + pub async fn post_beacon_rewards_sync_committee( + &self, + rewards: &[Option>], + ) -> Result<(), Error> { + let mut path = self.eth_path(V1)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("beacon") + .push("rewards") + .push("sync_committee"); + + self.post(path, &rewards).await?; + + Ok(()) + } + + /// `GET beacon/rewards/blocks` + pub async fn get_beacon_rewards_blocks(&self, epoch: Epoch) -> Result<(), Error> { + let mut path = self.eth_path(V1)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("beacon") + .push("rewards") + .push("blocks"); + + path.query_pairs_mut() + .append_pair("epoch", &epoch.to_string()); + + self.get(path).await + } + + /// `POST beacon/rewards/attestations` + pub async fn post_beacon_rewards_attestations( + &self, + attestations: &[ValidatorId], + ) -> Result<(), Error> { + let mut path = self.eth_path(V1)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("beacon") + .push("rewards") + .push("attestations"); + + self.post(path, &attestations).await?; + + Ok(()) + } + /// `POST validator/contribution_and_proofs` pub async fn post_validator_contribution_and_proofs( &self, diff --git a/common/eth2/src/lighthouse.rs b/common/eth2/src/lighthouse.rs index 2dced1c449..e50d9f4dc0 100644 --- a/common/eth2/src/lighthouse.rs +++ b/common/eth2/src/lighthouse.rs @@ -1,8 +1,11 @@ //! This module contains endpoints that are non-standard and only available on Lighthouse servers. mod attestation_performance; +pub mod attestation_rewards; mod block_packing_efficiency; mod block_rewards; +mod standard_block_rewards; +mod sync_committee_rewards; use crate::{ ok_or_error, @@ -22,11 +25,14 @@ use store::{AnchorInfo, Split, StoreConfig}; pub use attestation_performance::{ AttestationPerformance, AttestationPerformanceQuery, AttestationPerformanceStatistics, }; +pub use attestation_rewards::StandardAttestationRewards; pub use block_packing_efficiency::{ BlockPackingEfficiency, BlockPackingEfficiencyQuery, ProposerInfo, UniqueAttestation, }; pub use block_rewards::{AttestationRewards, BlockReward, BlockRewardMeta, BlockRewardsQuery}; pub use lighthouse_network::{types::SyncState, PeerInfo}; +pub use standard_block_rewards::StandardBlockReward; +pub use sync_committee_rewards::SyncCommitteeReward; // Define "legacy" implementations of `Option` which use four bytes for encoding the union // selector. diff --git a/common/eth2/src/lighthouse/attestation_rewards.rs b/common/eth2/src/lighthouse/attestation_rewards.rs new file mode 100644 index 0000000000..314ffb8512 --- /dev/null +++ b/common/eth2/src/lighthouse/attestation_rewards.rs @@ -0,0 +1,44 @@ +use serde::{Deserialize, Serialize}; + +// Details about the rewards paid for attestations +// All rewards in GWei + +#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] +pub struct IdealAttestationRewards { + // Validator's effective balance in gwei + #[serde(with = "eth2_serde_utils::quoted_u64")] + pub effective_balance: u64, + // Ideal attester's reward for head vote in gwei + #[serde(with = "eth2_serde_utils::quoted_u64")] + pub head: u64, + // Ideal attester's reward for target vote in gwei + #[serde(with = "eth2_serde_utils::quoted_u64")] + pub target: u64, + // Ideal attester's reward for source vote in gwei + #[serde(with = "eth2_serde_utils::quoted_u64")] + pub source: u64, +} + +#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] +pub struct TotalAttestationRewards { + // one entry for every validator based on their attestations in the epoch + #[serde(with = "eth2_serde_utils::quoted_u64")] + pub validator_index: u64, + // attester's reward for head vote in gwei + #[serde(with = "eth2_serde_utils::quoted_u64")] + pub head: u64, + // attester's reward for target vote in gwei + #[serde(with = "eth2_serde_utils::quoted_i64")] + pub target: i64, + // attester's reward for source vote in gwei + #[serde(with = "eth2_serde_utils::quoted_i64")] + pub source: i64, + // TBD attester's inclusion_delay reward in gwei (phase0 only) + // pub inclusion_delay: u64, +} + +#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] +pub struct StandardAttestationRewards { + pub ideal_rewards: Vec, + pub total_rewards: Vec, +} diff --git a/common/eth2/src/lighthouse/standard_block_rewards.rs b/common/eth2/src/lighthouse/standard_block_rewards.rs new file mode 100644 index 0000000000..502577500d --- /dev/null +++ b/common/eth2/src/lighthouse/standard_block_rewards.rs @@ -0,0 +1,26 @@ +use serde::{Deserialize, Serialize}; + +// Details about the rewards for a single block +// All rewards in GWei +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] +pub struct StandardBlockReward { + // proposer of the block, the proposer index who receives these rewards + #[serde(with = "eth2_serde_utils::quoted_u64")] + pub proposer_index: u64, + // total block reward in gwei, + // equal to attestations + sync_aggregate + proposer_slashings + attester_slashings + #[serde(with = "eth2_serde_utils::quoted_u64")] + pub total: u64, + // block reward component due to included attestations in gwei + #[serde(with = "eth2_serde_utils::quoted_u64")] + pub attestations: u64, + // block reward component due to included sync_aggregate in gwei + #[serde(with = "eth2_serde_utils::quoted_u64")] + pub sync_aggregate: u64, + // block reward component due to included proposer_slashings in gwei + #[serde(with = "eth2_serde_utils::quoted_u64")] + pub proposer_slashings: u64, + // block reward component due to included attester_slashings in gwei + #[serde(with = "eth2_serde_utils::quoted_u64")] + pub attester_slashings: u64, +} diff --git a/common/eth2/src/lighthouse/sync_committee_rewards.rs b/common/eth2/src/lighthouse/sync_committee_rewards.rs new file mode 100644 index 0000000000..e215d8e3e0 --- /dev/null +++ b/common/eth2/src/lighthouse/sync_committee_rewards.rs @@ -0,0 +1,13 @@ +use serde::{Deserialize, Serialize}; + +// Details about the rewards paid to sync committee members for attesting headers +// All rewards in GWei + +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] +pub struct SyncCommitteeReward { + #[serde(with = "eth2_serde_utils::quoted_u64")] + pub validator_index: u64, + // sync committee reward in gwei for the validator + #[serde(with = "eth2_serde_utils::quoted_i64")] + pub reward: i64, +} diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 7012972460..53cca49120 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -270,11 +270,20 @@ pub struct FinalityCheckpointsData { } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(try_from = "&str")] pub enum ValidatorId { PublicKey(PublicKeyBytes), Index(u64), } +impl TryFrom<&str> for ValidatorId { + type Error = String; + + fn try_from(s: &str) -> Result { + Self::from_str(s) + } +} + impl FromStr for ValidatorId { type Err = String; diff --git a/common/eth2_network_config/built_in_network_configs/gnosis/boot_enr.yaml b/common/eth2_network_config/built_in_network_configs/gnosis/boot_enr.yaml index 130c1fa1c3..e5896988c9 100644 --- a/common/eth2_network_config/built_in_network_configs/gnosis/boot_enr.yaml +++ b/common/eth2_network_config/built_in_network_configs/gnosis/boot_enr.yaml @@ -3,3 +3,7 @@ - enr:-Ly4QBf76jLiCA_pDXoZjhyRbuwzFOscFY-MIKkPnmHPQbvaKhIDZutfe38G9ibzgQP0RKrTo3vcWOy4hf_8wOZ-U5MBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpCCS-QxAgAAZP__________gmlkgnY0gmlwhBLGgjaJc2VjcDI1NmsxoQLGeo0Q4lDvxIjHjnkAqEuETTaFIjsNrEcSpdDhcHXWFYhzeW5jbmV0cwCDdGNwgiMog3VkcIIjKA - enr:-Ly4QLjZUWdqUO_RwyDqCAccIK5-MbLRD6A2c7oBuVbBgBnWDkEf0UKJVAaJqi2pO101WVQQLYSnYgz1Q3pRhYdrlFoBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpCCS-QxAgAAZP__________gmlkgnY0gmlwhANA8sSJc2VjcDI1NmsxoQK4TC_EK1jSs0VVPUpOjIo1rhJmff2SLBPFOWSXMwdLVYhzeW5jbmV0cwCDdGNwgiMog3VkcIIjKA - enr:-Ly4QKwX2rTFtKWKQHSGQFhquxsxL1jewO8JB1MG-jgHqAZVFWxnb3yMoQqnYSV1bk25-_jiLuhIulxar3RBWXEDm6EBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpCCS-QxAgAAZP__________gmlkgnY0gmlwhAN-qZeJc2VjcDI1NmsxoQI7EPGMpecl0QofLp4Wy_lYNCCChUFEH6kY7k-oBGkPFIhzeW5jbmV0cwCDdGNwgiMog3VkcIIjKA +- enr:-Ly4QPoChSQTleJROee1-k-4HOEgKqL9kLksE-tEiVqcY9kwF9V53aBg-MruD7Yx4Aks3LAeJpKXAS4ntMrIdqvQYc8Ch2F0dG5ldHOIAAAAAAAAAACEZXRoMpCCS-QxAgAAZP__________gmlkgnY0gmlwhGsWBHiJc2VjcDI1NmsxoQKwGQrwOSBJB_DtQOkFZVAY4YQfMAbUVxFpL5WgrzEddYhzeW5jbmV0cwCDdGNwgiMog3VkcIIjKA +- enr:-Ly4QBbaKRSX4SncCOxTTL611Kxlz-zYFrIn-k_63jGIPK_wbvFghVUHJICPCxufgTX5h79jvgfPr-2hEEQEdziGQ5MCh2F0dG5ldHOIAAAAAAAAAACEZXRoMpCCS-QxAgAAZP__________gmlkgnY0gmlwhAMazo6Jc2VjcDI1NmsxoQKt-kbM9isuWp8djhyEq6-4MLv1Sy7dOXeMOMdPgwu9LohzeW5jbmV0cwCDdGNwgiMog3VkcIIjKA +- enr:-Ly4QKJ5BzgFyJ6BaTlGY0C8ROzl508U3GA6qxdG5Gn2hxdke6nQO187pYlLvhp82Dez4PQn436Fts1F0WAm-_5l2LACh2F0dG5ldHOIAAAAAAAAAACEZXRoMpCCS-QxAgAAZP__________gmlkgnY0gmlwhA-YLVKJc2VjcDI1NmsxoQI8_Lvr6p_TkcAu8KorKacfUEnoOon0tdO0qWhriPdBP4hzeW5jbmV0cwCDdGNwgiMog3VkcIIjKA +- enr:-Ly4QJMtoiX2bPnVbiQOJCLbtUlqdqZk7kCJQln_W1bp1vOHcxWowE-iMXkKC4_uOb0o73wAW71WYi80Dlsg-7a5wiICh2F0dG5ldHOIAAAAAAAAAACEZXRoMpCCS-QxAgAAZP__________gmlkgnY0gmlwhDbP3KmJc2VjcDI1NmsxoQNvcfKYUqcemLFlpKxl7JcQJwQ3L9unYL44gY2aEiRnI4hzeW5jbmV0cwCDdGNwgiMog3VkcIIjKA diff --git a/common/lighthouse_version/src/lib.rs b/common/lighthouse_version/src/lib.rs index afcbae513b..3c136b18b9 100644 --- a/common/lighthouse_version/src/lib.rs +++ b/common/lighthouse_version/src/lib.rs @@ -17,8 +17,8 @@ pub const VERSION: &str = git_version!( // NOTE: using --match instead of --exclude for compatibility with old Git "--match=thiswillnevermatchlol" ], - prefix = "Lighthouse/v3.3.0-", - fallback = "Lighthouse/v3.3.0" + prefix = "Lighthouse/v3.4.0-", + fallback = "Lighthouse/v3.4.0" ); /// Returns `VERSION`, but with platform information appended to the end. diff --git a/common/malloc_utils/Cargo.toml b/common/malloc_utils/Cargo.toml index 569eed6082..c88ec0bd5a 100644 --- a/common/malloc_utils/Cargo.toml +++ b/common/malloc_utils/Cargo.toml @@ -4,13 +4,21 @@ version = "0.1.0" authors = ["Paul Hauner "] edition = "2021" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] lighthouse_metrics = { path = "../lighthouse_metrics" } lazy_static = "1.4.0" libc = "0.2.79" parking_lot = "0.12.0" +jemalloc-ctl = { version = "0.5.0", optional = true } + +# Jemalloc's background_threads feature requires Linux (pthreads). +[target.'cfg(target_os = "linux")'.dependencies] +jemallocator = { version = "0.5.0", optional = true, features = ["stats", "background_threads"] } + +[target.'cfg(not(target_os = "linux"))'.dependencies] +jemallocator = { version = "0.5.0", optional = true, features = ["stats"] } [features] mallinfo2 = [] +jemalloc = ["jemallocator", "jemalloc-ctl"] +jemalloc-profiling = ["jemallocator/profiling"] diff --git a/common/malloc_utils/src/jemalloc.rs b/common/malloc_utils/src/jemalloc.rs new file mode 100644 index 0000000000..c796ea39a1 --- /dev/null +++ b/common/malloc_utils/src/jemalloc.rs @@ -0,0 +1,52 @@ +//! Set the allocator to `jemalloc`. +//! +//! Due to `jemalloc` requiring configuration at compile time or immediately upon runtime +//! initialisation it is configured via a Cargo config file in `.cargo/config.toml`. +//! +//! The `jemalloc` tuning can be overriden by: +//! +//! A) `JEMALLOC_SYS_WITH_MALLOC_CONF` at compile-time. +//! B) `_RJEM_MALLOC_CONF` at runtime. +use jemalloc_ctl::{arenas, epoch, stats, Error}; +use lazy_static::lazy_static; +use lighthouse_metrics::{set_gauge, try_create_int_gauge, IntGauge}; + +#[global_allocator] +static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; + +// Metrics for jemalloc. +lazy_static! { + pub static ref NUM_ARENAS: lighthouse_metrics::Result = + try_create_int_gauge("jemalloc_num_arenas", "The number of arenas in use"); + pub static ref BYTES_ALLOCATED: lighthouse_metrics::Result = + try_create_int_gauge("jemalloc_bytes_allocated", "Equivalent to stats.allocated"); + pub static ref BYTES_ACTIVE: lighthouse_metrics::Result = + try_create_int_gauge("jemalloc_bytes_active", "Equivalent to stats.active"); + pub static ref BYTES_MAPPED: lighthouse_metrics::Result = + try_create_int_gauge("jemalloc_bytes_mapped", "Equivalent to stats.mapped"); + pub static ref BYTES_METADATA: lighthouse_metrics::Result = + try_create_int_gauge("jemalloc_bytes_metadata", "Equivalent to stats.metadata"); + pub static ref BYTES_RESIDENT: lighthouse_metrics::Result = + try_create_int_gauge("jemalloc_bytes_resident", "Equivalent to stats.resident"); + pub static ref BYTES_RETAINED: lighthouse_metrics::Result = + try_create_int_gauge("jemalloc_bytes_retained", "Equivalent to stats.retained"); +} + +pub fn scrape_jemalloc_metrics() { + scrape_jemalloc_metrics_fallible().unwrap() +} + +pub fn scrape_jemalloc_metrics_fallible() -> Result<(), Error> { + // Advance the epoch so that the underlying statistics are updated. + epoch::advance()?; + + set_gauge(&NUM_ARENAS, arenas::narenas::read()? as i64); + set_gauge(&BYTES_ALLOCATED, stats::allocated::read()? as i64); + set_gauge(&BYTES_ACTIVE, stats::active::read()? as i64); + set_gauge(&BYTES_MAPPED, stats::mapped::read()? as i64); + set_gauge(&BYTES_METADATA, stats::metadata::read()? as i64); + set_gauge(&BYTES_RESIDENT, stats::resident::read()? as i64); + set_gauge(&BYTES_RETAINED, stats::retained::read()? as i64); + + Ok(()) +} diff --git a/common/malloc_utils/src/lib.rs b/common/malloc_utils/src/lib.rs index b8aed948f8..3bb242369f 100644 --- a/common/malloc_utils/src/lib.rs +++ b/common/malloc_utils/src/lib.rs @@ -2,18 +2,18 @@ //! //! ## Conditional Compilation //! -//! Presently, only configuration for "The GNU Allocator" from `glibc` is supported. All other -//! allocators are ignored. +//! This crate can be compiled with different feature flags to support different allocators: //! -//! It is assumed that if the following two statements are correct then we should expect to -//! configure `glibc`: +//! - Jemalloc, via the `jemalloc` feature. +//! - GNU malloc, if no features are set and the system supports it. +//! - The system allocator, if no features are set and the allocator is not GNU malloc. +//! +//! It is assumed that if Jemalloc is not in use, and the following two statements are correct then +//! we should expect to configure `glibc`: //! //! - `target_os = linux` //! - `target_env != musl` //! -//! In all other cases this library will not attempt to do anything (i.e., all functions are -//! no-ops). -//! //! If the above conditions are fulfilled but `glibc` still isn't present at runtime then a panic //! may be triggered. It is understood that there's no way to be certain that a compatible `glibc` //! is present: https://github.com/rust-lang/rust/issues/33244. @@ -24,18 +24,42 @@ //! detecting `glibc` are best-effort. If this crate throws errors about undefined external //! functions, then try to compile with the `not_glibc_interface` module. -#[cfg(all(target_os = "linux", not(target_env = "musl")))] +#[cfg(all( + target_os = "linux", + not(target_env = "musl"), + not(feature = "jemalloc") +))] mod glibc; +#[cfg(feature = "jemalloc")] +mod jemalloc; + pub use interface::*; -#[cfg(all(target_os = "linux", not(target_env = "musl")))] +#[cfg(all( + target_os = "linux", + not(target_env = "musl"), + not(feature = "jemalloc") +))] mod interface { pub use crate::glibc::configure_glibc_malloc as configure_memory_allocator; pub use crate::glibc::scrape_mallinfo_metrics as scrape_allocator_metrics; } -#[cfg(any(not(target_os = "linux"), target_env = "musl"))] +#[cfg(feature = "jemalloc")] +mod interface { + #[allow(dead_code)] + pub fn configure_memory_allocator() -> Result<(), String> { + Ok(()) + } + + pub use crate::jemalloc::scrape_jemalloc_metrics as scrape_allocator_metrics; +} + +#[cfg(all( + any(not(target_os = "linux"), target_env = "musl"), + not(feature = "jemalloc") +))] mod interface { #[allow(dead_code, clippy::unnecessary_wraps)] pub fn configure_memory_allocator() -> Result<(), String> { diff --git a/common/validator_dir/src/builder.rs b/common/validator_dir/src/builder.rs index 22888e9d4d..bccf9086ac 100644 --- a/common/validator_dir/src/builder.rs +++ b/common/validator_dir/src/builder.rs @@ -209,7 +209,7 @@ impl<'a> Builder<'a> { if path.exists() { return Err(Error::DepositDataAlreadyExists(path)); } else { - let hex = format!("0x{}", hex::encode(&deposit_data)); + let hex = format!("0x{}", hex::encode(deposit_data)); File::options() .write(true) .read(true) diff --git a/consensus/cached_tree_hash/Cargo.toml b/consensus/cached_tree_hash/Cargo.toml index f9433e4a49..cd03b37a44 100644 --- a/consensus/cached_tree_hash/Cargo.toml +++ b/consensus/cached_tree_hash/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Michael Sproul "] edition = "2021" [dependencies] -ethereum-types = "0.12.1" +ethereum-types = "0.14.1" eth2_ssz_types = "0.2.2" eth2_hashing = "0.3.0" eth2_ssz_derive = "0.3.0" diff --git a/consensus/fork_choice/src/fork_choice.rs b/consensus/fork_choice/src/fork_choice.rs index 290cef78ab..afae7f058b 100644 --- a/consensus/fork_choice/src/fork_choice.rs +++ b/consensus/fork_choice/src/fork_choice.rs @@ -721,7 +721,7 @@ where op: &InvalidationOperation, ) -> Result<(), Error> { self.proto_array - .process_execution_payload_invalidation(op) + .process_execution_payload_invalidation::(op) .map_err(Error::FailedToProcessInvalidExecutionPayload) } @@ -1282,7 +1282,7 @@ where if store.best_justified_checkpoint().epoch > store.justified_checkpoint().epoch { let store = &self.fc_store; - if self.is_descendant_of_finalized(store.best_justified_checkpoint().root) { + if self.is_finalized_checkpoint_or_descendant(store.best_justified_checkpoint().root) { let store = &mut self.fc_store; store .set_justified_checkpoint(*store.best_justified_checkpoint()) @@ -1323,12 +1323,13 @@ where /// Returns `true` if the block is known **and** a descendant of the finalized root. pub fn contains_block(&self, block_root: &Hash256) -> bool { - self.proto_array.contains_block(block_root) && self.is_descendant_of_finalized(*block_root) + self.proto_array.contains_block(block_root) + && self.is_finalized_checkpoint_or_descendant(*block_root) } /// Returns a `ProtoBlock` if the block is known **and** a descendant of the finalized root. pub fn get_block(&self, block_root: &Hash256) -> Option { - if self.is_descendant_of_finalized(*block_root) { + if self.is_finalized_checkpoint_or_descendant(*block_root) { self.proto_array.get_block(block_root) } else { None @@ -1337,7 +1338,7 @@ where /// Returns an `ExecutionStatus` if the block is known **and** a descendant of the finalized root. pub fn get_block_execution_status(&self, block_root: &Hash256) -> Option { - if self.is_descendant_of_finalized(*block_root) { + if self.is_finalized_checkpoint_or_descendant(*block_root) { self.proto_array.get_block_execution_status(block_root) } else { None @@ -1372,10 +1373,10 @@ where }) } - /// Return `true` if `block_root` is equal to the finalized root, or a known descendant of it. - pub fn is_descendant_of_finalized(&self, block_root: Hash256) -> bool { + /// Return `true` if `block_root` is equal to the finalized checkpoint, or a known descendant of it. + pub fn is_finalized_checkpoint_or_descendant(&self, block_root: Hash256) -> bool { self.proto_array - .is_descendant(self.fc_store.finalized_checkpoint().root, block_root) + .is_finalized_checkpoint_or_descendant::(block_root) } /// Returns `Ok(true)` if `block_root` has been imported optimistically or deemed invalid. diff --git a/consensus/merkle_proof/Cargo.toml b/consensus/merkle_proof/Cargo.toml index 7400d4f54d..2c0dbf1a75 100644 --- a/consensus/merkle_proof/Cargo.toml +++ b/consensus/merkle_proof/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Michael Sproul "] edition = "2021" [dependencies] -ethereum-types = "0.12.1" +ethereum-types = "0.14.1" eth2_hashing = "0.3.0" lazy_static = "1.4.0" safe_arith = { path = "../safe_arith" } diff --git a/consensus/proto_array/src/fork_choice_test_definition.rs b/consensus/proto_array/src/fork_choice_test_definition.rs index 035fb799ee..68b3fb7198 100644 --- a/consensus/proto_array/src/fork_choice_test_definition.rs +++ b/consensus/proto_array/src/fork_choice_test_definition.rs @@ -273,7 +273,7 @@ impl ForkChoiceTestDefinition { } }; fork_choice - .process_execution_payload_invalidation(&op) + .process_execution_payload_invalidation::(&op) .unwrap() } Operation::AssertWeight { block_root, weight } => assert_eq!( diff --git a/consensus/proto_array/src/proto_array.rs b/consensus/proto_array/src/proto_array.rs index add84f5478..bf50c08026 100644 --- a/consensus/proto_array/src/proto_array.rs +++ b/consensus/proto_array/src/proto_array.rs @@ -451,7 +451,7 @@ impl ProtoArray { /// Invalidate zero or more blocks, as specified by the `InvalidationOperation`. /// /// See the documentation of `InvalidationOperation` for usage. - pub fn propagate_execution_payload_invalidation( + pub fn propagate_execution_payload_invalidation( &mut self, op: &InvalidationOperation, ) -> Result<(), Error> { @@ -482,7 +482,7 @@ impl ProtoArray { let latest_valid_ancestor_is_descendant = latest_valid_ancestor_root.map_or(false, |ancestor_root| { self.is_descendant(ancestor_root, head_block_root) - && self.is_descendant(self.finalized_checkpoint.root, ancestor_root) + && self.is_finalized_checkpoint_or_descendant::(ancestor_root) }); // Collect all *ancestors* which were declared invalid since they reside between the @@ -977,6 +977,12 @@ impl ProtoArray { /// ## Notes /// /// Still returns `true` if `ancestor_root` is known and `ancestor_root == descendant_root`. + /// + /// ## Warning + /// + /// Do not use this function to check if a block is a descendant of the + /// finalized checkpoint. Use `Self::is_finalized_checkpoint_or_descendant` + /// instead. pub fn is_descendant(&self, ancestor_root: Hash256, descendant_root: Hash256) -> bool { self.indices .get(&ancestor_root) @@ -990,6 +996,70 @@ impl ProtoArray { .unwrap_or(false) } + /// Returns `true` if `root` is equal to or a descendant of + /// `self.finalized_checkpoint`. + /// + /// Notably, this function is checking ancestory of the finalized + /// *checkpoint* not the finalized *block*. + pub fn is_finalized_checkpoint_or_descendant(&self, root: Hash256) -> bool { + let finalized_root = self.finalized_checkpoint.root; + let finalized_slot = self + .finalized_checkpoint + .epoch + .start_slot(E::slots_per_epoch()); + + let mut node = if let Some(node) = self + .indices + .get(&root) + .and_then(|index| self.nodes.get(*index)) + { + node + } else { + // An unknown root is not a finalized descendant. This line can only + // be reached if the user supplies a root that is not known to fork + // choice. + return false; + }; + + // The finalized and justified checkpoints represent a list of known + // ancestors of `node` that are likely to coincide with the store's + // finalized checkpoint. + // + // Run this check once, outside of the loop rather than inside the loop. + // If the conditions don't match for this node then they're unlikely to + // start matching for its ancestors. + for checkpoint in &[ + node.finalized_checkpoint, + node.justified_checkpoint, + node.unrealized_finalized_checkpoint, + node.unrealized_justified_checkpoint, + ] { + if checkpoint.map_or(false, |cp| cp == self.finalized_checkpoint) { + return true; + } + } + + loop { + // If `node` is less than or equal to the finalized slot then `node` + // must be the finalized block. + if node.slot <= finalized_slot { + return node.root == finalized_root; + } + + // Since `node` is from a higher slot that the finalized checkpoint, + // replace `node` with the parent of `node`. + if let Some(parent) = node.parent.and_then(|index| self.nodes.get(index)) { + node = parent + } else { + // If `node` is not the finalized block and its parent does not + // exist in fork choice, then the parent must have been pruned. + // Proto-array only prunes blocks prior to the finalized block, + // so this means the parent conflicts with finality. + return false; + }; + } + } + /// Returns the first *beacon block root* which contains an execution payload with the given /// `block_hash`, if any. pub fn execution_block_hash_to_beacon_block_root( diff --git a/consensus/proto_array/src/proto_array_fork_choice.rs b/consensus/proto_array/src/proto_array_fork_choice.rs index cbd369ae6e..0e0d806e76 100644 --- a/consensus/proto_array/src/proto_array_fork_choice.rs +++ b/consensus/proto_array/src/proto_array_fork_choice.rs @@ -358,12 +358,12 @@ impl ProtoArrayForkChoice { } /// See `ProtoArray::propagate_execution_payload_invalidation` for documentation. - pub fn process_execution_payload_invalidation( + pub fn process_execution_payload_invalidation( &mut self, op: &InvalidationOperation, ) -> Result<(), String> { self.proto_array - .propagate_execution_payload_invalidation(op) + .propagate_execution_payload_invalidation::(op) .map_err(|e| format!("Failed to process invalid payload: {:?}", e)) } @@ -748,6 +748,15 @@ impl ProtoArrayForkChoice { .is_descendant(ancestor_root, descendant_root) } + /// See `ProtoArray` documentation. + pub fn is_finalized_checkpoint_or_descendant( + &self, + descendant_root: Hash256, + ) -> bool { + self.proto_array + .is_finalized_checkpoint_or_descendant::(descendant_root) + } + pub fn latest_message(&self, validator_index: usize) -> Option<(Hash256, Epoch)> { if validator_index < self.votes.0.len() { let vote = &self.votes.0[validator_index]; @@ -928,6 +937,10 @@ mod test_compute_deltas { epoch: genesis_epoch, root: finalized_root, }; + let junk_checkpoint = Checkpoint { + epoch: Epoch::new(42), + root: Hash256::repeat_byte(42), + }; let mut fc = ProtoArrayForkChoice::new::( genesis_slot, @@ -973,8 +986,10 @@ mod test_compute_deltas { target_root: finalized_root, current_epoch_shuffling_id: junk_shuffling_id.clone(), next_epoch_shuffling_id: junk_shuffling_id, - justified_checkpoint: genesis_checkpoint, - finalized_checkpoint: genesis_checkpoint, + // Use the junk checkpoint for the next to values to prevent + // the loop-shortcutting mechanism from triggering. + justified_checkpoint: junk_checkpoint, + finalized_checkpoint: junk_checkpoint, execution_status, unrealized_justified_checkpoint: None, unrealized_finalized_checkpoint: None, @@ -993,6 +1008,11 @@ mod test_compute_deltas { assert!(!fc.is_descendant(finalized_root, not_finalized_desc)); assert!(!fc.is_descendant(finalized_root, unknown)); + assert!(fc.is_finalized_checkpoint_or_descendant::(finalized_root)); + assert!(fc.is_finalized_checkpoint_or_descendant::(finalized_desc)); + assert!(!fc.is_finalized_checkpoint_or_descendant::(not_finalized_desc)); + assert!(!fc.is_finalized_checkpoint_or_descendant::(unknown)); + assert!(!fc.is_descendant(finalized_desc, not_finalized_desc)); assert!(fc.is_descendant(finalized_desc, finalized_desc)); assert!(!fc.is_descendant(finalized_desc, finalized_root)); @@ -1004,6 +1024,171 @@ mod test_compute_deltas { assert!(!fc.is_descendant(not_finalized_desc, unknown)); } + /// This test covers an interesting case where a block can be a descendant + /// of the finalized *block*, but not a descenant of the finalized + /// *checkpoint*. + /// + /// ## Example + /// + /// Consider this block tree which has three blocks (`A`, `B` and `C`): + /// + /// ```ignore + /// [A] <--- [-] <--- [B] + /// | + /// |--[C] + /// ``` + /// + /// - `A` (slot 31) is the common descendant. + /// - `B` (slot 33) descends from `A`, but there is a single skip slot + /// between it and `A`. + /// - `C` (slot 32) descends from `A` and conflicts with `B`. + /// + /// Imagine that the `B` chain is finalized at epoch 1. This means that the + /// finalized checkpoint points to the skipped slot at 32. The root of the + /// finalized checkpoint is `A`. + /// + /// In this scenario, the block `C` has the finalized root (`A`) as an + /// ancestor whilst simultaneously conflicting with the finalized + /// checkpoint. + /// + /// This means that to ensure a block does not conflict with finality we + /// must check to ensure that it's an ancestor of the finalized + /// *checkpoint*, not just the finalized *block*. + #[test] + fn finalized_descendant_edge_case() { + let get_block_root = Hash256::from_low_u64_be; + let genesis_slot = Slot::new(0); + let junk_state_root = Hash256::zero(); + let junk_shuffling_id = + AttestationShufflingId::from_components(Epoch::new(0), Hash256::zero()); + let execution_status = ExecutionStatus::irrelevant(); + + let genesis_checkpoint = Checkpoint { + epoch: Epoch::new(0), + root: get_block_root(0), + }; + + let mut fc = ProtoArrayForkChoice::new::( + genesis_slot, + junk_state_root, + genesis_checkpoint, + genesis_checkpoint, + junk_shuffling_id.clone(), + junk_shuffling_id.clone(), + execution_status, + CountUnrealizedFull::default(), + ) + .unwrap(); + + struct TestBlock { + slot: u64, + root: u64, + parent_root: u64, + } + + let insert_block = |fc: &mut ProtoArrayForkChoice, block: TestBlock| { + fc.proto_array + .on_block::( + Block { + slot: Slot::from(block.slot), + root: get_block_root(block.root), + parent_root: Some(get_block_root(block.parent_root)), + state_root: Hash256::zero(), + target_root: Hash256::zero(), + current_epoch_shuffling_id: junk_shuffling_id.clone(), + next_epoch_shuffling_id: junk_shuffling_id.clone(), + justified_checkpoint: Checkpoint { + epoch: Epoch::new(0), + root: get_block_root(0), + }, + finalized_checkpoint: genesis_checkpoint, + execution_status, + unrealized_justified_checkpoint: Some(genesis_checkpoint), + unrealized_finalized_checkpoint: Some(genesis_checkpoint), + }, + Slot::from(block.slot), + ) + .unwrap(); + }; + + /* + * Start of interesting part of tests. + */ + + // Produce the 0th epoch of blocks. They should all form a chain from + // the genesis block. + for i in 1..MainnetEthSpec::slots_per_epoch() { + insert_block( + &mut fc, + TestBlock { + slot: i, + root: i, + parent_root: i - 1, + }, + ) + } + + let last_slot_of_epoch_0 = MainnetEthSpec::slots_per_epoch() - 1; + + // Produce a block that descends from the last block of epoch -. + // + // This block will be non-canonical. + let non_canonical_slot = last_slot_of_epoch_0 + 1; + insert_block( + &mut fc, + TestBlock { + slot: non_canonical_slot, + root: non_canonical_slot, + parent_root: non_canonical_slot - 1, + }, + ); + + // Produce a block that descends from the last block of the 0th epoch, + // that skips the 1st slot of the 1st epoch. + // + // This block will be canonical. + let canonical_slot = last_slot_of_epoch_0 + 2; + insert_block( + &mut fc, + TestBlock { + slot: canonical_slot, + root: canonical_slot, + parent_root: non_canonical_slot - 1, + }, + ); + + let finalized_root = get_block_root(last_slot_of_epoch_0); + + // Set the finalized checkpoint to finalize the first slot of epoch 1 on + // the canonical chain. + fc.proto_array.finalized_checkpoint = Checkpoint { + root: finalized_root, + epoch: Epoch::new(1), + }; + + assert!( + fc.proto_array + .is_finalized_checkpoint_or_descendant::(finalized_root), + "the finalized checkpoint is the finalized checkpoint" + ); + + assert!( + fc.proto_array + .is_finalized_checkpoint_or_descendant::(get_block_root( + canonical_slot + )), + "the canonical block is a descendant of the finalized checkpoint" + ); + assert!( + !fc.proto_array + .is_finalized_checkpoint_or_descendant::(get_block_root( + non_canonical_slot + )), + "although the non-canonical block is a descendant of the finalized block, \ + it's not a descendant of the finalized checkpoint" + ); + } + #[test] fn zero_hash() { let validator_count: usize = 16; diff --git a/consensus/serde_utils/Cargo.toml b/consensus/serde_utils/Cargo.toml index 1d258ec6ef..d4ba02765f 100644 --- a/consensus/serde_utils/Cargo.toml +++ b/consensus/serde_utils/Cargo.toml @@ -11,4 +11,4 @@ serde = { version = "1.0.116", features = ["derive"] } serde_derive = "1.0.116" serde_json = "1.0.58" hex = "0.4.2" -ethereum-types = "0.12.1" +ethereum-types = "0.14.1" diff --git a/consensus/serde_utils/src/hex.rs b/consensus/serde_utils/src/hex.rs index 1e6c02427f..9a2cd65c76 100644 --- a/consensus/serde_utils/src/hex.rs +++ b/consensus/serde_utils/src/hex.rs @@ -63,15 +63,15 @@ mod test { #[test] fn encoding() { let bytes = vec![0, 255]; - let hex = encode(&bytes); + let hex = encode(bytes); assert_eq!(hex.as_str(), "0x00ff"); let bytes = vec![]; - let hex = encode(&bytes); + let hex = encode(bytes); assert_eq!(hex.as_str(), "0x"); let bytes = vec![1, 2, 3]; - let hex = encode(&bytes); + let hex = encode(bytes); assert_eq!(hex.as_str(), "0x010203"); } } diff --git a/consensus/serde_utils/src/lib.rs b/consensus/serde_utils/src/lib.rs index 92b5966c9a..5c5dafc665 100644 --- a/consensus/serde_utils/src/lib.rs +++ b/consensus/serde_utils/src/lib.rs @@ -12,4 +12,4 @@ pub mod u64_hex_be; pub mod u8_hex; pub use fixed_bytes_hex::{bytes_4_hex, bytes_8_hex}; -pub use quoted_int::{quoted_u256, quoted_u32, quoted_u64, quoted_u8}; +pub use quoted_int::{quoted_i64, quoted_u256, quoted_u32, quoted_u64, quoted_u8}; diff --git a/consensus/serde_utils/src/quoted_int.rs b/consensus/serde_utils/src/quoted_int.rs index 822acb5ee8..0cc35aa318 100644 --- a/consensus/serde_utils/src/quoted_int.rs +++ b/consensus/serde_utils/src/quoted_int.rs @@ -11,7 +11,7 @@ use std::convert::TryFrom; use std::marker::PhantomData; macro_rules! define_mod { - ($int: ty, $visit_fn: ident) => { + ($int: ty) => { /// Serde support for deserializing quoted integers. /// /// Configurable so that quotes are either required or optional. @@ -140,19 +140,25 @@ macro_rules! define_mod { pub mod quoted_u8 { use super::*; - define_mod!(u8, visit_u8); + define_mod!(u8); } pub mod quoted_u32 { use super::*; - define_mod!(u32, visit_u32); + define_mod!(u32); } pub mod quoted_u64 { use super::*; - define_mod!(u64, visit_u64); + define_mod!(u64); +} + +pub mod quoted_i64 { + use super::*; + + define_mod!(i64); } pub mod quoted_u256 { @@ -216,4 +222,26 @@ mod test { fn u256_without_quotes() { serde_json::from_str::("1").unwrap_err(); } + + #[derive(Debug, PartialEq, Serialize, Deserialize)] + #[serde(transparent)] + struct WrappedI64(#[serde(with = "quoted_i64")] i64); + + #[test] + fn negative_i64_with_quotes() { + assert_eq!( + serde_json::from_str::("\"-200\"").unwrap().0, + -200 + ); + assert_eq!( + serde_json::to_string(&WrappedI64(-12_500)).unwrap(), + "\"-12500\"" + ); + } + + // It would be OK if this worked, but we don't need it to (i64s should always be quoted). + #[test] + fn negative_i64_without_quotes() { + serde_json::from_str::("-200").unwrap_err(); + } } diff --git a/consensus/serde_utils/src/u64_hex_be.rs b/consensus/serde_utils/src/u64_hex_be.rs index 6af8a75893..e3364a2d2c 100644 --- a/consensus/serde_utils/src/u64_hex_be.rs +++ b/consensus/serde_utils/src/u64_hex_be.rs @@ -36,7 +36,7 @@ impl<'de> Visitor<'de> for QuantityVisitor { } else if stripped.starts_with('0') { Err(de::Error::custom("cannot have leading zero")) } else if stripped.len() % 2 != 0 { - hex::decode(&format!("0{}", stripped)) + hex::decode(format!("0{}", stripped)) .map_err(|e| de::Error::custom(format!("invalid hex ({:?})", e))) } else { hex::decode(stripped).map_err(|e| de::Error::custom(format!("invalid hex ({:?})", e))) diff --git a/consensus/ssz/Cargo.toml b/consensus/ssz/Cargo.toml index a153c2efc1..1eef2a5557 100644 --- a/consensus/ssz/Cargo.toml +++ b/consensus/ssz/Cargo.toml @@ -13,7 +13,7 @@ name = "ssz" eth2_ssz_derive = "0.3.0" [dependencies] -ethereum-types = "0.12.1" +ethereum-types = "0.14.1" smallvec = { version = "1.6.1", features = ["const_generics"] } itertools = "0.10.3" diff --git a/consensus/state_processing/src/per_block_processing/altair/sync_committee.rs b/consensus/state_processing/src/per_block_processing/altair/sync_committee.rs index 306e86714c..a5dcd6e0b6 100644 --- a/consensus/state_processing/src/per_block_processing/altair/sync_committee.rs +++ b/consensus/state_processing/src/per_block_processing/altair/sync_committee.rs @@ -52,10 +52,10 @@ pub fn process_sync_aggregate( .zip(aggregate.sync_committee_bits.iter()) { if participation_bit { - increase_balance(state, participant_index as usize, participant_reward)?; + increase_balance(state, participant_index, participant_reward)?; increase_balance(state, proposer_index as usize, proposer_reward)?; } else { - decrease_balance(state, participant_index as usize, participant_reward)?; + decrease_balance(state, participant_index, participant_reward)?; } } diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 9f27c4c9a1..9aa1e6d376 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -9,9 +9,9 @@ use crate::VerifySignatures; use safe_arith::SafeArith; use types::consts::altair::{PARTICIPATION_FLAG_WEIGHTS, PROPOSER_WEIGHT, WEIGHT_DENOMINATOR}; -pub fn process_operations<'a, T: EthSpec, Payload: ExecPayload>( +pub fn process_operations>( state: &mut BeaconState, - block_body: BeaconBlockBodyRef<'a, T, Payload>, + block_body: BeaconBlockBodyRef, verify_signatures: VerifySignatures, ctxt: &mut ConsensusContext, spec: &ChainSpec, @@ -232,9 +232,9 @@ pub fn process_attester_slashings( } /// Wrapper function to handle calling the correct version of `process_attestations` based on /// the fork. -pub fn process_attestations<'a, T: EthSpec, Payload: ExecPayload>( +pub fn process_attestations>( state: &mut BeaconState, - block_body: BeaconBlockBodyRef<'a, T, Payload>, + block_body: BeaconBlockBodyRef, verify_signatures: VerifySignatures, ctxt: &mut ConsensusContext, spec: &ChainSpec, diff --git a/consensus/state_processing/src/per_epoch_processing/altair/rewards_and_penalties.rs b/consensus/state_processing/src/per_epoch_processing/altair/rewards_and_penalties.rs index ccebbcb3a2..e2aa67a619 100644 --- a/consensus/state_processing/src/per_epoch_processing/altair/rewards_and_penalties.rs +++ b/consensus/state_processing/src/per_epoch_processing/altair/rewards_and_penalties.rs @@ -76,7 +76,7 @@ pub fn get_flag_index_deltas( let base_reward = get_base_reward(state, index, base_reward_per_increment, spec)?; let mut delta = Delta::default(); - if unslashed_participating_indices.contains(index as usize)? { + if unslashed_participating_indices.contains(index)? { if !state.is_in_inactivity_leak(previous_epoch, spec) { let reward_numerator = base_reward .safe_mul(weight)? @@ -89,8 +89,8 @@ pub fn get_flag_index_deltas( delta.penalize(base_reward.safe_mul(weight)?.safe_div(WEIGHT_DENOMINATOR)?)?; } deltas - .get_mut(index as usize) - .ok_or(Error::DeltaOutOfBounds(index as usize))? + .get_mut(index) + .ok_or(Error::DeltaOutOfBounds(index))? .combine(delta)?; } Ok(()) diff --git a/consensus/state_processing/src/per_epoch_processing/base/rewards_and_penalties.rs b/consensus/state_processing/src/per_epoch_processing/base/rewards_and_penalties.rs index 87e4261e0a..e7a4d9c4dc 100644 --- a/consensus/state_processing/src/per_epoch_processing/base/rewards_and_penalties.rs +++ b/consensus/state_processing/src/per_epoch_processing/base/rewards_and_penalties.rs @@ -235,7 +235,7 @@ fn get_inclusion_delay_delta( let max_attester_reward = base_reward.safe_sub(proposer_reward)?; delta.reward(max_attester_reward.safe_div(inclusion_info.delay)?)?; - let proposer_index = inclusion_info.proposer_index as usize; + let proposer_index = inclusion_info.proposer_index; Ok((delta, Some((proposer_index, proposer_delta)))) } else { Ok((Delta::default(), None)) diff --git a/consensus/swap_or_not_shuffle/Cargo.toml b/consensus/swap_or_not_shuffle/Cargo.toml index 1c7ad5f02a..9a7d58b77d 100644 --- a/consensus/swap_or_not_shuffle/Cargo.toml +++ b/consensus/swap_or_not_shuffle/Cargo.toml @@ -13,7 +13,7 @@ criterion = "0.3.3" [dependencies] eth2_hashing = "0.3.0" -ethereum-types = "0.12.1" +ethereum-types = "0.14.1" [features] arbitrary = ["ethereum-types/arbitrary"] diff --git a/consensus/tree_hash/Cargo.toml b/consensus/tree_hash/Cargo.toml index ab080eac06..731e2f177e 100644 --- a/consensus/tree_hash/Cargo.toml +++ b/consensus/tree_hash/Cargo.toml @@ -15,7 +15,7 @@ eth2_ssz = "0.4.1" eth2_ssz_derive = "0.3.0" [dependencies] -ethereum-types = "0.12.1" +ethereum-types = "0.14.1" eth2_hashing = "0.3.0" smallvec = "1.6.1" diff --git a/consensus/types/Cargo.toml b/consensus/types/Cargo.toml index 1ccc8dba8b..6ae185f7ff 100644 --- a/consensus/types/Cargo.toml +++ b/consensus/types/Cargo.toml @@ -14,7 +14,7 @@ bls = { path = "../../crypto/bls" } compare_fields = { path = "../../common/compare_fields" } compare_fields_derive = { path = "../../common/compare_fields_derive" } eth2_interop_keypairs = { path = "../../common/eth2_interop_keypairs" } -ethereum-types = "0.12.1" +ethereum-types = "0.14.1" eth2_hashing = "0.3.0" hex = "0.4.2" int_to_bytes = { path = "../int_to_bytes" } @@ -45,6 +45,7 @@ lazy_static = "1.4.0" parking_lot = "0.12.0" itertools = "0.10.0" superstruct = "0.5.0" +metastruct = "0.1.0" serde_json = "1.0.74" smallvec = "1.8.0" serde_with = "1.13.0" diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index 79625c12e3..12d44741f9 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -482,7 +482,7 @@ impl BeaconState { /// Spec v0.12.1 pub fn get_committee_count_at_slot(&self, slot: Slot) -> Result { let cache = self.committee_cache_at_slot(slot)?; - Ok(cache.committees_per_slot() as u64) + Ok(cache.committees_per_slot()) } /// Compute the number of committees in an entire epoch. diff --git a/consensus/types/src/beacon_state/committee_cache.rs b/consensus/types/src/beacon_state/committee_cache.rs index 7a526acc58..03adaf3d44 100644 --- a/consensus/types/src/beacon_state/committee_cache.rs +++ b/consensus/types/src/beacon_state/committee_cache.rs @@ -144,7 +144,7 @@ impl CommitteeCache { self.committees_per_slot as usize, index as usize, ); - let committee = self.compute_committee(committee_index as usize)?; + let committee = self.compute_committee(committee_index)?; Some(BeaconCommittee { slot, diff --git a/consensus/types/src/beacon_state/tests.rs b/consensus/types/src/beacon_state/tests.rs index 5898bfe214..abca10e372 100644 --- a/consensus/types/src/beacon_state/tests.rs +++ b/consensus/types/src/beacon_state/tests.rs @@ -344,12 +344,7 @@ mod committees { let cache_epoch = cache_epoch.into_epoch(state_epoch); - execute_committee_consistency_test( - new_head_state, - cache_epoch, - validator_count as usize, - spec, - ); + execute_committee_consistency_test(new_head_state, cache_epoch, validator_count, spec); } async fn committee_consistency_test_suite(cached_epoch: RelativeEpoch) { @@ -361,18 +356,13 @@ mod committees { .mul(spec.target_committee_size) .add(1); - committee_consistency_test::(validator_count as usize, Epoch::new(0), cached_epoch) + committee_consistency_test::(validator_count, Epoch::new(0), cached_epoch).await; + + committee_consistency_test::(validator_count, T::genesis_epoch() + 4, cached_epoch) .await; committee_consistency_test::( - validator_count as usize, - T::genesis_epoch() + 4, - cached_epoch, - ) - .await; - - committee_consistency_test::( - validator_count as usize, + validator_count, T::genesis_epoch() + (T::slots_per_historical_root() as u64) .mul(T::slots_per_epoch()) diff --git a/consensus/types/src/execution_block_header.rs b/consensus/types/src/execution_block_header.rs new file mode 100644 index 0000000000..4baa5dd395 --- /dev/null +++ b/consensus/types/src/execution_block_header.rs @@ -0,0 +1,74 @@ +// Copyright (c) 2022 Reth Contributors +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +use crate::{Address, EthSpec, ExecutionPayload, Hash256, Hash64, Uint256}; +use metastruct::metastruct; + +/// Execution block header as used for RLP encoding and Keccak hashing. +/// +/// Credit to Reth for the type definition. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[metastruct(mappings(map_execution_block_header_fields()))] +pub struct ExecutionBlockHeader { + pub parent_hash: Hash256, + pub ommers_hash: Hash256, + pub beneficiary: Address, + pub state_root: Hash256, + pub transactions_root: Hash256, + pub receipts_root: Hash256, + pub logs_bloom: Vec, + pub difficulty: Uint256, + pub number: Uint256, + pub gas_limit: Uint256, + pub gas_used: Uint256, + pub timestamp: u64, + pub extra_data: Vec, + pub mix_hash: Hash256, + pub nonce: Hash64, + pub base_fee_per_gas: Uint256, +} + +impl ExecutionBlockHeader { + pub fn from_payload( + payload: &ExecutionPayload, + rlp_empty_list_root: Hash256, + rlp_transactions_root: Hash256, + ) -> Self { + // Most of these field mappings are defined in EIP-3675 except for `mixHash`, which is + // defined in EIP-4399. + ExecutionBlockHeader { + parent_hash: payload.parent_hash.into_root(), + ommers_hash: rlp_empty_list_root, + beneficiary: payload.fee_recipient, + state_root: payload.state_root, + transactions_root: rlp_transactions_root, + receipts_root: payload.receipts_root, + logs_bloom: payload.logs_bloom.clone().into(), + difficulty: Uint256::zero(), + number: payload.block_number.into(), + gas_limit: payload.gas_limit.into(), + gas_used: payload.gas_used.into(), + timestamp: payload.timestamp, + extra_data: payload.extra_data.clone().into(), + mix_hash: payload.prev_randao, + nonce: Hash64::zero(), + base_fee_per_gas: payload.base_fee_per_gas, + } + } +} diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index d37486a67f..0a0d461568 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -75,6 +75,7 @@ pub mod withdrawal_credentials; #[macro_use] pub mod slot_epoch_macros; pub mod config_and_preset; +pub mod execution_block_header; pub mod fork_context; pub mod participation_flags; pub mod participation_list; @@ -128,6 +129,7 @@ pub use crate::enr_fork_id::EnrForkId; pub use crate::eth1_data::Eth1Data; pub use crate::eth_spec::EthSpecId; pub use crate::execution_block_hash::ExecutionBlockHash; +pub use crate::execution_block_header::ExecutionBlockHeader; pub use crate::execution_payload::{ExecutionPayload, Transaction, Transactions}; pub use crate::execution_payload_header::ExecutionPayloadHeader; pub use crate::fork::Fork; @@ -181,6 +183,7 @@ pub type Hash256 = H256; pub type Uint256 = ethereum_types::U256; pub type Address = H160; pub type ForkVersion = [u8; 4]; +pub type Hash64 = ethereum_types::H64; pub use bls::{ AggregatePublicKey, AggregateSignature, Keypair, PublicKey, PublicKeyBytes, SecretKey, diff --git a/consensus/types/src/preset.rs b/consensus/types/src/preset.rs index 8ee38e46a6..fc5aa87300 100644 --- a/consensus/types/src/preset.rs +++ b/consensus/types/src/preset.rs @@ -202,7 +202,7 @@ mod test { } fn preset_from_file(preset_name: &str, filename: &str) -> T { - let f = File::open(&presets_base_path().join(preset_name).join(filename)) + let f = File::open(presets_base_path().join(preset_name).join(filename)) .expect("preset file exists"); serde_yaml::from_reader(f).unwrap() } diff --git a/crypto/bls/Cargo.toml b/crypto/bls/Cargo.toml index 9ac468d227..c3331824d9 100644 --- a/crypto/bls/Cargo.toml +++ b/crypto/bls/Cargo.toml @@ -14,7 +14,7 @@ serde_derive = "1.0.116" eth2_serde_utils = "0.1.1" hex = "0.4.2" eth2_hashing = "0.3.0" -ethereum-types = "0.12.1" +ethereum-types = "0.14.1" arbitrary = { version = "1.0", features = ["derive"], optional = true } zeroize = { version = "1.4.2", features = ["zeroize_derive"] } blst = { version = "0.3.3", optional = true } diff --git a/lcli/Cargo.toml b/lcli/Cargo.toml index 638ab46bfb..8ebac0ca61 100644 --- a/lcli/Cargo.toml +++ b/lcli/Cargo.toml @@ -1,13 +1,14 @@ [package] name = "lcli" description = "Lighthouse CLI (modeled after zcli)" -version = "3.3.0" +version = "3.4.0" authors = ["Paul Hauner "] edition = "2021" [features] portable = ["bls/supranational-portable"] fake_crypto = ['bls/fake_crypto'] +jemalloc = ["malloc_utils/jemalloc"] [dependencies] bls = { path = "../crypto/bls" } @@ -40,3 +41,7 @@ eth2 = { path = "../common/eth2" } snap = "1.0.1" beacon_chain = { path = "../beacon_node/beacon_chain" } store = { path = "../beacon_node/store" } +malloc_utils = { path = "../common/malloc_utils" } + +[package.metadata.cargo-udeps.ignore] +normal = ["malloc_utils"] diff --git a/lcli/src/main.rs b/lcli/src/main.rs index de6039f35a..137a4534b4 100644 --- a/lcli/src/main.rs +++ b/lcli/src/main.rs @@ -792,6 +792,7 @@ fn run( debug_level: String::from("trace"), logfile_debug_level: String::from("trace"), log_format: None, + logfile_format: None, log_color: false, disable_log_timestamp: false, max_log_size: 0, diff --git a/lighthouse/Cargo.toml b/lighthouse/Cargo.toml index 43b06f01df..dbd8399644 100644 --- a/lighthouse/Cargo.toml +++ b/lighthouse/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lighthouse" -version = "3.3.0" +version = "3.4.0" authors = ["Sigma Prime "] edition = "2021" autotests = false @@ -24,6 +24,8 @@ gnosis = [] slasher-mdbx = ["slasher/mdbx"] # Support slasher LMDB backend. slasher-lmdb = ["slasher/lmdb"] +# Use jemalloc. +jemalloc = ["malloc_utils/jemalloc"] [dependencies] beacon_node = { "path" = "../beacon_node" } diff --git a/lighthouse/environment/src/lib.rs b/lighthouse/environment/src/lib.rs index fad7edeb19..8ef67e82dd 100644 --- a/lighthouse/environment/src/lib.rs +++ b/lighthouse/environment/src/lib.rs @@ -50,6 +50,7 @@ pub struct LoggerConfig { pub debug_level: String, pub logfile_debug_level: String, pub log_format: Option, + pub logfile_format: Option, pub log_color: bool, pub disable_log_timestamp: bool, pub max_log_size: u64, @@ -64,6 +65,7 @@ impl Default for LoggerConfig { debug_level: String::from("info"), logfile_debug_level: String::from("debug"), log_format: None, + logfile_format: None, log_color: false, disable_log_timestamp: false, max_log_size: 200, @@ -252,7 +254,7 @@ impl EnvironmentBuilder { let file_logger = FileLoggerBuilder::new(&path) .level(logfile_level) .channel_size(LOG_CHANNEL_SIZE) - .format(match config.log_format.as_deref() { + .format(match config.logfile_format.as_deref() { Some("JSON") => Format::Json, _ => Format::default(), }) diff --git a/lighthouse/src/main.rs b/lighthouse/src/main.rs index 54dcc2291f..c4bed5195c 100644 --- a/lighthouse/src/main.rs +++ b/lighthouse/src/main.rs @@ -31,6 +31,14 @@ fn bls_library_name() -> &'static str { } } +fn allocator_name() -> &'static str { + if cfg!(feature = "jemalloc") { + "jemalloc" + } else { + "system" + } +} + fn main() { // Enable backtraces unless a RUST_BACKTRACE value has already been explicitly provided. if std::env::var("RUST_BACKTRACE").is_err() { @@ -51,10 +59,12 @@ fn main() { "{}\n\ BLS library: {}\n\ SHA256 hardware acceleration: {}\n\ + Allocator: {}\n\ Specs: mainnet (true), minimal ({}), gnosis ({})", VERSION.replace("Lighthouse/", ""), bls_library_name(), have_sha_extensions(), + allocator_name(), cfg!(feature = "spec-minimal"), cfg!(feature = "gnosis"), ).as_str() @@ -99,6 +109,15 @@ fn main() { .default_value("debug") .global(true), ) + .arg( + Arg::with_name("logfile-format") + .long("logfile-format") + .value_name("FORMAT") + .help("Specifies the log format used when emitting logs to the logfile.") + .possible_values(&["DEFAULT", "JSON"]) + .takes_value(true) + .global(true) + ) .arg( Arg::with_name("logfile-max-size") .long("logfile-max-size") @@ -403,6 +422,11 @@ fn run( .value_of("logfile-debug-level") .ok_or("Expected --logfile-debug-level flag")?; + let logfile_format = matches + .value_of("logfile-format") + // Ensure that `logfile-format` defaults to the value of `log-format`. + .or_else(|| matches.value_of("log-format")); + let logfile_max_size: u64 = matches .value_of("logfile-max-size") .ok_or("Expected --logfile-max-size flag")? @@ -453,6 +477,7 @@ fn run( debug_level: String::from(debug_level), logfile_debug_level: String::from(logfile_debug_level), log_format: log_format.map(String::from), + logfile_format: logfile_format.map(String::from), log_color, disable_log_timestamp, max_log_size: logfile_max_size * 1_024 * 1_024, diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 07c583da5c..053a04f879 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -1079,6 +1079,19 @@ fn http_port_flag() { .with_config(|config| assert_eq!(config.http_api.listen_port, port1)); } #[test] +fn empty_self_limiter_flag() { + // Test that empty rate limiter is accepted using the default rate limiting configurations. + CommandLineTest::new() + .flag("self-limiter", None) + .run_with_zero_port() + .with_config(|config| { + assert_eq!( + config.network.outbound_rate_limiter_config, + Some(lighthouse_network::rpc::config::OutboundRateLimiterConfig::default()) + ) + }); +} +#[test] fn http_allow_origin_flag() { CommandLineTest::new() .flag("http-allow-origin", Some("127.0.0.99")) @@ -1237,6 +1250,31 @@ fn validator_monitor_file_flag() { assert_eq!(config.validator_monitor_pubkeys[1].to_string(), "0xbeefdeadbeefdeaddeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef"); }); } +#[test] +fn validator_monitor_metrics_threshold_default() { + CommandLineTest::new() + .run_with_zero_port() + .with_config(|config| { + assert_eq!( + config.validator_monitor_individual_tracking_threshold, + // If this value changes make sure to update the help text for + // the CLI command. + 64 + ) + }); +} +#[test] +fn validator_monitor_metrics_threshold_custom() { + CommandLineTest::new() + .flag( + "validator-monitor-individual-tracking-threshold", + Some("42"), + ) + .run_with_zero_port() + .with_config(|config| { + assert_eq!(config.validator_monitor_individual_tracking_threshold, 42) + }); +} // Tests for Store flags. #[test] @@ -1637,7 +1675,24 @@ fn logfile_no_restricted_perms_flag() { assert!(config.logger_config.is_restricted == false); }); } - +#[test] +fn logfile_format_default() { + CommandLineTest::new() + .run_with_zero_port() + .with_config(|config| assert_eq!(config.logger_config.logfile_format, None)); +} +#[test] +fn logfile_format_flag() { + CommandLineTest::new() + .flag("logfile-format", Some("JSON")) + .run_with_zero_port() + .with_config(|config| { + assert_eq!( + config.logger_config.logfile_format, + Some("JSON".to_string()) + ) + }); +} #[test] fn sync_eth1_chain_default() { CommandLineTest::new() @@ -1697,3 +1752,22 @@ fn gui_flag() { assert!(config.validator_monitor_auto); }); } + +#[test] +fn optimistic_finalized_sync_default() { + CommandLineTest::new() + .run_with_zero_port() + .with_config(|config| { + assert!(config.chain.optimistic_finalized_sync); + }); +} + +#[test] +fn disable_optimistic_finalized_sync() { + CommandLineTest::new() + .flag("disable-optimistic-finalized-sync", None) + .run_with_zero_port() + .with_config(|config| { + assert!(!config.chain.optimistic_finalized_sync); + }); +} diff --git a/slasher/tests/attester_slashings.rs b/slasher/tests/attester_slashings.rs index 5cf3fe6c2a..40d9fa511c 100644 --- a/slasher/tests/attester_slashings.rs +++ b/slasher/tests/attester_slashings.rs @@ -39,8 +39,8 @@ fn double_vote_multi_vals() { fn double_vote_some_vals() { let v1 = vec![0, 1, 2, 3, 4, 5, 6]; let v2 = vec![0, 2, 4, 6]; - let att1 = indexed_att(&v1, 0, 1, 0); - let att2 = indexed_att(&v2, 0, 1, 1); + let att1 = indexed_att(v1, 0, 1, 0); + let att2 = indexed_att(v2, 0, 1, 1); let slashings = hashset![att_slashing(&att1, &att2)]; let attestations = vec![att1, att2]; slasher_test_indiv(&attestations, &slashings, 1); @@ -53,9 +53,9 @@ fn double_vote_some_vals_repeat() { let v1 = vec![0, 1, 2, 3, 4, 5, 6]; let v2 = vec![0, 2, 4, 6]; let v3 = vec![1, 3, 5]; - let att1 = indexed_att(&v1, 0, 1, 0); - let att2 = indexed_att(&v2, 0, 1, 1); - let att3 = indexed_att(&v3, 0, 1, 0); + let att1 = indexed_att(v1, 0, 1, 0); + let att2 = indexed_att(v2, 0, 1, 1); + let att3 = indexed_att(v3, 0, 1, 0); let slashings = hashset![att_slashing(&att1, &att2)]; let attestations = vec![att1, att2, att3]; slasher_test_indiv(&attestations, &slashings, 1); @@ -67,8 +67,8 @@ fn double_vote_some_vals_repeat() { fn no_double_vote_same_target() { let v1 = vec![0, 1, 2, 3, 4, 5, 6]; let v2 = vec![0, 1, 2, 3, 4, 5, 7, 8]; - let att1 = indexed_att(&v1, 0, 1, 0); - let att2 = indexed_att(&v2, 0, 1, 0); + let att1 = indexed_att(v1, 0, 1, 0); + let att2 = indexed_att(v2, 0, 1, 0); let attestations = vec![att1, att2]; slasher_test_indiv(&attestations, &hashset! {}, 1); slasher_test_indiv(&attestations, &hashset! {}, 1000); @@ -79,8 +79,8 @@ fn no_double_vote_same_target() { fn no_double_vote_distinct_vals() { let v1 = vec![0, 1, 2, 3]; let v2 = vec![4, 5, 6, 7]; - let att1 = indexed_att(&v1, 0, 1, 0); - let att2 = indexed_att(&v2, 0, 1, 1); + let att1 = indexed_att(v1, 0, 1, 0); + let att2 = indexed_att(v2, 0, 1, 1); let attestations = vec![att1, att2]; slasher_test_indiv(&attestations, &hashset! {}, 1); slasher_test_indiv(&attestations, &hashset! {}, 1000); @@ -89,7 +89,7 @@ fn no_double_vote_distinct_vals() { #[test] fn no_double_vote_repeated() { let v = vec![0, 1, 2, 3, 4]; - let att1 = indexed_att(&v, 0, 1, 0); + let att1 = indexed_att(v, 0, 1, 0); let att2 = att1.clone(); let attestations = vec![att1, att2]; slasher_test_indiv(&attestations, &hashset! {}, 1); diff --git a/testing/antithesis/Dockerfile.libvoidstar b/testing/antithesis/Dockerfile.libvoidstar index 32e2d5648d..bae1807329 100644 --- a/testing/antithesis/Dockerfile.libvoidstar +++ b/testing/antithesis/Dockerfile.libvoidstar @@ -1,11 +1,9 @@ -FROM rust:1.62.1-bullseye AS builder -RUN apt-get update && apt-get -y upgrade && apt-get install -y cmake libclang-dev +FROM rust:1.66.1-bullseye AS builder +RUN apt-get update && apt-get -y upgrade && apt-get install -y cmake libclang-dev protobuf-compiler COPY . lighthouse # Build lighthouse directly with a cargo build command, bypassing the Makefile. -# We have to use nightly in order to disable the new LLVM pass manager. -RUN rustup default nightly-2022-07-26 && cd lighthouse && LD_LIBRARY_PATH=/lighthouse/testing/antithesis/libvoidstar/ RUSTFLAGS="-Znew-llvm-pass-manager=no -Cpasses=sancov -Cllvm-args=-sanitizer-coverage-level=3 -Cllvm-args=-sanitizer-coverage-trace-pc-guard -Ccodegen-units=1 -Cdebuginfo=2 -L/lighthouse/testing/antithesis/libvoidstar/ -lvoidstar" cargo build --release --manifest-path lighthouse/Cargo.toml --target x86_64-unknown-linux-gnu --features modern --verbose --bin lighthouse - +RUN cd lighthouse && LD_LIBRARY_PATH=/lighthouse/testing/antithesis/libvoidstar/ RUSTFLAGS="-Cpasses=sancov-module -Cllvm-args=-sanitizer-coverage-level=3 -Cllvm-args=-sanitizer-coverage-trace-pc-guard -Ccodegen-units=1 -Cdebuginfo=2 -L/lighthouse/testing/antithesis/libvoidstar/ -lvoidstar" cargo build --release --manifest-path lighthouse/Cargo.toml --target x86_64-unknown-linux-gnu --features modern --verbose --bin lighthouse # build lcli binary directly with cargo install command, bypassing the makefile RUN cargo install --path /lighthouse/lcli --force --locked diff --git a/testing/ef_tests/Cargo.toml b/testing/ef_tests/Cargo.toml index 04a222c7af..d969c9727d 100644 --- a/testing/ef_tests/Cargo.toml +++ b/testing/ef_tests/Cargo.toml @@ -15,7 +15,7 @@ bls = { path = "../../crypto/bls", default-features = false } compare_fields = { path = "../../common/compare_fields" } compare_fields_derive = { path = "../../common/compare_fields_derive" } derivative = "2.1.1" -ethereum-types = "0.12.1" +ethereum-types = "0.14.1" hex = "0.4.2" rayon = "1.4.1" serde = "1.0.116" diff --git a/testing/ef_tests/Makefile b/testing/ef_tests/Makefile index e05ef0b06b..b2af490dd0 100644 --- a/testing/ef_tests/Makefile +++ b/testing/ef_tests/Makefile @@ -13,6 +13,8 @@ BLS_TARBALL = $(patsubst %,%-$(BLS_TEST_TAG).tar.gz,$(BLS_TEST)) BLS_OUTPUT_DIR := $(OUTPUT_DIR)/$(BLS_TEST_REPO_NAME) BLS_BASE_URL := https://github.com/ethereum/$(BLS_TEST_REPO_NAME)/releases/download/$(BLS_TEST_TAG) +WGET := $(if $(LIGHTHOUSE_GITHUB_TOKEN),wget --header="Authorization: $(LIGHTHOUSE_GITHUB_TOKEN)",wget) + all: make $(OUTPUT_DIR) make $(BLS_OUTPUT_DIR) @@ -25,11 +27,11 @@ $(OUTPUT_DIR): $(TARBALLS) $(BLS_OUTPUT_DIR): mkdir $(BLS_OUTPUT_DIR) - wget $(BLS_BASE_URL)/$(BLS_TEST).tar.gz -O $(BLS_TARBALL) + $(WGET) $(BLS_BASE_URL)/$(BLS_TEST).tar.gz -O $(BLS_TARBALL) tar -xzf $(BLS_TARBALL) -C $(BLS_OUTPUT_DIR) %-$(TESTS_TAG).tar.gz: - wget $(BASE_URL)/$*.tar.gz -O $@ + $(WGET) $(BASE_URL)/$*.tar.gz -O $@ clean-test-files: rm -rf $(OUTPUT_DIR) $(BLS_OUTPUT_DIR) diff --git a/testing/execution_engine_integration/Cargo.toml b/testing/execution_engine_integration/Cargo.toml index a85138be95..26b5f596f2 100644 --- a/testing/execution_engine_integration/Cargo.toml +++ b/testing/execution_engine_integration/Cargo.toml @@ -15,8 +15,8 @@ execution_layer = { path = "../../beacon_node/execution_layer" } sensitive_url = { path = "../../common/sensitive_url" } types = { path = "../../consensus/types" } unused_port = { path = "../../common/unused_port" } -ethers-core = "0.17.0" -ethers-providers = "0.17.0" +ethers-core = "1.0.2" +ethers-providers = "1.0.2" deposit_contract = { path = "../../common/deposit_contract" } reqwest = { version = "0.11.0", features = ["json"] } hex = "0.4.2" diff --git a/testing/execution_engine_integration/src/nethermind.rs b/testing/execution_engine_integration/src/nethermind.rs index f643fbd5f2..740d87ab8a 100644 --- a/testing/execution_engine_integration/src/nethermind.rs +++ b/testing/execution_engine_integration/src/nethermind.rs @@ -76,7 +76,7 @@ impl GenericExecutionEngine for NethermindEngine { fn init_datadir() -> TempDir { let datadir = TempDir::new().unwrap(); let genesis_json_path = datadir.path().join("genesis.json"); - let mut file = File::create(&genesis_json_path).unwrap(); + let mut file = File::create(genesis_json_path).unwrap(); let json = nethermind_genesis_json(); serde_json::to_writer(&mut file, &json).unwrap(); datadir diff --git a/testing/node_test_rig/Cargo.toml b/testing/node_test_rig/Cargo.toml index 2c9bd5939f..ea5d005c16 100644 --- a/testing/node_test_rig/Cargo.toml +++ b/testing/node_test_rig/Cargo.toml @@ -13,4 +13,4 @@ eth2 = { path = "../../common/eth2" } validator_client = { path = "../../validator_client" } validator_dir = { path = "../../common/validator_dir", features = ["insecure_keys"] } sensitive_url = { path = "../../common/sensitive_url" } -execution_layer = { path = "../../beacon_node/execution_layer" } \ No newline at end of file +execution_layer = { path = "../../beacon_node/execution_layer" } diff --git a/testing/node_test_rig/src/lib.rs b/testing/node_test_rig/src/lib.rs index d0a4ef9491..82a60cda2f 100644 --- a/testing/node_test_rig/src/lib.rs +++ b/testing/node_test_rig/src/lib.rs @@ -231,7 +231,7 @@ impl LocalExecutionNode { .tempdir() .expect("should create temp directory for client datadir"); let jwt_file_path = datadir.path().join("jwt.hex"); - if let Err(e) = std::fs::write(&jwt_file_path, config.jwt_key.hex_string()) { + if let Err(e) = std::fs::write(jwt_file_path, config.jwt_key.hex_string()) { panic!("Failed to write jwt file {}", e); } Self { diff --git a/testing/simulator/src/eth1_sim.rs b/testing/simulator/src/eth1_sim.rs index 8284bff609..42aefea7a5 100644 --- a/testing/simulator/src/eth1_sim.rs +++ b/testing/simulator/src/eth1_sim.rs @@ -62,6 +62,7 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> { debug_level: String::from("debug"), logfile_debug_level: String::from("debug"), log_format: None, + logfile_format: None, log_color: false, disable_log_timestamp: false, max_log_size: 0, diff --git a/testing/simulator/src/no_eth1_sim.rs b/testing/simulator/src/no_eth1_sim.rs index 53c4447da2..1a026ded46 100644 --- a/testing/simulator/src/no_eth1_sim.rs +++ b/testing/simulator/src/no_eth1_sim.rs @@ -47,6 +47,7 @@ pub fn run_no_eth1_sim(matches: &ArgMatches) -> Result<(), String> { debug_level: String::from("debug"), logfile_debug_level: String::from("debug"), log_format: None, + logfile_format: None, log_color: false, disable_log_timestamp: false, max_log_size: 0, diff --git a/testing/simulator/src/sync_sim.rs b/testing/simulator/src/sync_sim.rs index 1c8b41f057..9d759715eb 100644 --- a/testing/simulator/src/sync_sim.rs +++ b/testing/simulator/src/sync_sim.rs @@ -51,6 +51,7 @@ fn syncing_sim( debug_level: String::from(log_level), logfile_debug_level: String::from("debug"), log_format: log_format.map(String::from), + logfile_format: None, log_color: false, disable_log_timestamp: false, max_log_size: 0, diff --git a/testing/web3signer_tests/build.rs b/testing/web3signer_tests/build.rs index f62dff0b6f..a55c39376a 100644 --- a/testing/web3signer_tests/build.rs +++ b/testing/web3signer_tests/build.rs @@ -1,7 +1,10 @@ //! This build script downloads the latest Web3Signer release and places it in the `OUT_DIR` so it //! can be used for integration testing. -use reqwest::Client; +use reqwest::{ + header::{self, HeaderValue}, + Client, +}; use serde_json::Value; use std::env; use std::fs; @@ -15,10 +18,15 @@ const FIXED_VERSION_STRING: Option<&str> = None; #[tokio::main] async fn main() { let out_dir = env::var("OUT_DIR").unwrap(); - download_binary(out_dir.into()).await; + + // Read a Github API token from the environment. This is intended to prevent rate-limits on CI. + // We use a name that is unlikely to accidentally collide with anything the user has configured. + let github_token = env::var("LIGHTHOUSE_GITHUB_TOKEN"); + + download_binary(out_dir.into(), github_token.as_deref().unwrap_or("")).await; } -pub async fn download_binary(dest_dir: PathBuf) { +pub async fn download_binary(dest_dir: PathBuf, github_token: &str) { let version_file = dest_dir.join("version"); let client = Client::builder() @@ -33,8 +41,11 @@ pub async fn download_binary(dest_dir: PathBuf) { env_version } else { // Get the latest release of the web3 signer repo. + let mut token_header_value = HeaderValue::from_str(github_token).unwrap(); + token_header_value.set_sensitive(true); let latest_response: Value = client .get("https://api.github.com/repos/ConsenSys/web3signer/releases/latest") + .header(header::AUTHORIZATION, token_header_value) .send() .await .unwrap() diff --git a/validator_client/src/block_service.rs b/validator_client/src/block_service.rs index d4acbe7563..bef51a694a 100644 --- a/validator_client/src/block_service.rs +++ b/validator_client/src/block_service.rs @@ -333,6 +333,11 @@ impl BlockService { let proposer_index = self.validator_store.validator_index(&validator_pubkey); let validator_pubkey_ref = &validator_pubkey; + info!( + log, + "Requesting unsigned block"; + "slot" => slot.as_u64(), + ); // Request block from first responsive beacon node. let block = self .beacon_nodes @@ -383,6 +388,11 @@ impl BlockService { } }; + info!( + log, + "Received unsigned block"; + "slot" => slot.as_u64(), + ); if proposer_index != Some(block.proposer_index()) { return Err(BlockError::Recoverable( "Proposer index does not match block proposer. Beacon chain re-orged" @@ -401,6 +411,11 @@ impl BlockService { .await .map_err(|e| BlockError::Recoverable(format!("Unable to sign block: {:?}", e)))?; + info!( + log, + "Publishing signed block"; + "slot" => slot.as_u64(), + ); // Publish block with first available beacon node. self.beacon_nodes .first_success( diff --git a/validator_client/src/cli.rs b/validator_client/src/cli.rs index 5676b51afd..dfea548735 100644 --- a/validator_client/src/cli.rs +++ b/validator_client/src/cli.rs @@ -253,6 +253,15 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { address of this server (e.g., http://localhost:5064).") .takes_value(true), ) + .arg( + Arg::with_name("enable-high-validator-count-metrics") + .long("enable-high-validator-count-metrics") + .help("Enable per validator metrics for > 64 validators. \ + Note: This flag is automatically enabled for <= 64 validators. \ + Enabling this metric for higher validator counts will lead to higher volume \ + of prometheus metrics being collected.") + .takes_value(false), + ) /* * Explorer metrics */ diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index f503b3c01b..a7dac22fce 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -53,6 +53,11 @@ pub struct Config { /// If true, enable functionality that monitors the network for attestations or proposals from /// any of the validators managed by this client before starting up. pub enable_doppelganger_protection: bool, + /// If true, then we publish validator specific metrics (e.g next attestation duty slot) + /// for all our managed validators. + /// Note: We publish validator specific metrics for low validator counts without this flag + /// (<= 64 validators) + pub enable_high_validator_count_metrics: bool, /// Enable use of the blinded block endpoints during proposals. pub builder_proposals: bool, /// Overrides the timestamp field in builder api ValidatorRegistrationV1 @@ -99,6 +104,7 @@ impl Default for Config { http_metrics: <_>::default(), monitoring_api: None, enable_doppelganger_protection: false, + enable_high_validator_count_metrics: false, beacon_nodes_tls_certs: None, block_delay: None, builder_proposals: false, @@ -281,6 +287,10 @@ impl Config { config.http_metrics.enabled = true; } + if cli_args.is_present("enable-high-validator-count-metrics") { + config.enable_high_validator_count_metrics = true; + } + if let Some(address) = cli_args.value_of("metrics-address") { config.http_metrics.listen_addr = address .parse::() diff --git a/validator_client/src/doppelganger_service.rs b/validator_client/src/doppelganger_service.rs index e6934ed48b..558b9e199f 100644 --- a/validator_client/src/doppelganger_service.rs +++ b/validator_client/src/doppelganger_service.rs @@ -441,7 +441,7 @@ impl DoppelgangerService { } // Get a list of indices to provide to the BN API. - let indices_only = indices_map.iter().map(|(index, _)| *index).collect(); + let indices_only = indices_map.keys().copied().collect(); // Pull the liveness responses from the BN. let request_epoch = request_slot.epoch(E::slots_per_epoch()); @@ -971,16 +971,16 @@ mod test { LivenessResponses { current_epoch_responses: detection_indices .iter() - .map(|i| LivenessResponseData { - index: *i as u64, + .map(|&index| LivenessResponseData { + index, epoch: current_epoch, is_live: false, }) .collect(), previous_epoch_responses: detection_indices .iter() - .map(|i| LivenessResponseData { - index: *i as u64, + .map(|&index| LivenessResponseData { + index, epoch: current_epoch - 1, is_live: false, }) diff --git a/validator_client/src/duties_service.rs b/validator_client/src/duties_service.rs index 86b8ca870e..6ba2a2d1fd 100644 --- a/validator_client/src/duties_service.rs +++ b/validator_client/src/duties_service.rs @@ -9,6 +9,7 @@ mod sync; use crate::beacon_node_fallback::{BeaconNodeFallback, OfflineOnFailure, RequireSynced}; +use crate::http_metrics::metrics::{get_int_gauge, set_int_gauge, ATTESTATION_DUTY}; use crate::{ block_service::BlockServiceNotification, http_metrics::metrics, @@ -39,6 +40,11 @@ const SUBSCRIPTION_BUFFER_SLOTS: u64 = 2; /// Only retain `HISTORICAL_DUTIES_EPOCHS` duties prior to the current epoch. const HISTORICAL_DUTIES_EPOCHS: u64 = 2; +/// Minimum number of validators for which we auto-enable per-validator metrics. +/// For validators greater than this value, we need to manually set the `enable-per-validator-metrics` +/// flag in the cli to enable collection of per validator metrics. +const VALIDATOR_METRICS_MIN_COUNT: usize = 64; + #[derive(Debug)] pub enum Error { UnableToReadSlotClock, @@ -121,6 +127,7 @@ pub struct DutiesService { /// This functionality is a little redundant since most BNs will likely reject duties when they /// aren't synced, but we keep it around for an emergency. pub require_synced: RequireSynced, + pub enable_high_validator_count_metrics: bool, pub context: RuntimeContext, pub spec: ChainSpec, } @@ -220,6 +227,12 @@ impl DutiesService { .cloned() .collect() } + + /// Returns `true` if we should collect per validator metrics and `false` otherwise. + pub fn per_validator_metrics(&self) -> bool { + self.enable_high_validator_count_metrics + || self.total_validator_count() <= VALIDATOR_METRICS_MIN_COUNT + } } /// Start the service that periodically polls the beacon node for validator duties. This will start @@ -501,6 +514,7 @@ async fn poll_beacon_attesters( current_epoch, &local_indices, &local_pubkeys, + current_slot, ) .await { @@ -520,9 +534,14 @@ async fn poll_beacon_attesters( ); // Download the duties and update the duties for the next epoch. - if let Err(e) = - poll_beacon_attesters_for_epoch(duties_service, next_epoch, &local_indices, &local_pubkeys) - .await + if let Err(e) = poll_beacon_attesters_for_epoch( + duties_service, + next_epoch, + &local_indices, + &local_pubkeys, + current_slot, + ) + .await { error!( log, @@ -619,6 +638,7 @@ async fn poll_beacon_attesters_for_epoch( epoch: Epoch, local_indices: &[u64], local_pubkeys: &HashSet, + current_slot: Slot, ) -> Result<(), Error> { let log = duties_service.context.log(); @@ -671,6 +691,35 @@ async fn poll_beacon_attesters_for_epoch( .data .into_iter() .filter(|duty| { + if duties_service.per_validator_metrics() { + let validator_index = duty.validator_index; + let duty_slot = duty.slot; + if let Some(existing_slot_gauge) = + get_int_gauge(&ATTESTATION_DUTY, &[&validator_index.to_string()]) + { + let existing_slot = Slot::new(existing_slot_gauge.get() as u64); + let existing_epoch = existing_slot.epoch(E::slots_per_epoch()); + + // First condition ensures that we switch to the next epoch duty slot + // once the current epoch duty slot passes. + // Second condition is to ensure that next epoch duties don't override + // current epoch duties. + if existing_slot < current_slot + || (duty_slot.epoch(E::slots_per_epoch()) <= existing_epoch + && duty_slot > current_slot + && duty_slot != existing_slot) + { + existing_slot_gauge.set(duty_slot.as_u64() as i64); + } + } else { + set_int_gauge( + &ATTESTATION_DUTY, + &[&validator_index.to_string()], + duty_slot.as_u64() as i64, + ); + } + } + local_pubkeys.contains(&duty.pubkey) && { // Only update the duties if either is true: // diff --git a/validator_client/src/http_api/mod.rs b/validator_client/src/http_api/mod.rs index 898fa485ed..7402bff9c8 100644 --- a/validator_client/src/http_api/mod.rs +++ b/validator_client/src/http_api/mod.rs @@ -353,7 +353,7 @@ pub fn serve( .and(signer.clone()) .and_then(|sysinfo, app_start: std::time::Instant, val_dir, signer| { blocking_signed_json_task(signer, move || { - let app_uptime = app_start.elapsed().as_secs() as u64; + let app_uptime = app_start.elapsed().as_secs(); Ok(api_types::GenericResponse::from(observe_system_health_vc( sysinfo, val_dir, app_uptime, ))) diff --git a/validator_client/src/http_metrics/metrics.rs b/validator_client/src/http_metrics/metrics.rs index 146d008a57..0cb3417fc7 100644 --- a/validator_client/src/http_metrics/metrics.rs +++ b/validator_client/src/http_metrics/metrics.rs @@ -172,6 +172,12 @@ lazy_static::lazy_static! { "Duration to obtain a signature", &["type"] ); + + pub static ref ATTESTATION_DUTY: Result = try_create_int_gauge_vec( + "vc_attestation_duty_slot", + "Attestation duty slot for all managed validators", + &["validator"] + ); } pub fn gather_prometheus_metrics( diff --git a/validator_client/src/initialized_validators.rs b/validator_client/src/initialized_validators.rs index 0bb246b325..37dfcc2106 100644 --- a/validator_client/src/initialized_validators.rs +++ b/validator_client/src/initialized_validators.rs @@ -479,7 +479,7 @@ impl InitializedValidators { /// Iterate through all voting public keys in `self` that should be used when querying for duties. pub fn iter_voting_pubkeys(&self) -> impl Iterator { - self.validators.iter().map(|(pubkey, _)| pubkey) + self.validators.keys() } /// Returns the voting `Keypair` for a given voting `PublicKey`, if all are true: @@ -1020,17 +1020,17 @@ impl InitializedValidators { let mut disabled_uuids = HashSet::new(); for def in self.definitions.as_slice() { if def.enabled { + let pubkey_bytes = def.voting_public_key.compress(); + + if self.validators.contains_key(&pubkey_bytes) { + continue; + } + match &def.signing_definition { SigningDefinition::LocalKeystore { voting_keystore_path, .. } => { - let pubkey_bytes = def.voting_public_key.compress(); - - if self.validators.contains_key(&pubkey_bytes) { - continue; - } - if let Some(key_store) = key_stores.get(voting_keystore_path) { disabled_uuids.remove(key_store.uuid()); } diff --git a/validator_client/src/key_cache.rs b/validator_client/src/key_cache.rs index 2088aa6831..b7abaaed06 100644 --- a/validator_client/src/key_cache.rs +++ b/validator_client/src/key_cache.rs @@ -104,7 +104,7 @@ impl KeyCache { let file = File::options() .read(true) .create_new(false) - .open(&cache_path) + .open(cache_path) .map_err(Error::UnableToOpenFile)?; serde_json::from_reader(file).map_err(Error::UnableToParseFile) } diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index ecc85e2cc0..2cbf58380d 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -31,6 +31,7 @@ use crate::beacon_node_fallback::{ }; use crate::doppelganger_service::DoppelgangerService; use crate::graffiti_file::GraffitiFile; +use crate::initialized_validators::Error::UnableToOpenVotingKeystore; use account_utils::validator_definitions::ValidatorDefinitions; use attestation_service::{AttestationService, AttestationServiceBuilder}; use block_service::{BlockService, BlockServiceBuilder}; @@ -184,7 +185,16 @@ impl ProductionValidatorClient { log.clone(), ) .await - .map_err(|e| format!("Unable to initialize validators: {:?}", e))?; + .map_err(|e| { + match e { + UnableToOpenVotingKeystore(err) => { + format!("Unable to initialize validators: {:?}. If you have recently moved the location of your data directory \ + make sure to update the location of voting_keystore_path in your validator_definitions.yml", err) + }, + err => { + format!("Unable to initialize validators: {:?}", err)} + } + })?; let voting_pubkeys: Vec<_> = validators.iter_voting_pubkeys().collect(); @@ -412,6 +422,7 @@ impl ProductionValidatorClient { }, spec: context.eth2_config.spec.clone(), context: duties_context, + enable_high_validator_count_metrics: config.enable_high_validator_count_metrics, }); // Update the metrics server.