diff --git a/.github/mergify.yml b/.github/mergify.yml index ae01e3ffd2..4c4046cf67 100644 --- a/.github/mergify.yml +++ b/.github/mergify.yml @@ -4,6 +4,12 @@ queue_rules: batch_max_wait_time: 60 s checks_timeout: 10800 s merge_method: squash + commit_message_template: | + {{ title }} (#{{ number }}) + + {% for commit in commits %} + * {{ commit.commit_message }} + {% endfor %} queue_conditions: - "#approved-reviews-by >= 1" - "check-success=license/cla" diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 007070dbb5..bdd7b62653 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -152,10 +152,12 @@ jobs: - name: Dockerhub login run: | echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin - - name: Build lcli dockerfile (with push) - run: | - docker build \ - --build-arg PORTABLE=true \ - --tag ${LCLI_IMAGE_NAME}:${VERSION}${VERSION_SUFFIX} \ - --file ./lcli/Dockerfile . - docker push ${LCLI_IMAGE_NAME}:${VERSION}${VERSION_SUFFIX} + - name: Build lcli and push + uses: docker/build-push-action@v5 + with: + build-args: | + FEATURES=portable + context: . + push: true + file: ./lcli/Dockerfile + tags: ${{ env.LCLI_IMAGE_NAME }}:${{ env.VERSION }}${{ env.VERSION_SUFFIX }} diff --git a/Cargo.lock b/Cargo.lock index f58f51ed21..0b15faeb86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -299,9 +299,9 @@ dependencies = [ [[package]] name = "async-io" -version = "2.2.2" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6afaa937395a620e33dc6a742c593c01aced20aa376ffb0f628121198578ccc7" +checksum = "fb41eb19024a91746eba0773aa5e16036045bbf45733766661099e182ea6a744" dependencies = [ "async-lock", "cfg-if", @@ -583,7 +583,7 @@ dependencies = [ [[package]] name = "beacon_node" -version = "4.6.0-rc.0" +version = "4.6.0" dependencies = [ "beacon_chain", "clap", @@ -676,7 +676,7 @@ version = "0.66.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2b84e06fc203107bfbad243f4aba2af864eb7db3b1cf46ea0a023b0b433d2a7" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "cexpr", "clang-sys", "lazy_static", @@ -701,9 +701,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" [[package]] name = "bitvec" @@ -805,7 +805,7 @@ dependencies = [ [[package]] name = "boot_node" -version = "4.6.0-rc.0" +version = "4.6.0" dependencies = [ "beacon_node", "clap", @@ -1021,14 +1021,14 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.31" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "41daef31d7a747c5c847246f36de49ced6f7403b4cdabc807a97b5cc184cda7a" dependencies = [ "android-tzdata", "iana-time-zone", "num-traits", - "windows-targets 0.48.5", + "windows-targets 0.52.0", ] [[package]] @@ -1107,6 +1107,7 @@ dependencies = [ "eth2", "eth2_config", "execution_layer", + "futures", "genesis", "http_api", "http_metrics", @@ -1662,7 +1663,7 @@ version = "2.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62c6fcf842f17f8c78ecf7c81d75c5ce84436b41ee07e03f490fbb5f5a8731d8" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "byteorder", "diesel_derives", "itoa", @@ -1776,7 +1777,7 @@ dependencies = [ [[package]] name = "discv5" version = "0.4.0" -source = "git+https://github.com/sigp/discv5?rev=dbb4a718cd32eaed8127c3c8241bfd0fde9eb908#dbb4a718cd32eaed8127c3c8241bfd0fde9eb908" +source = "git+https://github.com/sigp/discv5?rev=e30a2c31b7ac0c57876458b971164654dfa4513b#e30a2c31b7ac0c57876458b971164654dfa4513b" dependencies = [ "aes 0.7.5", "aes-gcm 0.9.2", @@ -2837,7 +2838,7 @@ dependencies = [ [[package]] name = "futures-bounded" version = "0.2.3" -source = "git+https://github.com/sigp/rust-libp2p/?rev=b96b90894faab0a1eed78e1c82c6452138a3538a#b96b90894faab0a1eed78e1c82c6452138a3538a" +source = "git+https://github.com/sigp/rust-libp2p/?rev=cfa3275ca17e502799ed56e555b6c0611752e369#cfa3275ca17e502799ed56e555b6c0611752e369" dependencies = [ "futures-timer", "futures-util", @@ -3085,9 +3086,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.23" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b553656127a00601c8ae5590fcfdc118e4083a7924b6cf4ffc1ea4b99dc429d7" +checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" dependencies = [ "bytes", "fnv", @@ -3104,9 +3105,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "991910e35c615d8cab86b5ab04be67e6ad24d2bf5f4f11fdbbed26da999bbeab" +checksum = "31d030e59af851932b72ceebadf4a2b5986dba4c3b99dd2493f8273a0f151943" dependencies = [ "bytes", "fnv", @@ -3217,9 +3218,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f" [[package]] name = "hex" @@ -3500,7 +3501,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.3.23", + "h2 0.3.24", "http 0.2.11", "http-body 0.4.6", "httparse", @@ -3523,7 +3524,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.1", + "h2 0.4.2", "http 1.0.0", "http-body 1.0.0", "httparse", @@ -3836,7 +3837,7 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ - "hermit-abi 0.3.3", + "hermit-abi 0.3.4", "libc", "windows-sys 0.48.0", ] @@ -4016,7 +4017,7 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "lcli" -version = "4.6.0-rc.0" +version = "4.6.0" dependencies = [ "account_utils", "beacon_chain", @@ -4137,7 +4138,7 @@ dependencies = [ [[package]] name = "libp2p" version = "0.54.0" -source = "git+https://github.com/sigp/rust-libp2p/?rev=b96b90894faab0a1eed78e1c82c6452138a3538a#b96b90894faab0a1eed78e1c82c6452138a3538a" +source = "git+https://github.com/sigp/rust-libp2p/?rev=cfa3275ca17e502799ed56e555b6c0611752e369#cfa3275ca17e502799ed56e555b6c0611752e369" dependencies = [ "bytes", "either", @@ -4170,7 +4171,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.3.0" -source = "git+https://github.com/sigp/rust-libp2p/?rev=b96b90894faab0a1eed78e1c82c6452138a3538a#b96b90894faab0a1eed78e1c82c6452138a3538a" +source = "git+https://github.com/sigp/rust-libp2p/?rev=cfa3275ca17e502799ed56e555b6c0611752e369#cfa3275ca17e502799ed56e555b6c0611752e369" dependencies = [ "libp2p-core", "libp2p-identity", @@ -4181,7 +4182,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.3.1" -source = "git+https://github.com/sigp/rust-libp2p/?rev=b96b90894faab0a1eed78e1c82c6452138a3538a#b96b90894faab0a1eed78e1c82c6452138a3538a" +source = "git+https://github.com/sigp/rust-libp2p/?rev=cfa3275ca17e502799ed56e555b6c0611752e369#cfa3275ca17e502799ed56e555b6c0611752e369" dependencies = [ "libp2p-core", "libp2p-identity", @@ -4192,7 +4193,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.41.2" -source = "git+https://github.com/sigp/rust-libp2p/?rev=b96b90894faab0a1eed78e1c82c6452138a3538a#b96b90894faab0a1eed78e1c82c6452138a3538a" +source = "git+https://github.com/sigp/rust-libp2p/?rev=cfa3275ca17e502799ed56e555b6c0611752e369#cfa3275ca17e502799ed56e555b6c0611752e369" dependencies = [ "either", "fnv", @@ -4219,7 +4220,7 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.41.1" -source = "git+https://github.com/sigp/rust-libp2p/?rev=b96b90894faab0a1eed78e1c82c6452138a3538a#b96b90894faab0a1eed78e1c82c6452138a3538a" +source = "git+https://github.com/sigp/rust-libp2p/?rev=cfa3275ca17e502799ed56e555b6c0611752e369#cfa3275ca17e502799ed56e555b6c0611752e369" dependencies = [ "async-trait", "futures", @@ -4234,7 +4235,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.46.1" -source = "git+https://github.com/sigp/rust-libp2p/?rev=b96b90894faab0a1eed78e1c82c6452138a3538a#b96b90894faab0a1eed78e1c82c6452138a3538a" +source = "git+https://github.com/sigp/rust-libp2p/?rev=cfa3275ca17e502799ed56e555b6c0611752e369#cfa3275ca17e502799ed56e555b6c0611752e369" dependencies = [ "async-channel", "asynchronous-codec", @@ -4266,7 +4267,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.44.1" -source = "git+https://github.com/sigp/rust-libp2p/?rev=b96b90894faab0a1eed78e1c82c6452138a3538a#b96b90894faab0a1eed78e1c82c6452138a3538a" +source = "git+https://github.com/sigp/rust-libp2p/?rev=cfa3275ca17e502799ed56e555b6c0611752e369#cfa3275ca17e502799ed56e555b6c0611752e369" dependencies = [ "asynchronous-codec", "either", @@ -4311,7 +4312,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.45.1" -source = "git+https://github.com/sigp/rust-libp2p/?rev=b96b90894faab0a1eed78e1c82c6452138a3538a#b96b90894faab0a1eed78e1c82c6452138a3538a" +source = "git+https://github.com/sigp/rust-libp2p/?rev=cfa3275ca17e502799ed56e555b6c0611752e369#cfa3275ca17e502799ed56e555b6c0611752e369" dependencies = [ "data-encoding", "futures", @@ -4331,7 +4332,7 @@ dependencies = [ [[package]] name = "libp2p-metrics" version = "0.14.1" -source = "git+https://github.com/sigp/rust-libp2p/?rev=b96b90894faab0a1eed78e1c82c6452138a3538a#b96b90894faab0a1eed78e1c82c6452138a3538a" +source = "git+https://github.com/sigp/rust-libp2p/?rev=cfa3275ca17e502799ed56e555b6c0611752e369#cfa3275ca17e502799ed56e555b6c0611752e369" dependencies = [ "futures", "instant", @@ -4347,7 +4348,7 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.41.0" -source = "git+https://github.com/sigp/rust-libp2p/?rev=b96b90894faab0a1eed78e1c82c6452138a3538a#b96b90894faab0a1eed78e1c82c6452138a3538a" +source = "git+https://github.com/sigp/rust-libp2p/?rev=cfa3275ca17e502799ed56e555b6c0611752e369#cfa3275ca17e502799ed56e555b6c0611752e369" dependencies = [ "asynchronous-codec", "bytes", @@ -4365,7 +4366,7 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.44.0" -source = "git+https://github.com/sigp/rust-libp2p/?rev=b96b90894faab0a1eed78e1c82c6452138a3538a#b96b90894faab0a1eed78e1c82c6452138a3538a" +source = "git+https://github.com/sigp/rust-libp2p/?rev=cfa3275ca17e502799ed56e555b6c0611752e369#cfa3275ca17e502799ed56e555b6c0611752e369" dependencies = [ "asynchronous-codec", "bytes", @@ -4390,7 +4391,7 @@ dependencies = [ [[package]] name = "libp2p-plaintext" version = "0.41.0" -source = "git+https://github.com/sigp/rust-libp2p/?rev=b96b90894faab0a1eed78e1c82c6452138a3538a#b96b90894faab0a1eed78e1c82c6452138a3538a" +source = "git+https://github.com/sigp/rust-libp2p/?rev=cfa3275ca17e502799ed56e555b6c0611752e369#cfa3275ca17e502799ed56e555b6c0611752e369" dependencies = [ "asynchronous-codec", "bytes", @@ -4405,7 +4406,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.10.2" -source = "git+https://github.com/sigp/rust-libp2p/?rev=b96b90894faab0a1eed78e1c82c6452138a3538a#b96b90894faab0a1eed78e1c82c6452138a3538a" +source = "git+https://github.com/sigp/rust-libp2p/?rev=cfa3275ca17e502799ed56e555b6c0611752e369#cfa3275ca17e502799ed56e555b6c0611752e369" dependencies = [ "bytes", "futures", @@ -4428,7 +4429,7 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.45.0" -source = "git+https://github.com/sigp/rust-libp2p/?rev=b96b90894faab0a1eed78e1c82c6452138a3538a#b96b90894faab0a1eed78e1c82c6452138a3538a" +source = "git+https://github.com/sigp/rust-libp2p/?rev=cfa3275ca17e502799ed56e555b6c0611752e369#cfa3275ca17e502799ed56e555b6c0611752e369" dependencies = [ "either", "fnv", @@ -4450,7 +4451,7 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.34.1" -source = "git+https://github.com/sigp/rust-libp2p/?rev=b96b90894faab0a1eed78e1c82c6452138a3538a#b96b90894faab0a1eed78e1c82c6452138a3538a" +source = "git+https://github.com/sigp/rust-libp2p/?rev=cfa3275ca17e502799ed56e555b6c0611752e369#cfa3275ca17e502799ed56e555b6c0611752e369" dependencies = [ "heck", "proc-macro2", @@ -4461,7 +4462,7 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.41.0" -source = "git+https://github.com/sigp/rust-libp2p/?rev=b96b90894faab0a1eed78e1c82c6452138a3538a#b96b90894faab0a1eed78e1c82c6452138a3538a" +source = "git+https://github.com/sigp/rust-libp2p/?rev=cfa3275ca17e502799ed56e555b6c0611752e369#cfa3275ca17e502799ed56e555b6c0611752e369" dependencies = [ "futures", "futures-timer", @@ -4477,7 +4478,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.3.0" -source = "git+https://github.com/sigp/rust-libp2p/?rev=b96b90894faab0a1eed78e1c82c6452138a3538a#b96b90894faab0a1eed78e1c82c6452138a3538a" +source = "git+https://github.com/sigp/rust-libp2p/?rev=cfa3275ca17e502799ed56e555b6c0611752e369#cfa3275ca17e502799ed56e555b6c0611752e369" dependencies = [ "futures", "futures-rustls", @@ -4495,7 +4496,7 @@ dependencies = [ [[package]] name = "libp2p-upnp" version = "0.2.0" -source = "git+https://github.com/sigp/rust-libp2p/?rev=b96b90894faab0a1eed78e1c82c6452138a3538a#b96b90894faab0a1eed78e1c82c6452138a3538a" +source = "git+https://github.com/sigp/rust-libp2p/?rev=cfa3275ca17e502799ed56e555b6c0611752e369#cfa3275ca17e502799ed56e555b6c0611752e369" dependencies = [ "futures", "futures-timer", @@ -4510,7 +4511,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.45.1" -source = "git+https://github.com/sigp/rust-libp2p/?rev=b96b90894faab0a1eed78e1c82c6452138a3538a#b96b90894faab0a1eed78e1c82c6452138a3538a" +source = "git+https://github.com/sigp/rust-libp2p/?rev=cfa3275ca17e502799ed56e555b6c0611752e369#cfa3275ca17e502799ed56e555b6c0611752e369" dependencies = [ "either", "futures", @@ -4527,7 +4528,7 @@ version = "0.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "libc", "redox_syscall 0.4.1", ] @@ -4604,7 +4605,7 @@ dependencies = [ [[package]] name = "lighthouse" -version = "4.6.0-rc.0" +version = "4.6.0" dependencies = [ "account_manager", "account_utils", @@ -4729,9 +4730,9 @@ checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" [[package]] name = "linux-raw-sys" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "lmdb-rkv" @@ -5107,7 +5108,7 @@ dependencies = [ [[package]] name = "multistream-select" version = "0.13.0" -source = "git+https://github.com/sigp/rust-libp2p/?rev=b96b90894faab0a1eed78e1c82c6452138a3538a#b96b90894faab0a1eed78e1c82c6452138a3538a" +source = "git+https://github.com/sigp/rust-libp2p/?rev=cfa3275ca17e502799ed56e555b6c0611752e369#cfa3275ca17e502799ed56e555b6c0611752e369" dependencies = [ "bytes", "futures", @@ -5268,7 +5269,7 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "cfg-if", "libc", ] @@ -5389,7 +5390,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.3.3", + "hermit-abi 0.3.4", "libc", ] @@ -5472,11 +5473,11 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.62" +version = "0.10.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cde4d2d9200ad5909f8dac647e29482e07c3a35de8a13fce7c9c7747ad9f671" +checksum = "15c9d69dd87a29568d4d017cfe8ec518706046a05184e5aea92d0af890b803c8" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "cfg-if", "foreign-types", "libc", @@ -5513,9 +5514,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.98" +version = "0.9.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1665caf8ab2dc9aef43d1c0023bd904633a6a05cb30b0ad59bec2ae986e57a7" +checksum = "22e1bf214306098e4832460f797824c05d25aacdf896f64a985fb0fd992454ae" dependencies = [ "cc", "libc", @@ -5831,9 +5832,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d3587f8a9e599cc7ec2c00e331f71c4e69a5f9a4b8a6efd5b07466b9736f9a" +checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb" [[package]] name = "platforms" @@ -6073,9 +6074,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.76" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" dependencies = [ "unicode-ident", ] @@ -6189,7 +6190,7 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "git+https://github.com/sigp/rust-libp2p/?rev=b96b90894faab0a1eed78e1c82c6452138a3538a#b96b90894faab0a1eed78e1c82c6452138a3538a" +source = "git+https://github.com/sigp/rust-libp2p/?rev=cfa3275ca17e502799ed56e555b6c0611752e369#cfa3275ca17e502799ed56e555b6c0611752e369" dependencies = [ "asynchronous-codec", "bytes", @@ -6351,9 +6352,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +checksum = "fa7237101a77a10773db45d62004a272517633fbcc3df19d96455ede1122e051" dependencies = [ "either", "rayon-core", @@ -6361,9 +6362,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" dependencies = [ "crossbeam-deque", "crossbeam-utils", @@ -6412,13 +6413,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.2" +version = "1.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.3", + "regex-automata 0.4.4", "regex-syntax 0.8.2", ] @@ -6433,9 +6434,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +checksum = "3b7fa1134405e2ec9353fd416b17f8dacd46c473d7d3fd1cf202706a14eb792a" dependencies = [ "aho-corasick", "memchr", @@ -6465,7 +6466,7 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2 0.3.23", + "h2 0.3.24", "http 0.2.11", "http-body 0.4.6", "hyper 0.14.28", @@ -6681,10 +6682,10 @@ version = "0.38.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "errno", "libc", - "linux-raw-sys 0.4.12", + "linux-raw-sys 0.4.13", "windows-sys 0.52.0", ] @@ -6769,7 +6770,7 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "git+https://github.com/sigp/rust-libp2p/?rev=b96b90894faab0a1eed78e1c82c6452138a3538a#b96b90894faab0a1eed78e1c82c6452138a3538a" +source = "git+https://github.com/sigp/rust-libp2p/?rev=cfa3275ca17e502799ed56e555b6c0611752e369#cfa3275ca17e502799ed56e555b6c0611752e369" dependencies = [ "futures", "pin-project", @@ -7150,9 +7151,9 @@ dependencies = [ [[package]] name = "shlex" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook-registry" @@ -7409,9 +7410,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.12.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2593d31f82ead8df961d8bd23a64c2ccf2eb5dd34b0a34bfb4dd54011c72009e" +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" [[package]] name = "snap" @@ -8434,9 +8435,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" @@ -8710,7 +8711,7 @@ dependencies = [ [[package]] name = "warp" version = "0.3.6" -source = "git+https://github.com/seanmonstar/warp.git#724e767173132de7c435b323e12d6bec68038de2" +source = "git+https://github.com/seanmonstar/warp.git#7b07043cee0ca24e912155db4e8f6d9ab7c049ed" dependencies = [ "bytes", "futures-channel", @@ -8740,6 +8741,7 @@ name = "warp_utils" version = "0.1.0" dependencies = [ "beacon_chain", + "bytes", "eth2", "headers", "lazy_static", @@ -8747,6 +8749,7 @@ dependencies = [ "safe_arith", "serde", "serde_array_query", + "serde_json", "state_processing", "tokio", "types", diff --git a/Cargo.toml b/Cargo.toml index a3e1430f3f..ca55d00d4b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -105,7 +105,7 @@ criterion = "0.3" delay_map = "0.3" derivative = "2" dirs = "3" -discv5 = { git="https://github.com/sigp/discv5", rev="dbb4a718cd32eaed8127c3c8241bfd0fde9eb908", features = ["libp2p"] } +discv5 = { git="https://github.com/sigp/discv5", rev="e30a2c31b7ac0c57876458b971164654dfa4513b", features = ["libp2p"] } env_logger = "0.9" error-chain = "0.12" ethereum-types = "0.14" diff --git a/account_manager/src/validator/import.rs b/account_manager/src/validator/import.rs index 339d9a2914..bf000385f3 100644 --- a/account_manager/src/validator/import.rs +++ b/account_manager/src/validator/import.rs @@ -284,6 +284,8 @@ pub fn cli_run(matches: &ArgMatches, validator_dir: PathBuf) -> Result<(), Strin suggested_fee_recipient, None, None, + None, + None, ) .map_err(|e| format!("Unable to create new validator definition: {:?}", e))?; diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index d6d7de8b8d..8428a30a3b 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "beacon_node" -version = "4.6.0-rc.0" +version = "4.6.0" authors = [ "Paul Hauner ", "Age Manning { bls_to_execution_changes: Vec, } +pub type LightClientProducerEvent = (Hash256, Slot, SyncAggregate); + pub type BeaconForkChoice = ForkChoice< BeaconForkChoiceStore< ::EthSpec, @@ -420,10 +423,6 @@ pub struct BeaconChain { /// Maintains a record of which validators we've seen BLS to execution changes for. pub(crate) observed_bls_to_execution_changes: Mutex>, - /// The most recently validated light client finality update received on gossip. - pub latest_seen_finality_update: Mutex>>, - /// The most recently validated light client optimistic update received on gossip. - pub latest_seen_optimistic_update: Mutex>>, /// Provides information from the Ethereum 1 (PoW) chain. pub eth1_chain: Option>, /// Interfaces with the execution client. @@ -466,6 +465,10 @@ pub struct BeaconChain { pub block_times_cache: Arc>, /// A cache used to track pre-finalization block roots for quick rejection. pub pre_finalization_block_cache: PreFinalizationBlockCache, + /// A cache used to produce light_client server messages + pub light_client_server_cache: LightClientServerCache, + /// Sender to signal the light_client server to produce new updates + pub light_client_server_tx: Option>>, /// Sender given to tasks, so that if they encounter a state in which execution cannot /// continue they can request that everything shuts down. pub shutdown_sender: Sender, @@ -610,12 +613,19 @@ impl BeaconChain { let mut batch = vec![]; let _head_timer = metrics::start_timer(&metrics::PERSIST_HEAD); - batch.push(self.persist_head_in_batch()); + + // Hold a lock to head_tracker until it has been persisted to disk. Otherwise there's a race + // condition with the pruning thread which can result in a block present in the head tracker + // but absent in the DB. This inconsistency halts pruning and dramastically increases disk + // size. Ref: https://github.com/sigp/lighthouse/issues/4773 + let head_tracker = self.head_tracker.0.read(); + batch.push(self.persist_head_in_batch(&head_tracker)); let _fork_choice_timer = metrics::start_timer(&metrics::PERSIST_FORK_CHOICE); batch.push(self.persist_fork_choice_in_batch()); self.store.hot_db.do_atomically(batch)?; + drop(head_tracker); Ok(()) } @@ -623,25 +633,28 @@ impl BeaconChain { /// Return a `PersistedBeaconChain` without reference to a `BeaconChain`. pub fn make_persisted_head( genesis_block_root: Hash256, - head_tracker: &HeadTracker, + head_tracker_reader: &HeadTrackerReader, ) -> PersistedBeaconChain { PersistedBeaconChain { _canonical_head_block_root: DUMMY_CANONICAL_HEAD_BLOCK_ROOT, genesis_block_root, - ssz_head_tracker: head_tracker.to_ssz_container(), + ssz_head_tracker: SszHeadTracker::from_map(head_tracker_reader), } } /// Return a database operation for writing the beacon chain head to disk. - pub fn persist_head_in_batch(&self) -> KeyValueStoreOp { - Self::persist_head_in_batch_standalone(self.genesis_block_root, &self.head_tracker) + pub fn persist_head_in_batch( + &self, + head_tracker_reader: &HeadTrackerReader, + ) -> KeyValueStoreOp { + Self::persist_head_in_batch_standalone(self.genesis_block_root, head_tracker_reader) } pub fn persist_head_in_batch_standalone( genesis_block_root: Hash256, - head_tracker: &HeadTracker, + head_tracker_reader: &HeadTrackerReader, ) -> KeyValueStoreOp { - Self::make_persisted_head(genesis_block_root, head_tracker) + Self::make_persisted_head(genesis_block_root, head_tracker_reader) .as_kv_store_op(BEACON_CHAIN_DB_KEY) } @@ -1334,6 +1347,19 @@ impl BeaconChain { self.state_at_slot(load_slot, StateSkipConfig::WithoutStateRoots) } + pub fn recompute_and_cache_light_client_updates( + &self, + (parent_root, slot, sync_aggregate): LightClientProducerEvent, + ) -> Result<(), Error> { + self.light_client_server_cache.recompute_and_cache_updates( + &self.log, + self.store.clone(), + &parent_root, + slot, + &sync_aggregate, + ) + } + /// Returns the current heads of the `BeaconChain`. For the canonical head, see `Self::head`. /// /// Returns `(block_root, block_slot)`. @@ -1341,6 +1367,7 @@ impl BeaconChain { self.head_tracker.heads() } + /// Only used in tests. pub fn knows_head(&self, block_hash: &SignedBeaconBlockHash) -> bool { self.head_tracker.contains_head((*block_hash).into()) } @@ -3510,6 +3537,20 @@ impl BeaconChain { }; let current_finalized_checkpoint = state.finalized_checkpoint(); + // compute state proofs for light client updates before inserting the state into the + // snapshot cache. + if self.config.enable_light_client_server { + self.light_client_server_cache + .cache_state_data( + &self.spec, block, block_root, + // mutable reference on the state is needed to compute merkle proofs + &mut state, + ) + .unwrap_or_else(|e| { + error!(self.log, "error caching light_client data {:?}", e); + }); + } + self.snapshot_cache .try_write_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT) .ok_or(Error::SnapshotCacheLockTimeout) @@ -3882,6 +3923,28 @@ impl BeaconChain { })); } } + + // Do not trigger light_client server update producer for old blocks, to extra work + // during sync. + if self.config.enable_light_client_server + && block_delay_total < self.slot_clock.slot_duration() * 32 + { + if let Some(mut light_client_server_tx) = self.light_client_server_tx.clone() { + if let Ok(sync_aggregate) = block.body().sync_aggregate() { + if let Err(e) = light_client_server_tx.try_send(( + block.parent_root(), + block.slot(), + sync_aggregate.clone(), + )) { + warn!( + self.log, + "Failed to send light_client server event"; + "error" => ?e + ); + } + } + } + } } // For the current and next epoch of this state, ensure we have the shuffling from this @@ -4059,7 +4122,7 @@ impl BeaconChain { .task_executor .spawn_blocking_handle( move || chain.load_state_for_block_production(slot), - "produce_partial_beacon_block", + "load_state_for_block_production", ) .ok_or(BlockProductionError::ShuttingDown)? .await diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index 0cdf5b9fe9..e8df5b811e 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -722,7 +722,7 @@ impl IntoGossipVerifiedBlockContents for PublishBlockReq Ok::<_, BlockContentsError>(gossip_verified_blobs) }) .transpose()?; - let gossip_verified_block = GossipVerifiedBlock::new(Arc::new(block), chain)?; + let gossip_verified_block = GossipVerifiedBlock::new(block, chain)?; Ok((gossip_verified_block, gossip_verified_blobs)) } diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index d36d996f01..5e06692b8d 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -1,4 +1,6 @@ -use crate::beacon_chain::{CanonicalHead, BEACON_CHAIN_DB_KEY, ETH1_CACHE_DB_KEY, OP_POOL_DB_KEY}; +use crate::beacon_chain::{ + CanonicalHead, LightClientProducerEvent, BEACON_CHAIN_DB_KEY, ETH1_CACHE_DB_KEY, OP_POOL_DB_KEY, +}; use crate::beacon_proposer_cache::BeaconProposerCache; use crate::data_availability_checker::DataAvailabilityChecker; use crate::eth1_chain::{CachingEth1Backend, SszEth1}; @@ -6,6 +8,7 @@ use crate::eth1_finalization_cache::Eth1FinalizationCache; use crate::fork_choice_signal::ForkChoiceSignalTx; use crate::fork_revert::{reset_fork_choice_to_finalization, revert_to_fork_boundary}; use crate::head_tracker::HeadTracker; +use crate::light_client_server_cache::LightClientServerCache; use crate::migrate::{BackgroundMigrator, MigratorConfig}; use crate::persisted_beacon_chain::PersistedBeaconChain; use crate::shuffling_cache::{BlockShufflingIds, ShufflingCache}; @@ -87,6 +90,7 @@ pub struct BeaconChainBuilder { event_handler: Option>, slot_clock: Option, shutdown_sender: Option>, + light_client_server_tx: Option>>, head_tracker: Option, validator_pubkey_cache: Option>, spec: ChainSpec, @@ -129,6 +133,7 @@ where event_handler: None, slot_clock: None, shutdown_sender: None, + light_client_server_tx: None, head_tracker: None, validator_pubkey_cache: None, spec: TEthSpec::default_spec(), @@ -603,6 +608,15 @@ where self } + /// Sets a `Sender` to allow the beacon chain to trigger light_client update production. + pub fn light_client_server_tx( + mut self, + sender: Sender>, + ) -> Self { + self.light_client_server_tx = Some(sender); + self + } + /// Creates a new, empty operation pool. fn empty_op_pool(mut self) -> Self { self.op_pool = Some(OperationPool::new()); @@ -805,10 +819,11 @@ where // // This *must* be stored before constructing the `BeaconChain`, so that its `Drop` instance // doesn't write a `PersistedBeaconChain` without the rest of the batch. + let head_tracker_reader = head_tracker.0.read(); self.pending_io_batch.push(BeaconChain::< Witness, >::persist_head_in_batch_standalone( - genesis_block_root, &head_tracker + genesis_block_root, &head_tracker_reader )); self.pending_io_batch.push(BeaconChain::< Witness, @@ -819,6 +834,7 @@ where .hot_db .do_atomically(self.pending_io_batch) .map_err(|e| format!("Error writing chain & metadata to disk: {:?}", e))?; + drop(head_tracker_reader); let genesis_validators_root = head_snapshot.beacon_state.genesis_validators_root(); let genesis_time = head_snapshot.beacon_state.genesis_time(); @@ -885,8 +901,6 @@ where observed_proposer_slashings: <_>::default(), observed_attester_slashings: <_>::default(), observed_bls_to_execution_changes: <_>::default(), - latest_seen_finality_update: <_>::default(), - latest_seen_optimistic_update: <_>::default(), eth1_chain: self.eth1_chain, execution_layer: self.execution_layer, genesis_validators_root, @@ -914,6 +928,8 @@ where validator_pubkey_cache: TimeoutRwLock::new(validator_pubkey_cache), attester_cache: <_>::default(), early_attester_cache: <_>::default(), + light_client_server_cache: LightClientServerCache::new(), + light_client_server_tx: self.light_client_server_tx, shutdown_sender: self .shutdown_sender .ok_or("Cannot build without a shutdown sender.")?, diff --git a/beacon_node/beacon_chain/src/capella_readiness.rs b/beacon_node/beacon_chain/src/capella_readiness.rs index bb729d8999..cde71d462d 100644 --- a/beacon_node/beacon_chain/src/capella_readiness.rs +++ b/beacon_node/beacon_chain/src/capella_readiness.rs @@ -1,5 +1,4 @@ -//! Provides tools for checking if a node is ready for the Capella upgrade and following merge -//! transition. +//! Provides tools for checking if a node is ready for the Capella upgrade. use crate::{BeaconChain, BeaconChainTypes}; use execution_layer::http::{ diff --git a/beacon_node/beacon_chain/src/chain_config.rs b/beacon_node/beacon_chain/src/chain_config.rs index 7bcb764ab0..23e17a6efa 100644 --- a/beacon_node/beacon_chain/src/chain_config.rs +++ b/beacon_node/beacon_chain/src/chain_config.rs @@ -83,6 +83,8 @@ pub struct ChainConfig { pub progressive_balances_mode: ProgressiveBalancesMode, /// Number of epochs between each migration of data from the hot database to the freezer. pub epochs_per_migration: u64, + /// When set to true Light client server computes and caches state proofs for serving updates + pub enable_light_client_server: bool, } impl Default for ChainConfig { @@ -114,6 +116,7 @@ impl Default for ChainConfig { always_prepare_payload: false, progressive_balances_mode: ProgressiveBalancesMode::Fast, epochs_per_migration: crate::migrate::DEFAULT_EPOCHS_PER_MIGRATION, + enable_light_client_server: false, } } } diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 21cac9a264..48d505e9e7 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -545,6 +545,18 @@ pub struct AvailableBlock { } impl AvailableBlock { + pub fn __new_for_testing( + block_root: Hash256, + block: Arc>, + blobs: Option>, + ) -> Self { + Self { + block_root, + block, + blobs, + } + } + pub fn block(&self) -> &SignedBeaconBlock { &self.block } diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index 8b8368022a..34c9bc76f6 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -952,7 +952,7 @@ mod test { }; let availability_pending_block = AvailabilityPendingExecutedBlock { - block: Arc::new(block), + block, import_data, payload_verification_outcome, }; diff --git a/beacon_node/beacon_chain/src/deneb_readiness.rs b/beacon_node/beacon_chain/src/deneb_readiness.rs new file mode 100644 index 0000000000..1ba6fe3ea6 --- /dev/null +++ b/beacon_node/beacon_chain/src/deneb_readiness.rs @@ -0,0 +1,121 @@ +//! Provides tools for checking if a node is ready for the Deneb upgrade. + +use crate::{BeaconChain, BeaconChainTypes}; +use execution_layer::http::{ + ENGINE_FORKCHOICE_UPDATED_V3, ENGINE_GET_PAYLOAD_V3, ENGINE_NEW_PAYLOAD_V3, +}; +use serde::{Deserialize, Serialize}; +use std::fmt; +use std::time::Duration; +use types::*; + +/// The time before the Deneb fork when we will start issuing warnings about preparation. +use super::merge_readiness::SECONDS_IN_A_WEEK; +pub const DENEB_READINESS_PREPARATION_SECONDS: u64 = SECONDS_IN_A_WEEK * 2; +pub const ENGINE_CAPABILITIES_REFRESH_INTERVAL: u64 = 300; + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +#[serde(tag = "type")] +pub enum DenebReadiness { + /// The execution engine is deneb-enabled (as far as we can tell) + Ready, + /// We are connected to an execution engine which doesn't support the V3 engine api methods + V3MethodsNotSupported { error: String }, + /// The transition configuration with the EL failed, there might be a problem with + /// connectivity, authentication or a difference in configuration. + ExchangeCapabilitiesFailed { error: String }, + /// The user has not configured an execution endpoint + NoExecutionEndpoint, +} + +impl fmt::Display for DenebReadiness { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + DenebReadiness::Ready => { + write!(f, "This node appears ready for Deneb.") + } + DenebReadiness::ExchangeCapabilitiesFailed { error } => write!( + f, + "Could not exchange capabilities with the \ + execution endpoint: {}", + error + ), + DenebReadiness::NoExecutionEndpoint => write!( + f, + "The --execution-endpoint flag is not specified, this is a \ + requirement post-merge" + ), + DenebReadiness::V3MethodsNotSupported { error } => write!( + f, + "Execution endpoint does not support Deneb methods: {}", + error + ), + } + } +} + +impl BeaconChain { + /// Returns `true` if deneb epoch is set and Deneb fork has occurred or will + /// occur within `DENEB_READINESS_PREPARATION_SECONDS` + pub fn is_time_to_prepare_for_deneb(&self, current_slot: Slot) -> bool { + if let Some(deneb_epoch) = self.spec.deneb_fork_epoch { + let deneb_slot = deneb_epoch.start_slot(T::EthSpec::slots_per_epoch()); + let deneb_readiness_preparation_slots = + DENEB_READINESS_PREPARATION_SECONDS / self.spec.seconds_per_slot; + // Return `true` if Deneb has happened or is within the preparation time. + current_slot + deneb_readiness_preparation_slots > deneb_slot + } else { + // The Deneb fork epoch has not been defined yet, no need to prepare. + false + } + } + + /// Attempts to connect to the EL and confirm that it is ready for capella. + pub async fn check_deneb_readiness(&self) -> DenebReadiness { + if let Some(el) = self.execution_layer.as_ref() { + match el + .get_engine_capabilities(Some(Duration::from_secs( + ENGINE_CAPABILITIES_REFRESH_INTERVAL, + ))) + .await + { + Err(e) => { + // The EL was either unreachable or responded with an error + DenebReadiness::ExchangeCapabilitiesFailed { + error: format!("{:?}", e), + } + } + Ok(capabilities) => { + let mut missing_methods = String::from("Required Methods Unsupported:"); + let mut all_good = true; + if !capabilities.get_payload_v3 { + missing_methods.push(' '); + missing_methods.push_str(ENGINE_GET_PAYLOAD_V3); + all_good = false; + } + if !capabilities.forkchoice_updated_v3 { + missing_methods.push(' '); + missing_methods.push_str(ENGINE_FORKCHOICE_UPDATED_V3); + all_good = false; + } + if !capabilities.new_payload_v3 { + missing_methods.push(' '); + missing_methods.push_str(ENGINE_NEW_PAYLOAD_V3); + all_good = false; + } + + if all_good { + DenebReadiness::Ready + } else { + DenebReadiness::V3MethodsNotSupported { + error: missing_methods, + } + } + } + } + } else { + DenebReadiness::NoExecutionEndpoint + } + } +} diff --git a/beacon_node/beacon_chain/src/head_tracker.rs b/beacon_node/beacon_chain/src/head_tracker.rs index 3fa577ff93..71e2473cdc 100644 --- a/beacon_node/beacon_chain/src/head_tracker.rs +++ b/beacon_node/beacon_chain/src/head_tracker.rs @@ -1,4 +1,4 @@ -use parking_lot::RwLock; +use parking_lot::{RwLock, RwLockReadGuard}; use ssz_derive::{Decode, Encode}; use std::collections::HashMap; use types::{Hash256, Slot}; @@ -16,6 +16,8 @@ pub enum Error { #[derive(Default, Debug)] pub struct HeadTracker(pub RwLock>); +pub type HeadTrackerReader<'a> = RwLockReadGuard<'a, HashMap>; + impl HeadTracker { /// Register a block with `Self`, so it may or may not be included in a `Self::heads` call. /// @@ -44,6 +46,11 @@ impl HeadTracker { /// Returns a `SszHeadTracker`, which contains all necessary information to restore the state /// of `Self` at some later point. + /// + /// Should ONLY be used for tests, due to the potential for database races. + /// + /// See + #[cfg(test)] pub fn to_ssz_container(&self) -> SszHeadTracker { SszHeadTracker::from_map(&self.0.read()) } diff --git a/beacon_node/beacon_chain/src/historical_blocks.rs b/beacon_node/beacon_chain/src/historical_blocks.rs index b5e58e7ffa..85208c8ad6 100644 --- a/beacon_node/beacon_chain/src/historical_blocks.rs +++ b/beacon_node/beacon_chain/src/historical_blocks.rs @@ -101,8 +101,9 @@ impl BeaconChain { ChunkWriter::::new(&self.store.cold_db, prev_block_slot.as_usize())?; let mut new_oldest_blob_slot = blob_info.oldest_blob_slot; + let mut blob_batch = Vec::with_capacity(n_blobs_lists_to_import); let mut cold_batch = Vec::with_capacity(blocks_to_import.len()); - let mut hot_batch = Vec::with_capacity(blocks_to_import.len() + n_blobs_lists_to_import); + let mut hot_batch = Vec::with_capacity(blocks_to_import.len()); let mut signed_blocks = Vec::with_capacity(blocks_to_import.len()); for available_block in blocks_to_import.into_iter().rev() { @@ -124,7 +125,7 @@ impl BeaconChain { if let Some(blobs) = maybe_blobs { new_oldest_blob_slot = Some(block.slot()); self.store - .blobs_as_kv_store_ops(&block_root, blobs, &mut hot_batch); + .blobs_as_kv_store_ops(&block_root, blobs, &mut blob_batch); } // Store block roots, including at all skip slots in the freezer DB. @@ -134,20 +135,20 @@ impl BeaconChain { prev_block_slot = block.slot(); expected_block_root = block.message().parent_root(); + signed_blocks.push(block); // If we've reached genesis, add the genesis block root to the batch for all slots // between 0 and the first block slot, and set the anchor slot to 0 to indicate // completion. if expected_block_root == self.genesis_block_root { let genesis_slot = self.spec.genesis_slot; - for slot in genesis_slot.as_usize()..block.slot().as_usize() { + for slot in genesis_slot.as_usize()..prev_block_slot.as_usize() { chunk_writer.set(slot, self.genesis_block_root, &mut cold_batch)?; } prev_block_slot = genesis_slot; expected_block_root = Hash256::zero(); break; } - signed_blocks.push(block); } chunk_writer.write(&mut cold_batch)?; // these were pushed in reverse order so we reverse again @@ -199,6 +200,7 @@ impl BeaconChain { // Write the I/O batches to disk, writing the blocks themselves first, as it's better // for the hot DB to contain extra blocks than for the cold DB to point to blocks that // do not exist. + self.store.blobs_db.do_atomically(blob_batch)?; self.store.hot_db.do_atomically(hot_batch)?; self.store.cold_db.do_atomically(cold_batch)?; diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 40fd5e63d1..522009b1b2 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -18,6 +18,7 @@ pub mod canonical_head; pub mod capella_readiness; pub mod chain_config; pub mod data_availability_checker; +pub mod deneb_readiness; mod early_attester_cache; mod errors; pub mod eth1_chain; @@ -31,6 +32,7 @@ pub mod historical_blocks; pub mod kzg_utils; pub mod light_client_finality_update_verification; pub mod light_client_optimistic_update_verification; +mod light_client_server_cache; pub mod merge_readiness; pub mod metrics; pub mod migrate; @@ -60,8 +62,8 @@ pub mod validator_pubkey_cache; pub use self::beacon_chain::{ AttestationProcessingOutcome, AvailabilityProcessingStatus, BeaconBlockResponse, BeaconBlockResponseWrapper, BeaconChain, BeaconChainTypes, BeaconStore, ChainSegmentResult, - ForkChoiceError, OverrideForkchoiceUpdate, ProduceBlockVerification, StateSkipConfig, - WhenSlotSkipped, INVALID_FINALIZED_MERGE_TRANSITION_BLOCK_SHUTDOWN_REASON, + ForkChoiceError, LightClientProducerEvent, OverrideForkchoiceUpdate, ProduceBlockVerification, + StateSkipConfig, WhenSlotSkipped, INVALID_FINALIZED_MERGE_TRANSITION_BLOCK_SHUTDOWN_REASON, INVALID_JUSTIFIED_PAYLOAD_SHUTDOWN_REASON, }; pub use self::beacon_snapshot::BeaconSnapshot; diff --git a/beacon_node/beacon_chain/src/light_client_finality_update_verification.rs b/beacon_node/beacon_chain/src/light_client_finality_update_verification.rs index 791d63ccfe..35863aa05f 100644 --- a/beacon_node/beacon_chain/src/light_client_finality_update_verification.rs +++ b/beacon_node/beacon_chain/src/light_client_finality_update_verification.rs @@ -1,11 +1,9 @@ -use crate::{BeaconChain, BeaconChainError, BeaconChainTypes}; +use crate::{BeaconChain, BeaconChainTypes}; use derivative::Derivative; use slot_clock::SlotClock; use std::time::Duration; use strum::AsRefStr; -use types::{ - light_client_update::Error as LightClientUpdateError, LightClientFinalityUpdate, Slot, -}; +use types::LightClientFinalityUpdate; /// Returned when a light client finality update was not successfully verified. It might not have been verified for /// two reasons: @@ -16,8 +14,6 @@ use types::{ /// (the `BeaconChainError` variant) #[derive(Debug, AsRefStr)] pub enum Error { - /// Light client finality update message with a lower or equal finalized_header slot already forwarded. - FinalityUpdateAlreadySeen, /// The light client finality message was received is prior to one-third of slot duration passage. (with /// respect to the gossip clock disparity and slot clock duration). /// @@ -26,29 +22,11 @@ pub enum Error { /// Assuming the local clock is correct, the peer has sent an invalid message. TooEarly, /// Light client finality update message does not match the locally constructed one. - /// - /// ## Peer Scoring - /// InvalidLightClientFinalityUpdate, /// Signature slot start time is none. SigSlotStartIsNone, /// Failed to construct a LightClientFinalityUpdate from state. FailedConstructingUpdate, - /// Beacon chain error occurred. - BeaconChainError(BeaconChainError), - LightClientUpdateError(LightClientUpdateError), -} - -impl From for Error { - fn from(e: BeaconChainError) -> Self { - Error::BeaconChainError(e) - } -} - -impl From for Error { - fn from(e: LightClientUpdateError) -> Self { - Error::LightClientUpdateError(e) - } } /// Wraps a `LightClientFinalityUpdate` that has been verified for propagation on the gossip network. @@ -63,71 +41,34 @@ impl VerifiedLightClientFinalityUpdate { /// Returns `Ok(Self)` if the `light_client_finality_update` is valid to be (re)published on the gossip /// network. pub fn verify( - light_client_finality_update: LightClientFinalityUpdate, + rcv_finality_update: LightClientFinalityUpdate, chain: &BeaconChain, seen_timestamp: Duration, ) -> Result { - let gossiped_finality_slot = light_client_finality_update.finalized_header.beacon.slot; - let one_third_slot_duration = Duration::new(chain.spec.seconds_per_slot / 3, 0); - let signature_slot = light_client_finality_update.signature_slot; - let start_time = chain.slot_clock.start_of(signature_slot); - let mut latest_seen_finality_update = chain.latest_seen_finality_update.lock(); - - let head = chain.canonical_head.cached_head(); - let head_block = &head.snapshot.beacon_block; - let attested_block_root = head_block.message().parent_root(); - let attested_block = chain - .get_blinded_block(&attested_block_root)? - .ok_or(Error::FailedConstructingUpdate)?; - let mut attested_state = chain - .get_state(&attested_block.state_root(), Some(attested_block.slot()))? - .ok_or(Error::FailedConstructingUpdate)?; - - let finalized_block_root = attested_state.finalized_checkpoint().root; - let finalized_block = chain - .get_blinded_block(&finalized_block_root)? - .ok_or(Error::FailedConstructingUpdate)?; - let latest_seen_finality_update_slot = match latest_seen_finality_update.as_ref() { - Some(update) => update.finalized_header.beacon.slot, - None => Slot::new(0), - }; - - // verify that no other finality_update with a lower or equal - // finalized_header.slot was already forwarded on the network - if gossiped_finality_slot <= latest_seen_finality_update_slot { - return Err(Error::FinalityUpdateAlreadySeen); - } - // verify that enough time has passed for the block to have been propagated - match start_time { - Some(time) => { - if seen_timestamp + chain.spec.maximum_gossip_clock_disparity() - < time + one_third_slot_duration - { - return Err(Error::TooEarly); - } - } - None => return Err(Error::SigSlotStartIsNone), + let start_time = chain + .slot_clock + .start_of(rcv_finality_update.signature_slot) + .ok_or(Error::SigSlotStartIsNone)?; + let one_third_slot_duration = Duration::new(chain.spec.seconds_per_slot / 3, 0); + if seen_timestamp + chain.spec.maximum_gossip_clock_disparity() + < start_time + one_third_slot_duration + { + return Err(Error::TooEarly); } - let head_state = &head.snapshot.beacon_state; - let finality_update = LightClientFinalityUpdate::new( - &chain.spec, - head_state, - head_block, - &mut attested_state, - &finalized_block, - )?; + let latest_finality_update = chain + .light_client_server_cache + .get_latest_finality_update() + .ok_or(Error::FailedConstructingUpdate)?; // verify that the gossiped finality update is the same as the locally constructed one. - if finality_update != light_client_finality_update { + if latest_finality_update != rcv_finality_update { return Err(Error::InvalidLightClientFinalityUpdate); } - *latest_seen_finality_update = Some(light_client_finality_update.clone()); - Ok(Self { - light_client_finality_update, + light_client_finality_update: rcv_finality_update, seen_timestamp, }) } diff --git a/beacon_node/beacon_chain/src/light_client_optimistic_update_verification.rs b/beacon_node/beacon_chain/src/light_client_optimistic_update_verification.rs index 374cc9a775..813b112db5 100644 --- a/beacon_node/beacon_chain/src/light_client_optimistic_update_verification.rs +++ b/beacon_node/beacon_chain/src/light_client_optimistic_update_verification.rs @@ -1,12 +1,10 @@ -use crate::{BeaconChain, BeaconChainError, BeaconChainTypes}; +use crate::{BeaconChain, BeaconChainTypes}; use derivative::Derivative; use eth2::types::Hash256; use slot_clock::SlotClock; use std::time::Duration; use strum::AsRefStr; -use types::{ - light_client_update::Error as LightClientUpdateError, LightClientOptimisticUpdate, Slot, -}; +use types::LightClientOptimisticUpdate; /// Returned when a light client optimistic update was not successfully verified. It might not have been verified for /// two reasons: @@ -17,8 +15,6 @@ use types::{ /// (the `BeaconChainError` variant) #[derive(Debug, AsRefStr)] pub enum Error { - /// Light client optimistic update message with a lower or equal optimistic_header slot already forwarded. - OptimisticUpdateAlreadySeen, /// The light client optimistic message was received is prior to one-third of slot duration passage. (with /// respect to the gossip clock disparity and slot clock duration). /// @@ -27,9 +23,6 @@ pub enum Error { /// Assuming the local clock is correct, the peer has sent an invalid message. TooEarly, /// Light client optimistic update message does not match the locally constructed one. - /// - /// ## Peer Scoring - /// InvalidLightClientOptimisticUpdate, /// Signature slot start time is none. SigSlotStartIsNone, @@ -37,21 +30,6 @@ pub enum Error { FailedConstructingUpdate, /// Unknown block with parent root. UnknownBlockParentRoot(Hash256), - /// Beacon chain error occurred. - BeaconChainError(BeaconChainError), - LightClientUpdateError(LightClientUpdateError), -} - -impl From for Error { - fn from(e: BeaconChainError) -> Self { - Error::BeaconChainError(e) - } -} - -impl From for Error { - fn from(e: LightClientUpdateError) -> Self { - Error::LightClientUpdateError(e) - } } /// Wraps a `LightClientOptimisticUpdate` that has been verified for propagation on the gossip network. @@ -67,52 +45,27 @@ impl VerifiedLightClientOptimisticUpdate { /// Returns `Ok(Self)` if the `light_client_optimistic_update` is valid to be (re)published on the gossip /// network. pub fn verify( - light_client_optimistic_update: LightClientOptimisticUpdate, + rcv_optimistic_update: LightClientOptimisticUpdate, chain: &BeaconChain, seen_timestamp: Duration, ) -> Result { - let gossiped_optimistic_slot = light_client_optimistic_update.attested_header.beacon.slot; + // verify that enough time has passed for the block to have been propagated + let start_time = chain + .slot_clock + .start_of(rcv_optimistic_update.signature_slot) + .ok_or(Error::SigSlotStartIsNone)?; let one_third_slot_duration = Duration::new(chain.spec.seconds_per_slot / 3, 0); - let signature_slot = light_client_optimistic_update.signature_slot; - let start_time = chain.slot_clock.start_of(signature_slot); - let mut latest_seen_optimistic_update = chain.latest_seen_optimistic_update.lock(); + if seen_timestamp + chain.spec.maximum_gossip_clock_disparity() + < start_time + one_third_slot_duration + { + return Err(Error::TooEarly); + } let head = chain.canonical_head.cached_head(); let head_block = &head.snapshot.beacon_block; - let attested_block_root = head_block.message().parent_root(); - let attested_block = chain - .get_blinded_block(&attested_block_root)? - .ok_or(Error::FailedConstructingUpdate)?; - - let attested_state = chain - .get_state(&attested_block.state_root(), Some(attested_block.slot()))? - .ok_or(Error::FailedConstructingUpdate)?; - let latest_seen_optimistic_update_slot = match latest_seen_optimistic_update.as_ref() { - Some(update) => update.attested_header.beacon.slot, - None => Slot::new(0), - }; - - // verify that no other optimistic_update with a lower or equal - // optimistic_header.slot was already forwarded on the network - if gossiped_optimistic_slot <= latest_seen_optimistic_update_slot { - return Err(Error::OptimisticUpdateAlreadySeen); - } - - // verify that enough time has passed for the block to have been propagated - match start_time { - Some(time) => { - if seen_timestamp + chain.spec.maximum_gossip_clock_disparity() - < time + one_third_slot_duration - { - return Err(Error::TooEarly); - } - } - None => return Err(Error::SigSlotStartIsNone), - } - // check if we can process the optimistic update immediately // otherwise queue - let canonical_root = light_client_optimistic_update + let canonical_root = rcv_optimistic_update .attested_header .beacon .canonical_root(); @@ -121,19 +74,20 @@ impl VerifiedLightClientOptimisticUpdate { return Err(Error::UnknownBlockParentRoot(canonical_root)); } - let optimistic_update = - LightClientOptimisticUpdate::new(&chain.spec, head_block, &attested_state)?; + let latest_optimistic_update = chain + .light_client_server_cache + .get_latest_optimistic_update() + .ok_or(Error::FailedConstructingUpdate)?; // verify that the gossiped optimistic update is the same as the locally constructed one. - if optimistic_update != light_client_optimistic_update { + if latest_optimistic_update != rcv_optimistic_update { return Err(Error::InvalidLightClientOptimisticUpdate); } - *latest_seen_optimistic_update = Some(light_client_optimistic_update.clone()); - + let parent_root = rcv_optimistic_update.attested_header.beacon.parent_root; Ok(Self { - light_client_optimistic_update, - parent_root: canonical_root, + light_client_optimistic_update: rcv_optimistic_update, + parent_root, seen_timestamp, }) } diff --git a/beacon_node/beacon_chain/src/light_client_server_cache.rs b/beacon_node/beacon_chain/src/light_client_server_cache.rs new file mode 100644 index 0000000000..1397e3fc9d --- /dev/null +++ b/beacon_node/beacon_chain/src/light_client_server_cache.rs @@ -0,0 +1,256 @@ +use crate::errors::BeaconChainError; +use crate::{metrics, BeaconChainTypes, BeaconStore}; +use parking_lot::{Mutex, RwLock}; +use slog::{debug, Logger}; +use ssz_types::FixedVector; +use std::num::NonZeroUsize; +use types::light_client_update::{FinalizedRootProofLen, FINALIZED_ROOT_INDEX}; +use types::non_zero_usize::new_non_zero_usize; +use types::{ + BeaconBlockRef, BeaconState, ChainSpec, EthSpec, ForkName, Hash256, LightClientFinalityUpdate, + LightClientHeader, LightClientOptimisticUpdate, Slot, SyncAggregate, +}; + +/// A prev block cache miss requires to re-generate the state of the post-parent block. Items in the +/// prev block cache are very small 32 * (6 + 1) = 224 bytes. 32 is an arbitrary number that +/// represents unlikely re-orgs, while keeping the cache very small. +const PREV_BLOCK_CACHE_SIZE: NonZeroUsize = new_non_zero_usize(32); + +/// This cache computes light client messages ahead of time, required to satisfy p2p and API +/// requests. These messages include proofs on historical states, so on-demand computation is +/// expensive. +/// +pub struct LightClientServerCache { + /// Tracks a single global latest finality update out of all imported blocks. + /// + /// TODO: Active discussion with @etan-status if this cache should be fork aware to return + /// latest canonical (update with highest signature slot, where its attested header is part of + /// the head chain) instead of global latest (update with highest signature slot, out of all + /// branches). + latest_finality_update: RwLock>>, + /// Tracks a single global latest optimistic update out of all imported blocks. + latest_optimistic_update: RwLock>>, + /// Caches state proofs by block root + prev_block_cache: Mutex>, +} + +impl LightClientServerCache { + pub fn new() -> Self { + Self { + latest_finality_update: None.into(), + latest_optimistic_update: None.into(), + prev_block_cache: lru::LruCache::new(PREV_BLOCK_CACHE_SIZE).into(), + } + } + + /// Compute and cache state proofs for latter production of light-client messages. Does not + /// trigger block replay. + pub fn cache_state_data( + &self, + spec: &ChainSpec, + block: BeaconBlockRef, + block_root: Hash256, + block_post_state: &mut BeaconState, + ) -> Result<(), BeaconChainError> { + let _timer = metrics::start_timer(&metrics::LIGHT_CLIENT_SERVER_CACHE_STATE_DATA_TIMES); + + // Only post-altair + if spec.fork_name_at_slot::(block.slot()) == ForkName::Base { + return Ok(()); + } + + // Persist in memory cache for a descendent block + + let cached_data = LightClientCachedData::from_state(block_post_state)?; + self.prev_block_cache.lock().put(block_root, cached_data); + + Ok(()) + } + + /// Given a block with a SyncAggregte computes better or more recent light client updates. The + /// results are cached either on disk or memory to be served via p2p and rest API + pub fn recompute_and_cache_updates( + &self, + log: &Logger, + store: BeaconStore, + block_parent_root: &Hash256, + block_slot: Slot, + sync_aggregate: &SyncAggregate, + ) -> Result<(), BeaconChainError> { + let _timer = + metrics::start_timer(&metrics::LIGHT_CLIENT_SERVER_CACHE_RECOMPUTE_UPDATES_TIMES); + + let signature_slot = block_slot; + let attested_block_root = block_parent_root; + + let attested_block = store.get_blinded_block(attested_block_root)?.ok_or( + BeaconChainError::DBInconsistent(format!( + "Block not available {:?}", + attested_block_root + )), + )?; + + let cached_parts = self.get_or_compute_prev_block_cache( + store.clone(), + attested_block_root, + &attested_block.state_root(), + attested_block.slot(), + )?; + + let attested_slot = attested_block.slot(); + + // Spec: Full nodes SHOULD provide the LightClientOptimisticUpdate with the highest + // attested_header.beacon.slot (if multiple, highest signature_slot) as selected by fork choice + let is_latest_optimistic = match &self.latest_optimistic_update.read().clone() { + Some(latest_optimistic_update) => { + is_latest_optimistic_update(latest_optimistic_update, attested_slot, signature_slot) + } + None => true, + }; + if is_latest_optimistic { + // can create an optimistic update, that is more recent + *self.latest_optimistic_update.write() = Some(LightClientOptimisticUpdate { + attested_header: block_to_light_client_header(attested_block.message()), + sync_aggregate: sync_aggregate.clone(), + signature_slot, + }); + }; + + // Spec: Full nodes SHOULD provide the LightClientFinalityUpdate with the highest + // attested_header.beacon.slot (if multiple, highest signature_slot) as selected by fork choice + let is_latest_finality = match &self.latest_finality_update.read().clone() { + Some(latest_finality_update) => { + is_latest_finality_update(latest_finality_update, attested_slot, signature_slot) + } + None => true, + }; + if is_latest_finality & !cached_parts.finalized_block_root.is_zero() { + // Immediately after checkpoint sync the finalized block may not be available yet. + if let Some(finalized_block) = + store.get_blinded_block(&cached_parts.finalized_block_root)? + { + *self.latest_finality_update.write() = Some(LightClientFinalityUpdate { + // TODO: may want to cache this result from latest_optimistic_update if producing a + // light_client header becomes expensive + attested_header: block_to_light_client_header(attested_block.message()), + finalized_header: block_to_light_client_header(finalized_block.message()), + finality_branch: cached_parts.finality_branch.clone(), + sync_aggregate: sync_aggregate.clone(), + signature_slot, + }); + } else { + debug!( + log, + "Finalized block not available in store for light_client server"; + "finalized_block_root" => format!("{}", cached_parts.finalized_block_root), + ); + } + } + + Ok(()) + } + + /// Retrieves prev block cached data from cache. If not present re-computes by retrieving the + /// parent state, and inserts an entry to the cache. + /// + /// In separate function since FnOnce of get_or_insert can not be fallible. + fn get_or_compute_prev_block_cache( + &self, + store: BeaconStore, + block_root: &Hash256, + block_state_root: &Hash256, + block_slot: Slot, + ) -> Result { + // Attempt to get the value from the cache first. + if let Some(cached_parts) = self.prev_block_cache.lock().get(block_root) { + return Ok(cached_parts.clone()); + } + metrics::inc_counter(&metrics::LIGHT_CLIENT_SERVER_CACHE_PREV_BLOCK_CACHE_MISS); + + // Compute the value, handling potential errors. + let mut state = store + .get_state(block_state_root, Some(block_slot))? + .ok_or_else(|| { + BeaconChainError::DBInconsistent(format!("Missing state {:?}", block_state_root)) + })?; + let new_value = LightClientCachedData::from_state(&mut state)?; + + // Insert value and return owned + self.prev_block_cache + .lock() + .put(*block_root, new_value.clone()); + Ok(new_value) + } + + pub fn get_latest_finality_update(&self) -> Option> { + self.latest_finality_update.read().clone() + } + + pub fn get_latest_optimistic_update(&self) -> Option> { + self.latest_optimistic_update.read().clone() + } +} + +impl Default for LightClientServerCache { + fn default() -> Self { + Self::new() + } +} + +type FinalityBranch = FixedVector; + +#[derive(Clone)] +struct LightClientCachedData { + finality_branch: FinalityBranch, + finalized_block_root: Hash256, +} + +impl LightClientCachedData { + fn from_state(state: &mut BeaconState) -> Result { + Ok(Self { + finality_branch: state.compute_merkle_proof(FINALIZED_ROOT_INDEX)?.into(), + finalized_block_root: state.finalized_checkpoint().root, + }) + } +} + +// Implements spec priorization rules: +// > Full nodes SHOULD provide the LightClientFinalityUpdate with the highest attested_header.beacon.slot (if multiple, highest signature_slot) +// +// ref: https://github.com/ethereum/consensus-specs/blob/113c58f9bf9c08867f6f5f633c4d98e0364d612a/specs/altair/light-client/full-node.md#create_light_client_finality_update +fn is_latest_finality_update( + prev: &LightClientFinalityUpdate, + attested_slot: Slot, + signature_slot: Slot, +) -> bool { + if attested_slot > prev.attested_header.beacon.slot { + true + } else { + attested_slot == prev.attested_header.beacon.slot && signature_slot > prev.signature_slot + } +} + +// Implements spec priorization rules: +// > Full nodes SHOULD provide the LightClientOptimisticUpdate with the highest attested_header.beacon.slot (if multiple, highest signature_slot) +// +// ref: https://github.com/ethereum/consensus-specs/blob/113c58f9bf9c08867f6f5f633c4d98e0364d612a/specs/altair/light-client/full-node.md#create_light_client_optimistic_update +fn is_latest_optimistic_update( + prev: &LightClientOptimisticUpdate, + attested_slot: Slot, + signature_slot: Slot, +) -> bool { + if attested_slot > prev.attested_header.beacon.slot { + true + } else { + attested_slot == prev.attested_header.beacon.slot && signature_slot > prev.signature_slot + } +} + +fn block_to_light_client_header( + block: BeaconBlockRef>, +) -> LightClientHeader { + // TODO: make fork aware + LightClientHeader { + beacon: block.block_header(), + } +} diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index ad095b37b5..24c05e01f2 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -1128,6 +1128,22 @@ lazy_static! { // Create a custom bucket list for greater granularity in block delay Ok(vec![0.1, 0.2, 0.3,0.4,0.5,0.75,1.0,1.25,1.5,1.75,2.0,2.5,3.0,3.5,4.0,5.0,6.0,7.0,8.0,9.0,10.0,15.0,20.0]) ); + + /* + * light_client server metrics + */ + pub static ref LIGHT_CLIENT_SERVER_CACHE_STATE_DATA_TIMES: Result = try_create_histogram( + "beacon_light_client_server_cache_state_data_seconds", + "Time taken to produce and cache state data", + ); + pub static ref LIGHT_CLIENT_SERVER_CACHE_RECOMPUTE_UPDATES_TIMES: Result = try_create_histogram( + "beacon_light_client_server_cache_recompute_updates_seconds", + "Time taken to recompute and cache updates", + ); + pub static ref LIGHT_CLIENT_SERVER_CACHE_PREV_BLOCK_CACHE_MISS: Result = try_create_int_counter( + "beacon_light_client_server_cache_prev_block_cache_miss", + "Count of prev block cache misses", + ); } /// Scrape the `beacon_chain` for metrics that are not constantly updated (e.g., the present slot, diff --git a/beacon_node/beacon_chain/src/observed_aggregates.rs b/beacon_node/beacon_chain/src/observed_aggregates.rs index 18a761e29e..aa513da547 100644 --- a/beacon_node/beacon_chain/src/observed_aggregates.rs +++ b/beacon_node/beacon_chain/src/observed_aggregates.rs @@ -43,7 +43,7 @@ impl Consts for Attestation { /// We need to keep attestations for each slot of the current epoch. fn max_slot_capacity() -> usize { - T::slots_per_epoch() as usize + 2 * T::slots_per_epoch() as usize } /// As a DoS protection measure, the maximum number of distinct `Attestations` or diff --git a/beacon_node/beacon_chain/src/observed_attesters.rs b/beacon_node/beacon_chain/src/observed_attesters.rs index 605a134321..a1c6adc3e0 100644 --- a/beacon_node/beacon_chain/src/observed_attesters.rs +++ b/beacon_node/beacon_chain/src/observed_attesters.rs @@ -24,18 +24,16 @@ use types::{Epoch, EthSpec, Hash256, Slot, Unsigned}; /// The maximum capacity of the `AutoPruningEpochContainer`. /// -/// Fits the next, current and previous epochs. We require the next epoch due to the -/// `MAXIMUM_GOSSIP_CLOCK_DISPARITY`. We require the previous epoch since the specification -/// declares: +/// If the current epoch is N, this fits epoch N + 1, N, N - 1, and N - 2. We require the next epoch due +/// to the `MAXIMUM_GOSSIP_CLOCK_DISPARITY`. We require the N - 2 epoch since the specification declares: /// /// ```ignore -/// aggregate.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE -/// >= current_slot >= aggregate.data.slot +/// the epoch of `aggregate.data.slot` is either the current or previous epoch /// ``` /// -/// This means that during the current epoch we will always accept an attestation -/// from at least one slot in the previous epoch. -pub const MAX_CACHED_EPOCHS: u64 = 3; +/// This means that during the current epoch we will always accept an attestation from +/// at least one slot in the epoch prior to the previous epoch. +pub const MAX_CACHED_EPOCHS: u64 = 4; pub type ObservedAttesters = AutoPruningEpochContainer; pub type ObservedSyncContributors = diff --git a/beacon_node/beacon_chain/src/schema_change.rs b/beacon_node/beacon_chain/src/schema_change.rs index e42ee20c48..63eb72c43a 100644 --- a/beacon_node/beacon_chain/src/schema_change.rs +++ b/beacon_node/beacon_chain/src/schema_change.rs @@ -1,6 +1,7 @@ //! Utilities for managing database schema changes. mod migration_schema_v17; mod migration_schema_v18; +mod migration_schema_v19; use crate::beacon_chain::BeaconChainTypes; use crate::types::ChainSpec; @@ -69,6 +70,14 @@ pub fn migrate_schema( let ops = migration_schema_v18::downgrade_from_v18::(db.clone(), log)?; db.store_schema_version_atomically(to, ops) } + (SchemaVersion(18), SchemaVersion(19)) => { + let ops = migration_schema_v19::upgrade_to_v19::(db.clone(), log)?; + db.store_schema_version_atomically(to, ops) + } + (SchemaVersion(19), SchemaVersion(18)) => { + let ops = migration_schema_v19::downgrade_from_v19::(db.clone(), log)?; + db.store_schema_version_atomically(to, ops) + } // Anything else is an error. (_, _) => Err(HotColdDBError::UnsupportedSchemaVersion { target_version: to, diff --git a/beacon_node/beacon_chain/src/schema_change/migration_schema_v19.rs b/beacon_node/beacon_chain/src/schema_change/migration_schema_v19.rs new file mode 100644 index 0000000000..578e9bad31 --- /dev/null +++ b/beacon_node/beacon_chain/src/schema_change/migration_schema_v19.rs @@ -0,0 +1,65 @@ +use crate::beacon_chain::BeaconChainTypes; +use slog::{debug, info, Logger}; +use std::sync::Arc; +use store::{get_key_for_col, DBColumn, Error, HotColdDB, KeyValueStore, KeyValueStoreOp}; + +pub fn upgrade_to_v19( + db: Arc>, + log: Logger, +) -> Result, Error> { + let mut hot_delete_ops = vec![]; + let mut blob_keys = vec![]; + let column = DBColumn::BeaconBlob; + + debug!(log, "Migrating from v18 to v19"); + // Iterate through the blobs on disk. + for res in db.hot_db.iter_column_keys::>(column) { + let key = res?; + let key_col = get_key_for_col(column.as_str(), &key); + hot_delete_ops.push(KeyValueStoreOp::DeleteKey(key_col)); + blob_keys.push(key); + } + + let num_blobs = blob_keys.len(); + debug!(log, "Collected {} blob lists to migrate", num_blobs); + + let batch_size = 500; + let mut batch = Vec::with_capacity(batch_size); + + for key in blob_keys { + let next_blob = db.hot_db.get_bytes(column.as_str(), &key)?; + if let Some(next_blob) = next_blob { + let key_col = get_key_for_col(column.as_str(), &key); + batch.push(KeyValueStoreOp::PutKeyValue(key_col, next_blob)); + + if batch.len() >= batch_size { + db.blobs_db.do_atomically(batch.clone())?; + batch.clear(); + } + } + } + + // Process the remaining batch if it's not empty + if !batch.is_empty() { + db.blobs_db.do_atomically(batch)?; + } + + debug!(log, "Wrote {} blobs to the blobs db", num_blobs); + + // Delete all the blobs + info!(log, "Upgrading to v19 schema"); + Ok(hot_delete_ops) +} + +pub fn downgrade_from_v19( + _db: Arc>, + log: Logger, +) -> Result, Error> { + // No-op + info!( + log, + "Downgrading to v18 schema"; + ); + + Ok(vec![]) +} diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 7a66a6bdb4..6b85c8e493 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -822,7 +822,7 @@ where slot: Slot, ) -> (SignedBlindedBeaconBlock, BeaconState) { let (unblinded, new_state) = self.make_block(state, slot).await; - (unblinded.0.into(), new_state) + ((*unblinded.0).clone().into(), new_state) } /// Returns a newly created block, signed by the proposer for the given slot. @@ -866,14 +866,14 @@ where panic!("Should always be a full payload response"); }; - let signed_block = block_response.block.sign( + let signed_block = Arc::new(block_response.block.sign( &self.validator_keypairs[proposer_index].sk, &block_response.state.fork(), block_response.state.genesis_validators_root(), &self.spec, - ); + )); - let block_contents: SignedBlockContentsTuple = match &signed_block { + let block_contents: SignedBlockContentsTuple = match *signed_block { SignedBeaconBlock::Base(_) | SignedBeaconBlock::Altair(_) | SignedBeaconBlock::Merge(_) @@ -928,14 +928,14 @@ where panic!("Should always be a full payload response"); }; - let signed_block = block_response.block.sign( + let signed_block = Arc::new(block_response.block.sign( &self.validator_keypairs[proposer_index].sk, &block_response.state.fork(), block_response.state.genesis_validators_root(), &self.spec, - ); + )); - let block_contents: SignedBlockContentsTuple = match &signed_block { + let block_contents: SignedBlockContentsTuple = match *signed_block { SignedBeaconBlock::Base(_) | SignedBeaconBlock::Altair(_) | SignedBeaconBlock::Merge(_) @@ -1742,7 +1742,7 @@ where let ((block, blobs), state) = self.make_block_return_pre_state(state, slot).await; - let (mut block, _) = block.deconstruct(); + let (mut block, _) = (*block).clone().deconstruct(); block_modifier(&mut block); @@ -1754,7 +1754,7 @@ where state.genesis_validators_root(), &self.spec, ); - ((signed_block, blobs), state) + ((Arc::new(signed_block), blobs), state) } pub async fn make_blob_with_modifier( @@ -1768,7 +1768,7 @@ where let ((block, mut blobs), state) = self.make_block_return_pre_state(state, slot).await; - let (block, _) = block.deconstruct(); + let (block, _) = (*block).clone().deconstruct(); blob_modifier(&mut blobs.as_mut().unwrap().1); @@ -1780,7 +1780,7 @@ where state.genesis_validators_root(), &self.spec, ); - ((signed_block, blobs), state) + ((Arc::new(signed_block), blobs), state) } pub fn make_deposits<'a>( @@ -1873,7 +1873,7 @@ where .chain .process_block( block_root, - RpcBlock::new(Some(block_root), Arc::new(block), sidecars).unwrap(), + RpcBlock::new(Some(block_root), block, sidecars).unwrap(), NotifyExecutionLayer::Yes, || Ok(()), ) @@ -1899,7 +1899,7 @@ where .chain .process_block( block_root, - RpcBlock::new(Some(block_root), Arc::new(block), sidecars).unwrap(), + RpcBlock::new(Some(block_root), block, sidecars).unwrap(), NotifyExecutionLayer::Yes, || Ok(()), ) diff --git a/beacon_node/beacon_chain/tests/block_verification.rs b/beacon_node/beacon_chain/tests/block_verification.rs index 4344013b3c..541e97436d 100644 --- a/beacon_node/beacon_chain/tests/block_verification.rs +++ b/beacon_node/beacon_chain/tests/block_verification.rs @@ -1142,11 +1142,7 @@ async fn verify_block_for_gossip_slashing_detection() { let ((block1, blobs1), _) = harness.make_block(state.clone(), Slot::new(1)).await; let ((block2, _blobs2), _) = harness.make_block(state, Slot::new(1)).await; - let verified_block = harness - .chain - .verify_block_for_gossip(Arc::new(block1)) - .await - .unwrap(); + let verified_block = harness.chain.verify_block_for_gossip(block1).await.unwrap(); if let Some((kzg_proofs, blobs)) = blobs1 { let sidecars = @@ -1174,12 +1170,7 @@ async fn verify_block_for_gossip_slashing_detection() { ) .await .unwrap(); - unwrap_err( - harness - .chain - .verify_block_for_gossip(Arc::new(block2)) - .await, - ); + unwrap_err(harness.chain.verify_block_for_gossip(block2).await); // Slasher should have been handed the two conflicting blocks and crafted a slashing. slasher.process_queued(Epoch::new(0)).unwrap(); @@ -1198,11 +1189,7 @@ async fn verify_block_for_gossip_doppelganger_detection() { let state = harness.get_current_state(); let ((block, _), _) = harness.make_block(state.clone(), Slot::new(1)).await; - let verified_block = harness - .chain - .verify_block_for_gossip(Arc::new(block)) - .await - .unwrap(); + let verified_block = harness.chain.verify_block_for_gossip(block).await.unwrap(); let attestations = verified_block.block.message().body().attestations().clone(); harness .chain @@ -1564,7 +1551,6 @@ async fn import_duplicate_block_unrealized_justification() { let slot = harness.get_current_slot(); let (block_contents, _) = harness.make_block(state.clone(), slot).await; let (block, _) = block_contents; - let block = Arc::new(block); let block_root = block.canonical_root(); // Create two verified variants of the block, representing the same block being processed in diff --git a/beacon_node/beacon_chain/tests/payload_invalidation.rs b/beacon_node/beacon_chain/tests/payload_invalidation.rs index 5076a6c1a9..a0b7fbd365 100644 --- a/beacon_node/beacon_chain/tests/payload_invalidation.rs +++ b/beacon_node/beacon_chain/tests/payload_invalidation.rs @@ -319,7 +319,7 @@ impl InvalidPayloadRig { .get_full_block(&block_root) .unwrap() .unwrap(), - block, + *block, "block from db must match block imported" ); } @@ -700,7 +700,7 @@ async fn invalidates_all_descendants() { .chain .process_block( fork_block.canonical_root(), - Arc::new(fork_block), + fork_block, NotifyExecutionLayer::Yes, || Ok(()), ) @@ -800,7 +800,7 @@ async fn switches_heads() { .chain .process_block( fork_block.canonical_root(), - Arc::new(fork_block), + fork_block, NotifyExecutionLayer::Yes, || Ok(()), ) @@ -1044,8 +1044,7 @@ async fn invalid_parent() { // Produce another block atop the parent, but don't import yet. let slot = parent_block.slot() + 1; rig.harness.set_current_slot(slot); - let (block_tuple, state) = rig.harness.make_block(parent_state, slot).await; - let block = Arc::new(block_tuple.0); + let ((block, _), state) = rig.harness.make_block(parent_state, slot).await; let block_root = block.canonical_root(); assert_eq!(block.parent_root(), parent_root); @@ -1865,7 +1864,7 @@ impl InvalidHeadSetup { .state_at_slot(slot - 1, StateSkipConfig::WithStateRoots) .unwrap(); let (fork_block_tuple, _) = rig.harness.make_block(parent_state, slot).await; - opt_fork_block = Some(Arc::new(fork_block_tuple.0)); + opt_fork_block = Some(fork_block_tuple.0); } else { // Skipped slot. }; diff --git a/beacon_node/beacon_chain/tests/store_tests.rs b/beacon_node/beacon_chain/tests/store_tests.rs index 28770b93e2..ffd1b843a1 100644 --- a/beacon_node/beacon_chain/tests/store_tests.rs +++ b/beacon_node/beacon_chain/tests/store_tests.rs @@ -3,6 +3,7 @@ use beacon_chain::attestation_verification::Error as AttnError; use beacon_chain::block_verification_types::RpcBlock; use beacon_chain::builder::BeaconChainBuilder; +use beacon_chain::data_availability_checker::AvailableBlock; use beacon_chain::schema_change::migrate_schema; use beacon_chain::test_utils::{ mock_execution_layer_from_parts, test_spec, AttestationStrategy, BeaconChainHarness, @@ -2268,17 +2269,17 @@ async fn garbage_collect_temp_states_from_failed_block() { let block_slot = Slot::new(2 * slots_per_epoch); let ((signed_block, _), state) = harness.make_block(genesis_state, block_slot).await; - let (mut block, _) = signed_block.deconstruct(); + let (mut block, _) = (*signed_block).clone().deconstruct(); // Mutate the block to make it invalid, and re-sign it. *block.state_root_mut() = Hash256::repeat_byte(0xff); let proposer_index = block.proposer_index() as usize; - let block = block.sign( + let block = Arc::new(block.sign( &harness.validator_keypairs[proposer_index].sk, &state.fork(), state.genesis_validators_root(), &harness.spec, - ); + )); // The block should be rejected, but should store a bunch of temporary states. harness.set_current_slot(block_slot); @@ -2547,6 +2548,25 @@ async fn weak_subjectivity_sync_test(slots: Vec, checkpoint_slot: Slot) { } } + // Corrupt the signature on the 1st block to ensure that the backfill processor is checking + // signatures correctly. Regression test for https://github.com/sigp/lighthouse/pull/5120. + let mut batch_with_invalid_first_block = available_blocks.clone(); + batch_with_invalid_first_block[0] = { + let (block_root, block, blobs) = available_blocks[0].clone().deconstruct(); + let mut corrupt_block = (*block).clone(); + *corrupt_block.signature_mut() = Signature::empty(); + AvailableBlock::__new_for_testing(block_root, Arc::new(corrupt_block), blobs) + }; + + // Importing the invalid batch should error. + assert!(matches!( + beacon_chain + .import_historical_block_batch(batch_with_invalid_first_block) + .unwrap_err(), + BeaconChainError::HistoricalBlockError(HistoricalBlockError::InvalidSignature) + )); + + // Importing the batch with valid signatures should succeed. beacon_chain .import_historical_block_batch(available_blocks.clone()) .unwrap(); @@ -2677,7 +2697,7 @@ async fn process_blocks_and_attestations_for_unaligned_checkpoint() { .chain .process_block( invalid_fork_block.canonical_root(), - Arc::new(invalid_fork_block.clone()), + invalid_fork_block.clone(), NotifyExecutionLayer::Yes, || Ok(()), ) @@ -2690,7 +2710,7 @@ async fn process_blocks_and_attestations_for_unaligned_checkpoint() { .chain .process_block( valid_fork_block.canonical_root(), - Arc::new(valid_fork_block.clone()), + valid_fork_block.clone(), NotifyExecutionLayer::Yes, || Ok(()), ) diff --git a/beacon_node/beacon_processor/src/work_reprocessing_queue.rs b/beacon_node/beacon_processor/src/work_reprocessing_queue.rs index 9191509d39..20f3e21d08 100644 --- a/beacon_node/beacon_processor/src/work_reprocessing_queue.rs +++ b/beacon_node/beacon_processor/src/work_reprocessing_queue.rs @@ -82,12 +82,15 @@ pub enum ReprocessQueueMessage { /// A gossip block for hash `X` is being imported, we should queue the rpc block for the same /// hash until the gossip block is imported. RpcBlock(QueuedRpcBlock), - /// A block that was successfully processed. We use this to handle attestations and light client updates + /// A block that was successfully processed. We use this to handle attestations updates /// for unknown blocks. BlockImported { block_root: Hash256, parent_root: Hash256, }, + /// A new `LightClientOptimisticUpdate` has been produced. We use this to handle light client + /// updates for unknown parent blocks. + NewLightClientOptimisticUpdate { parent_root: Hash256 }, /// An unaggregated attestation that references an unknown block. UnknownBlockUnaggregate(QueuedUnaggregate), /// An aggregated attestation that references an unknown block. @@ -688,6 +691,8 @@ impl ReprocessQueue { ); } } + } + InboundEvent::Msg(NewLightClientOptimisticUpdate { parent_root }) => { // Unqueue the light client optimistic updates we have for this root, if any. if let Some(queued_lc_id) = self .awaiting_lc_updates_per_parent_root diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index 26c53154e3..0160cad4b2 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -25,6 +25,7 @@ serde = { workspace = true } error-chain = { workspace = true } slog = { workspace = true } tokio = { workspace = true } +futures = { workspace = true } dirs = { workspace = true } eth1 = { workspace = true } eth2 = { workspace = true } diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index 9c88eccc70..444a277509 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -1,4 +1,7 @@ use crate::address_change_broadcast::broadcast_address_changes_at_capella; +use crate::compute_light_client_updates::{ + compute_light_client_updates, LIGHT_CLIENT_SERVER_CHANNEL_CAPACITY, +}; use crate::config::{ClientGenesis, Config as ClientConfig}; use crate::notifier::spawn_notifier; use crate::Client; @@ -7,6 +10,7 @@ use beacon_chain::data_availability_checker::start_availability_cache_maintenanc use beacon_chain::otb_verification_service::start_otb_verification_service; use beacon_chain::proposer_prep_service::start_proposer_prep_service; use beacon_chain::schema_change::migrate_schema; +use beacon_chain::LightClientProducerEvent; use beacon_chain::{ builder::{BeaconChainBuilder, Witness}, eth1_chain::{CachingEth1Backend, Eth1Chain}, @@ -24,6 +28,7 @@ use eth2::{ BeaconNodeHttpClient, Error as ApiError, Timeouts, }; use execution_layer::ExecutionLayer; +use futures::channel::mpsc::Receiver; use genesis::{interop_genesis_state, Eth1GenesisService, DEFAULT_ETH1_BLOCK_HASH}; use lighthouse_network::{prometheus_client::registry::Registry, NetworkGlobals}; use monitoring_api::{MonitoringHttpClient, ProcessType}; @@ -83,6 +88,7 @@ pub struct ClientBuilder { slasher: Option>>, beacon_processor_config: Option, beacon_processor_channels: Option>, + light_client_server_rv: Option>>, eth_spec_instance: T::EthSpec, } @@ -118,6 +124,7 @@ where eth_spec_instance, beacon_processor_config: None, beacon_processor_channels: None, + light_client_server_rv: None, } } @@ -206,6 +213,16 @@ where builder }; + let builder = if config.network.enable_light_client_server { + let (tx, rv) = futures::channel::mpsc::channel::>( + LIGHT_CLIENT_SERVER_CHANNEL_CAPACITY, + ); + self.light_client_server_rv = Some(rv); + builder.light_client_server_tx(tx) + } else { + builder + }; + let chain_exists = builder.store_contains_beacon_chain().unwrap_or(false); // If the client is expect to resume but there's no beacon chain in the database, @@ -797,7 +814,7 @@ where } .spawn_manager( beacon_processor_channels.beacon_processor_rx, - beacon_processor_channels.work_reprocessing_tx, + beacon_processor_channels.work_reprocessing_tx.clone(), beacon_processor_channels.work_reprocessing_rx, None, beacon_chain.slot_clock.clone(), @@ -860,7 +877,7 @@ where } // Spawn a service to publish BLS to execution changes at the Capella fork. - if let Some(network_senders) = self.network_senders { + if let Some(network_senders) = self.network_senders.clone() { let inner_chain = beacon_chain.clone(); let broadcast_context = runtime_context.service_context("addr_bcast".to_string()); @@ -879,6 +896,26 @@ where } } + // Spawn service to publish light_client updates at some interval into the slot. + if let Some(light_client_server_rv) = self.light_client_server_rv { + let inner_chain = beacon_chain.clone(); + let light_client_update_context = + runtime_context.service_context("lc_update".to_string()); + let log = light_client_update_context.log().clone(); + light_client_update_context.executor.spawn( + async move { + compute_light_client_updates( + &inner_chain, + light_client_server_rv, + beacon_processor_channels.work_reprocessing_tx, + &log, + ) + .await + }, + "lc_update", + ); + } + start_proposer_prep_service(runtime_context.executor.clone(), beacon_chain.clone()); start_otb_verification_service(runtime_context.executor.clone(), beacon_chain.clone()); start_availability_cache_maintenance_service( diff --git a/beacon_node/client/src/compute_light_client_updates.rs b/beacon_node/client/src/compute_light_client_updates.rs new file mode 100644 index 0000000000..1eb977d421 --- /dev/null +++ b/beacon_node/client/src/compute_light_client_updates.rs @@ -0,0 +1,39 @@ +use beacon_chain::{BeaconChain, BeaconChainTypes, LightClientProducerEvent}; +use beacon_processor::work_reprocessing_queue::ReprocessQueueMessage; +use futures::channel::mpsc::Receiver; +use futures::StreamExt; +use slog::{error, Logger}; +use tokio::sync::mpsc::Sender; + +// Each `LightClientProducerEvent` is ~200 bytes. With the light_client server producing only recent +// updates it is okay to drop some events in case of overloading. In normal network conditions +// there's one event emitted per block at most every 12 seconds, while consuming the event should +// take a few milliseconds. 32 is a small enough arbitrary number. +pub(crate) const LIGHT_CLIENT_SERVER_CHANNEL_CAPACITY: usize = 32; + +pub async fn compute_light_client_updates( + chain: &BeaconChain, + mut light_client_server_rv: Receiver>, + reprocess_tx: Sender, + log: &Logger, +) { + // Should only receive events for recent blocks, import_block filters by blocks close to clock. + // + // Intents to process SyncAggregates of all recent blocks sequentially, without skipping. + // Uses a bounded receiver, so may drop some SyncAggregates if very overloaded. This is okay + // since only the most recent updates have value. + while let Some(event) = light_client_server_rv.next().await { + let parent_root = event.0; + + chain + .recompute_and_cache_light_client_updates(event) + .unwrap_or_else(|e| { + error!(log, "error computing light_client updates {:?}", e); + }); + + let msg = ReprocessQueueMessage::NewLightClientOptimisticUpdate { parent_root }; + if reprocess_tx.try_send(msg).is_err() { + error!(log, "Failed to inform light client update"; "parent_root" => %parent_root) + }; + } +} diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index 399aa06511..2f14d87efc 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -1,6 +1,7 @@ extern crate slog; mod address_change_broadcast; +mod compute_light_client_updates; pub mod config; mod metrics; mod notifier; diff --git a/beacon_node/client/src/notifier.rs b/beacon_node/client/src/notifier.rs index 2c7738e8fa..8a0e5ce223 100644 --- a/beacon_node/client/src/notifier.rs +++ b/beacon_node/client/src/notifier.rs @@ -1,6 +1,7 @@ use crate::metrics; use beacon_chain::{ capella_readiness::CapellaReadiness, + deneb_readiness::DenebReadiness, merge_readiness::{GenesisExecutionPayloadStatus, MergeConfig, MergeReadiness}, BeaconChain, BeaconChainTypes, ExecutionStatus, }; @@ -319,6 +320,7 @@ pub fn spawn_notifier( eth1_logging(&beacon_chain, &log); merge_readiness_logging(current_slot, &beacon_chain, &log).await; capella_readiness_logging(current_slot, &beacon_chain, &log).await; + deneb_readiness_logging(current_slot, &beacon_chain, &log).await; } }; @@ -356,8 +358,8 @@ async fn merge_readiness_logging( } if merge_completed && !has_execution_layer { + // Logging of the EE being offline is handled in the other readiness logging functions. if !beacon_chain.is_time_to_prepare_for_capella(current_slot) { - // logging of the EE being offline is handled in `capella_readiness_logging()` error!( log, "Execution endpoint required"; @@ -445,12 +447,15 @@ async fn capella_readiness_logging( } if capella_completed && !has_execution_layer { - error!( - log, - "Execution endpoint required"; - "info" => "you need a Capella enabled execution engine to validate blocks, see: \ - https://lighthouse-book.sigmaprime.io/merge-migration.html" - ); + // Logging of the EE being offline is handled in the other readiness logging functions. + if !beacon_chain.is_time_to_prepare_for_deneb(current_slot) { + error!( + log, + "Execution endpoint required"; + "info" => "you need a Capella enabled execution engine to validate blocks, see: \ + https://lighthouse-book.sigmaprime.io/merge-migration.html" + ); + } return; } @@ -479,6 +484,65 @@ async fn capella_readiness_logging( } } +/// Provides some helpful logging to users to indicate if their node is ready for Deneb +async fn deneb_readiness_logging( + current_slot: Slot, + beacon_chain: &BeaconChain, + log: &Logger, +) { + let deneb_completed = beacon_chain + .canonical_head + .cached_head() + .snapshot + .beacon_block + .message() + .body() + .execution_payload() + .map_or(false, |payload| payload.blob_gas_used().is_ok()); + + let has_execution_layer = beacon_chain.execution_layer.is_some(); + + if deneb_completed && has_execution_layer + || !beacon_chain.is_time_to_prepare_for_deneb(current_slot) + { + return; + } + + if deneb_completed && !has_execution_layer { + error!( + log, + "Execution endpoint required"; + "info" => "you need a Deneb enabled execution engine to validate blocks, see: \ + https://lighthouse-book.sigmaprime.io/merge-migration.html" + ); + return; + } + + match beacon_chain.check_deneb_readiness().await { + DenebReadiness::Ready => { + info!( + log, + "Ready for Deneb"; + "info" => "ensure the execution endpoint is updated to the latest Deneb/Cancun release" + ) + } + readiness @ DenebReadiness::ExchangeCapabilitiesFailed { error: _ } => { + error!( + log, + "Not ready for Deneb"; + "hint" => "the execution endpoint may be offline", + "info" => %readiness, + ) + } + readiness => warn!( + log, + "Not ready for Deneb"; + "hint" => "try updating the execution endpoint", + "info" => %readiness, + ), + } +} + async fn genesis_execution_payload_logging( beacon_chain: &BeaconChain, log: &Logger, diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index f1d7769e89..fe01f3c524 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -45,7 +45,7 @@ use eth2::types::{ PublishBlockRequest, ValidatorBalancesRequestBody, ValidatorId, ValidatorStatus, ValidatorsRequestBody, }; -use eth2::{CONTENT_TYPE_HEADER, SSZ_CONTENT_TYPE_HEADER}; +use eth2::{CONSENSUS_VERSION_HEADER, CONTENT_TYPE_HEADER, SSZ_CONTENT_TYPE_HEADER}; use lighthouse_network::{types::SyncState, EnrExt, NetworkGlobals, PeerId, PubsubMessage}; use lighthouse_version::version_with_platform; use logging::SSELoggingComponents; @@ -682,7 +682,7 @@ pub fn serve( .clone() .and(warp::path("validator_balances")) .and(warp::path::end()) - .and(warp::body::json()) + .and(warp_utils::json::json()) .then( |state_id: StateId, task_spawner: TaskSpawner, @@ -726,7 +726,7 @@ pub fn serve( .clone() .and(warp::path("validators")) .and(warp::path::end()) - .and(warp::body::json()) + .and(warp_utils::json::json()) .then( |state_id: StateId, task_spawner: TaskSpawner, @@ -1249,13 +1249,15 @@ pub fn serve( /* * beacon/blocks */ + let consensus_version_header_filter = + warp::header::header::(CONSENSUS_VERSION_HEADER); // POST beacon/blocks let post_beacon_blocks = eth_v1 .and(warp::path("beacon")) .and(warp::path("blocks")) .and(warp::path::end()) - .and(warp::body::json()) + .and(warp_utils::json::json()) .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and(network_tx_filter.clone()) @@ -1286,12 +1288,14 @@ pub fn serve( .and(warp::path("blocks")) .and(warp::path::end()) .and(warp::body::bytes()) + .and(consensus_version_header_filter) .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and(network_tx_filter.clone()) .and(log_filter.clone()) .then( move |block_bytes: Bytes, + consensus_version: ForkName, task_spawner: TaskSpawner, chain: Arc>, network_tx: UnboundedSender>, @@ -1299,7 +1303,7 @@ pub fn serve( task_spawner.spawn_async_with_rejection(Priority::P0, async move { let block_contents = PublishBlockRequest::::from_ssz_bytes( &block_bytes, - &chain.spec, + consensus_version, ) .map_err(|e| { warp_utils::reject::custom_bad_request(format!("invalid SSZ: {e:?}")) @@ -1323,7 +1327,7 @@ pub fn serve( .and(warp::path("blocks")) .and(warp::query::()) .and(warp::path::end()) - .and(warp::body::json()) + .and(warp_utils::json::json()) .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and(network_tx_filter.clone()) @@ -1356,6 +1360,7 @@ pub fn serve( .and(warp::query::()) .and(warp::path::end()) .and(warp::body::bytes()) + .and(consensus_version_header_filter) .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and(network_tx_filter.clone()) @@ -1363,6 +1368,7 @@ pub fn serve( .then( move |validation_level: api_types::BroadcastValidationQuery, block_bytes: Bytes, + consensus_version: ForkName, task_spawner: TaskSpawner, chain: Arc>, network_tx: UnboundedSender>, @@ -1370,7 +1376,7 @@ pub fn serve( task_spawner.spawn_async_with_rejection(Priority::P0, async move { let block_contents = PublishBlockRequest::::from_ssz_bytes( &block_bytes, - &chain.spec, + consensus_version, ) .map_err(|e| { warp_utils::reject::custom_bad_request(format!("invalid SSZ: {e:?}")) @@ -1398,13 +1404,13 @@ pub fn serve( .and(warp::path("beacon")) .and(warp::path("blinded_blocks")) .and(warp::path::end()) - .and(warp::body::json()) + .and(warp_utils::json::json()) .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and(network_tx_filter.clone()) .and(log_filter.clone()) .then( - move |block_contents: SignedBlindedBeaconBlock, + move |block_contents: Arc>, task_spawner: TaskSpawner, chain: Arc>, network_tx: UnboundedSender>, @@ -1444,6 +1450,7 @@ pub fn serve( &block_bytes, &chain.spec, ) + .map(Arc::new) .map_err(|e| { warp_utils::reject::custom_bad_request(format!("invalid SSZ: {e:?}")) })?; @@ -1465,14 +1472,14 @@ pub fn serve( .and(warp::path("blinded_blocks")) .and(warp::query::()) .and(warp::path::end()) - .and(warp::body::json()) + .and(warp_utils::json::json()) .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and(network_tx_filter.clone()) .and(log_filter.clone()) .then( move |validation_level: api_types::BroadcastValidationQuery, - blinded_block: SignedBlindedBeaconBlock, + blinded_block: Arc>, task_spawner: TaskSpawner, chain: Arc>, network_tx: UnboundedSender>, @@ -1513,6 +1520,7 @@ pub fn serve( &block_bytes, &chain.spec, ) + .map(Arc::new) .map_err(|e| { warp_utils::reject::custom_bad_request(format!("invalid SSZ: {e:?}")) })?; @@ -1696,18 +1704,19 @@ pub fn serve( .and(warp::path("beacon")) .and(warp::path("blob_sidecars")) .and(block_id_or_err) - .and(warp::query::()) .and(warp::path::end()) + .and(multi_key_query::()) .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and(warp::header::optional::("accept")) .then( |block_id: BlockId, - indices: api_types::BlobIndicesQuery, + indices_res: Result, task_spawner: TaskSpawner, chain: Arc>, accept_header: Option| { task_spawner.blocking_response_task(Priority::P1, move || { + let indices = indices_res?; let blob_sidecar_list_filtered = block_id.blob_sidecar_list_filtered(indices, &chain)?; match accept_header { @@ -1745,7 +1754,7 @@ pub fn serve( .clone() .and(warp::path("attestations")) .and(warp::path::end()) - .and(warp::body::json()) + .and(warp_utils::json::json()) .and(network_tx_filter.clone()) .and(log_filter.clone()) .then( @@ -1921,7 +1930,7 @@ pub fn serve( .clone() .and(warp::path("attester_slashings")) .and(warp::path::end()) - .and(warp::body::json()) + .and(warp_utils::json::json()) .and(network_tx_filter.clone()) .then( |task_spawner: TaskSpawner, @@ -1979,7 +1988,7 @@ pub fn serve( .clone() .and(warp::path("proposer_slashings")) .and(warp::path::end()) - .and(warp::body::json()) + .and(warp_utils::json::json()) .and(network_tx_filter.clone()) .then( |task_spawner: TaskSpawner, @@ -2037,7 +2046,7 @@ pub fn serve( .clone() .and(warp::path("voluntary_exits")) .and(warp::path::end()) - .and(warp::body::json()) + .and(warp_utils::json::json()) .and(network_tx_filter.clone()) .then( |task_spawner: TaskSpawner, @@ -2093,7 +2102,7 @@ pub fn serve( .clone() .and(warp::path("sync_committees")) .and(warp::path::end()) - .and(warp::body::json()) + .and(warp_utils::json::json()) .and(network_tx_filter.clone()) .and(log_filter.clone()) .then( @@ -2130,7 +2139,7 @@ pub fn serve( .clone() .and(warp::path("bls_to_execution_changes")) .and(warp::path::end()) - .and(warp::body::json()) + .and(warp_utils::json::json()) .and(network_tx_filter.clone()) .and(log_filter.clone()) .then( @@ -2425,9 +2434,8 @@ pub fn serve( accept_header: Option| { task_spawner.blocking_response_task(Priority::P1, move || { let update = chain - .latest_seen_optimistic_update - .lock() - .clone() + .light_client_server_cache + .get_latest_optimistic_update() .ok_or_else(|| { warp_utils::reject::custom_not_found( "No LightClientOptimisticUpdate is available".to_string(), @@ -2473,9 +2481,8 @@ pub fn serve( accept_header: Option| { task_spawner.blocking_response_task(Priority::P1, move || { let update = chain - .latest_seen_finality_update - .lock() - .clone() + .light_client_server_cache + .get_latest_finality_update() .ok_or_else(|| { warp_utils::reject::custom_not_found( "No LightClientFinalityUpdate is available".to_string(), @@ -2524,7 +2531,7 @@ pub fn serve( .and(warp::path("attestations")) .and(warp::path::param::()) .and(warp::path::end()) - .and(warp::body::json()) + .and(warp_utils::json::json()) .then( |task_spawner: TaskSpawner, chain: Arc>, @@ -2574,7 +2581,7 @@ pub fn serve( .and(warp::path("sync_committee")) .and(block_id_or_err) .and(warp::path::end()) - .and(warp::body::json()) + .and(warp_utils::json::json()) .and(log_filter.clone()) .then( |task_spawner: TaskSpawner, @@ -3317,7 +3324,7 @@ pub fn serve( })) .and(warp::path::end()) .and(not_while_syncing_filter.clone()) - .and(warp::body::json()) + .and(warp_utils::json::json()) .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .then( @@ -3343,7 +3350,7 @@ pub fn serve( })) .and(warp::path::end()) .and(not_while_syncing_filter.clone()) - .and(warp::body::json()) + .and(warp_utils::json::json()) .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .then( @@ -3397,7 +3404,7 @@ pub fn serve( .and(not_while_syncing_filter.clone()) .and(task_spawner_filter.clone()) .and(chain_filter.clone()) - .and(warp::body::json()) + .and(warp_utils::json::json()) .and(network_tx_filter.clone()) .and(log_filter.clone()) .then( @@ -3510,7 +3517,7 @@ pub fn serve( .and(not_while_syncing_filter.clone()) .and(task_spawner_filter.clone()) .and(chain_filter.clone()) - .and(warp::body::json()) + .and(warp_utils::json::json()) .and(network_tx_filter) .and(log_filter.clone()) .then( @@ -3536,7 +3543,7 @@ pub fn serve( .and(warp::path("validator")) .and(warp::path("beacon_committee_subscriptions")) .and(warp::path::end()) - .and(warp::body::json()) + .and(warp_utils::json::json()) .and(validator_subscription_tx_filter.clone()) .and(task_spawner_filter.clone()) .and(chain_filter.clone()) @@ -3592,7 +3599,7 @@ pub fn serve( .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and(log_filter.clone()) - .and(warp::body::json()) + .and(warp_utils::json::json()) .then( |task_spawner: TaskSpawner, chain: Arc>, @@ -3643,7 +3650,7 @@ pub fn serve( .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and(log_filter.clone()) - .and(warp::body::json()) + .and(warp_utils::json::json()) .then( |task_spawner: TaskSpawner, chain: Arc>, @@ -3817,7 +3824,7 @@ pub fn serve( .and(warp::path("validator")) .and(warp::path("sync_committee_subscriptions")) .and(warp::path::end()) - .and(warp::body::json()) + .and(warp_utils::json::json()) .and(validator_subscription_tx_filter) .and(task_spawner_filter.clone()) .and(chain_filter.clone()) @@ -3863,7 +3870,7 @@ pub fn serve( .and(warp::path("liveness")) .and(warp::path::param::()) .and(warp::path::end()) - .and(warp::body::json()) + .and(warp_utils::json::json()) .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .then( @@ -3904,7 +3911,7 @@ pub fn serve( let post_lighthouse_liveness = warp::path("lighthouse") .and(warp::path("liveness")) .and(warp::path::end()) - .and(warp::body::json()) + .and(warp_utils::json::json()) .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .then( @@ -4007,7 +4014,7 @@ pub fn serve( .and(warp::path("ui")) .and(warp::path("validator_metrics")) .and(warp::path::end()) - .and(warp::body::json()) + .and(warp_utils::json::json()) .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .then( @@ -4026,7 +4033,7 @@ pub fn serve( .and(warp::path("ui")) .and(warp::path("validator_info")) .and(warp::path::end()) - .and(warp::body::json()) + .and(warp_utils::json::json()) .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .then( @@ -4258,36 +4265,6 @@ pub fn serve( }, ); - // GET lighthouse/beacon/states/{state_id}/ssz - let get_lighthouse_beacon_states_ssz = warp::path("lighthouse") - .and(warp::path("beacon")) - .and(warp::path("states")) - .and(warp::path::param::()) - .and(warp::path("ssz")) - .and(warp::path::end()) - .and(task_spawner_filter.clone()) - .and(chain_filter.clone()) - .then( - |state_id: StateId, - task_spawner: TaskSpawner, - chain: Arc>| { - task_spawner.blocking_response_task(Priority::P1, move || { - // This debug endpoint provides no indication of optimistic status. - let (state, _execution_optimistic, _finalized) = state_id.state(&chain)?; - Response::builder() - .status(200) - .body(state.as_ssz_bytes().into()) - .map(|res: Response| add_ssz_content_type_header(res)) - .map_err(|e| { - warp_utils::reject::custom_server_error(format!( - "failed to create response: {}", - e - )) - }) - }) - }, - ); - // GET lighthouse/staking let get_lighthouse_staking = warp::path("lighthouse") .and(warp::path("staking")) @@ -4359,7 +4336,7 @@ pub fn serve( let post_lighthouse_block_rewards = warp::path("lighthouse") .and(warp::path("analysis")) .and(warp::path("block_rewards")) - .and(warp::body::json()) + .and(warp_utils::json::json()) .and(warp::path::end()) .and(task_spawner_filter.clone()) .and(chain_filter.clone()) @@ -4622,7 +4599,6 @@ pub fn serve( .uor(get_lighthouse_eth1_syncing) .uor(get_lighthouse_eth1_block_cache) .uor(get_lighthouse_eth1_deposit_cache) - .uor(get_lighthouse_beacon_states_ssz) .uor(get_lighthouse_staking) .uor(get_lighthouse_database_info) .uor(get_lighthouse_block_rewards) diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 8b03771540..8b85c2ac95 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -194,7 +194,7 @@ pub async fn publish_block { info!( @@ -291,7 +290,7 @@ pub async fn publish_block( - blinded_block: SignedBlindedBeaconBlock, + blinded_block: Arc>, chain: Arc>, network_tx: &UnboundedSender>, log: Logger, @@ -319,7 +318,7 @@ pub async fn publish_blinded_block( pub async fn reconstruct_block( chain: Arc>, block_root: Hash256, - block: SignedBlindedBeaconBlock, + block: Arc>, log: Logger, ) -> Result>, Rejection> { let full_payload_opt = if let Ok(payload_header) = block.message().body().execution_payload() { @@ -380,6 +379,10 @@ pub async fn reconstruct_block( None }; + // Perf: cloning the block here to unblind it is a little sub-optimal. This is considered an + // acceptable tradeoff to avoid passing blocks around on the stack (unarced), which blows up + // the size of futures. + let block = (*block).clone(); match full_payload_opt { // A block without a payload is pre-merge and we consider it locally // built. diff --git a/beacon_node/http_api/tests/broadcast_validation_tests.rs b/beacon_node/http_api/tests/broadcast_validation_tests.rs index eb39bdd115..6a3f7947e6 100644 --- a/beacon_node/http_api/tests/broadcast_validation_tests.rs +++ b/beacon_node/http_api/tests/broadcast_validation_tests.rs @@ -2,17 +2,16 @@ use beacon_chain::{ test_utils::{AttestationStrategy, BlockStrategy}, GossipVerifiedBlock, IntoGossipVerifiedBlockContents, }; -use eth2::types::{BroadcastValidation, PublishBlockRequest, SignedBeaconBlock}; +use eth2::reqwest::StatusCode; +use eth2::types::{BroadcastValidation, PublishBlockRequest}; use http_api::test_utils::InteractiveTester; use http_api::{publish_blinded_block, publish_block, reconstruct_block, ProvenancedBlock}; use std::sync::Arc; use tree_hash::TreeHash; -use types::{Hash256, MainnetEthSpec, Slot}; +use types::{Epoch, EthSpec, ForkName, Hash256, MainnetEthSpec, Slot}; use warp::Rejection; use warp_utils::reject::CustomBadRequest; -use eth2::reqwest::StatusCode; - type E = MainnetEthSpec; /* @@ -64,7 +63,7 @@ pub async fn gossip_invalid() { tester.harness.advance_slot(); - let ((block, blobs), _): ((SignedBeaconBlock, _), _) = tester + let ((block, blobs), _) = tester .harness .make_block_with_modifier(chain_state_before, slot, |b| { *b.state_root_mut() = Hash256::zero(); @@ -116,7 +115,7 @@ pub async fn gossip_partial_pass() { tester.harness.advance_slot(); - let ((block, blobs), _): ((SignedBeaconBlock, _), _) = tester + let ((block, blobs), _) = tester .harness .make_block_with_modifier(chain_state_before, slot, |b| { *b.state_root_mut() = Hash256::random() @@ -162,8 +161,7 @@ pub async fn gossip_full_pass() { let slot_b = slot_a + 1; let state_a = tester.harness.get_current_state(); - let ((block, blobs), _): ((SignedBeaconBlock, _), _) = - tester.harness.make_block(state_a, slot_b).await; + let ((block, blobs), _) = tester.harness.make_block(state_a, slot_b).await; let response: Result<(), eth2::Error> = tester .client @@ -190,7 +188,10 @@ pub async fn gossip_full_pass_ssz() { // `validator_count // 32`. let validator_count = 64; let num_initial: u64 = 31; - let tester = InteractiveTester::::new(None, validator_count).await; + // Deneb epoch set ahead of block slot, to test fork-based decoding + let mut spec = ForkName::Capella.make_genesis_spec(MainnetEthSpec::default_spec()); + spec.deneb_fork_epoch = Some(Epoch::new(4)); + let tester = InteractiveTester::::new(Some(spec), validator_count).await; // Create some chain depth. tester.harness.advance_slot(); @@ -250,7 +251,7 @@ pub async fn consensus_invalid() { tester.harness.advance_slot(); - let ((block, blobs), _): ((SignedBeaconBlock, _), _) = tester + let ((block, blobs), _) = tester .harness .make_block_with_modifier(chain_state_before, slot, |b| { *b.state_root_mut() = Hash256::zero(); @@ -302,7 +303,7 @@ pub async fn consensus_gossip() { let slot_b = slot_a + 1; let state_a = tester.harness.get_current_state(); - let ((block, blobs), _): ((SignedBeaconBlock, _), _) = tester + let ((block, blobs), _) = tester .harness .make_block_with_modifier(state_a, slot_b, |b| *b.state_root_mut() = Hash256::zero()) .await; @@ -416,8 +417,7 @@ pub async fn consensus_full_pass() { let slot_b = slot_a + 1; let state_a = tester.harness.get_current_state(); - let ((block, blobs), _): ((SignedBeaconBlock, _), _) = - tester.harness.make_block(state_a, slot_b).await; + let ((block, blobs), _) = tester.harness.make_block(state_a, slot_b).await; let response: Result<(), eth2::Error> = tester .client @@ -463,7 +463,7 @@ pub async fn equivocation_invalid() { tester.harness.advance_slot(); - let ((block, blobs), _): ((SignedBeaconBlock, _), _) = tester + let ((block, blobs), _) = tester .harness .make_block_with_modifier(chain_state_before, slot, |b| { *b.state_root_mut() = Hash256::zero(); @@ -516,10 +516,9 @@ pub async fn equivocation_consensus_early_equivocation() { let slot_b = slot_a + 1; let state_a = tester.harness.get_current_state(); - let ((block_a, blobs_a), state_after_a): ((SignedBeaconBlock, _), _) = + let ((block_a, blobs_a), state_after_a) = tester.harness.make_block(state_a.clone(), slot_b).await; - let ((block_b, blobs_b), state_after_b): ((SignedBeaconBlock, _), _) = - tester.harness.make_block(state_a, slot_b).await; + let ((block_b, blobs_b), state_after_b) = tester.harness.make_block(state_a, slot_b).await; /* check for `make_block` curios */ assert_eq!(block_a.state_root(), state_after_a.tree_hash_root()); @@ -588,7 +587,7 @@ pub async fn equivocation_gossip() { let slot_b = slot_a + 1; let state_a = tester.harness.get_current_state(); - let ((block, blobs), _): ((SignedBeaconBlock, _), _) = tester + let ((block, blobs), _) = tester .harness .make_block_with_modifier(state_a, slot_b, |b| *b.state_root_mut() = Hash256::zero()) .await; @@ -643,10 +642,9 @@ pub async fn equivocation_consensus_late_equivocation() { let slot_b = slot_a + 1; let state_a = tester.harness.get_current_state(); - let ((block_a, blobs_a), state_after_a): ((SignedBeaconBlock, _), _) = + let ((block_a, blobs_a), state_after_a) = tester.harness.make_block(state_a.clone(), slot_b).await; - let ((block_b, blobs_b), state_after_b): ((SignedBeaconBlock, _), _) = - tester.harness.make_block(state_a, slot_b).await; + let ((block_b, blobs_b), state_after_b) = tester.harness.make_block(state_a, slot_b).await; /* check for `make_block` curios */ assert_eq!(block_a.state_root(), state_after_a.tree_hash_root()); @@ -714,8 +712,7 @@ pub async fn equivocation_full_pass() { let slot_b = slot_a + 1; let state_a = tester.harness.get_current_state(); - let ((block, blobs), _): ((SignedBeaconBlock, _), _) = - tester.harness.make_block(state_a, slot_b).await; + let ((block, blobs), _) = tester.harness.make_block(state_a, slot_b).await; let response: Result<(), eth2::Error> = tester .client @@ -1267,6 +1264,7 @@ pub async fn blinded_equivocation_consensus_late_equivocation() { .make_blinded_block(state_a.clone(), slot_b) .await; let (block_b, state_after_b) = tester.harness.make_blinded_block(state_a, slot_b).await; + let block_b = Arc::new(block_b); /* check for `make_blinded_block` curios */ assert_eq!(block_a.state_root(), state_after_a.tree_hash_root()); @@ -1276,7 +1274,7 @@ pub async fn blinded_equivocation_consensus_late_equivocation() { let unblinded_block_a = reconstruct_block( tester.harness.chain.clone(), block_a.canonical_root(), - block_a, + Arc::new(block_a), test_logger.clone(), ) .await @@ -1299,15 +1297,11 @@ pub async fn blinded_equivocation_consensus_late_equivocation() { ProvenancedBlock::Builder(b, _) => b, }; - let gossip_block_b = GossipVerifiedBlock::new( - Arc::new(inner_block_b.clone().deconstruct().0), - &tester.harness.chain, - ); + let gossip_block_b = + GossipVerifiedBlock::new(inner_block_b.clone().deconstruct().0, &tester.harness.chain); assert!(gossip_block_b.is_ok()); - let gossip_block_a = GossipVerifiedBlock::new( - Arc::new(inner_block_a.clone().deconstruct().0), - &tester.harness.chain, - ); + let gossip_block_a = + GossipVerifiedBlock::new(inner_block_a.clone().deconstruct().0, &tester.harness.chain); assert!(gossip_block_a.is_err()); let channel = tokio::sync::mpsc::unbounded_channel(); diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index 210c4d2550..6fb197b41a 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -632,7 +632,7 @@ pub async fn proposer_boost_re_org_test( panic!("Should not be a blinded block"); } }; - let block_c = harness.sign_beacon_block(unsigned_block_c, &state_b); + let block_c = Arc::new(harness.sign_beacon_block(unsigned_block_c, &state_b)); if should_re_org { // Block C should build on A. diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 300c5a1060..2d946f3092 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -1587,6 +1587,39 @@ impl ApiTester { self } + pub async fn test_get_blob_sidecars(self, use_indices: bool) -> Self { + let block_id = BlockId(CoreBlockId::Finalized); + let (block_root, _, _) = block_id.root(&self.chain).unwrap(); + let (block, _, _) = block_id.full_block(&self.chain).await.unwrap(); + let num_blobs = block.num_expected_blobs(); + let blob_indices = if use_indices { + Some( + (0..num_blobs.saturating_sub(1) as u64) + .into_iter() + .collect::>(), + ) + } else { + None + }; + let result = match self + .client + .get_blobs::(CoreBlockId::Root(block_root), blob_indices.as_deref()) + .await + { + Ok(result) => result.unwrap().data, + Err(e) => panic!("query failed incorrectly: {e:?}"), + }; + + assert_eq!( + result.len(), + blob_indices.map_or(num_blobs, |indices| indices.len()) + ); + let expected = block.slot(); + assert_eq!(result.get(0).unwrap().slot(), expected); + + self + } + pub async fn test_beacon_blocks_attestations(self) -> Self { for block_id in self.interesting_block_ids() { let result = self @@ -1698,7 +1731,10 @@ impl ApiTester { Err(e) => panic!("query failed incorrectly: {e:?}"), }; - let expected = self.chain.latest_seen_optimistic_update.lock().clone(); + let expected = self + .chain + .light_client_server_cache + .get_latest_optimistic_update(); assert_eq!(result, expected); self @@ -1714,7 +1750,10 @@ impl ApiTester { Err(e) => panic!("query failed incorrectly: {e:?}"), }; - let expected = self.chain.latest_seen_finality_update.lock().clone(); + let expected = self + .chain + .light_client_server_cache + .get_latest_finality_update(); assert_eq!(result, expected); self @@ -2597,7 +2636,7 @@ impl ApiTester { let signed_block = block.sign(&sk, &fork, genesis_validators_root, &self.chain.spec); let signed_block_contents = - PublishBlockRequest::try_from(signed_block.clone()).unwrap(); + PublishBlockRequest::try_from(Arc::new(signed_block.clone())).unwrap(); self.client .post_beacon_blocks(&signed_block_contents) @@ -2670,8 +2709,8 @@ impl ApiTester { .unwrap(); assert_eq!( - self.chain.head_beacon_block().as_ref(), - signed_block_contents.signed_block() + self.chain.head_beacon_block(), + *signed_block_contents.signed_block() ); self.chain.slot_clock.set_slot(slot.as_u64() + 1); @@ -2763,8 +2802,8 @@ impl ApiTester { .unwrap(); assert_eq!( - self.chain.head_beacon_block().as_ref(), - signed_block_contents.signed_block() + self.chain.head_beacon_block(), + *signed_block_contents.signed_block() ); self.chain.slot_clock.set_slot(slot.as_u64() + 1); @@ -2994,7 +3033,7 @@ impl ApiTester { .data; let signed_block = signed_block_contents.signed_block(); - assert_eq!(&head_block, signed_block); + assert_eq!(head_block, **signed_block); self.chain.slot_clock.set_slot(slot.as_u64() + 1); } @@ -5024,26 +5063,6 @@ impl ApiTester { self } - pub async fn test_get_lighthouse_beacon_states_ssz(self) -> Self { - for state_id in self.interesting_state_ids() { - let result = self - .client - .get_lighthouse_beacon_states_ssz(&state_id.0, &self.chain.spec) - .await - .unwrap(); - - let mut expected = state_id - .state(&self.chain) - .ok() - .map(|(state, _execution_optimistic, _finalized)| state); - expected.as_mut().map(|state| state.drop_all_caches()); - - assert_eq!(result, expected, "{:?}", state_id); - } - - self - } - pub async fn test_get_lighthouse_staking(self) -> Self { let result = self.client.get_lighthouse_staking().await.unwrap(); @@ -6291,6 +6310,27 @@ async fn builder_works_post_deneb() { .await; } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn get_blob_sidecars() { + let mut config = ApiTesterConfig { + retain_historic_states: false, + spec: E::default_spec(), + }; + config.spec.altair_fork_epoch = Some(Epoch::new(0)); + config.spec.bellatrix_fork_epoch = Some(Epoch::new(0)); + config.spec.capella_fork_epoch = Some(Epoch::new(0)); + config.spec.deneb_fork_epoch = Some(Epoch::new(0)); + + ApiTester::new_from_config(config) + .await + .test_post_beacon_blocks_valid() + .await + .test_get_blob_sidecars(false) + .await + .test_get_blob_sidecars(true) + .await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn post_validator_liveness_epoch() { ApiTester::new() @@ -6319,8 +6359,6 @@ async fn lighthouse_endpoints() { .await .test_get_lighthouse_eth1_deposit_cache() .await - .test_get_lighthouse_beacon_states_ssz() - .await .test_get_lighthouse_staking() .await .test_get_lighthouse_database_info() diff --git a/beacon_node/lighthouse_network/Cargo.toml b/beacon_node/lighthouse_network/Cargo.toml index e1ae62be65..46acdeaded 100644 --- a/beacon_node/lighthouse_network/Cargo.toml +++ b/beacon_node/lighthouse_network/Cargo.toml @@ -43,11 +43,11 @@ prometheus-client = "0.22.0" unused_port = { workspace = true } delay_map = { workspace = true } void = "1" -libp2p-mplex = { git = "https://github.com/sigp/rust-libp2p/", rev = "b96b90894faab0a1eed78e1c82c6452138a3538a" } +libp2p-mplex = { git = "https://github.com/sigp/rust-libp2p/", rev = "cfa3275ca17e502799ed56e555b6c0611752e369" } [dependencies.libp2p] git = "https://github.com/sigp/rust-libp2p/" -rev = "b96b90894faab0a1eed78e1c82c6452138a3538a" +rev = "cfa3275ca17e502799ed56e555b6c0611752e369" default-features = false features = ["identify", "yamux", "noise", "gossipsub", "dns", "tcp", "tokio", "plaintext", "secp256k1", "macros", "ecdsa", "metrics", "quic"] diff --git a/beacon_node/lighthouse_network/src/peer_manager/mod.rs b/beacon_node/lighthouse_network/src/peer_manager/mod.rs index 3459548ec3..4316c0d07e 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/mod.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/mod.rs @@ -326,8 +326,10 @@ impl PeerManager { // considered a priority. We have pre-allocated some extra priority slots for these // peers as specified by PRIORITY_PEER_EXCESS. Therefore we dial these peers, even // if we are already at our max_peer limit. - if min_ttl.is_some() && connected_or_dialing + to_dial_peers < self.max_priority_peers() - || connected_or_dialing + to_dial_peers < self.max_peers() + if !self.peers_to_dial.contains(&enr) + && ((min_ttl.is_some() + && connected_or_dialing + to_dial_peers < self.max_priority_peers()) + || connected_or_dialing + to_dial_peers < self.max_peers()) { // This should be updated with the peer dialing. In fact created once the peer is // dialed @@ -337,9 +339,11 @@ impl PeerManager { .write() .update_min_ttl(&enr.peer_id(), min_ttl); } - debug!(self.log, "Dialing discovered peer"; "peer_id" => %enr.peer_id()); - self.dial_peer(enr); - to_dial_peers += 1; + let peer_id = enr.peer_id(); + if self.dial_peer(enr) { + debug!(self.log, "Dialing discovered peer"; "peer_id" => %peer_id); + to_dial_peers += 1; + } } } @@ -401,7 +405,8 @@ impl PeerManager { /* Notifications from the Swarm */ /// A peer is being dialed. - pub fn dial_peer(&mut self, peer: Enr) { + /// Returns true, if this peer will be dialed. + pub fn dial_peer(&mut self, peer: Enr) -> bool { if self .network_globals .peers @@ -409,6 +414,9 @@ impl PeerManager { .should_dial(&peer.peer_id()) { self.peers_to_dial.push(peer); + true + } else { + false } } @@ -916,7 +924,7 @@ impl PeerManager { { self.max_outbound_dialing_peers() .saturating_sub(dialing_peers) - - peer_count + .saturating_sub(peer_count) } else { 0 }; diff --git a/beacon_node/lighthouse_network/src/rpc/methods.rs b/beacon_node/lighthouse_network/src/rpc/methods.rs index 04ec6bac49..cd3579ad6e 100644 --- a/beacon_node/lighthouse_network/src/rpc/methods.rs +++ b/beacon_node/lighthouse_network/src/rpc/methods.rs @@ -384,7 +384,7 @@ pub enum RPCResponse { /// A response to a get BLOBS_BY_RANGE request BlobsByRange(Arc>), - /// A response to a get LIGHTCLIENT_BOOTSTRAP request. + /// A response to a get LIGHT_CLIENT_BOOTSTRAP request. LightClientBootstrap(LightClientBootstrap), /// A response to a get BLOBS_BY_ROOT request. @@ -426,7 +426,7 @@ pub enum RPCCodedResponse { StreamTermination(ResponseTermination), } -/// Request a light_client_bootstrap for lightclients peers. +/// Request a light_client_bootstrap for light_clients peers. #[derive(Encode, Decode, Clone, Debug, PartialEq)] pub struct LightClientBootstrapRequest { pub root: Hash256, diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index e85cf75fd8..2b20c76cf4 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -1161,9 +1161,11 @@ impl Network { // Remove the ENR from the cache to prevent continual re-dialing on disconnects for enr in peers_to_dial { - debug!(self.log, "Dialing cached ENR peer"; "peer_id" => %enr.peer_id()); self.discovery_mut().remove_cached_enr(&enr.peer_id()); - self.peer_manager_mut().dial_peer(enr); + let peer_id = enr.peer_id(); + if self.peer_manager_mut().dial_peer(enr) { + debug!(self.log, "Dialing cached ENR peer"; "peer_id" => %peer_id); + } } } diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 9d9b196e9b..07fc06bc37 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -1657,7 +1657,7 @@ impl NetworkBeaconProcessor { self.gossip_penalize_peer( peer_id, - PeerAction::LowToleranceError, + PeerAction::HighToleranceError, "light_client_gossip_error", ); } @@ -1675,15 +1675,7 @@ impl NetworkBeaconProcessor { "light_client_gossip_error", ); } - LightClientFinalityUpdateError::FinalityUpdateAlreadySeen => debug!( - self.log, - "Light client finality update already seen"; - "peer" => %peer_id, - "error" => ?e, - ), - LightClientFinalityUpdateError::BeaconChainError(_) - | LightClientFinalityUpdateError::LightClientUpdateError(_) - | LightClientFinalityUpdateError::SigSlotStartIsNone + LightClientFinalityUpdateError::SigSlotStartIsNone | LightClientFinalityUpdateError::FailedConstructingUpdate => debug!( self.log, "Light client error constructing finality update"; @@ -1801,19 +1793,7 @@ impl NetworkBeaconProcessor { "light_client_gossip_error", ); } - LightClientOptimisticUpdateError::OptimisticUpdateAlreadySeen => { - metrics::register_optimistic_update_error(&e); - - debug!( - self.log, - "Light client optimistic update already seen"; - "peer" => %peer_id, - "error" => ?e, - ) - } - LightClientOptimisticUpdateError::BeaconChainError(_) - | LightClientOptimisticUpdateError::LightClientUpdateError(_) - | LightClientOptimisticUpdateError::SigSlotStartIsNone + LightClientOptimisticUpdateError::SigSlotStartIsNone | LightClientOptimisticUpdateError::FailedConstructingUpdate => { metrics::register_optimistic_update_error(&e); diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 67fc2fabb1..e7d3a7ce21 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -589,7 +589,7 @@ impl NetworkBeaconProcessor { } /// Create a new work event to process `LightClientBootstrap`s from the RPC network. - pub fn send_lightclient_bootstrap_request( + pub fn send_light_client_bootstrap_request( self: &Arc, peer_id: PeerId, request_id: PeerRequestId, diff --git a/beacon_node/network/src/network_beacon_processor/sync_methods.rs b/beacon_node/network/src/network_beacon_processor/sync_methods.rs index 608d10d665..8894d5d9fd 100644 --- a/beacon_node/network/src/network_beacon_processor/sync_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/sync_methods.rs @@ -9,10 +9,8 @@ use beacon_chain::block_verification_types::{AsBlock, RpcBlock}; use beacon_chain::data_availability_checker::AvailabilityCheckError; use beacon_chain::data_availability_checker::MaybeAvailableBlock; use beacon_chain::{ - observed_block_producers::Error as ObserveError, - validator_monitor::{get_block_delay_ms, get_slot_delay_ms}, - AvailabilityProcessingStatus, BeaconChainError, BeaconChainTypes, BlockError, - ChainSegmentResult, HistoricalBlockError, NotifyExecutionLayer, + validator_monitor::get_slot_delay_ms, AvailabilityProcessingStatus, BeaconChainError, + BeaconChainTypes, BlockError, ChainSegmentResult, HistoricalBlockError, NotifyExecutionLayer, }; use beacon_processor::{ work_reprocessing_queue::{QueuedRpcBlock, ReprocessQueueMessage}, @@ -20,10 +18,8 @@ use beacon_processor::{ }; use lighthouse_network::PeerAction; use slog::{debug, error, info, warn}; -use slot_clock::SlotClock; use std::sync::Arc; use std::time::Duration; -use std::time::{SystemTime, UNIX_EPOCH}; use store::KzgCommitment; use tokio::sync::mpsc; use types::beacon_block_body::format_kzg_commitments; @@ -142,72 +138,6 @@ impl NetworkBeaconProcessor { return; }; - // Returns `true` if the time now is after the 4s attestation deadline. - let block_is_late = SystemTime::now() - .duration_since(UNIX_EPOCH) - // If we can't read the system time clock then indicate that the - // block is late (and therefore should *not* be requeued). This - // avoids infinite loops. - .map_or(true, |now| { - get_block_delay_ms(now, block.message(), &self.chain.slot_clock) - > self.chain.slot_clock.unagg_attestation_production_delay() - }); - - // Checks if a block from this proposer is already known. - let block_equivocates = || { - match self.chain.observed_slashable.read().is_slashable( - block.slot(), - block.message().proposer_index(), - block.canonical_root(), - ) { - Ok(is_slashable) => is_slashable, - //Both of these blocks will be rejected, so reject them now rather - // than re-queuing them. - Err(ObserveError::FinalizedBlock { .. }) - | Err(ObserveError::ValidatorIndexTooHigh { .. }) => false, - } - }; - - // If we've already seen a block from this proposer *and* the block - // arrived before the attestation deadline, requeue it to ensure it is - // imported late enough that it won't receive a proposer boost. - // - // Don't requeue blocks if they're already known to fork choice, just - // push them through to block processing so they can be handled through - // the normal channels. - if !block_is_late && block_equivocates() { - debug!( - self.log, - "Delaying processing of duplicate RPC block"; - "block_root" => ?block_root, - "proposer" => block.message().proposer_index(), - "slot" => block.slot() - ); - - // Send message to work reprocess queue to retry the block - let (process_fn, ignore_fn) = self.clone().generate_rpc_beacon_block_fns( - block_root, - block, - seen_timestamp, - process_type, - ); - let reprocess_msg = ReprocessQueueMessage::RpcBlock(QueuedRpcBlock { - beacon_block_root: block_root, - process_fn, - ignore_fn, - }); - - if reprocess_tx.try_send(reprocess_msg).is_err() { - error!( - self.log, - "Failed to inform block import"; - "source" => "rpc", - "block_root" => %block_root - ); - } - return; - } - let slot = block.slot(); let parent_root = block.message().parent_root(); let commitments_formatted = block.as_block().commitments_formatted(); diff --git a/beacon_node/network/src/network_beacon_processor/tests.rs b/beacon_node/network/src/network_beacon_processor/tests.rs index 48c5334357..dd58eb8355 100644 --- a/beacon_node/network/src/network_beacon_processor/tests.rs +++ b/beacon_node/network/src/network_beacon_processor/tests.rs @@ -250,7 +250,7 @@ impl TestRig { }; Self { chain, - next_block: Arc::new(block), + next_block: block, next_blobs: blob_sidecars, attestations, next_block_attestations, diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index f56a3b7445..a774c0e16f 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -218,7 +218,7 @@ impl Router { ), Request::LightClientBootstrap(request) => self.handle_beacon_processor_send_result( self.network_beacon_processor - .send_lightclient_bootstrap_request(peer_id, request_id, request), + .send_light_client_bootstrap_request(peer_id, request_id, request), ), } } diff --git a/beacon_node/network/src/sync/block_lookups/common.rs b/beacon_node/network/src/sync/block_lookups/common.rs index 78b10473df..d989fbb336 100644 --- a/beacon_node/network/src/sync/block_lookups/common.rs +++ b/beacon_node/network/src/sync/block_lookups/common.rs @@ -101,11 +101,10 @@ pub trait RequestState { fn build_request_and_send( &mut self, id: Id, - already_downloaded: bool, cx: &SyncNetworkContext, ) -> Result<(), LookupRequestError> { // Check if request is necessary. - if already_downloaded || !matches!(self.get_state().state, State::AwaitingDownload) { + if !matches!(self.get_state().state, State::AwaitingDownload) { return Ok(()); } diff --git a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs index 4e29816294..8c60621f1c 100644 --- a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs +++ b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs @@ -92,6 +92,10 @@ impl SingleBlockLookup { self.block_request_state.requested_block_root = block_root; self.block_request_state.state.state = State::AwaitingDownload; self.blob_request_state.state.state = State::AwaitingDownload; + self.block_request_state.state.component_downloaded = false; + self.blob_request_state.state.component_downloaded = false; + self.block_request_state.state.component_processed = false; + self.blob_request_state.state.component_processed = false; self.child_components = Some(ChildComponents::empty(block_root)); } @@ -110,19 +114,18 @@ impl SingleBlockLookup { &mut self, cx: &SyncNetworkContext, ) -> Result<(), LookupRequestError> { - let block_root = self.block_root(); let block_already_downloaded = self.block_already_downloaded(); let blobs_already_downloaded = self.blobs_already_downloaded(); - if block_already_downloaded && blobs_already_downloaded { - trace!(cx.log, "Lookup request already completed"; "block_root"=> ?block_root); - return Ok(()); + if !block_already_downloaded { + self.block_request_state + .build_request_and_send(self.id, cx)?; } - let id = self.id; - self.block_request_state - .build_request_and_send(id, block_already_downloaded, cx)?; - self.blob_request_state - .build_request_and_send(id, blobs_already_downloaded, cx) + if !blobs_already_downloaded { + self.blob_request_state + .build_request_and_send(self.id, cx)?; + } + Ok(()) } /// Returns a `CachedChild`, which is a wrapper around a `RpcBlock` that is either: diff --git a/beacon_node/network/src/sync/block_lookups/tests.rs b/beacon_node/network/src/sync/block_lookups/tests.rs index 83f0b26156..c506696b9d 100644 --- a/beacon_node/network/src/sync/block_lookups/tests.rs +++ b/beacon_node/network/src/sync/block_lookups/tests.rs @@ -1157,6 +1157,7 @@ mod deneb_only { use crate::sync::block_lookups::common::ResponseType; use beacon_chain::data_availability_checker::AvailabilityCheckError; use beacon_chain::test_utils::NumBlobs; + use ssz_types::VariableList; use std::ops::IndexMut; use std::str::FromStr; @@ -1164,10 +1165,12 @@ mod deneb_only { bl: BlockLookups, cx: SyncNetworkContext, rig: TestRig, - block: Option>>, + block: Arc>, blobs: Vec>>, - parent_block: Option>>, - parent_blobs: Vec>>, + parent_block: VecDeque>>, + parent_blobs: VecDeque>>>, + unknown_parent_block: Option>>, + unknown_parent_blobs: Option>>>, peer_id: PeerId, block_req_id: Option, parent_block_req_id: Option, @@ -1179,8 +1182,18 @@ mod deneb_only { enum RequestTrigger { AttestationUnknownBlock, - GossipUnknownParentBlock, - GossipUnknownParentBlob, + GossipUnknownParentBlock { num_parents: usize }, + GossipUnknownParentBlob { num_parents: usize }, + } + + impl RequestTrigger { + fn num_parents(&self) -> usize { + match self { + RequestTrigger::AttestationUnknownBlock => 0, + RequestTrigger::GossipUnknownParentBlock { num_parents } => *num_parents, + RequestTrigger::GossipUnknownParentBlob { num_parents } => *num_parents, + } + } } impl DenebTester { @@ -1194,14 +1207,33 @@ mod deneb_only { E::slots_per_epoch() * rig.harness.spec.deneb_fork_epoch.unwrap().as_u64(), ); let (block, blobs) = rig.rand_block_and_blobs(fork_name, NumBlobs::Random); - let block = Arc::new(block); - let slot = block.slot(); - let mut block_root = block.canonical_root(); - let mut block = Some(block); + let mut block = Arc::new(block); let mut blobs = blobs.into_iter().map(Arc::new).collect::>(); + let slot = block.slot(); - let mut parent_block = None; - let mut parent_blobs = vec![]; + let num_parents = request_trigger.num_parents(); + let mut parent_block_chain = VecDeque::with_capacity(num_parents); + let mut parent_blobs_chain = VecDeque::with_capacity(num_parents); + for _ in 0..num_parents { + // Set the current block as the parent. + let parent_root = block.canonical_root(); + let parent_block = block.clone(); + let parent_blobs = blobs.clone(); + parent_block_chain.push_front(parent_block); + parent_blobs_chain.push_front(parent_blobs); + + // Create the next block. + let (child_block, child_blobs) = + rig.block_with_parent_and_blobs(parent_root, get_fork_name(), NumBlobs::Random); + let mut child_block = Arc::new(child_block); + let mut child_blobs = child_blobs.into_iter().map(Arc::new).collect::>(); + + // Update the new block to the current block. + std::mem::swap(&mut child_block, &mut block); + std::mem::swap(&mut child_blobs, &mut blobs); + } + let block_root = block.canonical_root(); + let parent_root = block.parent_root(); let peer_id = PeerId::random(); @@ -1214,31 +1246,17 @@ mod deneb_only { let blob_req_id = rig.expect_lookup_request(ResponseType::Blob); (Some(block_req_id), Some(blob_req_id), None, None) } - RequestTrigger::GossipUnknownParentBlock => { - let (child_block, child_blobs) = rig.block_with_parent_and_blobs( - block_root, - get_fork_name(), - NumBlobs::Random, - ); - parent_block = Some(Arc::new(child_block)); - parent_blobs = child_blobs.into_iter().map(Arc::new).collect::>(); - std::mem::swap(&mut parent_block, &mut block); - std::mem::swap(&mut parent_blobs, &mut blobs); - - let child_block = block.as_ref().expect("block").clone(); - let child_root = child_block.canonical_root(); - let parent_root = block_root; - block_root = child_root; + RequestTrigger::GossipUnknownParentBlock { .. } => { bl.search_child_block( - child_root, - ChildComponents::new(child_root, Some(child_block), None), + block_root, + ChildComponents::new(block_root, Some(block.clone()), None), &[peer_id], &mut cx, ); let blob_req_id = rig.expect_lookup_request(ResponseType::Blob); rig.expect_empty_network(); // expect no block request - bl.search_parent(slot, child_root, parent_root, peer_id, &mut cx); + bl.search_parent(slot, block_root, parent_root, peer_id, &mut cx); let parent_block_req_id = rig.expect_parent_request(ResponseType::Block); let parent_blob_req_id = rig.expect_parent_request(ResponseType::Blob); ( @@ -1248,28 +1266,15 @@ mod deneb_only { Some(parent_blob_req_id), ) } - RequestTrigger::GossipUnknownParentBlob => { - let (child_block, child_blobs) = rig.block_with_parent_and_blobs( - block_root, - get_fork_name(), - NumBlobs::Random, - ); + RequestTrigger::GossipUnknownParentBlob { .. } => { + let single_blob = blobs.first().cloned().unwrap(); + let child_root = single_blob.block_root(); - parent_block = Some(Arc::new(child_block)); - parent_blobs = child_blobs.into_iter().map(Arc::new).collect::>(); - std::mem::swap(&mut parent_block, &mut block); - std::mem::swap(&mut parent_blobs, &mut blobs); - - let child_blob = blobs.first().cloned().unwrap(); - let parent_root = block_root; - let child_root = child_blob.block_root(); - block_root = child_root; - - let mut blobs = FixedBlobSidecarList::default(); - *blobs.index_mut(0) = Some(child_blob); + let mut lookup_blobs = FixedBlobSidecarList::default(); + *lookup_blobs.index_mut(0) = Some(single_blob); bl.search_child_block( child_root, - ChildComponents::new(child_root, None, Some(blobs)), + ChildComponents::new(child_root, None, Some(lookup_blobs)), &[peer_id], &mut cx, ); @@ -1295,8 +1300,10 @@ mod deneb_only { rig, block, blobs, - parent_block, - parent_blobs, + parent_block: parent_block_chain, + parent_blobs: parent_blobs_chain, + unknown_parent_block: None, + unknown_parent_blobs: None, peer_id, block_req_id, parent_block_req_id, @@ -1309,10 +1316,12 @@ mod deneb_only { fn parent_block_response(mut self) -> Self { self.rig.expect_empty_network(); + let block = self.parent_block.pop_front().unwrap().clone(); + let _ = self.unknown_parent_block.insert(block.clone()); self.bl.parent_lookup_response::>( self.parent_block_req_id.expect("parent request id"), self.peer_id, - self.parent_block.clone(), + Some(block), D, &self.cx, ); @@ -1322,7 +1331,9 @@ mod deneb_only { } fn parent_blob_response(mut self) -> Self { - for blob in &self.parent_blobs { + let blobs = self.parent_blobs.pop_front().unwrap(); + let _ = self.unknown_parent_blobs.insert(blobs.clone()); + for blob in &blobs { self.bl .parent_lookup_response::>( self.parent_blob_req_id.expect("parent blob request id"), @@ -1361,7 +1372,7 @@ mod deneb_only { .single_lookup_response::>( self.block_req_id.expect("block request id"), self.peer_id, - self.block.clone(), + Some(self.block.clone()), D, &self.cx, ); @@ -1483,12 +1494,16 @@ mod deneb_only { } fn parent_block_unknown_parent(mut self) -> Self { + let block = self.unknown_parent_block.take().unwrap(); + let block = RpcBlock::new( + Some(block.canonical_root()), + block, + self.unknown_parent_blobs.take().map(VariableList::from), + ) + .unwrap(); self.bl.parent_block_processed( self.block_root, - BlockProcessingResult::Err(BlockError::ParentUnknown(RpcBlock::new_without_blobs( - Some(self.block_root), - self.parent_block.clone().expect("parent block"), - ))), + BlockProcessingResult::Err(BlockError::ParentUnknown(block)), &mut self.cx, ); assert_eq!(self.bl.parent_lookups.len(), 1); @@ -1805,7 +1820,9 @@ mod deneb_only { #[test] fn parent_block_unknown_parent() { - let Some(tester) = DenebTester::new(RequestTrigger::GossipUnknownParentBlock) else { + let Some(tester) = + DenebTester::new(RequestTrigger::GossipUnknownParentBlock { num_parents: 1 }) + else { return; }; @@ -1823,7 +1840,9 @@ mod deneb_only { #[test] fn parent_block_invalid_parent() { - let Some(tester) = DenebTester::new(RequestTrigger::GossipUnknownParentBlock) else { + let Some(tester) = + DenebTester::new(RequestTrigger::GossipUnknownParentBlock { num_parents: 1 }) + else { return; }; @@ -1842,7 +1861,9 @@ mod deneb_only { #[test] fn parent_block_and_blob_lookup_parent_returned_first() { - let Some(tester) = DenebTester::new(RequestTrigger::GossipUnknownParentBlock) else { + let Some(tester) = + DenebTester::new(RequestTrigger::GossipUnknownParentBlock { num_parents: 1 }) + else { return; }; @@ -1857,7 +1878,9 @@ mod deneb_only { #[test] fn parent_block_and_blob_lookup_child_returned_first() { - let Some(tester) = DenebTester::new(RequestTrigger::GossipUnknownParentBlock) else { + let Some(tester) = + DenebTester::new(RequestTrigger::GossipUnknownParentBlock { num_parents: 1 }) + else { return; }; @@ -1875,7 +1898,9 @@ mod deneb_only { #[test] fn empty_parent_block_then_parent_blob() { - let Some(tester) = DenebTester::new(RequestTrigger::GossipUnknownParentBlock) else { + let Some(tester) = + DenebTester::new(RequestTrigger::GossipUnknownParentBlock { num_parents: 1 }) + else { return; }; @@ -1895,7 +1920,9 @@ mod deneb_only { #[test] fn empty_parent_blobs_then_parent_block() { - let Some(tester) = DenebTester::new(RequestTrigger::GossipUnknownParentBlock) else { + let Some(tester) = + DenebTester::new(RequestTrigger::GossipUnknownParentBlock { num_parents: 1 }) + else { return; }; @@ -1916,7 +1943,9 @@ mod deneb_only { #[test] fn parent_blob_unknown_parent() { - let Some(tester) = DenebTester::new(RequestTrigger::GossipUnknownParentBlob) else { + let Some(tester) = + DenebTester::new(RequestTrigger::GossipUnknownParentBlob { num_parents: 1 }) + else { return; }; @@ -1934,7 +1963,9 @@ mod deneb_only { #[test] fn parent_blob_invalid_parent() { - let Some(tester) = DenebTester::new(RequestTrigger::GossipUnknownParentBlob) else { + let Some(tester) = + DenebTester::new(RequestTrigger::GossipUnknownParentBlob { num_parents: 1 }) + else { return; }; @@ -1953,7 +1984,9 @@ mod deneb_only { #[test] fn parent_block_and_blob_lookup_parent_returned_first_blob_trigger() { - let Some(tester) = DenebTester::new(RequestTrigger::GossipUnknownParentBlob) else { + let Some(tester) = + DenebTester::new(RequestTrigger::GossipUnknownParentBlob { num_parents: 1 }) + else { return; }; @@ -1968,7 +2001,9 @@ mod deneb_only { #[test] fn parent_block_and_blob_lookup_child_returned_first_blob_trigger() { - let Some(tester) = DenebTester::new(RequestTrigger::GossipUnknownParentBlob) else { + let Some(tester) = + DenebTester::new(RequestTrigger::GossipUnknownParentBlob { num_parents: 1 }) + else { return; }; @@ -1986,7 +2021,9 @@ mod deneb_only { #[test] fn empty_parent_block_then_parent_blob_blob_trigger() { - let Some(tester) = DenebTester::new(RequestTrigger::GossipUnknownParentBlob) else { + let Some(tester) = + DenebTester::new(RequestTrigger::GossipUnknownParentBlob { num_parents: 1 }) + else { return; }; @@ -2006,7 +2043,9 @@ mod deneb_only { #[test] fn empty_parent_blobs_then_parent_block_blob_trigger() { - let Some(tester) = DenebTester::new(RequestTrigger::GossipUnknownParentBlob) else { + let Some(tester) = + DenebTester::new(RequestTrigger::GossipUnknownParentBlob { num_parents: 1 }) + else { return; }; @@ -2024,4 +2063,29 @@ mod deneb_only { .parent_block_imported() .expect_parent_chain_process(); } + + #[test] + fn parent_blob_unknown_parent_chain() { + let Some(tester) = + DenebTester::new(RequestTrigger::GossipUnknownParentBlob { num_parents: 2 }) + else { + return; + }; + + tester + .block_response() + .expect_empty_beacon_processor() + .parent_block_response() + .parent_blob_response() + .expect_no_penalty() + .expect_block_process() + .parent_block_unknown_parent() + .expect_parent_block_request() + .expect_parent_blobs_request() + .expect_empty_beacon_processor() + .parent_block_response() + .parent_blob_response() + .expect_no_penalty() + .expect_block_process(); + } } diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 2f67161ecf..fff384e195 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -62,6 +62,13 @@ pub fn get_config( fs::remove_dir_all(freezer_db) .map_err(|err| format!("Failed to remove freezer_db: {}", err))?; } + + // Remove the blobs db. + let blobs_db = client_config.get_blobs_db_path(); + if blobs_db.exists() { + fs::remove_dir_all(blobs_db) + .map_err(|err| format!("Failed to remove blobs_db: {}", err))?; + } } // Create `datadir` and any non-existing parent directories. @@ -156,6 +163,10 @@ pub fn get_config( cli_args.is_present("light-client-server"); } + if cli_args.is_present("light-client-server") { + client_config.chain.enable_light_client_server = true; + } + if let Some(cache_size) = clap_utils::parse_optional(cli_args, "shuffling-cache-size")? { client_config.chain.shuffling_cache_size = cache_size; } diff --git a/beacon_node/store/src/metadata.rs b/beacon_node/store/src/metadata.rs index 6fef74d7ff..1675051bd8 100644 --- a/beacon_node/store/src/metadata.rs +++ b/beacon_node/store/src/metadata.rs @@ -4,7 +4,7 @@ use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use types::{Checkpoint, Hash256, Slot}; -pub const CURRENT_SCHEMA_VERSION: SchemaVersion = SchemaVersion(18); +pub const CURRENT_SCHEMA_VERSION: SchemaVersion = SchemaVersion(19); // All the keys that get stored under the `BeaconMeta` column. // diff --git a/book/src/api-lighthouse.md b/book/src/api-lighthouse.md index 32c967c9e0..ce71450987 100644 --- a/book/src/api-lighthouse.md +++ b/book/src/api-lighthouse.md @@ -463,20 +463,6 @@ curl -X GET "http://localhost:5052/lighthouse/eth1/deposit_cache" -H "accept: a } ``` -### `/lighthouse/beacon/states/{state_id}/ssz` - -Obtains a `BeaconState` in SSZ bytes. Useful for obtaining a genesis state. - -The `state_id` parameter is identical to that used in the [Standard Beacon Node API -`beacon/state` -routes](https://ethereum.github.io/beacon-APIs/#/Beacon/getStateRoot). - -```bash -curl -X GET "http://localhost:5052/lighthouse/beacon/states/0/ssz" | jq -``` - -*Example omitted for brevity, the body simply contains SSZ bytes.* - ### `/lighthouse/liveness` POST request that checks if any of the given validators have attested in the given epoch. Returns a list @@ -515,7 +501,7 @@ curl "http://localhost:5052/lighthouse/database/info" | jq ```json { - "schema_version": 16, + "schema_version": 18, "config": { "slots_per_restore_point": 8192, "slots_per_restore_point_set_explicitly": false, @@ -523,18 +509,26 @@ curl "http://localhost:5052/lighthouse/database/info" | jq "historic_state_cache_size": 1, "compact_on_init": false, "compact_on_prune": true, - "prune_payloads": true + "prune_payloads": true, + "prune_blobs": true, + "epochs_per_blob_prune": 1, + "blob_prune_margin_epochs": 0 }, "split": { - "slot": "5485952", - "state_root": "0xcfe5d41e6ab5a9dab0de00d89d97ae55ecaeed3b08e4acda836e69b2bef698b4" + "slot": "7454656", + "state_root": "0xbecfb1c8ee209854c611ebc967daa77da25b27f1a8ef51402fdbe060587d7653", + "block_root": "0x8730e946901b0a406313d36b3363a1b7091604e1346a3410c1a7edce93239a68" }, "anchor": { - "anchor_slot": "5414688", - "oldest_block_slot": "0", - "oldest_block_parent": "0x0000000000000000000000000000000000000000000000000000000000000000", - "state_upper_limit": "5414912", - "state_lower_limit": "8192" + "anchor_slot": "7451168", + "oldest_block_slot": "3962593", + "oldest_block_parent": "0x4a39f21367b3b9cc272744d1e38817bda5daf38d190dc23dc091f09fb54acd97", + "state_upper_limit": "7454720", + "state_lower_limit": "0" + }, + "blob_info": { + "oldest_blob_slot": "7413769", + "blobs_db": true } } ``` diff --git a/book/src/api-vc-endpoints.md b/book/src/api-vc-endpoints.md index f41625ad88..d3b2edafe9 100644 --- a/book/src/api-vc-endpoints.md +++ b/book/src/api-vc-endpoints.md @@ -427,7 +427,7 @@ Example Response Body ## `PATCH /lighthouse/validators/:voting_pubkey` -Update some values for the validator with `voting_pubkey`. Possible fields: `enabled`, `gas_limit`, `builder_proposals`, +Update some values for the validator with `voting_pubkey`. Possible fields: `enabled`, `gas_limit`, `builder_proposals`, `builder_boost_factor`, `prefer_builder_proposals` and `graffiti`. The following example updates a validator from `enabled: true` to `enabled: false`. ### HTTP Specification diff --git a/book/src/builders.md b/book/src/builders.md index e48cc0a884..930d330d99 100644 --- a/book/src/builders.md +++ b/book/src/builders.md @@ -31,6 +31,18 @@ blinded blocks, you should use the following flag: lighthouse vc --builder-proposals ``` With the `--builder-proposals` flag, the validator client will ask for blinded blocks for all validators it manages. + +``` +lighthouse vc --prefer-builder-proposals +``` +With the `--prefer-builder-proposals` flag, the validator client will always prefer blinded blocks, regardless of the payload value, for all validators it manages. + +``` +lighthouse vc --builder-boost-factor +``` +With the `--builder-boost-factor` flag, a percentage multiplier is applied to the builder's payload value when choosing between a +builder payload header and payload from the paired execution node. For example, `--builder-boost-factor 50` will only use the builder payload if it is 2x more profitable than the local payload. + In order to configure whether a validator queries for blinded blocks check out [this section.](#validator-client-configuration) ## Multiple builders @@ -46,9 +58,9 @@ relays, run one of the following services and configure lighthouse to use it wit In the validator client you can configure gas limit and fee recipient on a per-validator basis. If no gas limit is configured, Lighthouse will use a default gas limit of 30,000,000, which is the current default value used in execution engines. You can also enable or disable use of external builders on a per-validator basis rather than using -`--builder-proposals`, which enables external builders for all validators. In order to manage these configurations -per-validator, you can either make updates to the `validator_definitions.yml` file or you can use the HTTP requests -described below. +`--builder-proposals`, `--builder-boost-factor` or `--prefer-builder-proposals`, which apply builder related preferences for all validators. +In order to manage these configurations per-validator, you can either make updates to the `validator_definitions.yml` file +or you can use the HTTP requests described below. Both the gas limit and fee recipient will be passed along as suggestions to connected builders. If there is a discrepancy in either, it will *not* keep you from proposing a block with the builder. This is because the bounds on gas limit are @@ -145,6 +157,7 @@ You can also directly configure these fields in the `validator_definitions.yml` suggested_fee_recipient: "0x6cc8dcbca744a6e4ffedb98e1d0df903b10abd21" gas_limit: 30000001 builder_proposals: true + builder_boost_factor: 50 - enabled: false voting_public_key: "0xa5566f9ec3c6e1fdf362634ebec9ef7aceb0e460e5079714808388e5d48f4ae1e12897fed1bea951c17fa389d511e477" type: local_keystore voting_keystore_path: /home/paul/.lighthouse/validators/0xa5566f9ec3c6e1fdf362634ebec9ef7aceb0e460e5079714808388e5d48f4ae1e12897fed1bea951c17fa389d511e477/voting-keystore.json @@ -152,6 +165,7 @@ You can also directly configure these fields in the `validator_definitions.yml` suggested_fee_recipient: "0xa2e334e71511686bcfe38bb3ee1ad8f6babcc03d" gas_limit: 33333333 builder_proposals: true + prefer_builder_proposals: true ``` ## Circuit breaker conditions diff --git a/book/src/checkpoint-sync.md b/book/src/checkpoint-sync.md index 0c375a5f00..00afea1567 100644 --- a/book/src/checkpoint-sync.md +++ b/book/src/checkpoint-sync.md @@ -1,9 +1,8 @@ # Checkpoint Sync -Since version 2.0.0 Lighthouse supports syncing from a recent finalized checkpoint. This is -substantially faster than syncing from genesis, while still providing all the same features. +Lighthouse supports syncing from a recent finalized checkpoint. This is substantially faster than syncing from genesis, while still providing all the same features. Checkpoint sync is also safer as it protects the node from long-range attacks. Since 4.6.0, checkpoint sync is required by default and genesis sync will no longer work without the use of `--allow-insecure-genesis-sync`. -If you would like to quickly get started with checkpoint sync, read the sections below on: +To quickly get started with checkpoint sync, read the sections below on: 1. [Automatic Checkpoint Sync](#automatic-checkpoint-sync) 2. [Backfilling Blocks](#backfilling-blocks) diff --git a/book/src/contributing.md b/book/src/contributing.md index 6b84843a69..5b0ab48e86 100644 --- a/book/src/contributing.md +++ b/book/src/contributing.md @@ -49,7 +49,7 @@ into the canonical spec. ## Rust Lighthouse adheres to Rust code conventions as outlined in the [**Rust -Styleguide**](https://github.com/rust-dev-tools/fmt-rfcs/blob/master/guide/guide.md). +Styleguide**](https://doc.rust-lang.org/nightly/style-guide/). Please use [clippy](https://github.com/rust-lang/rust-clippy) and [rustfmt](https://github.com/rust-lang/rustfmt) to detect common mistakes and diff --git a/book/src/database-migrations.md b/book/src/database-migrations.md index 5b7b4d4937..a4d28452d4 100644 --- a/book/src/database-migrations.md +++ b/book/src/database-migrations.md @@ -16,7 +16,8 @@ validator client or the slasher**. | Lighthouse version | Release date | Schema version | Downgrade available? | |--------------------|--------------|----------------|----------------------| -| v4.6.0 | Dec 2023 | v18 | yes before Deneb | +| v4.6.0 | Dec 2023 | v19 | yes before Deneb | +| v4.6.0-rc.0 | Dec 2023 | v18 | yes before Deneb | | v4.5.0 | Sep 2023 | v17 | yes | | v4.4.0 | Aug 2023 | v17 | yes | | v4.3.0 | Jul 2023 | v17 | yes | @@ -192,7 +193,8 @@ Here are the steps to prune historic states: | Lighthouse version | Release date | Schema version | Downgrade available? | |--------------------|--------------|----------------|-------------------------------------| -| v4.6.0 | Dec 2023 | v18 | yes before Deneb | +| v4.6.0 | Dec 2023 | v19 | yes before Deneb | +| v4.6.0-rc.0 | Dec 2023 | v18 | yes before Deneb | | v4.5.0 | Sep 2023 | v17 | yes | | v4.4.0 | Aug 2023 | v17 | yes | | v4.3.0 | Jul 2023 | v17 | yes | diff --git a/book/src/help_vc.md b/book/src/help_vc.md index 62b64efd41..bc6deec1e9 100644 --- a/book/src/help_vc.md +++ b/book/src/help_vc.md @@ -56,6 +56,9 @@ FLAGS: machine. Note that logs can often contain sensitive information about your validator and so this flag should be used with caution. For Windows users, the log file permissions will be inherited from the parent folder. --metrics Enable the Prometheus metrics HTTP server. Disabled by default. + --prefer-builder-proposals + If this flag is set, Lighthouse will always prefer blocks constructed by builders, regardless of payload + value. --produce-block-v3 Enable block production via the block v3 endpoint for this validator client. This should only be enabled when paired with a beacon node that has this endpoint implemented. This flag will be enabled by default in @@ -80,6 +83,9 @@ OPTIONS: Comma-separated list of beacon API topics to broadcast to all beacon nodes. Possible values are: none, attestations, blocks, subscriptions, sync-committee. Default (when flag is omitted) is to broadcast subscriptions only. + --builder-boost-factor + Defines the boost factor, a percentage multiplier to apply to the builder's payload value when choosing + between a builder payload header and payload from the local execution node. --builder-registration-timestamp-override This flag takes a unix timestamp value that will be used to override the timestamp used in the builder api registration diff --git a/book/src/help_vm_create.md b/book/src/help_vm_create.md index 505ea8638f..71db3cc599 100644 --- a/book/src/help_vm_create.md +++ b/book/src/help_vm_create.md @@ -42,6 +42,9 @@ OPTIONS: A HTTP(S) address of a beacon node using the beacon-API. If this value is provided, an error will be raised if any validator key here is already known as a validator by that beacon node. This helps prevent the same validator being created twice and therefore slashable conditions. + --builder-boost-factor + Defines the boost factor, a percentage multiplier to apply to the builder's payload value when choosing + between a builder payload header and payload from the local execution node. --builder-proposals When provided, all created validators will attempt to create blocks via builder rather than the local EL. [possible values: true, false] @@ -93,13 +96,18 @@ OPTIONS: --logfile-max-size The maximum size (in MB) each log file can grow to before rotating. If set to 0, background file logging is disabled. [default: 200] - --mnemonic-path If present, the mnemonic will be read in from this file. + --mnemonic-path + If present, the mnemonic will be read in from this file. + --network Name of the Eth2 chain Lighthouse will sync and follow. [possible values: mainnet, prater, goerli, gnosis, chiado, sepolia, holesky] --output-path The path to a directory where the validator and (optionally) deposits files will be created. The directory will be created if it does not exist. + --prefer-builder-proposals + If this flag is set, Lighthouse will always prefer blocks constructed by builders, regardless of payload + value. [possible values: true, false] --safe-slots-to-import-optimistically Used to coordinate manual overrides of the SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY parameter. This flag should only be used if the user has a clear understanding that the broad Ethereum community has elected to override diff --git a/book/src/help_vm_move.md b/book/src/help_vm_move.md index dea440dca9..a89af437a9 100644 --- a/book/src/help_vm_move.md +++ b/book/src/help_vm_move.md @@ -26,10 +26,13 @@ FLAGS: -V, --version Prints version information OPTIONS: + --builder-boost-factor + Defines the boost factor, a percentage multiplier to apply to the builder's payload value when choosing + between a builder payload header and payload from the local execution node. --builder-proposals When provided, all created validators will attempt to create blocks via builder rather than the local EL. [possible values: true, false] - --count The number of validators to move. + --count The number of validators to move. -d, --datadir Used to specify a custom root data directory for lighthouse keys and databases. Defaults to $HOME/.lighthouse/{network} where network is the value of the `network` flag Note: Users should specify @@ -75,6 +78,9 @@ OPTIONS: --network Name of the Eth2 chain Lighthouse will sync and follow. [possible values: mainnet, prater, goerli, gnosis, chiado, sepolia, holesky] + --prefer-builder-proposals + If this flag is set, Lighthouse will always prefer blocks constructed by builders, regardless of payload + value. [possible values: true, false] --safe-slots-to-import-optimistically Used to coordinate manual overrides of the SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY parameter. This flag should only be used if the user has a clear understanding that the broad Ethereum community has elected to override diff --git a/book/src/merge-migration.md b/book/src/merge-migration.md index bab520b569..e2dab9652f 100644 --- a/book/src/merge-migration.md +++ b/book/src/merge-migration.md @@ -207,6 +207,6 @@ guidance for specific setups. - [Ethereum.org: The Merge](https://ethereum.org/en/upgrades/merge/) - [Ethereum Staking Launchpad: Merge Readiness](https://launchpad.ethereum.org/en/merge-readiness). -- [CoinCashew: Ethereum Merge Upgrade Checklist](https://www.coincashew.com/coins/overview-eth/ethereum-merge-upgrade-checklist-for-home-stakers-and-validators) +- [CoinCashew: Ethereum Merge Upgrade Checklist](https://www.coincashew.com/coins/overview-eth/archived-guides/ethereum-merge-upgrade-checklist-for-home-stakers-and-validators) - [EthDocker: Merge Preparation](https://eth-docker.net/About/MergePrep/) - [Remy Roy: How to join the Goerli/Prater merge testnet](https://github.com/remyroy/ethstaker/blob/main/merge-goerli-prater.md) diff --git a/boot_node/Cargo.toml b/boot_node/Cargo.toml index 985c7661fd..bdf01914ad 100644 --- a/boot_node/Cargo.toml +++ b/boot_node/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "boot_node" -version = "4.6.0-rc.0" +version = "4.6.0" authors = ["Sigma Prime "] edition = { workspace = true } diff --git a/common/account_utils/src/validator_definitions.rs b/common/account_utils/src/validator_definitions.rs index 61c65a29ba..f228ce5fdf 100644 --- a/common/account_utils/src/validator_definitions.rs +++ b/common/account_utils/src/validator_definitions.rs @@ -157,6 +157,12 @@ pub struct ValidatorDefinition { #[serde(skip_serializing_if = "Option::is_none")] pub builder_proposals: Option, #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub builder_boost_factor: Option, + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub prefer_builder_proposals: Option, + #[serde(default)] pub description: String, #[serde(flatten)] pub signing_definition: SigningDefinition, @@ -169,6 +175,7 @@ impl ValidatorDefinition { /// ## Notes /// /// This function does not check the password against the keystore. + #[allow(clippy::too_many_arguments)] pub fn new_keystore_with_password>( voting_keystore_path: P, voting_keystore_password_storage: PasswordStorage, @@ -176,6 +183,8 @@ impl ValidatorDefinition { suggested_fee_recipient: Option
, gas_limit: Option, builder_proposals: Option, + builder_boost_factor: Option, + prefer_builder_proposals: Option, ) -> Result { let voting_keystore_path = voting_keystore_path.as_ref().into(); let keystore = @@ -196,6 +205,8 @@ impl ValidatorDefinition { suggested_fee_recipient, gas_limit, builder_proposals, + builder_boost_factor, + prefer_builder_proposals, signing_definition: SigningDefinition::LocalKeystore { voting_keystore_path, voting_keystore_password_path, @@ -344,6 +355,8 @@ impl ValidatorDefinitions { suggested_fee_recipient: None, gas_limit: None, builder_proposals: None, + builder_boost_factor: None, + prefer_builder_proposals: None, signing_definition: SigningDefinition::LocalKeystore { voting_keystore_path, voting_keystore_password_path, diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 05043b33e5..f79bb7462a 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -378,26 +378,27 @@ impl BeaconNodeHttpClient { if let Some(timeout) = timeout { builder = builder.timeout(timeout); } + let response = builder.json(body).send().await?; ok_or_error(response).await } /// Generic POST function supporting arbitrary responses and timeouts. - async fn post_generic_with_ssz_body, U: IntoUrl>( + /// Does not include Content-Type application/json in the request header. + async fn post_generic_json_without_content_type_header( &self, url: U, - body: T, + body: &T, timeout: Option, ) -> Result { let mut builder = self.client.post(url); if let Some(timeout) = timeout { builder = builder.timeout(timeout); } - let response = builder - .header("Content-Type", "application/octet-stream") - .body(body) - .send() - .await?; + + let serialized_body = serde_json::to_vec(body).map_err(Error::InvalidJson)?; + + let response = builder.body(serialized_body).send().await?; ok_or_error(response).await } @@ -871,10 +872,11 @@ impl BeaconNodeHttpClient { .push("beacon") .push("blocks"); - self.post_generic_with_ssz_body( + self.post_generic_with_consensus_version_and_ssz_body( path, block_contents.as_ssz_bytes(), Some(self.timeouts.proposal), + block_contents.signed_block().fork_name_unchecked(), ) .await?; @@ -915,8 +917,13 @@ impl BeaconNodeHttpClient { .push("beacon") .push("blinded_blocks"); - self.post_generic_with_ssz_body(path, block.as_ssz_bytes(), Some(self.timeouts.proposal)) - .await?; + self.post_generic_with_consensus_version_and_ssz_body( + path, + block.as_ssz_bytes(), + Some(self.timeouts.proposal), + block.fork_name_unchecked(), + ) + .await?; Ok(()) } @@ -1082,8 +1089,18 @@ impl BeaconNodeHttpClient { pub async fn get_blobs( &self, block_id: BlockId, + indices: Option<&[u64]>, ) -> Result>>, Error> { - let path = self.get_blobs_path(block_id)?; + let mut path = self.get_blobs_path(block_id)?; + if let Some(indices) = indices { + let indices_string = indices + .iter() + .map(|i| i.to_string()) + .collect::>() + .join(","); + path.query_pairs_mut() + .append_pair("indices", &indices_string); + } let Some(response) = self.get_response(path, |b| b).await.optional()? else { return Ok(None); }; @@ -1258,7 +1275,8 @@ impl BeaconNodeHttpClient { .push("pool") .push("attester_slashings"); - self.post(path, slashing).await?; + self.post_generic_json_without_content_type_header(path, slashing, None) + .await?; Ok(()) } diff --git a/common/eth2/src/lighthouse.rs b/common/eth2/src/lighthouse.rs index 11706f3094..538f1a42d1 100644 --- a/common/eth2/src/lighthouse.rs +++ b/common/eth2/src/lighthouse.rs @@ -8,15 +8,12 @@ mod standard_block_rewards; mod sync_committee_rewards; use crate::{ - ok_or_error, types::{ - BeaconState, ChainSpec, DepositTreeSnapshot, Epoch, EthSpec, FinalizedExecutionBlock, - GenericResponse, ValidatorId, + DepositTreeSnapshot, Epoch, EthSpec, FinalizedExecutionBlock, GenericResponse, ValidatorId, }, - BeaconNodeHttpClient, DepositData, Error, Eth1Data, Hash256, Slot, StateId, StatusCode, + BeaconNodeHttpClient, DepositData, Error, Eth1Data, Hash256, Slot, }; use proto_array::core::ProtoArray; -use reqwest::IntoUrl; use serde::{Deserialize, Serialize}; use ssz::four_byte_option_impl; use ssz_derive::{Decode, Encode}; @@ -371,27 +368,6 @@ pub struct DatabaseInfo { } impl BeaconNodeHttpClient { - /// Perform a HTTP GET request, returning `None` on a 404 error. - async fn get_bytes_opt(&self, url: U) -> Result>, Error> { - let response = self.client.get(url).send().await.map_err(Error::from)?; - match ok_or_error(response).await { - Ok(resp) => Ok(Some( - resp.bytes() - .await - .map_err(Error::from)? - .into_iter() - .collect::>(), - )), - Err(err) => { - if err.status() == Some(StatusCode::NOT_FOUND) { - Ok(None) - } else { - Err(err) - } - } - } - } - /// `GET lighthouse/health` pub async fn get_lighthouse_health(&self) -> Result, Error> { let mut path = self.server.full.clone(); @@ -516,28 +492,6 @@ impl BeaconNodeHttpClient { self.get(path).await } - /// `GET lighthouse/beacon/states/{state_id}/ssz` - pub async fn get_lighthouse_beacon_states_ssz( - &self, - state_id: &StateId, - spec: &ChainSpec, - ) -> Result>, Error> { - let mut path = self.server.full.clone(); - - path.path_segments_mut() - .map_err(|()| Error::InvalidUrl(self.server.clone()))? - .push("lighthouse") - .push("beacon") - .push("states") - .push(&state_id.to_string()) - .push("ssz"); - - self.get_bytes_opt(path) - .await? - .map(|bytes| BeaconState::from_ssz_bytes(&bytes, spec).map_err(Error::InvalidSsz)) - .transpose() - } - /// `GET lighthouse/staking` pub async fn get_lighthouse_staking(&self) -> Result { let mut path = self.server.full.clone(); diff --git a/common/eth2/src/lighthouse_vc/http_client.rs b/common/eth2/src/lighthouse_vc/http_client.rs index 2e6756c63e..83aeea4bfc 100644 --- a/common/eth2/src/lighthouse_vc/http_client.rs +++ b/common/eth2/src/lighthouse_vc/http_client.rs @@ -483,12 +483,15 @@ impl ValidatorClientHttpClient { } /// `PATCH lighthouse/validators/{validator_pubkey}` + #[allow(clippy::too_many_arguments)] pub async fn patch_lighthouse_validators( &self, voting_pubkey: &PublicKeyBytes, enabled: Option, gas_limit: Option, builder_proposals: Option, + builder_boost_factor: Option, + prefer_builder_proposals: Option, graffiti: Option, ) -> Result<(), Error> { let mut path = self.server.full.clone(); @@ -505,6 +508,8 @@ impl ValidatorClientHttpClient { enabled, gas_limit, builder_proposals, + builder_boost_factor, + prefer_builder_proposals, graffiti, }, ) diff --git a/common/eth2/src/lighthouse_vc/types.rs b/common/eth2/src/lighthouse_vc/types.rs index 230293f1b8..d903d7b73d 100644 --- a/common/eth2/src/lighthouse_vc/types.rs +++ b/common/eth2/src/lighthouse_vc/types.rs @@ -32,6 +32,12 @@ pub struct ValidatorRequest { #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub builder_proposals: Option, + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub builder_boost_factor: Option, + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub prefer_builder_proposals: Option, #[serde(with = "serde_utils::quoted_u64")] pub deposit_gwei: u64, } @@ -86,6 +92,12 @@ pub struct ValidatorPatchRequest { #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub graffiti: Option, + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub builder_boost_factor: Option, + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub prefer_builder_proposals: Option, } #[derive(Clone, PartialEq, Serialize, Deserialize)] @@ -105,6 +117,12 @@ pub struct KeystoreValidatorsPostRequest { #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub builder_proposals: Option, + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub builder_boost_factor: Option, + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub prefer_builder_proposals: Option, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -135,6 +153,12 @@ pub struct Web3SignerValidatorRequest { pub client_identity_path: Option, #[serde(skip_serializing_if = "Option::is_none")] pub client_identity_password: Option, + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub builder_boost_factor: Option, + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub prefer_builder_proposals: Option, } #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 98074615b8..a301055f34 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -15,6 +15,7 @@ use ssz_derive::{Decode, Encode}; use std::convert::TryFrom; use std::fmt::{self, Display}; use std::str::{from_utf8, FromStr}; +use std::sync::Arc; use std::time::Duration; use types::beacon_block_body::KzgCommitments; pub use types::*; @@ -1479,15 +1480,15 @@ mod tests { type E = MainnetEthSpec; let spec = ForkName::Capella.make_genesis_spec(E::default_spec()); - let block: PublishBlockRequest = SignedBeaconBlock::from_block( + let block: PublishBlockRequest = Arc::new(SignedBeaconBlock::from_block( BeaconBlock::::Capella(BeaconBlockCapella::empty(&spec)), Signature::empty(), - ) + )) .try_into() .expect("should convert into signed block contents"); let decoded: PublishBlockRequest = - PublishBlockRequest::from_ssz_bytes(&block.as_ssz_bytes(), &spec) + PublishBlockRequest::from_ssz_bytes(&block.as_ssz_bytes(), ForkName::Capella) .expect("should decode Block"); assert!(matches!(decoded, PublishBlockRequest::Block(_))); } @@ -1503,11 +1504,14 @@ mod tests { ); let blobs = BlobsList::::from(vec![Blob::::default()]); let kzg_proofs = KzgProofs::::from(vec![KzgProof::empty()]); - let signed_block_contents = PublishBlockRequest::new(block, Some((kzg_proofs, blobs))); + let signed_block_contents = + PublishBlockRequest::new(Arc::new(block), Some((kzg_proofs, blobs))); - let decoded: PublishBlockRequest = - PublishBlockRequest::from_ssz_bytes(&signed_block_contents.as_ssz_bytes(), &spec) - .expect("should decode BlockAndBlobSidecars"); + let decoded: PublishBlockRequest = PublishBlockRequest::from_ssz_bytes( + &signed_block_contents.as_ssz_bytes(), + ForkName::Deneb, + ) + .expect("should decode BlockAndBlobSidecars"); assert!(matches!(decoded, PublishBlockRequest::BlockContents(_))); } } @@ -1642,7 +1646,7 @@ impl FullBlockContents { ) -> PublishBlockRequest { let (block, maybe_blobs) = self.deconstruct(); let signed_block = block.sign(secret_key, fork, genesis_validators_root, spec); - PublishBlockRequest::new(signed_block, maybe_blobs) + PublishBlockRequest::new(Arc::new(signed_block), maybe_blobs) } } @@ -1673,7 +1677,10 @@ impl Into> for FullBlockContents { } } -pub type SignedBlockContentsTuple = (SignedBeaconBlock, Option<(KzgProofs, BlobsList)>); +pub type SignedBlockContentsTuple = ( + Arc>, + Option<(KzgProofs, BlobsList)>, +); fn parse_required_header( headers: &HeaderMap, @@ -1728,12 +1735,12 @@ impl TryFrom<&HeaderMap> for ProduceBlockV3Metadata { #[ssz(enum_behaviour = "transparent")] pub enum PublishBlockRequest { BlockContents(SignedBlockContents), - Block(SignedBeaconBlock), + Block(Arc>), } impl PublishBlockRequest { pub fn new( - block: SignedBeaconBlock, + block: Arc>, blob_items: Option<(KzgProofs, BlobsList)>, ) -> Self { match blob_items { @@ -1746,23 +1753,12 @@ impl PublishBlockRequest { } } - /// SSZ decode with fork variant determined by slot. - pub fn from_ssz_bytes(bytes: &[u8], spec: &ChainSpec) -> Result { - let slot_len = ::ssz_fixed_len(); - let slot_bytes = bytes - .get(0..slot_len) - .ok_or(DecodeError::InvalidByteLength { - len: bytes.len(), - expected: slot_len, - })?; - - let slot = Slot::from_ssz_bytes(slot_bytes)?; - let fork_at_slot = spec.fork_name_at_slot::(slot); - - match fork_at_slot { + /// SSZ decode with fork variant determined by `fork_name`. + pub fn from_ssz_bytes(bytes: &[u8], fork_name: ForkName) -> Result { + match fork_name { ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { - SignedBeaconBlock::from_ssz_bytes(bytes, spec) - .map(|block| PublishBlockRequest::Block(block)) + SignedBeaconBlock::from_ssz_bytes_for_fork(bytes, fork_name) + .map(|block| PublishBlockRequest::Block(Arc::new(block))) } ForkName::Deneb => { let mut builder = ssz::SszDecoderBuilder::new(bytes); @@ -1771,16 +1767,20 @@ impl PublishBlockRequest { builder.register_type::>()?; let mut decoder = builder.build()?; - let block = decoder - .decode_next_with(|bytes| SignedBeaconBlock::from_ssz_bytes(bytes, spec))?; + let block = decoder.decode_next_with(|bytes| { + SignedBeaconBlock::from_ssz_bytes_for_fork(bytes, fork_name) + })?; let kzg_proofs = decoder.decode_next()?; let blobs = decoder.decode_next()?; - Ok(PublishBlockRequest::new(block, Some((kzg_proofs, blobs)))) + Ok(PublishBlockRequest::new( + Arc::new(block), + Some((kzg_proofs, blobs)), + )) } } } - pub fn signed_block(&self) -> &SignedBeaconBlock { + pub fn signed_block(&self) -> &Arc> { match self { PublishBlockRequest::BlockContents(block_and_sidecars) => { &block_and_sidecars.signed_block @@ -1810,14 +1810,14 @@ pub fn into_full_block_and_blobs( let signed_block = blinded_block .try_into_full_block(None) .ok_or("Failed to build full block with payload".to_string())?; - Ok(PublishBlockRequest::new(signed_block, None)) + Ok(PublishBlockRequest::new(Arc::new(signed_block), None)) } // This variant implies a pre-deneb block Some(FullPayloadContents::Payload(execution_payload)) => { let signed_block = blinded_block .try_into_full_block(Some(execution_payload)) .ok_or("Failed to build full block with payload".to_string())?; - Ok(PublishBlockRequest::new(signed_block, None)) + Ok(PublishBlockRequest::new(Arc::new(signed_block), None)) } // This variant implies a post-deneb block Some(FullPayloadContents::PayloadAndBlobs(payload_and_blobs)) => { @@ -1826,7 +1826,7 @@ pub fn into_full_block_and_blobs( .ok_or("Failed to build full block with payload".to_string())?; Ok(PublishBlockRequest::new( - signed_block, + Arc::new(signed_block), Some(( payload_and_blobs.blobs_bundle.proofs, payload_and_blobs.blobs_bundle.blobs, @@ -1836,10 +1836,10 @@ pub fn into_full_block_and_blobs( } } -impl TryFrom> for PublishBlockRequest { +impl TryFrom>> for PublishBlockRequest { type Error = &'static str; - fn try_from(block: SignedBeaconBlock) -> Result { - match block { + fn try_from(block: Arc>) -> Result { + match *block { SignedBeaconBlock::Base(_) | SignedBeaconBlock::Altair(_) | SignedBeaconBlock::Merge(_) @@ -1860,7 +1860,7 @@ impl From> for PublishBlockRequest { #[derive(Debug, Clone, Serialize, Deserialize, Encode)] #[serde(bound = "T: EthSpec")] pub struct SignedBlockContents { - pub signed_block: SignedBeaconBlock, + pub signed_block: Arc>, pub kzg_proofs: KzgProofs, #[serde(with = "ssz_types::serde_utils::list_of_hex_fixed_vec")] pub blobs: BlobsList, diff --git a/common/eth2_network_config/built_in_network_configs/chiado/config.yaml b/common/eth2_network_config/built_in_network_configs/chiado/config.yaml index fcebaa9bd8..8064ea5556 100644 --- a/common/eth2_network_config/built_in_network_configs/chiado/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/chiado/config.yaml @@ -1,145 +1,38 @@ -# Extends the mainnet preset -PRESET_BASE: gnosis -# needs to exist because of Prysm. Otherwise it conflicts with mainnet genesis -CONFIG_NAME: chiado +PRESET_BASE: 'gnosis' + +# Free-form short name of the network that this configuration applies to - known +# canonical network names include: +# * 'mainnet' - there can be only one +# * 'prater' - testnet +# Must match the regex: [a-z0-9\-] +CONFIG_NAME: 'chiado' + +# Transition +# --------------------------------------------------------------- +# Projected time: 2022-11-04T15:00:00.000Z, block: 680928 +TERMINAL_TOTAL_DIFFICULTY: 231707791542740786049188744689299064356246512 +# By default, don't use these params +TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000 +TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615 + # Genesis +# --------------------------------------------------------------- +# *CUSTOM MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 6000 # 10 October 2022 10:00:00 GMT+0000 MIN_GENESIS_TIME: 1665396000 -GENESIS_DELAY: 300 - -# Projected time: 2022-11-04T15:00:00.000Z, block: 680928 -TERMINAL_TOTAL_DIFFICULTY: 231707791542740786049188744689299064356246512 - -# Deposit contract -# --------------------------------------------------------------- -# NOTE: Don't use a value too high, or Teku rejects it (4294906129 NOK) -DEPOSIT_CHAIN_ID: 10200 -DEPOSIT_NETWORK_ID: 10200 -DEPOSIT_CONTRACT_ADDRESS: 0xb97036A26259B7147018913bD58a774cf91acf25 - -# Misc -# --------------------------------------------------------------- -# 2**6 (= 64) -MAX_COMMITTEES_PER_SLOT: 64 -# 2**7 (= 128) -TARGET_COMMITTEE_SIZE: 128 -# 2**11 (= 2,048) -MAX_VALIDATORS_PER_COMMITTEE: 2048 -# 2**2 (= 4) -MIN_PER_EPOCH_CHURN_LIMIT: 4 -# 2**3 (= 8) -MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: 8 -# 2**12 (= 4096) -CHURN_LIMIT_QUOTIENT: 4096 -# See issue 563 -SHUFFLE_ROUND_COUNT: 90 -# 4 -HYSTERESIS_QUOTIENT: 4 -# 1 (minus 0.25) -HYSTERESIS_DOWNWARD_MULTIPLIER: 1 -# 5 (plus 1.25) -HYSTERESIS_UPWARD_MULTIPLIER: 5 -# Validator -# --------------------------------------------------------------- -# 2**10 (= 1024) ~1.4 hour -ETH1_FOLLOW_DISTANCE: 1024 -# 2**4 (= 16) -TARGET_AGGREGATORS_PER_COMMITTEE: 16 -# 2**0 (= 1) -RANDOM_SUBNETS_PER_VALIDATOR: 1 -# 2**8 (= 256) -EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION: 256 -# 6 (estimate from xDai mainnet) -SECONDS_PER_ETH1_BLOCK: 6 - -# Gwei values -# --------------------------------------------------------------- -# 2**0 * 10**9 (= 1,000,000,000) Gwei -MIN_DEPOSIT_AMOUNT: 1000000000 -# 2**5 * 10**9 (= 32,000,000,000) Gwei -MAX_EFFECTIVE_BALANCE: 32000000000 -# 2**4 * 10**9 (= 16,000,000,000) Gwei -EJECTION_BALANCE: 16000000000 -# 2**0 * 10**9 (= 1,000,000,000) Gwei -EFFECTIVE_BALANCE_INCREMENT: 1000000000 -# Initial values -# --------------------------------------------------------------- # GBC area code GENESIS_FORK_VERSION: 0x0000006f -BLS_WITHDRAWAL_PREFIX: 0x00 -# Time parameters -# --------------------------------------------------------------- -# 5 seconds -SECONDS_PER_SLOT: 5 -# 2**0 (= 1) slots 12 seconds -MIN_ATTESTATION_INCLUSION_DELAY: 1 -# 2**4 (= 16) slots 1.87 minutes -SLOTS_PER_EPOCH: 16 -# 2**0 (= 1) epochs 1.87 minutes -MIN_SEED_LOOKAHEAD: 1 -# 2**2 (= 4) epochs 7.47 minutes -MAX_SEED_LOOKAHEAD: 4 -# 2**6 (= 64) epochs ~2 hours -EPOCHS_PER_ETH1_VOTING_PERIOD: 64 -# 2**13 (= 8,192) slots ~15.9 hours -SLOTS_PER_HISTORICAL_ROOT: 8192 -# 2**8 (= 256) epochs ~8 hours -MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 -# 2**8 (= 256) epochs ~8 hours -SHARD_COMMITTEE_PERIOD: 256 -# 2**2 (= 4) epochs 7.47 minutes -MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 +# *CUSTOM +GENESIS_DELAY: 300 -# State vector lengths + +# Forking # --------------------------------------------------------------- -# 2**16 (= 65,536) epochs ~85 days -EPOCHS_PER_HISTORICAL_VECTOR: 65536 -# 2**13 (= 8,192) epochs ~10.6 days -EPOCHS_PER_SLASHINGS_VECTOR: 8192 -# 2**24 (= 16,777,216) historical roots, ~15,243 years -HISTORICAL_ROOTS_LIMIT: 16777216 -# 2**40 (= 1,099,511,627,776) validator spots -VALIDATOR_REGISTRY_LIMIT: 1099511627776 -# Reward and penalty quotients -# --------------------------------------------------------------- -# 25 -BASE_REWARD_FACTOR: 25 -# 2**9 (= 512) -WHISTLEBLOWER_REWARD_QUOTIENT: 512 -# 2**3 (= 8) -PROPOSER_REWARD_QUOTIENT: 8 -# 2**26 (= 67,108,864) -INACTIVITY_PENALTY_QUOTIENT: 67108864 -# 2**7 (= 128) (lower safety margin at Phase 0 genesis) -MIN_SLASHING_PENALTY_QUOTIENT: 128 -# 1 (lower safety margin at Phase 0 genesis) -PROPORTIONAL_SLASHING_MULTIPLIER: 1 -# Max operations per block -# --------------------------------------------------------------- -# 2**4 (= 16) -MAX_PROPOSER_SLASHINGS: 16 -# 2**1 (= 2) -MAX_ATTESTER_SLASHINGS: 2 -# 2**7 (= 128) -MAX_ATTESTATIONS: 128 -# 2**4 (= 16) -MAX_DEPOSITS: 16 -# 2**4 (= 16) -MAX_VOLUNTARY_EXITS: 16 -# Signature domains -# --------------------------------------------------------------- -DOMAIN_BEACON_PROPOSER: 0x00000000 -DOMAIN_BEACON_ATTESTER: 0x01000000 -DOMAIN_RANDAO: 0x02000000 -DOMAIN_DEPOSIT: 0x03000000 -DOMAIN_VOLUNTARY_EXIT: 0x04000000 -DOMAIN_SELECTION_PROOF: 0x05000000 -DOMAIN_AGGREGATE_AND_PROOF: 0x06000000 -DOMAIN_SYNC_COMMITTEE: 0x07000000 -DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF: 0x08000000 -DOMAIN_CONTRIBUTION_AND_PROOF: 0x09000000 +# Some forks are disabled for now: +# - These may be re-assigned to another fork-version later +# - Temporarily set to max uint64 value: 2**64 - 1 # Altair ALTAIR_FORK_VERSION: 0x0100006f @@ -150,7 +43,95 @@ BELLATRIX_FORK_EPOCH: 180 # Mon Oct 10 2022 14:00:00 GMT+0000 # Capella CAPELLA_FORK_VERSION: 0x0300006f CAPELLA_FORK_EPOCH: 244224 # Wed May 24 2023 13:12:00 GMT+0000 +# Deneb +DENEB_FORK_VERSION: 0x0400006f +DENEB_FORK_EPOCH: 516608 # Wed Jan 31 2024 18:15:40 GMT+0000 + +# Time parameters +# --------------------------------------------------------------- +# 5 seconds +SECONDS_PER_SLOT: 5 +# 6 (estimate from xDai mainnet) +SECONDS_PER_ETH1_BLOCK: 6 +# 2**8 (= 256) epochs ~5.7 hours +MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 +# 2**8 (= 256) epochs ~5.7 hours +SHARD_COMMITTEE_PERIOD: 256 +# 2**10 (= 1024) ~1.4 hour +ETH1_FOLLOW_DISTANCE: 1024 + + +# Validator cycle +# --------------------------------------------------------------- +# 2**2 (= 4) INACTIVITY_SCORE_BIAS: 4 # 2**4 (= 16) INACTIVITY_SCORE_RECOVERY_RATE: 16 +# 2**4 * 10**9 (= 16,000,000,000) Gwei +EJECTION_BALANCE: 16000000000 +# 2**2 (= 4) +MIN_PER_EPOCH_CHURN_LIMIT: 4 +# 2**12 (= 4096) +CHURN_LIMIT_QUOTIENT: 4096 +# [New in Deneb:EIP7514] 2* +MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: 2 + +# Fork choice +# --------------------------------------------------------------- +# 40% +PROPOSER_SCORE_BOOST: 40 +# 20% +REORG_HEAD_WEIGHT_THRESHOLD: 20 +# 160% +REORG_PARENT_WEIGHT_THRESHOLD: 160 +# `2` epochs +REORG_MAX_EPOCHS_SINCE_FINALIZATION: 2 + + +# Deposit contract +# --------------------------------------------------------------- +# xDai Mainnet +DEPOSIT_CHAIN_ID: 10200 +DEPOSIT_NETWORK_ID: 10200 +DEPOSIT_CONTRACT_ADDRESS: 0xb97036A26259B7147018913bD58a774cf91acf25 + +# Networking +# --------------------------------------------------------------- +# `10 * 2**20` (= 10485760, 10 MiB) +GOSSIP_MAX_SIZE: 10485760 +# `2**10` (= 1024) +MAX_REQUEST_BLOCKS: 1024 +# `2**8` (= 256) +EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 +# 33024, ~31 days +MIN_EPOCHS_FOR_BLOCK_REQUESTS: 33024 +# `10 * 2**20` (=10485760, 10 MiB) +MAX_CHUNK_SIZE: 10485760 +# 5s +TTFB_TIMEOUT: 5 +# 10s +RESP_TIMEOUT: 10 +ATTESTATION_PROPAGATION_SLOT_RANGE: 32 +# 500ms +MAXIMUM_GOSSIP_CLOCK_DISPARITY: 500 +MESSAGE_DOMAIN_INVALID_SNAPPY: 0x00000000 +MESSAGE_DOMAIN_VALID_SNAPPY: 0x01000000 +# 2 subnets per node +SUBNETS_PER_NODE: 2 +# 2**8 (= 64) +ATTESTATION_SUBNET_COUNT: 64 +ATTESTATION_SUBNET_EXTRA_BITS: 0 +# ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS +ATTESTATION_SUBNET_PREFIX_BITS: 6 + +# Deneb +# `2**7` (=128) +MAX_REQUEST_BLOCKS_DENEB: 128 +# MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK +MAX_REQUEST_BLOB_SIDECARS: 768 +# `2**14` (= 16384 epochs, ~15 days) +MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 16384 +# `6` +BLOB_SIDECAR_SUBNET_COUNT: 6 + diff --git a/common/eth2_network_config/built_in_network_configs/holesky/config.yaml b/common/eth2_network_config/built_in_network_configs/holesky/config.yaml index d699b9a39f..0bb72ebd88 100644 --- a/common/eth2_network_config/built_in_network_configs/holesky/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/holesky/config.yaml @@ -33,6 +33,10 @@ TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615 CAPELLA_FORK_VERSION: 0x04017000 CAPELLA_FORK_EPOCH: 256 +# Deneb +DENEB_FORK_VERSION: 0x05017000 +DENEB_FORK_EPOCH: 29696 + # Time parameters # --------------------------------------------------------------- # 12 seconds @@ -57,10 +61,10 @@ INACTIVITY_SCORE_RECOVERY_RATE: 16 EJECTION_BALANCE: 28000000000 # 2**2 (= 4) MIN_PER_EPOCH_CHURN_LIMIT: 4 -# 2**3 (= 8) -MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: 8 # 2**16 (= 65,536) CHURN_LIMIT_QUOTIENT: 65536 +# [New in Deneb:EIP7514] 2**3 (= 8) +MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: 8 # Fork choice # --------------------------------------------------------------- diff --git a/common/eth2_network_config/built_in_network_configs/sepolia/config.yaml b/common/eth2_network_config/built_in_network_configs/sepolia/config.yaml index b3dbb8a115..33a5ccb3fe 100644 --- a/common/eth2_network_config/built_in_network_configs/sepolia/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/sepolia/config.yaml @@ -33,12 +33,8 @@ CAPELLA_FORK_VERSION: 0x90000072 CAPELLA_FORK_EPOCH: 56832 # Deneb -DENEB_FORK_VERSION: 0x03001020 -DENEB_FORK_EPOCH: 18446744073709551615 - -# Sharding -SHARDING_FORK_VERSION: 0x04001020 -SHARDING_FORK_EPOCH: 18446744073709551615 +DENEB_FORK_VERSION: 0x90000073 +DENEB_FORK_EPOCH: 132608 # Time parameters # --------------------------------------------------------------- @@ -64,11 +60,10 @@ INACTIVITY_SCORE_RECOVERY_RATE: 16 EJECTION_BALANCE: 16000000000 # 2**2 (= 4) MIN_PER_EPOCH_CHURN_LIMIT: 4 -# 2**3 (= 8) -MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: 8 # 2**16 (= 65,536) CHURN_LIMIT_QUOTIENT: 65536 - +# [New in Deneb:EIP7514] 2**3 (= 8) +MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: 8 # Fork choice # --------------------------------------------------------------- @@ -81,16 +76,41 @@ DEPOSIT_CHAIN_ID: 11155111 DEPOSIT_NETWORK_ID: 11155111 DEPOSIT_CONTRACT_ADDRESS: 0x7f02C3E3c98b133055B8B348B2Ac625669Ed295D -# Network +# Networking # --------------------------------------------------------------- -SUBNETS_PER_NODE: 2 +# `10 * 2**20` (= 10485760, 10 MiB) GOSSIP_MAX_SIZE: 10485760 +# `2**10` (= 1024) +MAX_REQUEST_BLOCKS: 1024 +# `2**8` (= 256) +EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 +# `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months) MIN_EPOCHS_FOR_BLOCK_REQUESTS: 33024 +# `10 * 2**20` (=10485760, 10 MiB) MAX_CHUNK_SIZE: 10485760 +# 5s TTFB_TIMEOUT: 5 +# 10s RESP_TIMEOUT: 10 +ATTESTATION_PROPAGATION_SLOT_RANGE: 32 +# 500ms +MAXIMUM_GOSSIP_CLOCK_DISPARITY: 500 MESSAGE_DOMAIN_INVALID_SNAPPY: 0x00000000 MESSAGE_DOMAIN_VALID_SNAPPY: 0x01000000 +# 2 subnets per node +SUBNETS_PER_NODE: 2 +# 2**8 (= 64) ATTESTATION_SUBNET_COUNT: 64 ATTESTATION_SUBNET_EXTRA_BITS: 0 +# ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS ATTESTATION_SUBNET_PREFIX_BITS: 6 + +# Deneb +# `2**7` (=128) +MAX_REQUEST_BLOCKS_DENEB: 128 +# MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK +MAX_REQUEST_BLOB_SIDECARS: 768 +# `2**12` (= 4096 epochs, ~18 days) +MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 +# `6` +BLOB_SIDECAR_SUBNET_COUNT: 6 diff --git a/common/lighthouse_version/src/lib.rs b/common/lighthouse_version/src/lib.rs index db31826eca..af11723487 100644 --- a/common/lighthouse_version/src/lib.rs +++ b/common/lighthouse_version/src/lib.rs @@ -17,8 +17,8 @@ pub const VERSION: &str = git_version!( // NOTE: using --match instead of --exclude for compatibility with old Git "--match=thiswillnevermatchlol" ], - prefix = "Lighthouse/v4.6.0-rc.0-", - fallback = "Lighthouse/v4.6.0-rc.0" + prefix = "Lighthouse/v4.6.0-", + fallback = "Lighthouse/v4.6.0" ); /// Returns `VERSION`, but with platform information appended to the end. diff --git a/common/logging/src/lib.rs b/common/logging/src/lib.rs index 5217d541cf..caf3e1d2fb 100644 --- a/common/logging/src/lib.rs +++ b/common/logging/src/lib.rs @@ -10,6 +10,7 @@ use std::io::{Result, Write}; use std::path::PathBuf; use std::time::{Duration, Instant}; use tracing_appender::non_blocking::NonBlocking; +use tracing_appender::rolling::{RollingFileAppender, Rotation}; use tracing_logging_layer::LoggingLayer; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; @@ -21,7 +22,6 @@ mod tracing_logging_layer; mod tracing_metrics_layer; pub use sse_logging_components::SSELoggingComponents; -pub use tracing_logging_layer::cleanup_logging_task; pub use tracing_metrics_layer::MetricsLayer; /// The minimum interval between log messages indicating that a queue is full. @@ -234,10 +234,27 @@ pub fn create_tracing_layer(base_tracing_log_path: PathBuf, turn_on_terminal_log } }; - let libp2p_writer = - tracing_appender::rolling::daily(base_tracing_log_path.clone(), "libp2p.log"); - let discv5_writer = - tracing_appender::rolling::daily(base_tracing_log_path.clone(), "discv5.log"); + let Ok(libp2p_writer) = RollingFileAppender::builder() + .rotation(Rotation::DAILY) + .max_log_files(2) + .filename_prefix("libp2p") + .filename_suffix("log") + .build(base_tracing_log_path.clone()) + else { + eprintln!("Failed to initialize libp2p rolling file appender"); + return; + }; + + let Ok(discv5_writer) = RollingFileAppender::builder() + .rotation(Rotation::DAILY) + .max_log_files(2) + .filename_prefix("discv5") + .filename_suffix("log") + .build(base_tracing_log_path.clone()) + else { + eprintln!("Failed to initialize discv5 rolling file appender"); + return; + }; let (libp2p_non_blocking_writer, libp2p_guard) = NonBlocking::new(libp2p_writer); let (discv5_non_blocking_writer, discv5_guard) = NonBlocking::new(discv5_writer); diff --git a/common/logging/src/tracing_logging_layer.rs b/common/logging/src/tracing_logging_layer.rs index a74e24bdb6..e7d9109beb 100644 --- a/common/logging/src/tracing_logging_layer.rs +++ b/common/logging/src/tracing_logging_layer.rs @@ -1,5 +1,4 @@ -use chrono::{naive::Days, prelude::*}; -use slog::{debug, warn}; +use chrono::prelude::*; use std::io::Write; use tracing::Subscriber; use tracing_appender::non_blocking::{NonBlocking, WorkerGuard}; @@ -55,61 +54,3 @@ impl tracing_core::field::Visit for LogMessageExtractor { self.message = format!("{} {:?}", self.message, value); } } - -/// Creates a long lived async task that routinely deletes old tracing log files -pub async fn cleanup_logging_task(path: std::path::PathBuf, log: slog::Logger) { - loop { - // Delay for 1 day and then prune old logs - tokio::time::sleep(std::time::Duration::from_secs(60 * 60 * 24)).await; - - let Some(yesterday_date) = chrono::prelude::Local::now() - .naive_local() - .checked_sub_days(Days::new(1)) - else { - warn!(log, "Could not calculate the current date"); - return; - }; - - // Search for old log files - let dir = path.as_path(); - - if dir.is_dir() { - let Ok(files) = std::fs::read_dir(dir) else { - warn!(log, "Could not read log directory contents"; "path" => ?dir); - break; - }; - - for file in files { - let Ok(dir_entry) = file else { - warn!(log, "Could not read file"); - continue; - }; - - let Ok(file_name) = dir_entry.file_name().into_string() else { - warn!(log, "Could not read file"; "file" => ?dir_entry); - continue; - }; - - if file_name.starts_with("libp2p.log") | file_name.starts_with("discv5.log") { - let log_file_date = file_name.split('.').collect::>(); - if log_file_date.len() == 3 { - let Ok(log_file_date_type) = - NaiveDate::parse_from_str(log_file_date[2], "%Y-%m-%d") - else { - warn!(log, "Could not parse log file date"; "file" => file_name); - continue; - }; - - if log_file_date_type < yesterday_date.into() { - // Delete the file, its too old - debug!(log, "Removing old log file"; "file" => &file_name); - if let Err(e) = std::fs::remove_file(dir_entry.path()) { - warn!(log, "Failed to remove log file"; "file" => file_name, "error" => %e); - } - } - } - } - } - } - } -} diff --git a/common/warp_utils/Cargo.toml b/common/warp_utils/Cargo.toml index 85c1901bad..0d33de998e 100644 --- a/common/warp_utils/Cargo.toml +++ b/common/warp_utils/Cargo.toml @@ -14,8 +14,10 @@ beacon_chain = { workspace = true } state_processing = { workspace = true } safe_arith = { workspace = true } serde = { workspace = true } +serde_json = { workspace = true } tokio = { workspace = true } headers = "0.3.2" lighthouse_metrics = { workspace = true } lazy_static = { workspace = true } serde_array_query = "0.1.0" +bytes = { workspace = true } diff --git a/common/warp_utils/src/json.rs b/common/warp_utils/src/json.rs new file mode 100644 index 0000000000..203a6495a4 --- /dev/null +++ b/common/warp_utils/src/json.rs @@ -0,0 +1,22 @@ +use bytes::Bytes; +use serde::de::DeserializeOwned; +use std::error::Error as StdError; +use warp::{Filter, Rejection}; + +use crate::reject; + +struct Json; + +type BoxError = Box; + +impl Json { + fn decode(bytes: Bytes) -> Result { + serde_json::from_slice(&bytes).map_err(Into::into) + } +} + +pub fn json() -> impl Filter + Copy { + warp::body::bytes().and_then(|bytes: Bytes| async move { + Json::decode(bytes).map_err(|err| reject::custom_deserialize_error(format!("{:?}", err))) + }) +} diff --git a/common/warp_utils/src/lib.rs b/common/warp_utils/src/lib.rs index 77d61251f2..55ee423fa4 100644 --- a/common/warp_utils/src/lib.rs +++ b/common/warp_utils/src/lib.rs @@ -2,6 +2,7 @@ //! Lighthouse project. E.g., the `http_api` and `http_metrics` crates. pub mod cors; +pub mod json; pub mod metrics; pub mod query; pub mod reject; diff --git a/common/warp_utils/src/reject.rs b/common/warp_utils/src/reject.rs index cf3d11af8d..b6bb5ace3d 100644 --- a/common/warp_utils/src/reject.rs +++ b/common/warp_utils/src/reject.rs @@ -82,6 +82,15 @@ pub fn custom_bad_request(msg: String) -> warp::reject::Rejection { warp::reject::custom(CustomBadRequest(msg)) } +#[derive(Debug)] +pub struct CustomDeserializeError(pub String); + +impl Reject for CustomDeserializeError {} + +pub fn custom_deserialize_error(msg: String) -> warp::reject::Rejection { + warp::reject::custom(CustomDeserializeError(msg)) +} + #[derive(Debug)] pub struct CustomServerError(pub String); @@ -161,6 +170,9 @@ pub async fn handle_rejection(err: warp::Rejection) -> Result() { + message = format!("BAD_REQUEST: body deserialize error: {}", e.0); + code = StatusCode::BAD_REQUEST; } else if let Some(e) = err.find::() { message = format!("BAD_REQUEST: body deserialize error: {}", e); code = StatusCode::BAD_REQUEST; diff --git a/consensus/fork_choice/tests/tests.rs b/consensus/fork_choice/tests/tests.rs index c21b8ed567..649fbcc555 100644 --- a/consensus/fork_choice/tests/tests.rs +++ b/consensus/fork_choice/tests/tests.rs @@ -323,8 +323,9 @@ impl ForkChoiceTest { ) .unwrap(); let slot = self.harness.get_current_slot(); - let (mut block_tuple, mut state) = self.harness.make_block(state, slot).await; - func(&mut block_tuple.0, &mut state); + let ((block_arc, _block_blobs), mut state) = self.harness.make_block(state, slot).await; + let mut block = (*block_arc).clone(); + func(&mut block, &mut state); let current_slot = self.harness.get_current_slot(); self.harness .chain @@ -332,8 +333,8 @@ impl ForkChoiceTest { .fork_choice_write_lock() .on_block( current_slot, - block_tuple.0.message(), - block_tuple.0.canonical_root(), + block.message(), + block.canonical_root(), Duration::from_secs(0), &state, PayloadVerificationStatus::Verified, @@ -366,8 +367,9 @@ impl ForkChoiceTest { ) .unwrap(); let slot = self.harness.get_current_slot(); - let (mut block_tuple, mut state) = self.harness.make_block(state, slot).await; - mutation_func(&mut block_tuple.0, &mut state); + let ((block_arc, _block_blobs), mut state) = self.harness.make_block(state, slot).await; + let mut block = (*block_arc).clone(); + mutation_func(&mut block, &mut state); let current_slot = self.harness.get_current_slot(); let err = self .harness @@ -376,8 +378,8 @@ impl ForkChoiceTest { .fork_choice_write_lock() .on_block( current_slot, - block_tuple.0.message(), - block_tuple.0.canonical_root(), + block.message(), + block.canonical_root(), Duration::from_secs(0), &state, PayloadVerificationStatus::Verified, diff --git a/consensus/state_processing/src/per_block_processing/tests.rs b/consensus/state_processing/src/per_block_processing/tests.rs index 75eb438967..83fd0f232c 100644 --- a/consensus/state_processing/src/per_block_processing/tests.rs +++ b/consensus/state_processing/src/per_block_processing/tests.rs @@ -90,7 +90,7 @@ async fn invalid_block_header_state_slot() { let slot = state.slot() + Slot::new(1); let ((signed_block, _), mut state) = harness.make_block_return_pre_state(state, slot).await; - let (mut block, signature) = signed_block.deconstruct(); + let (mut block, signature) = (*signed_block).clone().deconstruct(); *block.slot_mut() = slot + Slot::new(1); let mut ctxt = ConsensusContext::new(block.slot()); @@ -123,7 +123,7 @@ async fn invalid_parent_block_root() { let ((signed_block, _), mut state) = harness .make_block_return_pre_state(state, slot + Slot::new(1)) .await; - let (mut block, signature) = signed_block.deconstruct(); + let (mut block, signature) = (*signed_block).clone().deconstruct(); *block.parent_root_mut() = Hash256::from([0xAA; 32]); let mut ctxt = ConsensusContext::new(block.slot()); @@ -158,7 +158,7 @@ async fn invalid_block_signature() { let ((signed_block, _), mut state) = harness .make_block_return_pre_state(state, slot + Slot::new(1)) .await; - let (block, _) = signed_block.deconstruct(); + let (block, _) = (*signed_block).clone().deconstruct(); let mut ctxt = ConsensusContext::new(block.slot()); let result = per_block_processing( diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index e2e25f24b8..bdc3be9ece 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -1866,6 +1866,7 @@ impl BeaconState { }; // 2. Get all `BeaconState` leaves. + self.initialize_tree_hash_cache(); let mut cache = self .tree_hash_cache_mut() .take() diff --git a/consensus/types/src/light_client_bootstrap.rs b/consensus/types/src/light_client_bootstrap.rs index 616aced483..6660783abd 100644 --- a/consensus/types/src/light_client_bootstrap.rs +++ b/consensus/types/src/light_client_bootstrap.rs @@ -9,7 +9,7 @@ use ssz_derive::{Decode, Encode}; use std::sync::Arc; use test_random_derive::TestRandom; -/// A LightClientBootstrap is the initializer we send over to lightclient nodes +/// A LightClientBootstrap is the initializer we send over to light_client nodes /// that are trying to generate their basic storage when booting up. #[derive( Debug, diff --git a/consensus/types/src/light_client_finality_update.rs b/consensus/types/src/light_client_finality_update.rs index 87601b8156..494e68b63f 100644 --- a/consensus/types/src/light_client_finality_update.rs +++ b/consensus/types/src/light_client_finality_update.rs @@ -11,7 +11,7 @@ use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; use tree_hash::TreeHash; -/// A LightClientFinalityUpdate is the update lightclient request or received by a gossip that +/// A LightClientFinalityUpdate is the update light_client request or received by a gossip that /// signal a new finalized beacon block header for the light client sync protocol. #[derive( Debug, diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index fa7745ad97..2f7975161c 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -39,6 +39,7 @@ pub trait ExecPayload: Debug + Clone + PartialEq + Hash + TreeHash + fn transactions(&self) -> Option<&Transactions>; /// fork-specific fields fn withdrawals_root(&self) -> Result; + fn blob_gas_used(&self) -> Result; /// Is this a default payload with 0x0 roots for transactions and withdrawals? fn is_default_with_zero_roots(&self) -> bool; @@ -254,6 +255,13 @@ impl ExecPayload for FullPayload { } } + fn blob_gas_used(&self) -> Result { + match self { + FullPayload::Merge(_) | FullPayload::Capella(_) => Err(Error::IncorrectStateVariant), + FullPayload::Deneb(ref inner) => Ok(inner.execution_payload.blob_gas_used), + } + } + fn is_default_with_zero_roots<'a>(&'a self) -> bool { map_full_payload_ref!(&'a _, self.to_ref(), move |payload, cons| { cons(payload); @@ -372,6 +380,15 @@ impl<'b, T: EthSpec> ExecPayload for FullPayloadRef<'b, T> { } } + fn blob_gas_used(&self) -> Result { + match self { + FullPayloadRef::Merge(_) | FullPayloadRef::Capella(_) => { + Err(Error::IncorrectStateVariant) + } + FullPayloadRef::Deneb(inner) => Ok(inner.execution_payload.blob_gas_used), + } + } + fn is_default_with_zero_roots<'a>(&'a self) -> bool { map_full_payload_ref!(&'a _, self, move |payload, cons| { cons(payload); @@ -533,6 +550,15 @@ impl ExecPayload for BlindedPayload { } } + fn blob_gas_used(&self) -> Result { + match self { + BlindedPayload::Merge(_) | BlindedPayload::Capella(_) => { + Err(Error::IncorrectStateVariant) + } + BlindedPayload::Deneb(ref inner) => Ok(inner.execution_payload_header.blob_gas_used), + } + } + fn is_default_with_zero_roots(&self) -> bool { self.to_ref().is_default_with_zero_roots() } @@ -621,6 +647,15 @@ impl<'b, T: EthSpec> ExecPayload for BlindedPayloadRef<'b, T> { } } + fn blob_gas_used(&self) -> Result { + match self { + BlindedPayloadRef::Merge(_) | BlindedPayloadRef::Capella(_) => { + Err(Error::IncorrectStateVariant) + } + BlindedPayloadRef::Deneb(inner) => Ok(inner.execution_payload_header.blob_gas_used), + } + } + fn is_default_with_zero_roots<'a>(&'a self) -> bool { map_blinded_payload_ref!(&'b _, self, move |payload, cons| { cons(payload); @@ -646,7 +681,8 @@ macro_rules! impl_exec_payload_common { $block_type_variant:ident, // Blinded | Full $is_default_with_empty_roots:block, $f:block, - $g:block) => { + $g:block, + $h:block) => { impl ExecPayload for $wrapper_type { fn block_type() -> BlockType { BlockType::$block_type_variant @@ -704,6 +740,11 @@ macro_rules! impl_exec_payload_common { let g = $g; g(self) } + + fn blob_gas_used(&self) -> Result { + let h = $h; + h(self) + } } impl From<$wrapped_type> for $wrapper_type { @@ -741,6 +782,14 @@ macro_rules! impl_exec_payload_for_fork { wrapper_ref_type.withdrawals_root() }; c + }, + { + let c: for<'a> fn(&'a $wrapper_type_header) -> Result = + |payload: &$wrapper_type_header| { + let wrapper_ref_type = BlindedPayloadRef::$fork_variant(&payload); + wrapper_ref_type.blob_gas_used() + }; + c } ); @@ -820,6 +869,14 @@ macro_rules! impl_exec_payload_for_fork { wrapper_ref_type.withdrawals_root() }; c + }, + { + let c: for<'a> fn(&'a $wrapper_type_full) -> Result = + |payload: &$wrapper_type_full| { + let wrapper_ref_type = FullPayloadRef::$fork_variant(&payload); + wrapper_ref_type.blob_gas_used() + }; + c } ); diff --git a/consensus/types/src/signed_beacon_block.rs b/consensus/types/src/signed_beacon_block.rs index 24b5e27c8c..37304de1f1 100644 --- a/consensus/types/src/signed_beacon_block.rs +++ b/consensus/types/src/signed_beacon_block.rs @@ -104,6 +104,16 @@ impl> SignedBeaconBlock Self::from_ssz_bytes_with(bytes, |bytes| BeaconBlock::from_ssz_bytes(bytes, spec)) } + /// SSZ decode with explicit fork variant. + pub fn from_ssz_bytes_for_fork( + bytes: &[u8], + fork_name: ForkName, + ) -> Result { + Self::from_ssz_bytes_with(bytes, |bytes| { + BeaconBlock::from_ssz_bytes_for_fork(bytes, fork_name) + }) + } + /// SSZ decode which attempts to decode all variants (slow). pub fn any_from_ssz_bytes(bytes: &[u8]) -> Result { Self::from_ssz_bytes_with(bytes, BeaconBlock::any_from_ssz_bytes) diff --git a/lcli/Cargo.toml b/lcli/Cargo.toml index fb980ac7f2..1bba8242c5 100644 --- a/lcli/Cargo.toml +++ b/lcli/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lcli" description = "Lighthouse CLI (modeled after zcli)" -version = "4.6.0-rc.0" +version = "4.6.0" authors = ["Paul Hauner "] edition = { workspace = true } diff --git a/lcli/Dockerfile b/lcli/Dockerfile index 2ff4706a91..aed3628cf3 100644 --- a/lcli/Dockerfile +++ b/lcli/Dockerfile @@ -4,10 +4,10 @@ FROM rust:1.73.0-bullseye AS builder RUN apt-get update && apt-get -y upgrade && apt-get install -y cmake libclang-dev COPY . lighthouse -ARG PORTABLE -ENV PORTABLE $PORTABLE +ARG FEATURES +ENV FEATURES $FEATURES RUN cd lighthouse && make install-lcli FROM ubuntu:22.04 RUN apt-get update && apt-get -y upgrade && apt-get clean && rm -rf /var/lib/apt/lists/* -COPY --from=builder /usr/local/cargo/bin/lcli /usr/local/bin/lcli \ No newline at end of file +COPY --from=builder /usr/local/cargo/bin/lcli /usr/local/bin/lcli diff --git a/lighthouse/Cargo.toml b/lighthouse/Cargo.toml index c402e5b932..8517c66c38 100644 --- a/lighthouse/Cargo.toml +++ b/lighthouse/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lighthouse" -version = "4.6.0-rc.0" +version = "4.6.0" authors = ["Sigma Prime "] edition = { workspace = true } autotests = false diff --git a/lighthouse/src/main.rs b/lighthouse/src/main.rs index b8cedfde0d..06eb06fc08 100644 --- a/lighthouse/src/main.rs +++ b/lighthouse/src/main.rs @@ -542,13 +542,6 @@ fn run( let turn_on_terminal_logs = matches.is_present("env_log"); - // Run a task to clean up old tracing logs. - let log_cleaner_context = environment.service_context("log_cleaner".to_string()); - log_cleaner_context.executor.spawn( - logging::cleanup_logging_task(path.clone(), log.clone()), - "log_cleaner", - ); - logging::create_tracing_layer(path, turn_on_terminal_logs); // Allow Prometheus to export the time at which the process was started. diff --git a/lighthouse/tests/account_manager.rs b/lighthouse/tests/account_manager.rs index 63d79fceb2..f82e3ec713 100644 --- a/lighthouse/tests/account_manager.rs +++ b/lighthouse/tests/account_manager.rs @@ -492,6 +492,8 @@ fn validator_import_launchpad() { suggested_fee_recipient: None, gas_limit: None, builder_proposals: None, + builder_boost_factor: None, + prefer_builder_proposals: None, voting_public_key: keystore.public_key().unwrap(), signing_definition: SigningDefinition::LocalKeystore { voting_keystore_path, @@ -614,6 +616,8 @@ fn validator_import_launchpad_no_password_then_add_password() { suggested_fee_recipient: None, gas_limit: None, builder_proposals: None, + builder_boost_factor: None, + prefer_builder_proposals: None, voting_public_key: keystore.public_key().unwrap(), signing_definition: SigningDefinition::LocalKeystore { voting_keystore_path, @@ -640,6 +644,8 @@ fn validator_import_launchpad_no_password_then_add_password() { suggested_fee_recipient: None, gas_limit: None, builder_proposals: None, + builder_boost_factor: None, + prefer_builder_proposals: None, voting_public_key: keystore.public_key().unwrap(), signing_definition: SigningDefinition::LocalKeystore { voting_keystore_path: dst_keystore_dir.join(KEYSTORE_NAME), @@ -742,6 +748,8 @@ fn validator_import_launchpad_password_file() { suggested_fee_recipient: None, gas_limit: None, builder_proposals: None, + builder_boost_factor: None, + prefer_builder_proposals: None, signing_definition: SigningDefinition::LocalKeystore { voting_keystore_path, voting_keystore_password_path: None, diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 2a88770cdd..3efcb2d0e5 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -2372,6 +2372,7 @@ fn light_client_server_default() { .run_with_zero_port() .with_config(|config| { assert_eq!(config.network.enable_light_client_server, false); + assert_eq!(config.chain.enable_light_client_server, false); assert_eq!(config.http_api.enable_light_client_server, false); }); } @@ -2383,6 +2384,7 @@ fn light_client_server_enabled() { .run_with_zero_port() .with_config(|config| { assert_eq!(config.network.enable_light_client_server, true); + assert_eq!(config.chain.enable_light_client_server, true); }); } diff --git a/lighthouse/tests/validator_client.rs b/lighthouse/tests/validator_client.rs index c713347297..98c34b3a9b 100644 --- a/lighthouse/tests/validator_client.rs +++ b/lighthouse/tests/validator_client.rs @@ -464,6 +464,32 @@ fn builder_proposals_flag() { .with_config(|config| assert!(config.builder_proposals)); } #[test] +fn builder_boost_factor_flag() { + CommandLineTest::new() + .flag("builder-boost-factor", Some("150")) + .run() + .with_config(|config| assert_eq!(config.builder_boost_factor, Some(150))); +} +#[test] +fn no_builder_boost_factor_flag() { + CommandLineTest::new() + .run() + .with_config(|config| assert_eq!(config.builder_boost_factor, None)); +} +#[test] +fn prefer_builder_proposals_flag() { + CommandLineTest::new() + .flag("prefer-builder-proposals", None) + .run() + .with_config(|config| assert!(config.prefer_builder_proposals)); +} +#[test] +fn no_prefer_builder_proposals_flag() { + CommandLineTest::new() + .run() + .with_config(|config| assert!(!config.prefer_builder_proposals)); +} +#[test] fn no_builder_registration_timestamp_override_flag() { CommandLineTest::new() .run() diff --git a/lighthouse/tests/validator_manager.rs b/lighthouse/tests/validator_manager.rs index e0a1e92d6a..fab1cfebf4 100644 --- a/lighthouse/tests/validator_manager.rs +++ b/lighthouse/tests/validator_manager.rs @@ -122,6 +122,8 @@ pub fn validator_create_defaults() { specify_voting_keystore_password: false, eth1_withdrawal_address: None, builder_proposals: None, + builder_boost_factor: None, + prefer_builder_proposals: None, fee_recipient: None, gas_limit: None, bn_url: None, @@ -143,6 +145,8 @@ pub fn validator_create_misc_flags() { .flag("--specify-voting-keystore-password", None) .flag("--eth1-withdrawal-address", Some(EXAMPLE_ETH1_ADDRESS)) .flag("--builder-proposals", Some("true")) + .flag("--prefer-builder-proposals", Some("true")) + .flag("--builder-boost-factor", Some("150")) .flag("--suggested-fee-recipient", Some(EXAMPLE_ETH1_ADDRESS)) .flag("--gas-limit", Some("1337")) .flag("--beacon-node", Some("http://localhost:1001")) @@ -159,6 +163,8 @@ pub fn validator_create_misc_flags() { specify_voting_keystore_password: true, eth1_withdrawal_address: Some(Address::from_str(EXAMPLE_ETH1_ADDRESS).unwrap()), builder_proposals: Some(true), + builder_boost_factor: Some(150), + prefer_builder_proposals: Some(true), fee_recipient: Some(Address::from_str(EXAMPLE_ETH1_ADDRESS).unwrap()), gas_limit: Some(1337), bn_url: Some(SensitiveUrl::parse("http://localhost:1001").unwrap()), @@ -244,6 +250,8 @@ pub fn validator_move_defaults() { dest_vc_token_path: PathBuf::from("./2.json"), validators: Validators::All, builder_proposals: None, + builder_boost_factor: None, + prefer_builder_proposals: None, fee_recipient: None, gas_limit: None, password_source: PasswordSource::Interactive { @@ -280,6 +288,8 @@ pub fn validator_move_misc_flags_0() { PublicKeyBytes::from_str(EXAMPLE_PUBKEY_1).unwrap(), ]), builder_proposals: Some(true), + builder_boost_factor: None, + prefer_builder_proposals: None, fee_recipient: Some(Address::from_str(EXAMPLE_ETH1_ADDRESS).unwrap()), gas_limit: Some(1337), password_source: PasswordSource::Interactive { stdin_inputs: true }, @@ -297,6 +307,7 @@ pub fn validator_move_misc_flags_1() { .flag("--dest-vc-token", Some("./2.json")) .flag("--validators", Some(&format!("{}", EXAMPLE_PUBKEY_0))) .flag("--builder-proposals", Some("false")) + .flag("--prefer-builder-proposals", Some("false")) .assert_success(|config| { let expected = MoveConfig { src_vc_url: SensitiveUrl::parse("http://localhost:1").unwrap(), @@ -307,6 +318,40 @@ pub fn validator_move_misc_flags_1() { PublicKeyBytes::from_str(EXAMPLE_PUBKEY_0).unwrap() ]), builder_proposals: Some(false), + builder_boost_factor: None, + prefer_builder_proposals: Some(false), + fee_recipient: None, + gas_limit: None, + password_source: PasswordSource::Interactive { + stdin_inputs: cfg!(windows) || false, + }, + }; + assert_eq!(expected, config); + }); +} + +#[test] +pub fn validator_move_misc_flags_2() { + CommandLineTest::validators_move() + .flag("--src-vc-url", Some("http://localhost:1")) + .flag("--src-vc-token", Some("./1.json")) + .flag("--dest-vc-url", Some("http://localhost:2")) + .flag("--dest-vc-token", Some("./2.json")) + .flag("--validators", Some(&format!("{}", EXAMPLE_PUBKEY_0))) + .flag("--builder-proposals", Some("false")) + .flag("--builder-boost-factor", Some("100")) + .assert_success(|config| { + let expected = MoveConfig { + src_vc_url: SensitiveUrl::parse("http://localhost:1").unwrap(), + src_vc_token_path: PathBuf::from("./1.json"), + dest_vc_url: SensitiveUrl::parse("http://localhost:2").unwrap(), + dest_vc_token_path: PathBuf::from("./2.json"), + validators: Validators::Specific(vec![ + PublicKeyBytes::from_str(EXAMPLE_PUBKEY_0).unwrap() + ]), + builder_proposals: Some(false), + builder_boost_factor: Some(100), + prefer_builder_proposals: None, fee_recipient: None, gas_limit: None, password_source: PasswordSource::Interactive { @@ -333,6 +378,8 @@ pub fn validator_move_count() { dest_vc_token_path: PathBuf::from("./2.json"), validators: Validators::Count(42), builder_proposals: None, + builder_boost_factor: None, + prefer_builder_proposals: None, fee_recipient: None, gas_limit: None, password_source: PasswordSource::Interactive { diff --git a/scripts/cli.sh b/scripts/cli.sh index 768ec7b301..7ba98d08ba 100755 --- a/scripts/cli.sh +++ b/scripts/cli.sh @@ -90,7 +90,7 @@ rm -f help_general.md help_bn.md help_vc.md help_am.md help_vm.md help_vm_create # only exit at the very end if [[ $changes == true ]]; then - echo "Exiting with error to indicate changes occurred. To fix, run `make cli-local` or `make cli` and commit the changes." + echo "Exiting with error to indicate changes occurred. To fix, run 'make cli-local' or 'make cli' and commit the changes." exit 1 else echo "CLI help texts are up to date." diff --git a/scripts/local_testnet/README.md b/scripts/local_testnet/README.md index 2862fde075..74dc4739b4 100644 --- a/scripts/local_testnet/README.md +++ b/scripts/local_testnet/README.md @@ -193,3 +193,9 @@ Update the genesis time to now using: ./start_local_testnet.sh -p genesis.json ``` 4. Block production using builder flow will start at epoch 4. + +### Testing sending a transaction + +Some addresses in the local testnet are seeded with testnet ETH, allowing users to carry out transactions. To send a transaction, we first add the address to a wallet, such as [Metamask](https://metamask.io/). The private keys for the addresses are listed [here](https://github.com/sigp/lighthouse/blob/441fc1691b69f9edc4bbdc6665f3efab16265c9b/testing/execution_engine_integration/src/execution_engine.rs#L13-L14). + +Next, we add the local testnet to Metamask, a brief guide can be found [here](https://support.metamask.io/hc/en-us/articles/360043227612-How-to-add-a-custom-network-RPC). If you start the local testnet with default settings, the network RPC is: http://localhost:6001 and the `Chain ID` is `4242`, as defined in [`vars.env`](https://github.com/sigp/lighthouse/blob/441fc1691b69f9edc4bbdc6665f3efab16265c9b/scripts/local_testnet/vars.env#L42). Once the network and account are added, you should see that the account contains testnet ETH which allow us to carry out transactions. \ No newline at end of file diff --git a/scripts/local_testnet/beacon_node.sh b/scripts/local_testnet/beacon_node.sh index 940fe2b858..2660dfa3c0 100755 --- a/scripts/local_testnet/beacon_node.sh +++ b/scripts/local_testnet/beacon_node.sh @@ -67,4 +67,5 @@ exec $lighthouse_binary \ --target-peers $((BN_COUNT - 1)) \ --execution-endpoint $execution_endpoint \ --execution-jwt $execution_jwt \ + --http-allow-sync-stalled \ $BN_ARGS diff --git a/testing/simulator/src/checks.rs b/testing/simulator/src/checks.rs index c0be219d30..3f603af669 100644 --- a/testing/simulator/src/checks.rs +++ b/testing/simulator/src/checks.rs @@ -1,6 +1,6 @@ use crate::local_network::LocalNetwork; use crate::ACCEPTABLE_FALLBACK_ATTESTATION_HIT_PERCENTAGE; -use node_test_rig::eth2::types::{BlockId, StateId}; +use node_test_rig::eth2::types::{BlockId, FinalityCheckpointsData, StateId}; use std::time::Duration; use types::{Epoch, EthSpec, ExecPayload, ExecutionBlockHash, Hash256, Slot, Unsigned}; @@ -245,6 +245,96 @@ pub async fn verify_transition_block_finalized( } } +pub(crate) async fn verify_light_client_updates( + network: LocalNetwork, + start_slot: Slot, + end_slot: Slot, + slot_duration: Duration, +) -> Result<(), String> { + slot_delay(start_slot, slot_duration).await; + + // Tolerance of 2 slot allows for 1 single missed slot. + let light_client_update_slot_tolerance = Slot::new(2); + let remote_nodes = network.remote_nodes()?; + let client = remote_nodes.first().unwrap(); + let mut have_seen_block = false; + let mut have_achieved_finality = false; + + for slot in start_slot.as_u64()..=end_slot.as_u64() { + slot_delay(Slot::new(1), slot_duration).await; + let slot = Slot::new(slot); + let previous_slot = slot - 1; + + let previous_slot_block = client + .get_beacon_blocks::(BlockId::Slot(previous_slot)) + .await + .map_err(|e| { + format!("Unable to get beacon block for previous slot {previous_slot:?}: {e:?}") + })?; + let previous_slot_has_block = previous_slot_block.is_some(); + + if !have_seen_block { + // Make sure we have seen the first block in Altair, to make sure we have sync aggregates available. + if previous_slot_has_block { + have_seen_block = true; + } + // Wait for another slot before we check the first update to avoid race condition. + continue; + } + + // Make sure previous slot has a block, otherwise skip checking for the signature slot distance + if !previous_slot_has_block { + continue; + } + + // Verify light client optimistic update. `signature_slot_distance` should be 1 in the ideal scenario. + let signature_slot = client + .get_beacon_light_client_optimistic_update::() + .await + .map_err(|e| format!("Error while getting light client updates: {:?}", e))? + .ok_or(format!("Light client optimistic update not found {slot:?}"))? + .data + .signature_slot; + let signature_slot_distance = slot - signature_slot; + if signature_slot_distance > light_client_update_slot_tolerance { + return Err(format!("Existing optimistic update too old: signature slot {signature_slot}, current slot {slot:?}")); + } + + // Verify light client finality update. `signature_slot_distance` should be 1 in the ideal scenario. + // NOTE: Currently finality updates are produced as long as the finalized block is known, even if the finalized header + // sync committee period does not match the signature slot committee period. + // TODO: This complies with the current spec, but we should check if this is a bug. + if !have_achieved_finality { + let FinalityCheckpointsData { finalized, .. } = client + .get_beacon_states_finality_checkpoints(StateId::Head) + .await + .map_err(|e| format!("Unable to get beacon state finality checkpoint: {e:?}"))? + .ok_or("Unable to get head state".to_string())? + .data; + if !finalized.root.is_zero() { + // Wait for another slot before we check the first finality update to avoid race condition. + have_achieved_finality = true; + } + continue; + } + let signature_slot = client + .get_beacon_light_client_finality_update::() + .await + .map_err(|e| format!("Error while getting light client updates: {:?}", e))? + .ok_or(format!("Light client finality update not found {slot:?}"))? + .data + .signature_slot; + let signature_slot_distance = slot - signature_slot; + if signature_slot_distance > light_client_update_slot_tolerance { + return Err(format!( + "Existing finality update too old: signature slot {signature_slot}, current slot {slot:?}" + )); + } + } + + Ok(()) +} + // Causes the execution node at `node_index` to disconnect from the execution layer 1 epoch after // the merge transition. pub async fn disconnect_from_execution_layer( diff --git a/testing/simulator/src/common.rs b/testing/simulator/src/common.rs index 27548384ad..318356f47e 100644 --- a/testing/simulator/src/common.rs +++ b/testing/simulator/src/common.rs @@ -78,6 +78,9 @@ pub async fn create_local_network( beacon_config.network.target_peers = node_count + proposer_nodes - 1; beacon_config.network.enr_address = (Some(Ipv4Addr::LOCALHOST), None); + beacon_config.network.enable_light_client_server = true; + beacon_config.chain.enable_light_client_server = true; + beacon_config.http_api.enable_light_client_server = true; if post_merge_sim { let el_config = execution_layer::Config { diff --git a/testing/simulator/src/eth1_sim.rs b/testing/simulator/src/eth1_sim.rs index cf0a66578d..96f454bd88 100644 --- a/testing/simulator/src/eth1_sim.rs +++ b/testing/simulator/src/eth1_sim.rs @@ -214,6 +214,7 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> { fork, sync_aggregate, transition, + light_client_update, ) = futures::join!( // Check that the chain finalizes at the first given opportunity. checks::verify_first_finalization(network.clone(), slot_duration), @@ -266,6 +267,13 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> { Epoch::new(TERMINAL_BLOCK / MinimalEthSpec::slots_per_epoch()), slot_duration, post_merge_sim + ), + checks::verify_light_client_updates( + network.clone(), + // Sync aggregate available from slot 1 after Altair fork transition. + Epoch::new(ALTAIR_FORK_EPOCH).start_slot(MinimalEthSpec::slots_per_epoch()) + 1, + Epoch::new(END_EPOCH).start_slot(MinimalEthSpec::slots_per_epoch()), + slot_duration ) ); @@ -276,6 +284,7 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> { fork?; sync_aggregate?; transition?; + light_client_update?; // The `final_future` either completes immediately or never completes, depending on the value // of `continue_after_checks`. diff --git a/testing/state_transition_vectors/src/exit.rs b/testing/state_transition_vectors/src/exit.rs index 29f5c015e3..50b98d3066 100644 --- a/testing/state_transition_vectors/src/exit.rs +++ b/testing/state_transition_vectors/src/exit.rs @@ -57,7 +57,7 @@ impl ExitTest { block_modifier(&harness, block); }) .await; - (signed_block.0, state) + ((*signed_block.0).clone(), state) } fn process( diff --git a/testing/web3signer_tests/src/lib.rs b/testing/web3signer_tests/src/lib.rs index 463de0c8b3..6f3536fe46 100644 --- a/testing/web3signer_tests/src/lib.rs +++ b/testing/web3signer_tests/src/lib.rs @@ -391,6 +391,8 @@ mod tests { suggested_fee_recipient: None, gas_limit: None, builder_proposals: None, + builder_boost_factor: None, + prefer_builder_proposals: None, description: String::default(), signing_definition: SigningDefinition::LocalKeystore { voting_keystore_path: signer_rig.keystore_path.clone(), @@ -409,6 +411,8 @@ mod tests { suggested_fee_recipient: None, gas_limit: None, builder_proposals: None, + builder_boost_factor: None, + prefer_builder_proposals: None, description: String::default(), signing_definition: SigningDefinition::Web3Signer(Web3SignerDefinition { url: signer_rig.url.to_string(), diff --git a/validator_client/src/block_service.rs b/validator_client/src/block_service.rs index 3b3ed6be80..2150b2417f 100644 --- a/validator_client/src/block_service.rs +++ b/validator_client/src/block_service.rs @@ -300,14 +300,7 @@ impl BlockService { if self.validator_store.produce_block_v3() { for validator_pubkey in proposers { - let builder_proposals = self - .validator_store - .get_builder_proposals(&validator_pubkey); - // Translate `builder_proposals` to a boost factor. Builder proposals set to `true` - // requires no boost factor, it just means "use a builder proposal if the BN returns - // one". On the contrary, `builder_proposals: false` indicates a preference for - // local payloads, so we set the builder boost factor to 0. - let builder_boost_factor = if !builder_proposals { Some(0) } else { None }; + let builder_boost_factor = self.get_builder_boost_factor(&validator_pubkey); let service = self.clone(); let log = log.clone(); self.inner.context.executor.spawn( @@ -425,12 +418,13 @@ impl BlockService { self.validator_store .sign_block(*validator_pubkey, block, slot) .await - .map(|b| SignedBlock::Full(PublishBlockRequest::new(b, maybe_blobs))) + .map(|b| SignedBlock::Full(PublishBlockRequest::new(Arc::new(b), maybe_blobs))) } UnsignedBlock::Blinded(block) => self .validator_store .sign_block(*validator_pubkey, block, slot) .await + .map(Arc::new) .map(SignedBlock::Blinded), }; @@ -815,6 +809,36 @@ impl BlockService { Ok::<_, BlockError>(unsigned_block) } + + /// Returns the builder boost factor of the given public key. + /// The priority order for fetching this value is: + /// + /// 1. validator_definitions.yml + /// 2. process level flag + fn get_builder_boost_factor(&self, validator_pubkey: &PublicKeyBytes) -> Option { + // Apply per validator configuration first. + let validator_builder_boost_factor = self + .validator_store + .determine_validator_builder_boost_factor(validator_pubkey); + + // Fallback to process-wide configuration if needed. + let maybe_builder_boost_factor = validator_builder_boost_factor.or_else(|| { + self.validator_store + .determine_default_builder_boost_factor() + }); + + if let Some(builder_boost_factor) = maybe_builder_boost_factor { + // if builder boost factor is set to 100 it should be treated + // as None to prevent unnecessary calculations that could + // lead to loss of information. + if builder_boost_factor == 100 { + return None; + } + return Some(builder_boost_factor); + } + + None + } } pub enum UnsignedBlock { @@ -833,7 +857,7 @@ impl UnsignedBlock { pub enum SignedBlock { Full(PublishBlockRequest), - Blinded(SignedBlindedBeaconBlock), + Blinded(Arc>), } impl SignedBlock { diff --git a/validator_client/src/cli.rs b/validator_client/src/cli.rs index e9d185f50e..28dce1c6df 100644 --- a/validator_client/src/cli.rs +++ b/validator_client/src/cli.rs @@ -349,6 +349,24 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .default_value("500") .takes_value(true), ) + .arg( + Arg::with_name("builder-boost-factor") + .long("builder-boost-factor") + .value_name("UINT64") + .help("Defines the boost factor, \ + a percentage multiplier to apply to the builder's payload value \ + when choosing between a builder payload header and payload from \ + the local execution node.") + .conflicts_with("prefer-builder-proposals") + .takes_value(true), + ) + .arg( + Arg::with_name("prefer-builder-proposals") + .long("prefer-builder-proposals") + .help("If this flag is set, Lighthouse will always prefer blocks \ + constructed by builders, regardless of payload value.") + .takes_value(false), + ) .arg( Arg::with_name("beacon-node-sync-tolerance") .long("beacon-node-sync-tolerance") @@ -364,7 +382,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { can result in suboptimal fallback behaviour. Sets the size (in slots) of the \ `small` sync distance range when calculating the health tiers of connected \ Beacon Nodes. The range falls immediately after the end of the `synced` range.") - .takes_value(true) + .takes_value(true), ) .arg( Arg::with_name("beacon-node-medium-sync-distance-modifier") @@ -373,6 +391,6 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { can result in suboptimal fallback behaviour. Sets the size (in slots) of the \ `medium` sync distance range when calculating the health tiers of connected \ Beacon Nodes. The range falls immediately after the end of the `small` range.") - .takes_value(true) + .takes_value(true), ) } diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index f1708188df..a89c035c44 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -79,6 +79,10 @@ pub struct Config { pub validator_registration_batch_size: usize, /// Enables block production via the block v3 endpoint. This configuration option can be removed post deneb. pub produce_block_v3: bool, + /// Specifies the boost factor, a percentage multiplier to apply to the builder's payload value. + pub builder_boost_factor: Option, + /// If true, Lighthouse will prefer builder proposals, if available. + pub prefer_builder_proposals: bool, } impl Default for Config { @@ -121,6 +125,8 @@ impl Default for Config { enable_latency_measurement_service: true, validator_registration_batch_size: 500, produce_block_v3: false, + builder_boost_factor: None, + prefer_builder_proposals: false, } } } @@ -381,6 +387,10 @@ impl Config { config.produce_block_v3 = true; } + if cli_args.is_present("prefer-builder-proposals") { + config.prefer_builder_proposals = true; + } + config.gas_limit = cli_args .value_of("gas-limit") .map(|gas_limit| { @@ -400,6 +410,8 @@ impl Config { ); } + config.builder_boost_factor = parse_optional(cli_args, "builder-boost-factor")?; + config.enable_latency_measurement_service = parse_optional(cli_args, "latency-measurement-service")?.unwrap_or(true); diff --git a/validator_client/src/http_api/create_validator.rs b/validator_client/src/http_api/create_validator.rs index 52336afa59..afa5d4fed1 100644 --- a/validator_client/src/http_api/create_validator.rs +++ b/validator_client/src/http_api/create_validator.rs @@ -148,6 +148,8 @@ pub async fn create_validators_mnemonic, T: 'static + SlotClock, request.suggested_fee_recipient, request.gas_limit, request.builder_proposals, + request.builder_boost_factor, + request.prefer_builder_proposals, ) .await .map_err(|e| { diff --git a/validator_client/src/http_api/keystores.rs b/validator_client/src/http_api/keystores.rs index c2d9b4d67f..074c578347 100644 --- a/validator_client/src/http_api/keystores.rs +++ b/validator_client/src/http_api/keystores.rs @@ -224,6 +224,8 @@ fn import_single_keystore( None, None, None, + None, + None, )) .map_err(|e| format!("failed to initialize validator: {:?}", e))?; diff --git a/validator_client/src/http_api/mod.rs b/validator_client/src/http_api/mod.rs index c4401ebd1c..ddf7cce3eb 100644 --- a/validator_client/src/http_api/mod.rs +++ b/validator_client/src/http_api/mod.rs @@ -602,6 +602,8 @@ pub fn serve( let suggested_fee_recipient = body.suggested_fee_recipient; let gas_limit = body.gas_limit; let builder_proposals = body.builder_proposals; + let builder_boost_factor = body.builder_boost_factor; + let prefer_builder_proposals = body.prefer_builder_proposals; let validator_def = { if let Some(handle) = task_executor.handle() { @@ -614,6 +616,8 @@ pub fn serve( suggested_fee_recipient, gas_limit, builder_proposals, + builder_boost_factor, + prefer_builder_proposals, )) .map_err(|e| { warp_utils::reject::custom_server_error(format!( @@ -662,6 +666,8 @@ pub fn serve( suggested_fee_recipient: web3signer.suggested_fee_recipient, gas_limit: web3signer.gas_limit, builder_proposals: web3signer.builder_proposals, + builder_boost_factor: web3signer.builder_boost_factor, + prefer_builder_proposals: web3signer.prefer_builder_proposals, description: web3signer.description, signing_definition: SigningDefinition::Web3Signer( Web3SignerDefinition { @@ -728,8 +734,12 @@ pub fn serve( (Some(is_enabled), Some(initialized_validator)) if Some(is_enabled) == body.enabled && initialized_validator.get_gas_limit() == body.gas_limit + && initialized_validator.get_builder_boost_factor() + == body.builder_boost_factor && initialized_validator.get_builder_proposals() == body.builder_proposals + && initialized_validator.get_prefer_builder_proposals() + == body.prefer_builder_proposals && initialized_validator.get_graffiti() == maybe_graffiti => { Ok(()) @@ -743,6 +753,8 @@ pub fn serve( body.enabled, body.gas_limit, body.builder_proposals, + body.builder_boost_factor, + body.prefer_builder_proposals, body.graffiti, ), ) diff --git a/validator_client/src/http_api/remotekeys.rs b/validator_client/src/http_api/remotekeys.rs index 991dfb8bf7..053bbcb4b2 100644 --- a/validator_client/src/http_api/remotekeys.rs +++ b/validator_client/src/http_api/remotekeys.rs @@ -125,6 +125,8 @@ fn import_single_remotekey( suggested_fee_recipient: None, gas_limit: None, builder_proposals: None, + builder_boost_factor: None, + prefer_builder_proposals: None, description: String::from("Added by remotekey API"), signing_definition: SigningDefinition::Web3Signer(Web3SignerDefinition { url, diff --git a/validator_client/src/http_api/test_utils.rs b/validator_client/src/http_api/test_utils.rs index 360d74fbbf..d61d88e84f 100644 --- a/validator_client/src/http_api/test_utils.rs +++ b/validator_client/src/http_api/test_utils.rs @@ -316,6 +316,8 @@ impl ApiTester { suggested_fee_recipient: None, gas_limit: None, builder_proposals: None, + builder_boost_factor: None, + prefer_builder_proposals: None, deposit_gwei: E::default_spec().max_effective_balance, }) .collect::>(); @@ -448,6 +450,8 @@ impl ApiTester { suggested_fee_recipient: None, gas_limit: None, builder_proposals: None, + builder_boost_factor: None, + prefer_builder_proposals: None, }; self.client @@ -468,6 +472,8 @@ impl ApiTester { suggested_fee_recipient: None, gas_limit: None, builder_proposals: None, + builder_boost_factor: None, + prefer_builder_proposals: None, }; let response = self @@ -512,6 +518,8 @@ impl ApiTester { request_timeout_ms: None, client_identity_path: None, client_identity_password: None, + builder_boost_factor: None, + prefer_builder_proposals: None, } }) .collect(); @@ -535,7 +543,15 @@ impl ApiTester { let validator = &self.client.get_lighthouse_validators().await.unwrap().data[index]; self.client - .patch_lighthouse_validators(&validator.voting_pubkey, Some(enabled), None, None, None) + .patch_lighthouse_validators( + &validator.voting_pubkey, + Some(enabled), + None, + None, + None, + None, + None, + ) .await .unwrap(); @@ -583,6 +599,8 @@ impl ApiTester { Some(gas_limit), None, None, + None, + None, ) .await .unwrap(); @@ -611,6 +629,8 @@ impl ApiTester { None, Some(builder_proposals), None, + None, + None, ) .await .unwrap(); diff --git a/validator_client/src/http_api/tests.rs b/validator_client/src/http_api/tests.rs index 076482f690..7ac88bb499 100644 --- a/validator_client/src/http_api/tests.rs +++ b/validator_client/src/http_api/tests.rs @@ -52,6 +52,12 @@ struct ApiTester { impl ApiTester { pub async fn new() -> Self { + let mut config = Config::default(); + config.fee_recipient = Some(TEST_DEFAULT_FEE_RECIPIENT); + Self::new_with_config(config).await + } + + pub async fn new_with_config(mut config: Config) -> Self { let log = test_logger(); let validator_dir = tempdir().unwrap(); @@ -70,10 +76,8 @@ impl ApiTester { let api_secret = ApiSecret::create_or_open(validator_dir.path()).unwrap(); let api_pubkey = api_secret.api_token(); - let mut config = Config::default(); config.validator_dir = validator_dir.path().into(); config.secrets_dir = secrets_dir.path().into(); - config.fee_recipient = Some(TEST_DEFAULT_FEE_RECIPIENT); let spec = E::default_spec(); @@ -272,6 +276,8 @@ impl ApiTester { suggested_fee_recipient: None, gas_limit: None, builder_proposals: None, + builder_boost_factor: None, + prefer_builder_proposals: None, deposit_gwei: E::default_spec().max_effective_balance, }) .collect::>(); @@ -405,6 +411,8 @@ impl ApiTester { suggested_fee_recipient: None, gas_limit: None, builder_proposals: None, + builder_boost_factor: None, + prefer_builder_proposals: None, }; self.client @@ -425,6 +433,8 @@ impl ApiTester { suggested_fee_recipient: None, gas_limit: None, builder_proposals: None, + builder_boost_factor: None, + prefer_builder_proposals: None, }; let response = self @@ -463,6 +473,8 @@ impl ApiTester { suggested_fee_recipient: None, gas_limit: None, builder_proposals: None, + builder_boost_factor: None, + prefer_builder_proposals: None, voting_public_key: kp.pk, url: format!("http://signer_{}.com/", i), root_certificate_path: None, @@ -519,7 +531,15 @@ impl ApiTester { let validator = &self.client.get_lighthouse_validators().await.unwrap().data[index]; self.client - .patch_lighthouse_validators(&validator.voting_pubkey, Some(enabled), None, None, None) + .patch_lighthouse_validators( + &validator.voting_pubkey, + Some(enabled), + None, + None, + None, + None, + None, + ) .await .unwrap(); @@ -567,6 +587,8 @@ impl ApiTester { Some(gas_limit), None, None, + None, + None, ) .await .unwrap(); @@ -595,6 +617,50 @@ impl ApiTester { None, Some(builder_proposals), None, + None, + None, + ) + .await + .unwrap(); + + self + } + + pub async fn set_builder_boost_factor(self, index: usize, builder_boost_factor: u64) -> Self { + let validator = &self.client.get_lighthouse_validators().await.unwrap().data[index]; + + self.client + .patch_lighthouse_validators( + &validator.voting_pubkey, + None, + None, + None, + Some(builder_boost_factor), + None, + None, + ) + .await + .unwrap(); + + self + } + + pub async fn set_prefer_builder_proposals( + self, + index: usize, + prefer_builder_proposals: bool, + ) -> Self { + let validator = &self.client.get_lighthouse_validators().await.unwrap().data[index]; + + self.client + .patch_lighthouse_validators( + &validator.voting_pubkey, + None, + None, + None, + None, + Some(prefer_builder_proposals), + None, ) .await .unwrap(); @@ -614,6 +680,64 @@ impl ApiTester { self } + pub async fn assert_builder_boost_factor( + self, + index: usize, + builder_boost_factor: Option, + ) -> Self { + let validator = &self.client.get_lighthouse_validators().await.unwrap().data[index]; + + assert_eq!( + self.validator_store + .get_builder_boost_factor(&validator.voting_pubkey), + builder_boost_factor + ); + + self + } + + pub async fn assert_validator_derived_builder_boost_factor( + self, + index: usize, + builder_boost_factor: Option, + ) -> Self { + let validator = &self.client.get_lighthouse_validators().await.unwrap().data[index]; + + assert_eq!( + self.validator_store + .determine_validator_builder_boost_factor(&validator.voting_pubkey), + builder_boost_factor + ); + + self + } + + pub fn assert_default_builder_boost_factor(self, builder_boost_factor: Option) -> Self { + assert_eq!( + self.validator_store + .determine_default_builder_boost_factor(), + builder_boost_factor + ); + + self + } + + pub async fn assert_prefer_builder_proposals( + self, + index: usize, + prefer_builder_proposals: bool, + ) -> Self { + let validator = &self.client.get_lighthouse_validators().await.unwrap().data[index]; + + assert_eq!( + self.validator_store + .get_prefer_builder_proposals(&validator.voting_pubkey), + prefer_builder_proposals + ); + + self + } + pub async fn set_graffiti(self, index: usize, graffiti: &str) -> Self { let validator = &self.client.get_lighthouse_validators().await.unwrap().data[index]; let graffiti_str = GraffitiString::from_str(graffiti).unwrap(); @@ -623,6 +747,8 @@ impl ApiTester { None, None, None, + None, + None, Some(graffiti_str), ) .await @@ -742,6 +868,8 @@ async fn routes_with_invalid_auth() { gas_limit: <_>::default(), builder_proposals: <_>::default(), deposit_gwei: <_>::default(), + builder_boost_factor: <_>::default(), + prefer_builder_proposals: <_>::default(), }]) .await }) @@ -772,6 +900,8 @@ async fn routes_with_invalid_auth() { suggested_fee_recipient: <_>::default(), gas_limit: <_>::default(), builder_proposals: <_>::default(), + builder_boost_factor: <_>::default(), + prefer_builder_proposals: <_>::default(), }) .await }) @@ -784,6 +914,8 @@ async fn routes_with_invalid_auth() { None, None, None, + None, + None, ) .await }) @@ -981,6 +1113,152 @@ async fn validator_builder_proposals() { .await; } +#[tokio::test] +async fn validator_builder_boost_factor() { + ApiTester::new() + .await + .create_hd_validators(HdValidatorScenario { + count: 2, + specify_mnemonic: false, + key_derivation_path_offset: 0, + disabled: vec![], + }) + .await + .assert_enabled_validators_count(2) + .assert_validators_count(2) + .set_builder_boost_factor(0, 120) + .await + // Test setting builder proposals while the validator is disabled + .set_validator_enabled(0, false) + .await + .assert_enabled_validators_count(1) + .assert_validators_count(2) + .set_builder_boost_factor(0, 80) + .await + .set_validator_enabled(0, true) + .await + .assert_enabled_validators_count(2) + .assert_builder_boost_factor(0, Some(80)) + .await; +} + +/// Verifies the builder boost factors translated from the `builder_proposals`, +/// `prefer_builder_proposals` and `builder_boost_factor` values. +#[tokio::test] +async fn validator_derived_builder_boost_factor_with_process_defaults() { + let config = Config { + builder_proposals: true, + prefer_builder_proposals: false, + builder_boost_factor: Some(80), + ..Config::default() + }; + ApiTester::new_with_config(config) + .await + .create_hd_validators(HdValidatorScenario { + count: 3, + specify_mnemonic: false, + key_derivation_path_offset: 0, + disabled: vec![], + }) + .await + .assert_default_builder_boost_factor(Some(80)) + .assert_validator_derived_builder_boost_factor(0, None) + .await + .set_builder_proposals(0, false) + .await + .assert_validator_derived_builder_boost_factor(0, Some(0)) + .await + .set_builder_boost_factor(1, 120) + .await + .assert_validator_derived_builder_boost_factor(1, Some(120)) + .await + .set_prefer_builder_proposals(2, true) + .await + .assert_validator_derived_builder_boost_factor(2, Some(u64::MAX)) + .await; +} + +#[tokio::test] +async fn validator_builder_boost_factor_global_builder_proposals_true() { + let config = Config { + builder_proposals: true, + prefer_builder_proposals: false, + builder_boost_factor: None, + ..Config::default() + }; + ApiTester::new_with_config(config) + .await + .assert_default_builder_boost_factor(None); +} + +#[tokio::test] +async fn validator_builder_boost_factor_global_builder_proposals_false() { + let config = Config { + builder_proposals: false, + prefer_builder_proposals: false, + builder_boost_factor: None, + ..Config::default() + }; + ApiTester::new_with_config(config) + .await + .assert_default_builder_boost_factor(Some(0)); +} + +#[tokio::test] +async fn validator_builder_boost_factor_global_prefer_builder_proposals_true() { + let config = Config { + builder_proposals: true, + prefer_builder_proposals: true, + builder_boost_factor: None, + ..Config::default() + }; + ApiTester::new_with_config(config) + .await + .assert_default_builder_boost_factor(Some(u64::MAX)); +} + +#[tokio::test] +async fn validator_builder_boost_factor_global_prefer_builder_proposals_true_override() { + let config = Config { + builder_proposals: false, + prefer_builder_proposals: true, + builder_boost_factor: None, + ..Config::default() + }; + ApiTester::new_with_config(config) + .await + .assert_default_builder_boost_factor(Some(u64::MAX)); +} + +#[tokio::test] +async fn prefer_builder_proposals_validator() { + ApiTester::new() + .await + .create_hd_validators(HdValidatorScenario { + count: 2, + specify_mnemonic: false, + key_derivation_path_offset: 0, + disabled: vec![], + }) + .await + .assert_enabled_validators_count(2) + .assert_validators_count(2) + .set_prefer_builder_proposals(0, false) + .await + // Test setting builder proposals while the validator is disabled + .set_validator_enabled(0, false) + .await + .assert_enabled_validators_count(1) + .assert_validators_count(2) + .set_prefer_builder_proposals(0, true) + .await + .set_validator_enabled(0, true) + .await + .assert_enabled_validators_count(2) + .assert_prefer_builder_proposals(0, true) + .await; +} + #[tokio::test] async fn validator_graffiti() { ApiTester::new() diff --git a/validator_client/src/http_api/tests/keystores.rs b/validator_client/src/http_api/tests/keystores.rs index f301af1c21..fe58393bb8 100644 --- a/validator_client/src/http_api/tests/keystores.rs +++ b/validator_client/src/http_api/tests/keystores.rs @@ -43,6 +43,8 @@ fn web3signer_validator_with_pubkey(pubkey: PublicKey) -> Web3SignerValidatorReq suggested_fee_recipient: None, gas_limit: None, builder_proposals: None, + builder_boost_factor: None, + prefer_builder_proposals: None, voting_public_key: pubkey, url: web3_signer_url(), root_certificate_path: None, @@ -468,7 +470,7 @@ async fn import_and_delete_conflicting_web3_signer_keystores() { for pubkey in &pubkeys { tester .client - .patch_lighthouse_validators(pubkey, Some(false), None, None, None) + .patch_lighthouse_validators(pubkey, Some(false), None, None, None, None, None) .await .unwrap(); } diff --git a/validator_client/src/initialized_validators.rs b/validator_client/src/initialized_validators.rs index b65dad4c47..7e4331dc85 100644 --- a/validator_client/src/initialized_validators.rs +++ b/validator_client/src/initialized_validators.rs @@ -131,6 +131,8 @@ pub struct InitializedValidator { suggested_fee_recipient: Option
, gas_limit: Option, builder_proposals: Option, + builder_boost_factor: Option, + prefer_builder_proposals: Option, /// The validators index in `state.validators`, to be updated by an external service. index: Option, } @@ -159,6 +161,14 @@ impl InitializedValidator { self.gas_limit } + pub fn get_builder_boost_factor(&self) -> Option { + self.builder_boost_factor + } + + pub fn get_prefer_builder_proposals(&self) -> Option { + self.prefer_builder_proposals + } + pub fn get_builder_proposals(&self) -> Option { self.builder_proposals } @@ -335,6 +345,8 @@ impl InitializedValidator { suggested_fee_recipient: def.suggested_fee_recipient, gas_limit: def.gas_limit, builder_proposals: def.builder_proposals, + builder_boost_factor: def.builder_boost_factor, + prefer_builder_proposals: def.prefer_builder_proposals, index: None, }) } @@ -815,6 +827,22 @@ impl InitializedValidators { .and_then(|v| v.builder_proposals) } + /// Returns the `builder_boost_factor` for a given public key specified in the + /// `ValidatorDefinitions`. + pub fn builder_boost_factor(&self, public_key: &PublicKeyBytes) -> Option { + self.validators + .get(public_key) + .and_then(|v| v.builder_boost_factor) + } + + /// Returns the `prefer_builder_proposals` for a given public key specified in the + /// `ValidatorDefinitions`. + pub fn prefer_builder_proposals(&self, public_key: &PublicKeyBytes) -> Option { + self.validators + .get(public_key) + .and_then(|v| v.prefer_builder_proposals) + } + /// Returns an `Option` of a reference to an `InitializedValidator` for a given public key specified in the /// `ValidatorDefinitions`. pub fn validator(&self, public_key: &PublicKeyBytes) -> Option<&InitializedValidator> { @@ -835,12 +863,15 @@ impl InitializedValidators { /// or `InitializedValidator`. The same logic applies to `builder_proposals` and `graffiti`. /// /// Saves the `ValidatorDefinitions` to file, even if no definitions were changed. + #[allow(clippy::too_many_arguments)] pub async fn set_validator_definition_fields( &mut self, voting_public_key: &PublicKey, enabled: Option, gas_limit: Option, builder_proposals: Option, + builder_boost_factor: Option, + prefer_builder_proposals: Option, graffiti: Option, ) -> Result<(), Error> { if let Some(def) = self @@ -862,6 +893,12 @@ impl InitializedValidators { if let Some(graffiti) = graffiti.clone() { def.graffiti = Some(graffiti); } + if let Some(builder_boost_factor) = builder_boost_factor { + def.builder_boost_factor = Some(builder_boost_factor); + } + if let Some(prefer_builder_proposals) = prefer_builder_proposals { + def.prefer_builder_proposals = Some(prefer_builder_proposals); + } } self.update_validators().await?; @@ -880,6 +917,12 @@ impl InitializedValidators { if let Some(graffiti) = graffiti { val.graffiti = Some(graffiti.into()); } + if let Some(builder_boost_factor) = builder_boost_factor { + val.builder_boost_factor = Some(builder_boost_factor); + } + if let Some(prefer_builder_proposals) = prefer_builder_proposals { + val.prefer_builder_proposals = Some(prefer_builder_proposals); + } } self.definitions diff --git a/validator_client/src/validator_store.rs b/validator_client/src/validator_store.rs index 19726c2aec..a2298d3038 100644 --- a/validator_client/src/validator_store.rs +++ b/validator_client/src/validator_store.rs @@ -98,6 +98,8 @@ pub struct ValidatorStore { gas_limit: Option, builder_proposals: bool, produce_block_v3: bool, + prefer_builder_proposals: bool, + builder_boost_factor: Option, task_executor: TaskExecutor, _phantom: PhantomData, } @@ -130,6 +132,8 @@ impl ValidatorStore { gas_limit: config.gas_limit, builder_proposals: config.builder_proposals, produce_block_v3: config.produce_block_v3, + prefer_builder_proposals: config.prefer_builder_proposals, + builder_boost_factor: config.builder_boost_factor, task_executor, _phantom: PhantomData, } @@ -178,6 +182,8 @@ impl ValidatorStore { suggested_fee_recipient: Option
, gas_limit: Option, builder_proposals: Option, + builder_boost_factor: Option, + prefer_builder_proposals: Option, ) -> Result { let mut validator_def = ValidatorDefinition::new_keystore_with_password( voting_keystore_path, @@ -186,6 +192,8 @@ impl ValidatorStore { suggested_fee_recipient, gas_limit, builder_proposals, + builder_boost_factor, + prefer_builder_proposals, ) .map_err(|e| format!("failed to create validator definitions: {:?}", e))?; @@ -474,7 +482,7 @@ impl ValidatorStore { .unwrap_or(DEFAULT_GAS_LIMIT) } - /// Returns a `bool` for the given public key that denotes whther this validator should use the + /// Returns a `bool` for the given public key that denotes whether this validator should use the /// builder API. The priority order for fetching this value is: /// /// 1. validator_definitions.yml @@ -487,12 +495,91 @@ impl ValidatorStore { ) } + /// Returns a `u64` for the given public key that denotes the builder boost factor. The priority order for fetching this value is: + /// + /// 1. validator_definitions.yml + /// 2. process level flag + pub fn get_builder_boost_factor(&self, validator_pubkey: &PublicKeyBytes) -> Option { + self.validators + .read() + .builder_boost_factor(validator_pubkey) + .or(self.builder_boost_factor) + } + + /// Returns a `bool` for the given public key that denotes whether this validator should prefer a + /// builder payload. The priority order for fetching this value is: + /// + /// 1. validator_definitions.yml + /// 2. process level flag + pub fn get_prefer_builder_proposals(&self, validator_pubkey: &PublicKeyBytes) -> bool { + self.validators + .read() + .prefer_builder_proposals(validator_pubkey) + .unwrap_or(self.prefer_builder_proposals) + } + fn get_builder_proposals_defaulting(&self, builder_proposals: Option) -> bool { builder_proposals // If there's nothing in the file, try the process-level default value. .unwrap_or(self.builder_proposals) } + /// Translate the per validator `builder_proposals`, `builder_boost_factor` and + /// `prefer_builder_proposals` to a boost factor, if available. + /// - If `prefer_builder_proposals` is true, set boost factor to `u64::MAX` to indicate a + /// preference for builder payloads. + /// - If `builder_boost_factor` is a value other than None, return its value as the boost factor. + /// - If `builder_proposals` is set to false, set boost factor to 0 to indicate a preference for + /// local payloads. + /// - Else return `None` to indicate no preference between builder and local payloads. + pub fn determine_validator_builder_boost_factor( + &self, + validator_pubkey: &PublicKeyBytes, + ) -> Option { + let validator_prefer_builder_proposals = self + .validators + .read() + .prefer_builder_proposals(validator_pubkey); + + if matches!(validator_prefer_builder_proposals, Some(true)) { + return Some(u64::MAX); + } + + self.validators + .read() + .builder_boost_factor(validator_pubkey) + .or_else(|| { + if matches!( + self.validators.read().builder_proposals(validator_pubkey), + Some(false) + ) { + return Some(0); + } + None + }) + } + + /// Translate the process-wide `builder_proposals`, `builder_boost_factor` and + /// `prefer_builder_proposals` configurations to a boost factor. + /// - If `prefer_builder_proposals` is true, set boost factor to `u64::MAX` to indicate a + /// preference for builder payloads. + /// - If `builder_boost_factor` is a value other than None, return its value as the boost factor. + /// - If `builder_proposals` is set to false, set boost factor to 0 to indicate a preference for + /// local payloads. + /// - Else return `None` to indicate no preference between builder and local payloads. + pub fn determine_default_builder_boost_factor(&self) -> Option { + if self.prefer_builder_proposals { + return Some(u64::MAX); + } + self.builder_boost_factor.or({ + if !self.builder_proposals { + Some(0) + } else { + None + } + }) + } + pub async fn sign_block>( &self, validator_pubkey: PublicKeyBytes, diff --git a/validator_manager/src/common.rs b/validator_manager/src/common.rs index 6a3f93a3f7..871c536203 100644 --- a/validator_manager/src/common.rs +++ b/validator_manager/src/common.rs @@ -46,6 +46,8 @@ pub struct ValidatorSpecification { pub fee_recipient: Option
, pub gas_limit: Option, pub builder_proposals: Option, + pub builder_boost_factor: Option, + pub prefer_builder_proposals: Option, pub enabled: Option, } @@ -64,6 +66,8 @@ impl ValidatorSpecification { gas_limit, builder_proposals, enabled, + builder_boost_factor, + prefer_builder_proposals, } = self; let voting_public_key = voting_keystore @@ -136,6 +140,8 @@ impl ValidatorSpecification { enabled, gas_limit, builder_proposals, + builder_boost_factor, + prefer_builder_proposals, None, // Grafitti field is not maintained between validator moves. ) .await diff --git a/validator_manager/src/create_validators.rs b/validator_manager/src/create_validators.rs index 8ea740ff5b..8ab3303d36 100644 --- a/validator_manager/src/create_validators.rs +++ b/validator_manager/src/create_validators.rs @@ -25,6 +25,8 @@ pub const ETH1_WITHDRAWAL_ADDRESS_FLAG: &str = "eth1-withdrawal-address"; pub const GAS_LIMIT_FLAG: &str = "gas-limit"; pub const FEE_RECIPIENT_FLAG: &str = "suggested-fee-recipient"; pub const BUILDER_PROPOSALS_FLAG: &str = "builder-proposals"; +pub const BUILDER_BOOST_FACTOR_FLAG: &str = "builder-boost-factor"; +pub const PREFER_BUILDER_PROPOSALS_FLAG: &str = "prefer-builder-proposals"; pub const BEACON_NODE_FLAG: &str = "beacon-node"; pub const FORCE_BLS_WITHDRAWAL_CREDENTIALS: &str = "force-bls-withdrawal-credentials"; @@ -183,6 +185,30 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { address. This is not recommended.", ), ) + .arg( + Arg::with_name(BUILDER_BOOST_FACTOR_FLAG) + .long(BUILDER_BOOST_FACTOR_FLAG) + .takes_value(true) + .value_name("UINT64") + .required(false) + .help( + "Defines the boost factor, \ + a percentage multiplier to apply to the builder's payload value \ + when choosing between a builder payload header and payload from \ + the local execution node.", + ), + ) + .arg( + Arg::with_name(PREFER_BUILDER_PROPOSALS_FLAG) + .long(PREFER_BUILDER_PROPOSALS_FLAG) + .help( + "If this flag is set, Lighthouse will always prefer blocks \ + constructed by builders, regardless of payload value.", + ) + .required(false) + .possible_values(&["true", "false"]) + .takes_value(true), + ) } /// The CLI arguments are parsed into this struct before running the application. This step of @@ -199,6 +225,8 @@ pub struct CreateConfig { pub specify_voting_keystore_password: bool, pub eth1_withdrawal_address: Option
, pub builder_proposals: Option, + pub builder_boost_factor: Option, + pub prefer_builder_proposals: Option, pub fee_recipient: Option
, pub gas_limit: Option, pub bn_url: Option, @@ -223,6 +251,11 @@ impl CreateConfig { ETH1_WITHDRAWAL_ADDRESS_FLAG, )?, builder_proposals: clap_utils::parse_optional(matches, BUILDER_PROPOSALS_FLAG)?, + builder_boost_factor: clap_utils::parse_optional(matches, BUILDER_BOOST_FACTOR_FLAG)?, + prefer_builder_proposals: clap_utils::parse_optional( + matches, + PREFER_BUILDER_PROPOSALS_FLAG, + )?, fee_recipient: clap_utils::parse_optional(matches, FEE_RECIPIENT_FLAG)?, gas_limit: clap_utils::parse_optional(matches, GAS_LIMIT_FLAG)?, bn_url: clap_utils::parse_optional(matches, BEACON_NODE_FLAG)?, @@ -254,6 +287,8 @@ impl ValidatorsAndDeposits { gas_limit, bn_url, force_bls_withdrawal_credentials, + builder_boost_factor, + prefer_builder_proposals, } = config; // Since Capella, it really doesn't make much sense to use BLS @@ -456,6 +491,8 @@ impl ValidatorsAndDeposits { fee_recipient, gas_limit, builder_proposals, + builder_boost_factor, + prefer_builder_proposals, // Allow the VC to choose a default "enabled" state. Since "enabled" is not part of // the standard API, leaving this as `None` means we are not forced to use the // non-standard API. @@ -585,6 +622,8 @@ pub mod tests { specify_voting_keystore_password: false, eth1_withdrawal_address: junk_execution_address(), builder_proposals: None, + builder_boost_factor: None, + prefer_builder_proposals: None, fee_recipient: None, gas_limit: None, bn_url: None, diff --git a/validator_manager/src/move_validators.rs b/validator_manager/src/move_validators.rs index fa886e8f94..5826f2756b 100644 --- a/validator_manager/src/move_validators.rs +++ b/validator_manager/src/move_validators.rs @@ -32,6 +32,8 @@ pub const VALIDATORS_FLAG: &str = "validators"; pub const GAS_LIMIT_FLAG: &str = "gas-limit"; pub const FEE_RECIPIENT_FLAG: &str = "suggested-fee-recipient"; pub const BUILDER_PROPOSALS_FLAG: &str = "builder-proposals"; +pub const BUILDER_BOOST_FACTOR_FLAG: &str = "builder-boost-factor"; +pub const PREFER_BUILDER_PROPOSALS_FLAG: &str = "prefer-builder-proposals"; const NO_VALIDATORS_MSG: &str = "No validators present on source validator client"; @@ -170,6 +172,30 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .long(STDIN_INPUTS_FLAG) .help("If present, read all user inputs from stdin instead of tty."), ) + .arg( + Arg::with_name(BUILDER_BOOST_FACTOR_FLAG) + .long(BUILDER_BOOST_FACTOR_FLAG) + .takes_value(true) + .value_name("UINT64") + .required(false) + .help( + "Defines the boost factor, \ + a percentage multiplier to apply to the builder's payload value \ + when choosing between a builder payload header and payload from \ + the local execution node.", + ), + ) + .arg( + Arg::with_name(PREFER_BUILDER_PROPOSALS_FLAG) + .long(PREFER_BUILDER_PROPOSALS_FLAG) + .help( + "If this flag is set, Lighthouse will always prefer blocks \ + constructed by builders, regardless of payload value.", + ) + .required(false) + .possible_values(&["true", "false"]) + .takes_value(true), + ) } #[derive(Clone, PartialEq, Debug, Serialize, Deserialize)] @@ -187,6 +213,8 @@ pub struct MoveConfig { pub dest_vc_token_path: PathBuf, pub validators: Validators, pub builder_proposals: Option, + pub builder_boost_factor: Option, + pub prefer_builder_proposals: Option, pub fee_recipient: Option
, pub gas_limit: Option, pub password_source: PasswordSource, @@ -221,6 +249,11 @@ impl MoveConfig { dest_vc_token_path: clap_utils::parse_required(matches, DEST_VC_TOKEN_FLAG)?, validators, builder_proposals: clap_utils::parse_optional(matches, BUILDER_PROPOSALS_FLAG)?, + builder_boost_factor: clap_utils::parse_optional(matches, BUILDER_BOOST_FACTOR_FLAG)?, + prefer_builder_proposals: clap_utils::parse_optional( + matches, + PREFER_BUILDER_PROPOSALS_FLAG, + )?, fee_recipient: clap_utils::parse_optional(matches, FEE_RECIPIENT_FLAG)?, gas_limit: clap_utils::parse_optional(matches, GAS_LIMIT_FLAG)?, password_source: PasswordSource::Interactive { @@ -253,6 +286,8 @@ async fn run<'a>(config: MoveConfig) -> Result<(), String> { fee_recipient, gas_limit, mut password_source, + builder_boost_factor, + prefer_builder_proposals, } = config; // Moving validators between the same VC is unlikely to be useful and probably indicates a user @@ -488,13 +523,15 @@ async fn run<'a>(config: MoveConfig) -> Result<(), String> { let keystore_derivation_path = voting_keystore.0.path(); - let validator_specification = ValidatorSpecification { + let validator_specification: ValidatorSpecification = ValidatorSpecification { voting_keystore, voting_keystore_password, slashing_protection: Some(InterchangeJsonStr(slashing_protection)), fee_recipient, gas_limit, builder_proposals, + builder_boost_factor, + prefer_builder_proposals, // Allow the VC to choose a default "enabled" state. Since "enabled" is not part of // the standard API, leaving this as `None` means we are not forced to use the // non-standard API. @@ -758,6 +795,8 @@ mod test { dest_vc_token_path: dest_vc_token_path.clone(), validators: validators.clone(), builder_proposals: None, + builder_boost_factor: None, + prefer_builder_proposals: None, fee_recipient: None, gas_limit: None, password_source: PasswordSource::Testing(self.passwords.clone()),