diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000000..f9478d1369 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +beacon_node/network/ @jxs +beacon_node/lighthouse_network/ @jxs diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index 031a88b03c..e9db3b6ab1 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -11,7 +11,7 @@ concurrency: jobs: build-and-upload-to-s3: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 45f3b757e7..0ee9dbb622 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -392,6 +392,10 @@ jobs: cache: false env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Fetch libssl1.1 + run: wget https://nz2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb + - name: Install libssl1.1 + run: sudo dpkg -i libssl1.1_1.1.1f-1ubuntu2_amd64.deb - name: Create Cargo config dir run: mkdir -p .cargo - name: Install custom Cargo config diff --git a/Cargo.lock b/Cargo.lock index c62e9fbc87..20d2548d09 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -43,7 +43,6 @@ dependencies = [ name = "account_utils" version = "0.1.0" dependencies = [ - "directory", "eth2_keystore", "eth2_wallet", "filesystem", @@ -150,9 +149,9 @@ dependencies = [ [[package]] name = "allocator-api2" -version = "0.2.20" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45862d1c77f2228b9e10bc609d5bc203d86ebc9b87ad8d5d5167a6c9abf739d9" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "alloy-consensus" @@ -205,9 +204,9 @@ dependencies = [ [[package]] name = "alloy-primitives" -version = "0.8.12" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fce5dbd6a4f118eecc4719eaa9c7ffc31c315e6c5ccde3642db927802312425" +checksum = "ec878088ec6283ce1e90d280316aadd3d6ce3de06ff63d68953c855e7e447e92" dependencies = [ "alloy-rlp", "arbitrary", @@ -217,10 +216,9 @@ dependencies = [ "derive_arbitrary", "derive_more 1.0.0", "foldhash", - "getrandom", - "hashbrown 0.15.1", - "hex-literal", - "indexmap 2.6.0", + "getrandom 0.2.15", + "hashbrown 0.15.2", + "indexmap 2.7.1", "itoa", "k256 0.13.4", "keccak-asm", @@ -229,7 +227,7 @@ dependencies = [ "proptest-derive", "rand", "ruint", - "rustc-hash 2.0.0", + "rustc-hash 2.1.0", "serde", "sha3 0.10.8", "tiny-keccak", @@ -237,9 +235,9 @@ dependencies = [ [[package]] name = "alloy-rlp" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0822426598f95e45dd1ea32a738dac057529a709ee645fcc516ffa4cbde08f" +checksum = "3d6c1d995bff8d011f7cd6c81820d51825e6e06d6db73914c1630ecf544d83d6" dependencies = [ "alloy-rlp-derive", "arrayvec", @@ -248,13 +246,13 @@ dependencies = [ [[package]] name = "alloy-rlp-derive" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b09cae092c27b6f1bde952653a22708691802e57bfef4a2973b80bea21efd3f" +checksum = "a40e1ef334153322fd878d07e86af7a529bcb86b2439525920a88eba87bcf943" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] @@ -319,19 +317,20 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "3.0.6" +version = "3.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" dependencies = [ "anstyle", + "once_cell", "windows-sys 0.59.0", ] [[package]] name = "anyhow" -version = "1.0.93" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" +checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" [[package]] name = "arbitrary" @@ -523,7 +522,7 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", "synstructure", ] @@ -535,7 +534,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] @@ -568,7 +567,7 @@ dependencies = [ "futures-lite", "parking", "polling", - "rustix 0.38.41", + "rustix 0.38.44", "slab", "tracing", "windows-sys 0.59.0", @@ -580,20 +579,31 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" dependencies = [ - "event-listener 5.3.1", + "event-listener 5.4.0", "event-listener-strategy", "pin-project-lite", ] [[package]] -name = "async-trait" -version = "0.1.83" +name = "async-recursion" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" +checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", +] + +[[package]] +name = "async-trait" +version = "0.1.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f934833b4b7233644e5848f235df3f57ed8c80f1528a26c3dfa13d2147fa056" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", ] [[package]] @@ -620,6 +630,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "attohttpc" version = "0.24.1" @@ -644,13 +660,13 @@ dependencies = [ [[package]] name = "auto_impl" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c87f3f15e7794432337fc718554eaa4dc8f04c9677a950ffe366f20a162ae42" +checksum = "e12882f59de5360c748c4cbf569a042d5fb0eb515f7bea9c1f470b47f6ffbd73" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] @@ -669,10 +685,10 @@ dependencies = [ "axum-core", "bytes", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", - "hyper 1.5.1", + "hyper 1.6.0", "hyper-util", "itoa", "matchit", @@ -702,7 +718,7 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", "mime", @@ -849,7 +865,7 @@ dependencies = [ "genesis", "hex", "http_api", - "hyper 1.5.1", + "hyper 1.6.0", "lighthouse_network", "monitoring_api", "node_test_rig", @@ -918,7 +934,7 @@ version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "cexpr", "clang-sys", "itertools 0.12.1", @@ -931,7 +947,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.89", + "syn 2.0.96", "which", ] @@ -958,9 +974,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.6.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" [[package]] name = "bitvec" @@ -1128,9 +1144,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.16.0" +version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" [[package]] name = "byte-slice-cast" @@ -1146,9 +1162,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" +checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" dependencies = [ "serde", ] @@ -1200,9 +1216,9 @@ dependencies = [ [[package]] name = "cargo-platform" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24b1f0365a6c6bb4020cd05806fd0d33c44d38046b8bd7f0e40814b9763cabfc" +checksum = "e35af189006b9c0f00a064685c727031e3ed2d8020f7ba284d78cc2671bd36ea" dependencies = [ "serde", ] @@ -1215,7 +1231,7 @@ checksum = "eee4243f1f26fc7a42710e7439c149e2b10b05472f88090acce52632f231a73a" dependencies = [ "camino", "cargo-platform", - "semver 1.0.23", + "semver 1.0.25", "serde", "serde_json", "thiserror 1.0.69", @@ -1229,9 +1245,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.1" +version = "1.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd9de9f2205d5ef3fd67e685b0df337994ddd4495e2a28d185500d0e1edfea47" +checksum = "13208fcbb66eaeffe09b99fffbe1af420f00a7b35aa99ad683dfc1aa76145229" dependencies = [ "jobserver", "libc", @@ -1285,9 +1301,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1355,9 +1371,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.21" +version = "4.5.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb3b4b9e5a7c7514dfa52869339ee98b3156b0bfb4e8a77c4ff4babb64b1604f" +checksum = "769b0145982b4b48713e01ec42d61614425f27b7058bda7180a3a41f30104796" dependencies = [ "clap_builder", "clap_derive", @@ -1365,9 +1381,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.21" +version = "4.5.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b17a95aa67cc7b5ebd32aa5370189aa0d79069ef1c64ce893bd30fb24bff20ec" +checksum = "1b26884eb4b57140e4d2d93652abfa49498b938b3c9179f9fc487b0acc3edad7" dependencies = [ "anstream", "anstyle", @@ -1378,21 +1394,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.18" +version = "4.5.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +checksum = "54b755194d6389280185988721fffba69495eed5ee9feeee9a599b53db80318c" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] name = "clap_lex" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" [[package]] name = "clap_utils" @@ -1453,9 +1469,9 @@ dependencies = [ [[package]] name = "cmake" -version = "0.1.51" +version = "0.1.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb1e43aa7fd152b1f968787f7dbcdeb306d1867ff373c69955211876c053f91a" +checksum = "e24a03c8b52922d68a1589ad61032f2c1aa5a8158d2aa0d93c6e9534944bbad6" dependencies = [ "cc", ] @@ -1493,9 +1509,9 @@ dependencies = [ [[package]] name = "const-hex" -version = "1.13.2" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "487981fa1af147182687064d0a2c336586d337a606595ced9ffb0c685c250c73" +checksum = "4b0485bab839b018a8f1723fc5391819fea5f8f0f32288ef8a735fd096b6160c" dependencies = [ "cfg-if", "cpufeatures", @@ -1549,9 +1565,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ "libc", ] @@ -1655,18 +1671,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.13" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -1683,15 +1699,15 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crunchy" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" [[package]] name = "crypto-bigint" @@ -1800,7 +1816,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] @@ -1848,7 +1864,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] @@ -1870,7 +1886,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core 0.20.10", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] @@ -1901,15 +1917,15 @@ checksum = "04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728" [[package]] name = "data-encoding" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" +checksum = "0e60eed09d8c01d3cee5b7d30acb059b76614c918fa0f992e0dd6eeb10daad6f" [[package]] name = "data-encoding-macro" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1559b6cba622276d6d63706db152618eeb15b89b3e4041446b05876e352e639" +checksum = "5b16d9d0d88a5273d830dac8b78ceb217ffc9b1d5404e5597a3542515329405b" dependencies = [ "data-encoding", "data-encoding-macro-internal", @@ -1917,12 +1933,12 @@ dependencies = [ [[package]] name = "data-encoding-macro-internal" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "332d754c0af53bc87c108fed664d121ecf59207ec4196041f04d6ab9002ad33f" +checksum = "1145d32e826a7748b69ee8fc62d3e6355ff7f1051df53141e7048162fc90481b" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.96", ] [[package]] @@ -2036,7 +2052,7 @@ checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] @@ -2049,7 +2065,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version 0.4.1", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] @@ -2069,17 +2085,17 @@ checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", "unicode-xid", ] [[package]] name = "diesel" -version = "2.2.5" +version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf9649c05e0a9dbd6d0b0b8301db5182b972d0fd02f0a7c6736cf632d7c0fd5" +checksum = "ccf1bedf64cdb9643204a36dd15b19a6ce8e7aa7f7b105868e9f1fad5ffa7d12" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "byteorder", "diesel_derives", "itoa", @@ -2097,7 +2113,7 @@ dependencies = [ "dsl_auto_type", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] @@ -2117,7 +2133,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "209c735641a413bc68c4923a9d6ad4bcb3ca306b794edaa7eb0b3228a99ffb25" dependencies = [ - "syn 2.0.89", + "syn 2.0.96", ] [[package]] @@ -2193,9 +2209,9 @@ dependencies = [ [[package]] name = "discv5" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "898d136ecb64116ec68aecf14d889bd30f8b1fe0c19e262953f7388dbe77052e" +checksum = "c4b4e7798d2ff74e29cee344dc490af947ae657d6ab5273dde35d58ce06a4d71" dependencies = [ "aes 0.8.4", "aes-gcm", @@ -2232,7 +2248,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] @@ -2263,7 +2279,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] @@ -2443,7 +2459,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] @@ -2507,12 +2523,12 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2563,6 +2579,7 @@ name = "eth2" version = "0.1.0" dependencies = [ "derivative", + "either", "eth2_keystore", "ethereum_serde_utils", "ethereum_ssz", @@ -2572,9 +2589,7 @@ dependencies = [ "lighthouse_network", "mediatype", "pretty_reqwest_error", - "procfs", "proto_array", - "psutil", "reqwest", "reqwest-eventsource", "sensitive_url", @@ -2639,7 +2654,7 @@ dependencies = [ "sha2 0.9.9", "tempfile", "unicode-normalization", - "uuid", + "uuid 0.8.2", "zeroize", ] @@ -2679,7 +2694,7 @@ dependencies = [ "serde_repr", "tempfile", "tiny-bip39", - "uuid", + "uuid 0.8.2", ] [[package]] @@ -2825,7 +2840,7 @@ dependencies = [ "darling 0.20.10", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] @@ -2858,7 +2873,7 @@ dependencies = [ "dunce", "ethers-core", "eyre", - "getrandom", + "getrandom 0.2.15", "hex", "proc-macro2", "quote", @@ -2929,7 +2944,7 @@ dependencies = [ "futures-core", "futures-timer", "futures-util", - "getrandom", + "getrandom 0.2.15", "hashers", "hex", "http 0.2.12", @@ -2959,9 +2974,9 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "event-listener" -version = "5.3.1" +version = "5.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" +checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae" dependencies = [ "concurrent-queue", "parking", @@ -2970,11 +2985,11 @@ dependencies = [ [[package]] name = "event-listener-strategy" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f214dc438f977e6d4e3500aaa277f5ad94ca83fbbd9b1a15713ce2344ccc5a1" +checksum = "3c3e4e0dd3673c1139bf041f3008816d9cf2946bbfac2945c09e523b8d7b05b2" dependencies = [ - "event-listener 5.3.1", + "event-listener 5.4.0", "pin-project-lite", ] @@ -3089,9 +3104,9 @@ checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" [[package]] name = "fastrand" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "fastrlp" @@ -3104,6 +3119,17 @@ dependencies = [ "bytes", ] +[[package]] +name = "fastrlp" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce8dba4714ef14b8274c371879b175aa55b16b30f269663f19d576f380018dc4" +dependencies = [ + "arrayvec", + "auto_impl", + "bytes", +] + [[package]] name = "fdlimit" version = "0.3.0" @@ -3216,9 +3242,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "foldhash" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" +checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" [[package]] name = "foreign-types" @@ -3343,9 +3369,9 @@ checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-lite" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cef40d21ae2c515b51041df9ed313ed21e572df340ea58a922a0aefe7e8891a1" +checksum = "f5edaec856126859abb19ed65f39e90fea3a9574b9707f13539acf4abf7eb532" dependencies = [ "futures-core", "pin-project-lite", @@ -3359,7 +3385,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] @@ -3369,7 +3395,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f2f12607f92c69b12ed746fabf9ca4f5c482cba46679c1a75b874ed7c26adb" dependencies = [ "futures-io", - "rustls 0.23.18", + "rustls 0.23.21", "rustls-pki-types", ] @@ -3422,6 +3448,19 @@ dependencies = [ "byteorder", ] +[[package]] +name = "generator" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bd114ceda131d3b1d665eba35788690ad37f5916457286b32ab6fd3c438dd" +dependencies = [ + "cfg-if", + "libc", + "log", + "rustversion", + "windows 0.58.0", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -3463,10 +3502,22 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.13.3+wasi-0.2.2", + "windows-targets 0.52.6", +] + [[package]] name = "ghash" version = "0.5.1" @@ -3500,14 +3551,14 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] name = "glob" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "gloo-timers" @@ -3534,7 +3585,7 @@ dependencies = [ "fnv", "futures", "futures-timer", - "getrandom", + "getrandom 0.2.15", "hashlink 0.9.1", "hex_fmt", "libp2p", @@ -3599,7 +3650,26 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.6.0", + "indexmap 2.7.1", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.2.0", + "indexmap 2.7.1", "slab", "tokio", "tokio-util", @@ -3649,9 +3719,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.1" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" dependencies = [ "allocator-api2", "equivalent", @@ -3710,6 +3780,16 @@ dependencies = [ "http 0.2.12", ] +[[package]] +name = "health_metrics" +version = "0.1.0" +dependencies = [ + "eth2", + "metrics", + "procfs", + "psutil", +] + [[package]] name = "heck" version = "0.4.1" @@ -3752,12 +3832,6 @@ dependencies = [ "serde", ] -[[package]] -name = "hex-literal" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fe2267d4ed49bc07b63801559be28c718ea06c4738b7a03c94df7386d2cde46" - [[package]] name = "hex_fmt" version = "0.3.0" @@ -3766,10 +3840,11 @@ checksum = "b07f60793ff0a4d9cef0f18e63b5357e06209987153a64648c972c1e5aff336f" [[package]] name = "hickory-proto" -version = "0.24.1" +version = "0.25.0-alpha.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07698b8420e2f0d6447a436ba999ec85d8fbf2a398bbd737b82cac4a2e96e512" +checksum = "d063c0692ee669aa6d261988aa19ca5510f1cc40e4f211024f50c888499a35d7" dependencies = [ + "async-recursion", "async-trait", "cfg-if", "data-encoding", @@ -3777,12 +3852,12 @@ dependencies = [ "futures-channel", "futures-io", "futures-util", - "idna 0.4.0", + "idna", "ipnet", "once_cell", "rand", "socket2", - "thiserror 1.0.69", + "thiserror 2.0.11", "tinyvec", "tokio", "tracing", @@ -3791,21 +3866,21 @@ dependencies = [ [[package]] name = "hickory-resolver" -version = "0.24.1" +version = "0.25.0-alpha.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28757f23aa75c98f254cf0405e6d8c25b831b32921b050a66692427679b1f243" +checksum = "42bc352e4412fb657e795f79b4efcf2bd60b59ee5ca0187f3554194cd1107a27" dependencies = [ "cfg-if", "futures-util", "hickory-proto", "ipconfig", - "lru-cache", + "moka", "once_cell", "parking_lot 0.12.3", "rand", "resolv-conf", "smallvec", - "thiserror 1.0.69", + "thiserror 2.0.11", "tokio", "tracing", ] @@ -3861,11 +3936,11 @@ dependencies = [ [[package]] name = "home" -version = "0.5.9" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3892,9 +3967,9 @@ dependencies = [ [[package]] name = "http" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" dependencies = [ "bytes", "fnv", @@ -3919,7 +3994,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.1.0", + "http 1.2.0", ] [[package]] @@ -3930,7 +4005,7 @@ checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "pin-project-lite", ] @@ -3944,6 +4019,7 @@ dependencies = [ "bs58 0.4.0", "bytes", "directory", + "either", "eth1", "eth2", "ethereum_serde_utils", @@ -3951,6 +4027,7 @@ dependencies = [ "execution_layer", "futures", "genesis", + "health_metrics", "hex", "lighthouse_network", "lighthouse_version", @@ -3986,6 +4063,7 @@ name = "http_metrics" version = "0.1.0" dependencies = [ "beacon_chain", + "health_metrics", "lighthouse_network", "lighthouse_version", "logging", @@ -4004,9 +4082,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.9.5" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" +checksum = "f2d708df4e7140240a16cd6ab0ab65c972d7433ab77819ea693fde9c43811e2a" [[package]] name = "httpdate" @@ -4022,15 +4100,15 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.31" +version = "0.14.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c08302e8fa335b151b788c775ff56e7a03ae64ff85c548ee820fecb70356e85" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" dependencies = [ "bytes", "futures-channel", "futures-core", "futures-util", - "h2", + "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", "httparse", @@ -4046,14 +4124,15 @@ dependencies = [ [[package]] name = "hyper" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f" +checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.1.0", + "h2 0.4.7", + "http 1.2.0", "http-body 1.0.1", "httparse", "httpdate", @@ -4061,6 +4140,7 @@ dependencies = [ "pin-project-lite", "smallvec", "tokio", + "want", ] [[package]] @@ -4071,7 +4151,7 @@ checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http 0.2.12", - "hyper 0.14.31", + "hyper 0.14.32", "rustls 0.21.12", "tokio", "tokio-rustls 0.24.1", @@ -4084,7 +4164,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes", - "hyper 0.14.31", + "hyper 0.14.32", "native-tls", "tokio", "tokio-native-tls", @@ -4097,13 +4177,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" dependencies = [ "bytes", + "futures-channel", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", - "hyper 1.5.1", + "hyper 1.6.0", "pin-project-lite", + "socket2", "tokio", "tower-service", + "tracing", ] [[package]] @@ -4244,7 +4327,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] @@ -4253,16 +4336,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" -[[package]] -name = "idna" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "idna" version = "1.0.3" @@ -4314,21 +4387,44 @@ dependencies = [ "rtnetlink", "system-configuration 0.6.1", "tokio", - "windows", + "windows 0.53.0", ] [[package]] name = "igd-next" -version = "0.14.3" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "064d90fec10d541084e7b39ead8875a5a80d9114a2b18791565253bae25f49e4" +checksum = "76b0d7d4541def58a37bf8efc559683f21edce7c82f0d866c93ac21f7e098f93" dependencies = [ "async-trait", "attohttpc", "bytes", "futures", - "http 0.2.12", - "hyper 0.14.31", + "http 1.2.0", + "http-body-util", + "hyper 1.6.0", + "hyper-util", + "log", + "rand", + "tokio", + "url", + "xmltree", +] + +[[package]] +name = "igd-next" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2830127baaaa55dae9aa5ee03158d5aa3687a9c2c11ce66870452580cc695df4" +dependencies = [ + "async-trait", + "attohttpc", + "bytes", + "futures", + "http 1.2.0", + "http-body-util", + "hyper 1.6.0", + "hyper-util", "log", "rand", "tokio", @@ -4351,7 +4447,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba6a270039626615617f3f36d15fc827041df3b78c439da2cadfa47455a77f2f" dependencies = [ - "parity-scale-codec 3.7.0", + "parity-scale-codec 3.6.12", ] [[package]] @@ -4389,7 +4485,7 @@ checksum = "a0eb5a3343abf848c0984fe4604b2b105da9539376e24fc0a3b0007411ae4fd9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] @@ -4410,13 +4506,13 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.6.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" dependencies = [ "arbitrary", "equivalent", - "hashbrown 0.15.1", + "hashbrown 0.15.2", "serde", ] @@ -4510,19 +4606,19 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.10.1" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" [[package]] name = "is-terminal" -version = "0.4.13" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" +checksum = "e19b23d53f35ce9f56aebc7d1bb4e6ac1e9c0db7ac85c8d1760c04379edced37" dependencies = [ "hermit-abi 0.4.0", "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4560,9 +4656,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "540654e97a3f4470a492cd30ff187bc95d89557a903a2bbf112e2fae98104ef2" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] name = "jobserver" @@ -4575,10 +4671,11 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.72" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -4659,6 +4756,11 @@ version = "0.1.0" dependencies = [ "arbitrary", "c-kzg", + "crate_crypto_internal_eth_kzg_bls12_381", + "crate_crypto_internal_eth_kzg_erasure_codes", + "crate_crypto_internal_eth_kzg_maybe_rayon", + "crate_crypto_internal_eth_kzg_polynomial", + "crate_crypto_kzg_multi_open_fk20", "criterion", "derivative", "ethereum_hashing", @@ -4748,9 +4850,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.164" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433bfe06b8c75da9b2e3fbea6e5329ff87748f0b144ef75306e674c3f6f7c13f" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "libflate" @@ -4778,9 +4880,9 @@ dependencies = [ [[package]] name = "libloading" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", "windows-targets 0.52.6", @@ -4809,15 +4911,15 @@ dependencies = [ [[package]] name = "libp2p" -version = "0.54.1" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbbe80f9c7e00526cd6b838075b9c171919404a4732cb2fa8ece0a093223bfc4" +checksum = "b72dc443ddd0254cb49a794ed6b6728400ee446a0f7ab4a07d0209ee98de20e9" dependencies = [ "bytes", "either", "futures", "futures-timer", - "getrandom", + "getrandom 0.2.15", "libp2p-allow-block-list", "libp2p-connection-limits", "libp2p-core", @@ -4836,38 +4938,36 @@ dependencies = [ "multiaddr", "pin-project", "rw-stream-sink", - "thiserror 1.0.69", + "thiserror 2.0.11", ] [[package]] name = "libp2p-allow-block-list" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1027ccf8d70320ed77e984f273bc8ce952f623762cb9bf2d126df73caef8041" +checksum = "38944b7cb981cc93f2f0fb411ff82d0e983bd226fbcc8d559639a3a73236568b" dependencies = [ "libp2p-core", "libp2p-identity", "libp2p-swarm", - "void", ] [[package]] name = "libp2p-connection-limits" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d003540ee8baef0d254f7b6bfd79bac3ddf774662ca0abf69186d517ef82ad8" +checksum = "efe9323175a17caa8a2ed4feaf8a548eeef5e0b72d03840a0eab4bcb0210ce1c" dependencies = [ "libp2p-core", "libp2p-identity", "libp2p-swarm", - "void", ] [[package]] name = "libp2p-core" -version = "0.42.0" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a61f26c83ed111104cd820fe9bc3aaabbac5f1652a1d213ed6e900b7918a1298" +checksum = "193c75710ba43f7504ad8f58a62ca0615b1d7e572cb0f1780bc607252c39e9ef" dependencies = [ "either", "fnv", @@ -4883,19 +4983,17 @@ dependencies = [ "quick-protobuf", "rand", "rw-stream-sink", - "smallvec", - "thiserror 1.0.69", + "thiserror 2.0.11", "tracing", "unsigned-varint 0.8.0", - "void", "web-time", ] [[package]] name = "libp2p-dns" -version = "0.42.0" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97f37f30d5c7275db282ecd86e54f29dd2176bd3ac656f06abf43bedb21eb8bd" +checksum = "1b780a1150214155b0ed1cdf09fbd2e1b0442604f9146a431d1b21d23eef7bd7" dependencies = [ "async-trait", "futures", @@ -4909,9 +5007,9 @@ dependencies = [ [[package]] name = "libp2p-identify" -version = "0.45.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1711b004a273be4f30202778856368683bd9a83c4c7dcc8f848847606831a4e3" +checksum = "e8c06862544f02d05d62780ff590cc25a75f5c2b9df38ec7a370dcae8bb873cf" dependencies = [ "asynchronous-codec", "either", @@ -4921,13 +5019,11 @@ dependencies = [ "libp2p-core", "libp2p-identity", "libp2p-swarm", - "lru", "quick-protobuf", "quick-protobuf-codec", "smallvec", - "thiserror 1.0.69", + "thiserror 2.0.11", "tracing", - "void", ] [[package]] @@ -4954,11 +5050,10 @@ dependencies = [ [[package]] name = "libp2p-mdns" -version = "0.46.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14b8546b6644032565eb29046b42744aee1e9f261ed99671b2c93fb140dba417" +checksum = "11d0ba095e1175d797540e16b62e7576846b883cb5046d4159086837b36846cc" dependencies = [ - "data-encoding", "futures", "hickory-proto", "if-watch", @@ -4970,14 +5065,13 @@ dependencies = [ "socket2", "tokio", "tracing", - "void", ] [[package]] name = "libp2p-metrics" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ebafa94a717c8442d8db8d3ae5d1c6a15e30f2d347e0cd31d057ca72e42566" +checksum = "2ce58c64292e87af624fcb86465e7dd8342e46a388d71e8fec0ab37ee789630a" dependencies = [ "futures", "libp2p-core", @@ -4991,9 +5085,9 @@ dependencies = [ [[package]] name = "libp2p-mplex" -version = "0.42.0" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41187ab8f6c835ad864edf94224f666f636ee2d270601422c1441f739e0abccc" +checksum = "8aaa6fee3722e355443058472fc4705d78681bc2d8e447a0bdeb3fecf40cd197" dependencies = [ "asynchronous-codec", "bytes", @@ -5010,13 +5104,12 @@ dependencies = [ [[package]] name = "libp2p-noise" -version = "0.45.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36b137cb1ae86ee39f8e5d6245a296518912014eaa87427d24e6ff58cfc1b28c" +checksum = "afcc133e0f3cea07acde6eb8a9665cb11b600bd61110b010593a0210b8153b16" dependencies = [ "asynchronous-codec", "bytes", - "curve25519-dalek", "futures", "libp2p-core", "libp2p-identity", @@ -5025,10 +5118,9 @@ dependencies = [ "once_cell", "quick-protobuf", "rand", - "sha2 0.10.8", "snow", "static_assertions", - "thiserror 1.0.69", + "thiserror 2.0.11", "tracing", "x25519-dalek", "zeroize", @@ -5036,9 +5128,9 @@ dependencies = [ [[package]] name = "libp2p-plaintext" -version = "0.42.0" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b63d926c6be56a2489e0e7316b17fe95a70bc5c4f3e85740bb3e67c0f3c6a44" +checksum = "7e659439578fc6d305da8303834beb9d62f155f40e7f5b9d81c9f2b2c69d1926" dependencies = [ "asynchronous-codec", "bytes", @@ -5052,33 +5144,31 @@ dependencies = [ [[package]] name = "libp2p-quic" -version = "0.11.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46352ac5cd040c70e88e7ff8257a2ae2f891a4076abad2c439584a31c15fd24e" +checksum = "41432a159b00424a0abaa2c80d786cddff81055ac24aa127e0cf375f7858d880" dependencies = [ - "bytes", "futures", "futures-timer", "if-watch", "libp2p-core", "libp2p-identity", "libp2p-tls", - "parking_lot 0.12.3", "quinn", "rand", "ring 0.17.8", - "rustls 0.23.18", + "rustls 0.23.21", "socket2", - "thiserror 1.0.69", + "thiserror 2.0.11", "tokio", "tracing", ] [[package]] name = "libp2p-swarm" -version = "0.45.1" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7dd6741793d2c1fb2088f67f82cf07261f25272ebe3c0b0c311e0c6b50e851a" +checksum = "803399b4b6f68adb85e63ab573ac568154b193e9a640f03e0f2890eabbcb37f8" dependencies = [ "either", "fnv", @@ -5094,7 +5184,6 @@ dependencies = [ "smallvec", "tokio", "tracing", - "void", "web-time", ] @@ -5107,21 +5196,20 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] name = "libp2p-tcp" -version = "0.42.0" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad964f312c59dcfcac840acd8c555de8403e295d39edf96f5240048b5fcaa314" +checksum = "65346fb4d36035b23fec4e7be4c320436ba53537ce9b6be1d1db1f70c905cad0" dependencies = [ "futures", "futures-timer", "if-watch", "libc", "libp2p-core", - "libp2p-identity", "socket2", "tokio", "tracing", @@ -5129,9 +5217,9 @@ dependencies = [ [[package]] name = "libp2p-tls" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b23dddc2b9c355f73c1e36eb0c3ae86f7dc964a3715f0731cfad352db4d847" +checksum = "dcaebc1069dea12c5b86a597eaaddae0317c2c2cb9ec99dc94f82fd340f5c78b" dependencies = [ "futures", "futures-rustls", @@ -5139,39 +5227,38 @@ dependencies = [ "libp2p-identity", "rcgen", "ring 0.17.8", - "rustls 0.23.18", + "rustls 0.23.21", "rustls-webpki 0.101.7", - "thiserror 1.0.69", + "thiserror 2.0.11", "x509-parser", "yasna", ] [[package]] name = "libp2p-upnp" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01bf2d1b772bd3abca049214a3304615e6a36fa6ffc742bdd1ba774486200b8f" +checksum = "d457b9ecceb66e7199f049926fad447f1f17f040e8d29d690c086b4cab8ed14a" dependencies = [ "futures", "futures-timer", - "igd-next", + "igd-next 0.15.1", "libp2p-core", "libp2p-swarm", "tokio", "tracing", - "void", ] [[package]] name = "libp2p-yamux" -version = "0.46.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "788b61c80789dba9760d8c669a5bedb642c8267555c803fabd8396e4ca5c5882" +checksum = "f15df094914eb4af272acf9adaa9e287baa269943f32ea348ba29cfb9bfc60d8" dependencies = [ "either", "futures", "libp2p-core", - "thiserror 1.0.69", + "thiserror 2.0.11", "tracing", "yamux 0.12.1", "yamux 0.13.4", @@ -5183,7 +5270,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "libc", ] @@ -5248,9 +5335,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.20" +version = "1.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472" +checksum = "df9b68e50e6e0b26f672573834882eb57759f6db9b3be2ea3c35c91188bb4eaa" dependencies = [ "cc", "pkg-config", @@ -5291,6 +5378,7 @@ dependencies = [ "slasher", "slashing_protection", "slog", + "store", "task_executor", "tempfile", "types", @@ -5365,12 +5453,6 @@ dependencies = [ "target_info", ] -[[package]] -name = "linked-hash-map" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" - [[package]] name = "linux-raw-sys" version = "0.1.4" @@ -5379,9 +5461,9 @@ checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" [[package]] name = "linux-raw-sys" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "litemap" @@ -5430,9 +5512,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.22" +version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" [[package]] name = "logging" @@ -5455,22 +5537,26 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + [[package]] name = "lru" version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ - "hashbrown 0.15.1", -] - -[[package]] -name = "lru-cache" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" -dependencies = [ - "linked-hash-map", + "hashbrown 0.15.2", ] [[package]] @@ -5686,22 +5772,21 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.0" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +checksum = "b8402cab7aefae129c6977bb0ff1b8fd9a04eb5b51efc50a70bea51cda0c7924" dependencies = [ "adler2", ] [[package]] name = "mio" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ - "hermit-abi 0.3.9", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.52.0", ] @@ -5711,11 +5796,31 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9366861eb2a2c436c20b12c8dbec5f798cea6b47ad99216be0282942e2c81ea0" +[[package]] +name = "moka" +version = "0.12.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9321642ca94a4282428e6ea4af8cc2ca4eac48ac7a6a4ea8f33f76d0ce70926" +dependencies = [ + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "loom", + "parking_lot 0.12.3", + "portable-atomic", + "rustc_version 0.4.1", + "smallvec", + "tagptr", + "thiserror 1.0.69", + "uuid 1.12.1", +] + [[package]] name = "monitoring_api" version = "0.1.0" dependencies = [ "eth2", + "health_metrics", "lighthouse_version", "metrics", "regex", @@ -5767,9 +5872,9 @@ dependencies = [ [[package]] name = "multihash" -version = "0.19.2" +version = "0.19.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc41f430805af9d1cf4adae4ed2149c759b877b01d909a1f40256188d09345d2" +checksum = "6b430e7953c29dd6a09afc29ff0bb69c6e306329ee6794700aee27b76a1aea8d" dependencies = [ "core2", "unsigned-varint 0.8.0", @@ -5791,9 +5896,9 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" +checksum = "0dab59f8e050d5df8e4dd87d9206fb6f65a483e20ac9fda365ade4fab353196c" dependencies = [ "libc", "log", @@ -5845,24 +5950,23 @@ dependencies = [ [[package]] name = "netlink-proto" -version = "0.11.3" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b33524dc0968bfad349684447bfce6db937a9ac3332a1fe60c0c5a5ce63f21" +checksum = "72452e012c2f8d612410d89eea01e2d9b56205274abb35d53f60200b2ec41d60" dependencies = [ "bytes", "futures", "log", "netlink-packet-core", "netlink-sys", - "thiserror 1.0.69", - "tokio", + "thiserror 2.0.11", ] [[package]] name = "netlink-sys" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "416060d346fbaf1f23f9512963e3e878f1a78e707cb699ba9215761754244307" +checksum = "16c903aa70590cb93691bf97a767c8d1d6122d2cc9070433deb3bbf36ce8bd23" dependencies = [ "bytes", "futures", @@ -5893,7 +5997,7 @@ dependencies = [ "genesis", "gossipsub", "hex", - "igd-next", + "igd-next 0.16.0", "itertools 0.10.5", "kzg", "lighthouse_network", @@ -5948,7 +6052,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "cfg-if", "cfg_aliases", "libc", @@ -6082,9 +6186,9 @@ dependencies = [ [[package]] name = "object" -version = "0.36.5" +version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" dependencies = [ "memchr", ] @@ -6150,11 +6254,11 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.68" +version = "0.10.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" +checksum = "f5e534d133a060a3c19daec1eb3e98ec6f4685978834f2dbadfe2ec215bab64e" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "cfg-if", "foreign-types", "libc", @@ -6171,14 +6275,14 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] name = "openssl-probe" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "openssl-src" @@ -6267,16 +6371,15 @@ dependencies = [ [[package]] name = "parity-scale-codec" -version = "3.7.0" +version = "3.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be4817d39f3272f69c59fe05d0535ae6456c2dc2fa1ba02910296c7e0a5c590" +checksum = "306800abfa29c7f16596b5970a588435e3d5b3149683d00c12b699cc19f895ee" dependencies = [ "arrayvec", "bitvec 1.0.1", "byte-slice-cast", "impl-trait-for-tuples", - "parity-scale-codec-derive 3.7.0", - "rustversion", + "parity-scale-codec-derive 3.6.12", "serde", ] @@ -6294,14 +6397,14 @@ dependencies = [ [[package]] name = "parity-scale-codec-derive" -version = "3.7.0" +version = "3.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8781a75c6205af67215f382092b6e0a4ff3734798523e69073d4bcd294ec767b" +checksum = "d830939c76d294956402033aee57a6da7b438f2294eb94864c37b0569053a42c" dependencies = [ "proc-macro-crate 3.2.0", "proc-macro2", "quote", - "syn 2.0.89", + "syn 1.0.109", ] [[package]] @@ -6353,7 +6456,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.7", + "redox_syscall 0.5.8", "smallvec", "windows-targets 0.52.6", ] @@ -6423,12 +6526,12 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.14" +version = "2.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879952a81a83930934cbf1786752d6dedc3b1f29e8f8fb2ad1d0a36f377cf442" +checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" dependencies = [ "memchr", - "thiserror 1.0.69", + "thiserror 2.0.11", "ucd-trie", ] @@ -6444,47 +6547,47 @@ dependencies = [ [[package]] name = "phf" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" dependencies = [ "phf_shared", ] [[package]] name = "phf_shared" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" dependencies = [ "siphasher", ] [[package]] name = "pin-project" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" +checksum = "1e2ec53ad785f4d35dac0adea7f7dc6f1bb277ad84a680c7afefeae05d1f5916" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" +checksum = "d56a66c0c55993aa927429d0f8a0abfd74f084e4d9c192cffed01e418d83eefb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] name = "pin-project-lite" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" @@ -6562,7 +6665,7 @@ dependencies = [ "concurrent-queue", "hermit-abi 0.4.0", "pin-project-lite", - "rustix 0.38.41", + "rustix 0.38.44", "tracing", "windows-sys 0.59.0", ] @@ -6590,6 +6693,12 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "portable-atomic" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" + [[package]] name = "postgres-protocol" version = "0.6.7" @@ -6653,12 +6762,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.25" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" +checksum = "6924ced06e1f7dfe3fa48d57b9f74f55d8915f5036121bef647ef4b204895fac" dependencies = [ "proc-macro2", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] @@ -6718,9 +6827,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.92" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" dependencies = [ "unicode-ident", ] @@ -6775,7 +6884,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] @@ -6786,7 +6895,7 @@ checksum = "b4c2511913b88df1637da85cc8d96ec8e43a3f8bb8ccb71ee1ac240d6f3df58d" dependencies = [ "bit-set", "bit-vec", - "bitflags 2.6.0", + "bitflags 2.8.0", "lazy_static", "num-traits", "rand", @@ -6800,13 +6909,13 @@ dependencies = [ [[package]] name = "proptest-derive" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff7ff745a347b87471d859a377a9a404361e7efc2a971d73424a6d183c0fc77" +checksum = "4ee1c9ac207483d5e7db4940700de86a9aae46ef90c48b57f99fe7edb8345e49" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] @@ -6907,10 +7016,10 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.0.0", - "rustls 0.23.18", + "rustc-hash 2.1.0", + "rustls 0.23.21", "socket2", - "thiserror 2.0.3", + "thiserror 2.0.11", "tokio", "tracing", ] @@ -6922,14 +7031,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" dependencies = [ "bytes", - "getrandom", + "getrandom 0.2.15", "rand", "ring 0.17.8", - "rustc-hash 2.0.0", - "rustls 0.23.18", + "rustc-hash 2.1.0", + "rustls 0.23.21", "rustls-pki-types", "slab", - "thiserror 2.0.3", + "thiserror 2.0.11", "tinyvec", "tracing", "web-time", @@ -6937,9 +7046,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.7" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5a626c6807713b15cac82a6acaccd6043c9a5408c24baae07611fec3f243da" +checksum = "1c40286217b4ba3a71d644d752e6a0b71f13f1b6a2c5311acfcbe0c2418ed904" dependencies = [ "cfg_aliases", "libc", @@ -6951,9 +7060,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] @@ -7019,7 +7128,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.15", ] [[package]] @@ -7065,9 +7174,9 @@ dependencies = [ [[package]] name = "redb" -version = "2.2.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84b1de48a7cf7ba193e81e078d17ee2b786236eed1d3f7c60f8a09545efc4925" +checksum = "ea0a72cd7140de9fc3e318823b883abf819c20d478ec89ce880466dc2ef263c6" dependencies = [ "libc", ] @@ -7083,11 +7192,11 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" +checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", ] [[package]] @@ -7096,7 +7205,7 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ - "getrandom", + "getrandom 0.2.15", "libredox", "thiserror 1.0.69", ] @@ -7156,10 +7265,10 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2", + "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.31", + "hyper 0.14.32", "hyper-rustls", "hyper-tls", "ipnet", @@ -7261,7 +7370,7 @@ checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", "cfg-if", - "getrandom", + "getrandom 0.2.15", "libc", "spin 0.9.8", "untrusted 0.9.0", @@ -7334,19 +7443,21 @@ dependencies = [ [[package]] name = "ruint" -version = "1.12.3" +version = "1.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c3cc4c2511671f327125da14133d0c5c5d137f006a1017a16f557bc85b16286" +checksum = "f5ef8fb1dd8de3870cb8400d51b4c2023854bbafd5431a3ac7e7317243e22d2f" dependencies = [ "alloy-rlp", "arbitrary", "ark-ff 0.3.0", "ark-ff 0.4.2", "bytes", - "fastrlp", + "fastrlp 0.3.1", + "fastrlp 0.4.0", "num-bigint", + "num-integer", "num-traits", - "parity-scale-codec 3.7.0", + "parity-scale-codec 3.6.12", "primitive-types 0.12.2", "proptest", "rand", @@ -7405,9 +7516,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc-hash" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" +checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" [[package]] name = "rustc-hex" @@ -7430,7 +7541,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ - "semver 1.0.23", + "semver 1.0.25", ] [[package]] @@ -7458,15 +7569,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.41" +version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "errno", "libc", - "linux-raw-sys 0.4.14", - "windows-sys 0.52.0", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", ] [[package]] @@ -7497,9 +7608,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.18" +version = "0.23.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c9cc1d47e243d655ace55ed38201c19ae02c148ae56412ab8750e8f0166ab7f" +checksum = "8f287924602bf649d949c63dc8ac8b235fa5387d394020705b80c4eb597ce5b8" dependencies = [ "once_cell", "ring 0.17.8", @@ -7529,9 +7640,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" dependencies = [ "web-time", ] @@ -7559,9 +7670,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" +checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" [[package]] name = "rusty-fork" @@ -7588,9 +7699,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd" [[package]] name = "safe_arith" @@ -7622,7 +7733,7 @@ checksum = "346a3b32eba2640d17a9cb5927056b08f3de90f65b72fe09402c2ad07d684d0b" dependencies = [ "cfg-if", "derive_more 1.0.0", - "parity-scale-codec 3.7.0", + "parity-scale-codec 3.6.12", "scale-info-derive", ] @@ -7635,7 +7746,7 @@ dependencies = [ "proc-macro-crate 3.2.0", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] @@ -7724,7 +7835,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "core-foundation", "core-foundation-sys", "libc", @@ -7733,9 +7844,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.12.1" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" dependencies = [ "core-foundation-sys", "libc", @@ -7752,9 +7863,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.23" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +checksum = "f79dfe2d285b0488816f30e700a7438c5a73d816b5b7d3ac72fbc48b0d185e03" dependencies = [ "serde", ] @@ -7790,9 +7901,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.215" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] @@ -7809,20 +7920,20 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.215" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] name = "serde_json" -version = "1.0.133" +version = "1.0.138" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" +checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949" dependencies = [ "itoa", "memchr", @@ -7848,7 +7959,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] @@ -7900,7 +8011,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.6.0", + "indexmap 2.7.1", "itoa", "ryu", "serde", @@ -8036,13 +8147,13 @@ dependencies = [ [[package]] name = "simple_asn1" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085" +checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb" dependencies = [ "num-bigint", "num-traits", - "thiserror 1.0.69", + "thiserror 2.0.11", "time", ] @@ -8067,9 +8178,9 @@ dependencies = [ [[package]] name = "siphasher" -version = "0.3.11" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" [[package]] name = "slab" @@ -8292,9 +8403,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" dependencies = [ "libc", "windows-sys 0.52.0", @@ -8419,6 +8530,7 @@ dependencies = [ "metrics", "parking_lot 0.12.3", "rand", + "redb", "safe_arith", "serde", "slog", @@ -8521,9 +8633,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.89" +version = "2.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d46482f1c1c87acd84dea20c1bf5ebff4c757009ed6bf19cfd36fb10e92c4e" +checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" dependencies = [ "proc-macro2", "quote", @@ -8550,7 +8662,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] @@ -8585,7 +8697,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "core-foundation", "system-configuration-sys 0.6.0", ] @@ -8621,6 +8733,12 @@ dependencies = [ "types", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "take_mut" version = "0.2.2" @@ -8662,14 +8780,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.14.0" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" +checksum = "38c246215d7d24f48ae091a2902398798e05d978b24315d6efbc00ede9a8bb91" dependencies = [ "cfg-if", "fastrand", + "getrandom 0.3.1", "once_cell", - "rustix 0.38.41", + "rustix 0.38.44", "windows-sys 0.59.0", ] @@ -8695,11 +8814,11 @@ dependencies = [ [[package]] name = "terminal_size" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f599bd7ca042cfdf8f4512b277c02ba102247820f9d9d4a9f521f496751a6ef" +checksum = "5352447f921fda68cf61b4101566c0bdb5104eff6804d0678e5227580ab6a4e9" dependencies = [ - "rustix 0.38.41", + "rustix 0.38.44", "windows-sys 0.59.0", ] @@ -8747,11 +8866,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.3" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c006c85c7651b3cf2ada4584faa36773bd07bac24acfb39f3c431b36d7e667aa" +checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" dependencies = [ - "thiserror-impl 2.0.3", + "thiserror-impl 2.0.11", ] [[package]] @@ -8762,18 +8881,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] name = "thiserror-impl" -version = "2.0.3" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" +checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] @@ -8828,9 +8947,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.36" +version = "0.3.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" dependencies = [ "deranged", "itoa", @@ -8849,9 +8968,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" dependencies = [ "num-conv", "time-core", @@ -8918,9 +9037,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +checksum = "022db8904dfa342efe721985167e9fcd16c29b226db4397ed752a761cfce81e8" dependencies = [ "tinyvec_macros", ] @@ -8933,9 +9052,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.41.1" +version = "1.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cfb5bee7a6a52939ca9224d6ac897bb669134078daa8735560897f69de4d33" +checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" dependencies = [ "backtrace", "bytes", @@ -8960,13 +9079,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] @@ -9028,9 +9147,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" dependencies = [ "futures-core", "pin-project-lite", @@ -9040,9 +9159,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" +checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" dependencies = [ "bytes", "futures-core", @@ -9089,7 +9208,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.6.0", + "indexmap 2.7.1", "toml_datetime", "winnow 0.5.40", ] @@ -9100,23 +9219,23 @@ version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.6.0", + "indexmap 2.7.1", "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.20", + "winnow 0.6.25", ] [[package]] name = "tower" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ "futures-core", "futures-util", "pin-project-lite", - "sync_wrapper 0.1.2", + "sync_wrapper 1.0.2", "tokio", "tower-layer", "tower-service", @@ -9137,9 +9256,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.40" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ "log", "pin-project-lite", @@ -9161,20 +9280,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] name = "tracing-core" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", "valuable", @@ -9203,9 +9322,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ "matchers", "nu-ansi-term", @@ -9258,7 +9377,7 @@ dependencies = [ "darling 0.20.10", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] @@ -9390,21 +9509,21 @@ checksum = "ccb97dac3243214f8d8507998906ca3e2e0b900bf9bf4870477f125b82e68f6e" [[package]] name = "unicase" -version = "2.8.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" +checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" [[package]] name = "unicode-bidi" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" -version = "1.0.14" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" +checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034" [[package]] name = "unicode-normalization" @@ -9487,7 +9606,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" dependencies = [ "form_urlencoded", - "idna 1.0.3", + "idna", "percent-encoding", ] @@ -9515,10 +9634,19 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" dependencies = [ - "getrandom", + "getrandom 0.2.15", "serde", ] +[[package]] +name = "uuid" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3758f5e68192bb96cc8f9b7e2c2cfdabb435499a28499a42f8f984092adad4b" +dependencies = [ + "getrandom 0.2.15", +] + [[package]] name = "validator_client" version = "0.3.5" @@ -9534,7 +9662,7 @@ dependencies = [ "eth2", "fdlimit", "graffiti_file", - "hyper 1.5.1", + "hyper 1.6.0", "initialized_validators", "metrics", "monitoring_api", @@ -9561,7 +9689,6 @@ dependencies = [ "bls", "deposit_contract", "derivative", - "directory", "eth2_keystore", "filesystem", "hex", @@ -9589,6 +9716,7 @@ dependencies = [ "filesystem", "futures", "graffiti_file", + "health_metrics", "initialized_validators", "itertools 0.10.5", "lighthouse_version", @@ -9621,6 +9749,7 @@ dependencies = [ name = "validator_http_metrics" version = "0.1.0" dependencies = [ + "health_metrics", "lighthouse_version", "malloc_utils", "metrics", @@ -9675,6 +9804,7 @@ dependencies = [ "beacon_node_fallback", "bls", "doppelganger_service", + "either", "environment", "eth2", "futures", @@ -9710,9 +9840,9 @@ dependencies = [ [[package]] name = "valuable" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" [[package]] name = "vcpkg" @@ -9777,7 +9907,7 @@ dependencies = [ "futures-util", "headers", "http 0.2.12", - "hyper 0.14.31", + "hyper 0.14.32", "log", "mime", "mime_guess", @@ -9799,7 +9929,6 @@ dependencies = [ name = "warp_utils" version = "0.1.0" dependencies = [ - "beacon_chain", "bytes", "eth2", "headers", @@ -9808,7 +9937,6 @@ dependencies = [ "serde", "serde_array_query", "serde_json", - "state_processing", "tokio", "types", "warp", @@ -9820,6 +9948,15 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi" +version = "0.13.3+wasi-0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "wasite" version = "0.1.0" @@ -9828,47 +9965,48 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.95" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", "once_cell", + "rustversion", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.95" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.45" +version = "0.4.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" dependencies = [ "cfg-if", "js-sys", + "once_cell", "wasm-bindgen", "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.95" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -9876,22 +10014,25 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.95" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.95" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] [[package]] name = "wasm-streams" @@ -9936,7 +10077,7 @@ dependencies = [ "env_logger 0.9.3", "eth2", "http_api", - "hyper 1.5.1", + "hyper 1.6.0", "log", "logging", "network", @@ -9957,9 +10098,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.72" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" dependencies = [ "js-sys", "wasm-bindgen", @@ -10018,7 +10159,7 @@ dependencies = [ "either", "home", "once_cell", - "rustix 0.38.41", + "rustix 0.38.44", ] [[package]] @@ -10027,7 +10168,7 @@ version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d" dependencies = [ - "redox_syscall 0.5.7", + "redox_syscall 0.5.8", "wasite", "web-sys", ] @@ -10085,6 +10226,16 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6" +dependencies = [ + "windows-core 0.58.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows-acl" version = "0.3.0" @@ -10112,10 +10263,45 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9dcc5b895a6377f1ab9fa55acedab1fd5ac0db66ad1e6c7f47e28a22e446a5dd" dependencies = [ - "windows-result", + "windows-result 0.1.2", "windows-targets 0.52.6", ] +[[package]] +name = "windows-core" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-result 0.2.0", + "windows-strings", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-implement" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + +[[package]] +name = "windows-interface" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + [[package]] name = "windows-result" version = "0.1.2" @@ -10125,6 +10311,25 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-result" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +dependencies = [ + "windows-result 0.2.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.45.0" @@ -10350,9 +10555,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.20" +version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +checksum = "ad699df48212c6cc6eb4435f35500ac6fd3b9913324f938aea302022ce19d310" dependencies = [ "memchr", ] @@ -10367,6 +10572,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "wit-bindgen-rt" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +dependencies = [ + "bitflags 2.8.0", +] + [[package]] name = "write16" version = "1.0.0" @@ -10458,9 +10672,9 @@ dependencies = [ [[package]] name = "xml-rs" -version = "0.8.23" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af310deaae937e48a26602b730250b4949e125f468f11e6990be3e5304ddd96f" +checksum = "c5b940ebc25896e71dd073bad2dbaa2abfe97b0a391415e22ad1326d9c54e3c4" [[package]] name = "xmltree" @@ -10542,7 +10756,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", "synstructure", ] @@ -10564,7 +10778,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] @@ -10584,7 +10798,7 @@ checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", "synstructure", ] @@ -10606,7 +10820,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] @@ -10628,7 +10842,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.96", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 23e52a306b..e30b6aa2b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,11 +9,13 @@ members = [ "beacon_node/client", "beacon_node/eth1", "beacon_node/execution_layer", + "beacon_node/genesis", "beacon_node/http_api", "beacon_node/http_metrics", "beacon_node/lighthouse_network", "beacon_node/lighthouse_network/gossipsub", "beacon_node/network", + "beacon_node/operation_pool", "beacon_node/store", "beacon_node/timer", @@ -30,6 +32,8 @@ members = [ "common/eth2_interop_keypairs", "common/eth2_network_config", "common/eth2_wallet_manager", + "common/filesystem", + "common/health_metrics", "common/lighthouse_version", "common/lockfile", "common/logging", @@ -48,14 +52,16 @@ members = [ "common/unused_port", "common/validator_dir", "common/warp_utils", + "consensus/fixed_bytes", "consensus/fork_choice", - "consensus/int_to_bytes", + "consensus/merkle_proof", "consensus/proto_array", "consensus/safe_arith", "consensus/state_processing", "consensus/swap_or_not_shuffle", + "consensus/types", "crypto/bls", "crypto/eth2_key_derivation", @@ -247,6 +253,7 @@ filesystem = { path = "common/filesystem" } fork_choice = { path = "consensus/fork_choice" } genesis = { path = "beacon_node/genesis" } gossipsub = { path = "beacon_node/lighthouse_network/gossipsub/" } +health_metrics = { path = "common/health_metrics" } http_api = { path = "beacon_node/http_api" } initialized_validators = { path = "validator_client/initialized_validators" } int_to_bytes = { path = "consensus/int_to_bytes" } diff --git a/Cross.toml b/Cross.toml index 871391253d..8181967f32 100644 --- a/Cross.toml +++ b/Cross.toml @@ -3,3 +3,14 @@ pre-build = ["apt-get install -y cmake clang-5.0"] [target.aarch64-unknown-linux-gnu] pre-build = ["apt-get install -y cmake clang-5.0"] + +# Allow setting page size limits for jemalloc at build time: +# For certain architectures (like aarch64), we must compile +# jemalloc with support for large page sizes, otherwise the host's +# system page size will be used, which may not work on the target systems. +# JEMALLOC_SYS_WITH_LG_PAGE=16 tells jemalloc to support up to 64-KiB +# pages. See: https://github.com/sigp/lighthouse/issues/5244 +[build.env] +passthrough = [ + "JEMALLOC_SYS_WITH_LG_PAGE", +] diff --git a/Dockerfile b/Dockerfile index 0f334e2ac8..437c864c30 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.80.0-bullseye AS builder +FROM rust:1.84.0-bullseye AS builder RUN apt-get update && apt-get -y upgrade && apt-get install -y cmake libclang-dev COPY . lighthouse ARG FEATURES diff --git a/Makefile b/Makefile index 8faf8a2e54..81477634fe 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ BUILD_PATH_AARCH64 = "target/$(AARCH64_TAG)/release" PINNED_NIGHTLY ?= nightly # List of features to use when cross-compiling. Can be overridden via the environment. -CROSS_FEATURES ?= gnosis,slasher-lmdb,slasher-mdbx,slasher-redb,jemalloc +CROSS_FEATURES ?= gnosis,slasher-lmdb,slasher-mdbx,slasher-redb,jemalloc,beacon-node-leveldb,beacon-node-redb # Cargo profile for Cross builds. Default is for local builds, CI uses an override. CROSS_PROFILE ?= release @@ -30,7 +30,7 @@ PROFILE ?= release # List of all hard forks. This list is used to set env variables for several tests so that # they run for different forks. -FORKS=phase0 altair bellatrix capella deneb electra +FORKS=phase0 altair bellatrix capella deneb electra fulu # Extra flags for Cargo CARGO_INSTALL_EXTRA_FLAGS?= @@ -63,12 +63,18 @@ install-lcli: build-x86_64: cross build --bin lighthouse --target x86_64-unknown-linux-gnu --features "portable,$(CROSS_FEATURES)" --profile "$(CROSS_PROFILE)" --locked build-aarch64: - cross build --bin lighthouse --target aarch64-unknown-linux-gnu --features "portable,$(CROSS_FEATURES)" --profile "$(CROSS_PROFILE)" --locked + # JEMALLOC_SYS_WITH_LG_PAGE=16 tells jemalloc to support up to 64-KiB + # pages, which are commonly used by aarch64 systems. + # See: https://github.com/sigp/lighthouse/issues/5244 + JEMALLOC_SYS_WITH_LG_PAGE=16 cross build --bin lighthouse --target aarch64-unknown-linux-gnu --features "portable,$(CROSS_FEATURES)" --profile "$(CROSS_PROFILE)" --locked build-lcli-x86_64: cross build --bin lcli --target x86_64-unknown-linux-gnu --features "portable" --profile "$(CROSS_PROFILE)" --locked build-lcli-aarch64: - cross build --bin lcli --target aarch64-unknown-linux-gnu --features "portable" --profile "$(CROSS_PROFILE)" --locked + # JEMALLOC_SYS_WITH_LG_PAGE=16 tells jemalloc to support up to 64-KiB + # pages, which are commonly used by aarch64 systems. + # See: https://github.com/sigp/lighthouse/issues/5244 + JEMALLOC_SYS_WITH_LG_PAGE=16 cross build --bin lcli --target aarch64-unknown-linux-gnu --features "portable" --profile "$(CROSS_PROFILE)" --locked # Create a `.tar.gz` containing a binary for a specific target. define tarball_release_binary @@ -222,7 +228,7 @@ lint-fix: # Also run the lints on the optimized-only tests lint-full: - RUSTFLAGS="-C debug-assertions=no $(RUSTFLAGS)" $(MAKE) lint + TEST_FEATURES="beacon-node-leveldb,beacon-node-redb,${TEST_FEATURES}" RUSTFLAGS="-C debug-assertions=no $(RUSTFLAGS)" $(MAKE) lint # Runs the makefile in the `ef_tests` repo. # diff --git a/account_manager/src/validator/create.rs b/account_manager/src/validator/create.rs index 73e0ad54d4..3db8c3f152 100644 --- a/account_manager/src/validator/create.rs +++ b/account_manager/src/validator/create.rs @@ -6,14 +6,13 @@ use account_utils::{ }; use clap::{Arg, ArgAction, ArgMatches, Command}; use clap_utils::FLAG_HEADER; -use directory::{ - ensure_dir_exists, parse_path_or_default_with_flag, DEFAULT_SECRET_DIR, DEFAULT_WALLET_DIR, -}; +use directory::{parse_path_or_default_with_flag, DEFAULT_SECRET_DIR, DEFAULT_WALLET_DIR}; use environment::Environment; use eth2_wallet_manager::WalletManager; use slashing_protection::{SlashingDatabase, SLASHING_PROTECTION_FILENAME}; use std::ffi::OsStr; use std::fs; +use std::fs::create_dir_all; use std::path::{Path, PathBuf}; use types::EthSpec; use validator_dir::Builder as ValidatorDirBuilder; @@ -156,8 +155,10 @@ pub fn cli_run( )); } - ensure_dir_exists(&validator_dir)?; - ensure_dir_exists(&secrets_dir)?; + create_dir_all(&validator_dir) + .map_err(|e| format!("Could not create validator dir at {validator_dir:?}: {e:?}"))?; + create_dir_all(&secrets_dir) + .map_err(|e| format!("Could not create secrets dir at {secrets_dir:?}: {e:?}"))?; eprintln!("secrets-dir path {:?}", secrets_dir); eprintln!("wallets-dir path {:?}", wallet_base_dir); diff --git a/account_manager/src/validator/recover.rs b/account_manager/src/validator/recover.rs index ddf754edac..19d161a468 100644 --- a/account_manager/src/validator/recover.rs +++ b/account_manager/src/validator/recover.rs @@ -5,10 +5,10 @@ use account_utils::eth2_keystore::{keypair_from_secret, Keystore, KeystoreBuilde use account_utils::{random_password, read_mnemonic_from_cli, STDIN_INPUTS_FLAG}; use clap::{Arg, ArgAction, ArgMatches, Command}; use clap_utils::FLAG_HEADER; -use directory::ensure_dir_exists; use directory::{parse_path_or_default_with_flag, DEFAULT_SECRET_DIR}; use eth2_wallet::bip39::Seed; use eth2_wallet::{recover_validator_secret_from_mnemonic, KeyType, ValidatorKeystores}; +use std::fs::create_dir_all; use std::path::PathBuf; use validator_dir::Builder as ValidatorDirBuilder; pub const CMD: &str = "recover"; @@ -91,8 +91,10 @@ pub fn cli_run(matches: &ArgMatches, validator_dir: PathBuf) -> Result<(), Strin eprintln!("secrets-dir path: {:?}", secrets_dir); - ensure_dir_exists(&validator_dir)?; - ensure_dir_exists(&secrets_dir)?; + create_dir_all(&validator_dir) + .map_err(|e| format!("Could not create validator dir at {validator_dir:?}: {e:?}"))?; + create_dir_all(&secrets_dir) + .map_err(|e| format!("Could not create secrets dir at {secrets_dir:?}: {e:?}"))?; eprintln!(); eprintln!("WARNING: KEY RECOVERY CAN LEAD TO DUPLICATING VALIDATORS KEYS, WHICH CAN LEAD TO SLASHING."); diff --git a/account_manager/src/wallet/mod.rs b/account_manager/src/wallet/mod.rs index 020858db77..c34f0363a4 100644 --- a/account_manager/src/wallet/mod.rs +++ b/account_manager/src/wallet/mod.rs @@ -5,7 +5,8 @@ pub mod recover; use crate::WALLETS_DIR_FLAG; use clap::{Arg, ArgAction, ArgMatches, Command}; use clap_utils::FLAG_HEADER; -use directory::{ensure_dir_exists, parse_path_or_default_with_flag, DEFAULT_WALLET_DIR}; +use directory::{parse_path_or_default_with_flag, DEFAULT_WALLET_DIR}; +use std::fs::create_dir_all; use std::path::PathBuf; pub const CMD: &str = "wallet"; @@ -44,7 +45,7 @@ pub fn cli_run(matches: &ArgMatches) -> Result<(), String> { } else { parse_path_or_default_with_flag(matches, WALLETS_DIR_FLAG, DEFAULT_WALLET_DIR)? }; - ensure_dir_exists(&wallet_base_dir)?; + create_dir_all(&wallet_base_dir).map_err(|_| "Could not create wallet base dir")?; eprintln!("wallet-dir path: {:?}", wallet_base_dir); diff --git a/beacon_node/beacon_chain/src/attestation_rewards.rs b/beacon_node/beacon_chain/src/attestation_rewards.rs index 87b7384ea6..3b37b09e40 100644 --- a/beacon_node/beacon_chain/src/attestation_rewards.rs +++ b/beacon_node/beacon_chain/src/attestation_rewards.rs @@ -51,13 +51,10 @@ impl BeaconChain { .get_state(&state_root, Some(state_slot))? .ok_or(BeaconChainError::MissingBeaconState(state_root))?; - match state { - BeaconState::Base(_) => self.compute_attestation_rewards_base(state, validators), - BeaconState::Altair(_) - | BeaconState::Bellatrix(_) - | BeaconState::Capella(_) - | BeaconState::Deneb(_) - | BeaconState::Electra(_) => self.compute_attestation_rewards_altair(state, validators), + if state.fork_name_unchecked().altair_enabled() { + self.compute_attestation_rewards_altair(state, validators) + } else { + self.compute_attestation_rewards_base(state, validators) } } diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index c3dea3dbb4..a69eb99a51 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -62,7 +62,7 @@ use tree_hash::TreeHash; use types::{ Attestation, AttestationRef, BeaconCommittee, BeaconStateError::NoCommitteeFound, ChainSpec, CommitteeIndex, Epoch, EthSpec, Hash256, IndexedAttestation, SelectionProof, - SignedAggregateAndProof, Slot, SubnetId, + SignedAggregateAndProof, SingleAttestation, Slot, SubnetId, }; pub use batch::{batch_verify_aggregated_attestations, batch_verify_unaggregated_attestations}; @@ -317,12 +317,22 @@ pub struct VerifiedUnaggregatedAttestation<'a, T: BeaconChainTypes> { attestation: AttestationRef<'a, T::EthSpec>, indexed_attestation: IndexedAttestation, subnet_id: SubnetId, + validator_index: usize, } impl VerifiedUnaggregatedAttestation<'_, T> { pub fn into_indexed_attestation(self) -> IndexedAttestation { self.indexed_attestation } + + pub fn single_attestation(&self) -> Option { + Some(SingleAttestation { + committee_index: self.attestation.committee_index()?, + attester_index: self.validator_index as u64, + data: self.attestation.data().clone(), + signature: self.attestation.signature().clone(), + }) + } } /// Custom `Clone` implementation is to avoid the restrictive trait bounds applied by the usual derive @@ -1035,6 +1045,7 @@ impl<'a, T: BeaconChainTypes> VerifiedUnaggregatedAttestation<'a, T> { attestation, indexed_attestation, subnet_id, + validator_index: validator_index as usize, }) } diff --git a/beacon_node/beacon_chain/src/beacon_block_streamer.rs b/beacon_node/beacon_chain/src/beacon_block_streamer.rs index b76dba88fd..32ec776868 100644 --- a/beacon_node/beacon_chain/src/beacon_block_streamer.rs +++ b/beacon_node/beacon_chain/src/beacon_block_streamer.rs @@ -15,7 +15,7 @@ use types::{ }; use types::{ ExecutionPayload, ExecutionPayloadBellatrix, ExecutionPayloadCapella, ExecutionPayloadElectra, - ExecutionPayloadHeader, + ExecutionPayloadFulu, ExecutionPayloadHeader, }; #[derive(PartialEq)] @@ -99,6 +99,7 @@ fn reconstruct_default_header_block( ForkName::Capella => ExecutionPayloadCapella::default().into(), ForkName::Deneb => ExecutionPayloadDeneb::default().into(), ForkName::Electra => ExecutionPayloadElectra::default().into(), + ForkName::Fulu => ExecutionPayloadFulu::default().into(), ForkName::Base | ForkName::Altair => { return Err(Error::PayloadReconstruction(format!( "Block with fork variant {} has execution payload", @@ -742,13 +743,14 @@ mod tests { } #[tokio::test] - async fn check_all_blocks_from_altair_to_electra() { + async fn check_all_blocks_from_altair_to_fulu() { let slots_per_epoch = MinimalEthSpec::slots_per_epoch() as usize; - let num_epochs = 10; + let num_epochs = 12; let bellatrix_fork_epoch = 2usize; let capella_fork_epoch = 4usize; let deneb_fork_epoch = 6usize; let electra_fork_epoch = 8usize; + let fulu_fork_epoch = 10usize; let num_blocks_produced = num_epochs * slots_per_epoch; let mut spec = test_spec::(); @@ -757,6 +759,7 @@ mod tests { spec.capella_fork_epoch = Some(Epoch::new(capella_fork_epoch as u64)); spec.deneb_fork_epoch = Some(Epoch::new(deneb_fork_epoch as u64)); spec.electra_fork_epoch = Some(Epoch::new(electra_fork_epoch as u64)); + spec.fulu_fork_epoch = Some(Epoch::new(fulu_fork_epoch as u64)); let spec = Arc::new(spec); let harness = get_harness(VALIDATOR_COUNT, spec.clone()); diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 14d902d50f..9f3f06ab8c 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -34,6 +34,7 @@ use crate::execution_payload::{get_execution_payload, NotifyExecutionLayer, Prep use crate::fork_choice_signal::{ForkChoiceSignalRx, ForkChoiceSignalTx, ForkChoiceWaitResult}; use crate::graffiti_calculator::GraffitiCalculator; use crate::head_tracker::{HeadTracker, HeadTrackerReader, SszHeadTracker}; +use crate::kzg_utils::reconstruct_blobs; use crate::light_client_finality_update_verification::{ Error as LightClientFinalityUpdateError, VerifiedLightClientFinalityUpdate, }; @@ -117,10 +118,11 @@ use std::sync::Arc; use std::time::Duration; use store::iter::{BlockRootsIterator, ParentRootBlockIterator, StateRootsIterator}; use store::{ - DatabaseBlock, Error as DBError, HotColdDB, KeyValueStore, KeyValueStoreOp, StoreItem, StoreOp, + BlobSidecarListFromRoot, DatabaseBlock, Error as DBError, HotColdDB, KeyValueStore, + KeyValueStoreOp, StoreItem, StoreOp, }; use task_executor::{ShutdownReason, TaskExecutor}; -use tokio::sync::mpsc::Receiver; +use tokio::sync::oneshot; use tokio_stream::Stream; use tree_hash::TreeHash; use types::blob_sidecar::FixedBlobSidecarList; @@ -575,7 +577,7 @@ impl BeaconChain { .start_slot(T::EthSpec::slots_per_epoch()); let is_canonical = self .block_root_at_slot(block_slot, WhenSlotSkipped::None)? - .map_or(false, |canonical_root| block_root == &canonical_root); + .is_some_and(|canonical_root| block_root == &canonical_root); Ok(block_slot <= finalized_slot && is_canonical) } @@ -606,7 +608,7 @@ impl BeaconChain { let slot_is_finalized = state_slot <= finalized_slot; let canonical = self .state_root_at_slot(state_slot)? - .map_or(false, |canonical_root| state_root == &canonical_root); + .is_some_and(|canonical_root| state_root == &canonical_root); Ok(FinalizationAndCanonicity { slot_is_finalized, canonical, @@ -1149,9 +1151,10 @@ impl BeaconChain { pub fn get_blobs_checking_early_attester_cache( &self, block_root: &Hash256, - ) -> Result, Error> { + ) -> Result, Error> { self.early_attester_cache .get_blobs(*block_root) + .map(Into::into) .map_or_else(|| self.get_blobs(block_root), Ok) } @@ -1242,10 +1245,59 @@ impl BeaconChain { /// /// ## Errors /// May return a database error. - pub fn get_blobs(&self, block_root: &Hash256) -> Result, Error> { - match self.store.get_blobs(block_root)? { - Some(blobs) => Ok(blobs), - None => Ok(BlobSidecarList::default()), + pub fn get_blobs( + &self, + block_root: &Hash256, + ) -> Result, Error> { + self.store.get_blobs(block_root).map_err(Error::from) + } + + /// Returns the data columns at the given root, if any. + /// + /// ## Errors + /// May return a database error. + pub fn get_data_columns( + &self, + block_root: &Hash256, + ) -> Result>, Error> { + self.store.get_data_columns(block_root).map_err(Error::from) + } + + /// Returns the blobs at the given root, if any. + /// + /// Uses the `block.epoch()` to determine whether to retrieve blobs or columns from the store. + /// + /// If at least 50% of columns are retrieved, blobs will be reconstructed and returned, + /// otherwise an error `InsufficientColumnsToReconstructBlobs` is returned. + /// + /// ## Errors + /// May return a database error. + pub fn get_or_reconstruct_blobs( + &self, + block_root: &Hash256, + ) -> Result>, Error> { + let Some(block) = self.store.get_blinded_block(block_root)? else { + return Ok(None); + }; + + if self.spec.is_peer_das_enabled_for_epoch(block.epoch()) { + if let Some(columns) = self.store.get_data_columns(block_root)? { + let num_required_columns = self.spec.number_of_columns / 2; + let reconstruction_possible = columns.len() >= num_required_columns as usize; + if reconstruction_possible { + reconstruct_blobs(&self.kzg, &columns, None, &block, &self.spec) + .map(Some) + .map_err(Error::FailedToReconstructBlobs) + } else { + Err(Error::InsufficientColumnsToReconstructBlobs { + columns_found: columns.len(), + }) + } + } else { + Ok(None) + } + } else { + self.get_blobs(block_root).map(|b| b.blobs()) } } @@ -2161,10 +2213,30 @@ impl BeaconChain { |v| { // This method is called for API and gossip attestations, so this covers all unaggregated attestation events if let Some(event_handler) = self.event_handler.as_ref() { + if event_handler.has_single_attestation_subscribers() { + let current_fork = self + .spec + .fork_name_at_slot::(v.attestation().data().slot); + if current_fork.electra_enabled() { + // I don't see a situation where this could return None. The upstream unaggregated attestation checks + // should have already verified that this is an attestation with a single committee bit set. + if let Some(single_attestation) = v.single_attestation() { + event_handler.register(EventKind::SingleAttestation(Box::new( + single_attestation, + ))); + } + } + } + if event_handler.has_attestation_subscribers() { - event_handler.register(EventKind::Attestation(Box::new( - v.attestation().clone_as_attestation(), - ))); + let current_fork = self + .spec + .fork_name_at_slot::(v.attestation().data().slot); + if !current_fork.electra_enabled() { + event_handler.register(EventKind::Attestation(Box::new( + v.attestation().clone_as_attestation(), + ))); + } } } metrics::inc_counter(&metrics::UNAGGREGATED_ATTESTATION_PROCESSING_SUCCESSES); @@ -3214,7 +3286,7 @@ impl BeaconChain { slot: Slot, block_root: Hash256, blobs: FixedBlobSidecarList, - data_column_recv: Option>>, + data_column_recv: Option>>, ) -> Result { // If this block has already been imported to forkchoice it must have been available, so // we don't need to process its blobs again. @@ -3342,7 +3414,7 @@ impl BeaconChain { }; let r = self - .process_availability(slot, availability, None, || Ok(())) + .process_availability(slot, availability, || Ok(())) .await; self.remove_notified(&block_root, r) .map(|availability_processing_status| { @@ -3470,7 +3542,7 @@ impl BeaconChain { match executed_block { ExecutedBlock::Available(block) => { - self.import_available_block(Box::new(block), None).await + self.import_available_block(Box::new(block)).await } ExecutedBlock::AvailabilityPending(block) => { self.check_block_availability_and_import(block).await @@ -3602,7 +3674,7 @@ impl BeaconChain { let availability = self .data_availability_checker .put_pending_executed_block(block)?; - self.process_availability(slot, availability, None, || Ok(())) + self.process_availability(slot, availability, || Ok(())) .await } @@ -3618,7 +3690,7 @@ impl BeaconChain { } let availability = self.data_availability_checker.put_gossip_blob(blob)?; - self.process_availability(slot, availability, None, || Ok(())) + self.process_availability(slot, availability, || Ok(())) .await } @@ -3641,7 +3713,7 @@ impl BeaconChain { .data_availability_checker .put_gossip_data_columns(block_root, data_columns)?; - self.process_availability(slot, availability, None, publish_fn) + self.process_availability(slot, availability, publish_fn) .await } @@ -3685,7 +3757,7 @@ impl BeaconChain { .data_availability_checker .put_rpc_blobs(block_root, blobs)?; - self.process_availability(slot, availability, None, || Ok(())) + self.process_availability(slot, availability, || Ok(())) .await } @@ -3694,14 +3766,14 @@ impl BeaconChain { slot: Slot, block_root: Hash256, blobs: FixedBlobSidecarList, - data_column_recv: Option>>, + data_column_recv: Option>>, ) -> Result { self.check_blobs_for_slashability(block_root, &blobs)?; - let availability = self - .data_availability_checker - .put_engine_blobs(block_root, blobs)?; + let availability = + self.data_availability_checker + .put_engine_blobs(block_root, blobs, data_column_recv)?; - self.process_availability(slot, availability, data_column_recv, || Ok(())) + self.process_availability(slot, availability, || Ok(())) .await } @@ -3741,7 +3813,7 @@ impl BeaconChain { .data_availability_checker .put_rpc_custody_columns(block_root, custody_columns)?; - self.process_availability(slot, availability, None, || Ok(())) + self.process_availability(slot, availability, || Ok(())) .await } @@ -3753,14 +3825,13 @@ impl BeaconChain { self: &Arc, slot: Slot, availability: Availability, - recv: Option>>, publish_fn: impl FnOnce() -> Result<(), BlockError>, ) -> Result { match availability { Availability::Available(block) => { publish_fn()?; // Block is fully available, import into fork choice - self.import_available_block(block, recv).await + self.import_available_block(block).await } Availability::MissingComponents(block_root) => Ok( AvailabilityProcessingStatus::MissingComponents(slot, block_root), @@ -3771,7 +3842,6 @@ impl BeaconChain { pub async fn import_available_block( self: &Arc, block: Box>, - data_column_recv: Option>>, ) -> Result { let AvailableExecutedBlock { block, @@ -3786,6 +3856,7 @@ impl BeaconChain { parent_eth1_finalization_data, confirmed_state_roots, consensus_context, + data_column_recv, } = import_data; // Record the time at which this block's blobs became available. @@ -3852,7 +3923,7 @@ impl BeaconChain { parent_block: SignedBlindedBeaconBlock, parent_eth1_finalization_data: Eth1FinalizationData, mut consensus_context: ConsensusContext, - data_column_recv: Option>>, + data_column_recv: Option>>, ) -> Result { // ----------------------------- BLOCK NOT YET ATTESTABLE ---------------------------------- // Everything in this initial section is on the hot path between processing the block and @@ -4020,44 +4091,32 @@ impl BeaconChain { // end up with blocks in fork choice that are missing from disk. // See https://github.com/sigp/lighthouse/issues/2028 let (_, signed_block, blobs, data_columns) = signed_block.deconstruct(); - // TODO(das) we currently store all subnet sampled columns. Tracking issue to exclude non - // custody columns: https://github.com/sigp/lighthouse/issues/6465 - let custody_columns_count = self.data_availability_checker.get_sampling_column_count(); - // if block is made available via blobs, dropped the data columns. - let data_columns = data_columns.filter(|columns| columns.len() == custody_columns_count); - let data_columns = match (data_columns, data_column_recv) { - // If the block was made available via custody columns received from gossip / rpc, use them - // since we already have them. - (Some(columns), _) => Some(columns), - // Otherwise, it means blobs were likely available via fetching from EL, in this case we - // wait for the data columns to be computed (blocking). - (None, Some(mut data_column_recv)) => { - let _column_recv_timer = - metrics::start_timer(&metrics::BLOCK_PROCESSING_DATA_COLUMNS_WAIT); - // Unable to receive data columns from sender, sender is either dropped or - // failed to compute data columns from blobs. We restore fork choice here and - // return to avoid inconsistency in database. - if let Some(columns) = data_column_recv.blocking_recv() { - Some(columns) - } else { - let err_msg = "Did not receive data columns from sender"; - error!( - self.log, - "Failed to store data columns into the database"; - "msg" => "Restoring fork choice from disk", - "error" => err_msg, - ); - return Err(self - .handle_import_block_db_write_error(fork_choice) - .err() - .unwrap_or(BlockError::InternalError(err_msg.to_string()))); - } + match self.get_blobs_or_columns_store_op( + block_root, + signed_block.epoch(), + blobs, + data_columns, + data_column_recv, + ) { + Ok(Some(blobs_or_columns_store_op)) => { + ops.push(blobs_or_columns_store_op); } - // No data columns present and compute data columns task was not spawned. - // Could either be no blobs in the block or before PeerDAS activation. - (None, None) => None, - }; + Ok(None) => {} + Err(e) => { + error!( + self.log, + "Failed to store data columns into the database"; + "msg" => "Restoring fork choice from disk", + "error" => &e, + "block_root" => ?block_root + ); + return Err(self + .handle_import_block_db_write_error(fork_choice) + .err() + .unwrap_or(BlockError::InternalError(e))); + } + } let block = signed_block.message(); let db_write_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_DB_WRITE); @@ -4069,30 +4128,6 @@ impl BeaconChain { ops.push(StoreOp::PutBlock(block_root, signed_block.clone())); ops.push(StoreOp::PutState(block.state_root(), &state)); - if let Some(blobs) = blobs { - if !blobs.is_empty() { - debug!( - self.log, "Writing blobs to store"; - "block_root" => %block_root, - "count" => blobs.len(), - ); - ops.push(StoreOp::PutBlobs(block_root, blobs)); - } - } - - if let Some(data_columns) = data_columns { - // TODO(das): `available_block includes all sampled columns, but we only need to store - // custody columns. To be clarified in spec. - if !data_columns.is_empty() { - debug!( - self.log, "Writing data_columns to store"; - "block_root" => %block_root, - "count" => data_columns.len(), - ); - ops.push(StoreOp::PutDataColumns(block_root, data_columns)); - } - } - let txn_lock = self.store.hot_db.begin_rw_transaction(); if let Err(e) = self.store.do_atomically_with_block_and_blobs_cache(ops) { @@ -5246,9 +5281,9 @@ impl BeaconChain { .start_of(slot) .unwrap_or_else(|| Duration::from_secs(0)), ); - block_delays.observed.map_or(false, |delay| { - delay >= self.slot_clock.unagg_attestation_production_delay() - }) + block_delays + .observed + .is_some_and(|delay| delay >= self.slot_clock.unagg_attestation_production_delay()) } /// Produce a block for some `slot` upon the given `state`. @@ -5445,23 +5480,19 @@ impl BeaconChain { // If required, start the process of loading an execution payload from the EL early. This // allows it to run concurrently with things like attestation packing. - let prepare_payload_handle = match &state { - BeaconState::Base(_) | BeaconState::Altair(_) => None, - BeaconState::Bellatrix(_) - | BeaconState::Capella(_) - | BeaconState::Deneb(_) - | BeaconState::Electra(_) => { - let prepare_payload_handle = get_execution_payload( - self.clone(), - &state, - parent_root, - proposer_index, - builder_params, - builder_boost_factor, - block_production_version, - )?; - Some(prepare_payload_handle) - } + let prepare_payload_handle = if state.fork_name_unchecked().bellatrix_enabled() { + let prepare_payload_handle = get_execution_payload( + self.clone(), + &state, + parent_root, + proposer_index, + builder_params, + builder_boost_factor, + block_production_version, + )?; + Some(prepare_payload_handle) + } else { + None }; let (mut proposer_slashings, mut attester_slashings, mut voluntary_exits) = @@ -5879,6 +5910,48 @@ impl BeaconChain { execution_payload_value, ) } + BeaconState::Fulu(_) => { + let ( + payload, + kzg_commitments, + maybe_blobs_and_proofs, + maybe_requests, + execution_payload_value, + ) = block_contents + .ok_or(BlockProductionError::MissingExecutionPayload)? + .deconstruct(); + + ( + BeaconBlock::Fulu(BeaconBlockFulu { + slot, + proposer_index, + parent_root, + state_root: Hash256::zero(), + body: BeaconBlockBodyFulu { + randao_reveal, + eth1_data, + graffiti, + proposer_slashings: proposer_slashings.into(), + attester_slashings: attester_slashings_electra.into(), + attestations: attestations_electra.into(), + deposits: deposits.into(), + voluntary_exits: voluntary_exits.into(), + sync_aggregate: sync_aggregate + .ok_or(BlockProductionError::MissingSyncAggregate)?, + execution_payload: payload + .try_into() + .map_err(|_| BlockProductionError::InvalidPayloadFork)?, + bls_to_execution_changes: bls_to_execution_changes.into(), + blob_kzg_commitments: kzg_commitments + .ok_or(BlockProductionError::InvalidPayloadFork)?, + execution_requests: maybe_requests + .ok_or(BlockProductionError::MissingExecutionRequests)?, + }, + }), + maybe_blobs_and_proofs, + execution_payload_value, + ) + } }; let block = SignedBeaconBlock::from_block( @@ -5955,6 +6028,7 @@ impl BeaconChain { let kzg = self.kzg.as_ref(); + // TODO(fulu): we no longer need blob proofs from PeerDAS and could avoid computing. kzg_utils::validate_blobs::( kzg, expected_kzg_commitments, @@ -7307,6 +7381,68 @@ impl BeaconChain { reqresp_pre_import_cache_len: self.reqresp_pre_import_cache.read().len(), } } + + fn get_blobs_or_columns_store_op( + &self, + block_root: Hash256, + block_epoch: Epoch, + blobs: Option>, + data_columns: Option>, + data_column_recv: Option>>, + ) -> Result>, String> { + if self.spec.is_peer_das_enabled_for_epoch(block_epoch) { + // TODO(das) we currently store all subnet sampled columns. Tracking issue to exclude non + // custody columns: https://github.com/sigp/lighthouse/issues/6465 + let custody_columns_count = self.data_availability_checker.get_sampling_column_count(); + + let custody_columns_available = data_columns + .as_ref() + .as_ref() + .is_some_and(|columns| columns.len() == custody_columns_count); + + let data_columns_to_persist = if custody_columns_available { + // If the block was made available via custody columns received from gossip / rpc, use them + // since we already have them. + data_columns + } else if let Some(data_column_recv) = data_column_recv { + // Blobs were available from the EL, in this case we wait for the data columns to be computed (blocking). + let _column_recv_timer = + metrics::start_timer(&metrics::BLOCK_PROCESSING_DATA_COLUMNS_WAIT); + // Unable to receive data columns from sender, sender is either dropped or + // failed to compute data columns from blobs. We restore fork choice here and + // return to avoid inconsistency in database. + let computed_data_columns = data_column_recv + .blocking_recv() + .map_err(|e| format!("Did not receive data columns from sender: {e:?}"))?; + Some(computed_data_columns) + } else { + // No blobs in the block. + None + }; + + if let Some(data_columns) = data_columns_to_persist { + if !data_columns.is_empty() { + debug!( + self.log, "Writing data_columns to store"; + "block_root" => %block_root, + "count" => data_columns.len(), + ); + return Ok(Some(StoreOp::PutDataColumns(block_root, data_columns))); + } + } + } else if let Some(blobs) = blobs { + if !blobs.is_empty() { + debug!( + self.log, "Writing blobs to store"; + "block_root" => %block_root, + "count" => blobs.len(), + ); + return Ok(Some(StoreOp::PutBlobs(block_root, blobs))); + } + } + + Ok(None) + } } impl Drop for BeaconChain { diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index 6c87deb826..786b627bb7 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -400,7 +400,7 @@ pub fn validate_blob_sidecar_for_gossip= T::EthSpec::max_blobs_per_block() as u64 { + if blob_index >= chain.spec.max_blobs_per_block(blob_epoch) { return Err(GossipBlobError::InvalidSubnet { expected: subnet, received: blob_index, diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index ddb7bb614a..1265276376 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -208,24 +208,18 @@ pub enum BlockError { /// /// The block is invalid and the peer is faulty. IncorrectBlockProposer { block: u64, local_shuffling: u64 }, - /// The proposal signature in invalid. - /// - /// ## Peer scoring - /// - /// The block is invalid and the peer is faulty. - ProposalSignatureInvalid, /// The `block.proposal_index` is not known. /// /// ## Peer scoring /// /// The block is invalid and the peer is faulty. UnknownValidator(u64), - /// A signature in the block is invalid (exactly which is unknown). + /// A signature in the block is invalid /// /// ## Peer scoring /// /// The block is invalid and the peer is faulty. - InvalidSignature, + InvalidSignature(InvalidSignature), /// The provided block is not from a later slot than its parent. /// /// ## Peer scoring @@ -329,6 +323,17 @@ pub enum BlockError { InternalError(String), } +/// Which specific signature(s) are invalid in a SignedBeaconBlock +#[derive(Debug)] +pub enum InvalidSignature { + // The outer signature in a SignedBeaconBlock + ProposerSignature, + // One or more signatures in BeaconBlockBody + BlockBodySignatures, + // One or more signatures in SignedBeaconBlock + Unknown, +} + impl From for BlockError { fn from(e: AvailabilityCheckError) -> Self { Self::AvailabilityCheck(e) @@ -523,7 +528,9 @@ pub enum BlockSlashInfo { impl BlockSlashInfo { pub fn from_early_error_block(header: SignedBeaconBlockHeader, e: BlockError) -> Self { match e { - BlockError::ProposalSignatureInvalid => BlockSlashInfo::SignatureInvalid(e), + BlockError::InvalidSignature(InvalidSignature::ProposerSignature) => { + BlockSlashInfo::SignatureInvalid(e) + } // `InvalidSignature` could indicate any signature in the block, so we want // to recheck the proposer signature alone. _ => BlockSlashInfo::SignatureNotChecked(header, e), @@ -652,7 +659,7 @@ pub fn signature_verify_chain_segment( } if signature_verifier.verify().is_err() { - return Err(BlockError::InvalidSignature); + return Err(BlockError::InvalidSignature(InvalidSignature::Unknown)); } drop(pubkey_cache); @@ -964,7 +971,9 @@ impl GossipVerifiedBlock { }; if !signature_is_valid { - return Err(BlockError::ProposalSignatureInvalid); + return Err(BlockError::InvalidSignature( + InvalidSignature::ProposerSignature, + )); } chain @@ -1098,7 +1107,26 @@ impl SignatureVerifiedBlock { parent: Some(parent), }) } else { - Err(BlockError::InvalidSignature) + // Re-verify the proposer signature in isolation to attribute fault + let pubkey = pubkey_cache + .get(block.message().proposer_index() as usize) + .ok_or_else(|| BlockError::UnknownValidator(block.message().proposer_index()))?; + if block.as_block().verify_signature( + Some(block_root), + pubkey, + &state.fork(), + chain.genesis_validators_root, + &chain.spec, + ) { + // Proposer signature is valid, the invalid signature must be in the body + Err(BlockError::InvalidSignature( + InvalidSignature::BlockBodySignatures, + )) + } else { + Err(BlockError::InvalidSignature( + InvalidSignature::ProposerSignature, + )) + } } } @@ -1153,7 +1181,9 @@ impl SignatureVerifiedBlock { consensus_context, }) } else { - Err(BlockError::InvalidSignature) + Err(BlockError::InvalidSignature( + InvalidSignature::BlockBodySignatures, + )) } } @@ -1677,6 +1707,7 @@ impl ExecutionPendingBlock { parent_eth1_finalization_data, confirmed_state_roots, consensus_context, + data_column_recv: None, }, payload_verification_handle, }) @@ -1980,7 +2011,7 @@ impl BlockBlobError for BlockError { } fn proposer_signature_invalid() -> Self { - BlockError::ProposalSignatureInvalid + BlockError::InvalidSignature(InvalidSignature::ProposerSignature) } } diff --git a/beacon_node/beacon_chain/src/block_verification_types.rs b/beacon_node/beacon_chain/src/block_verification_types.rs index 420c83081c..38d0fc708c 100644 --- a/beacon_node/beacon_chain/src/block_verification_types.rs +++ b/beacon_node/beacon_chain/src/block_verification_types.rs @@ -4,14 +4,14 @@ use crate::data_column_verification::{CustodyDataColumn, CustodyDataColumnList}; use crate::eth1_finalization_cache::Eth1FinalizationData; use crate::{get_block_root, PayloadVerificationOutcome}; use derivative::Derivative; -use ssz_types::VariableList; use state_processing::ConsensusContext; use std::fmt::{Debug, Formatter}; use std::sync::Arc; -use types::blob_sidecar::{BlobIdentifier, FixedBlobSidecarList}; +use tokio::sync::oneshot; +use types::blob_sidecar::BlobIdentifier; use types::{ - BeaconBlockRef, BeaconState, BlindedPayload, BlobSidecarList, ChainSpec, Epoch, EthSpec, - Hash256, RuntimeVariableList, SignedBeaconBlock, SignedBeaconBlockHeader, Slot, + BeaconBlockRef, BeaconState, BlindedPayload, BlobSidecarList, ChainSpec, DataColumnSidecarList, + Epoch, EthSpec, Hash256, RuntimeVariableList, SignedBeaconBlock, SignedBeaconBlockHeader, Slot, }; /// A block that has been received over RPC. It has 2 internal variants: @@ -165,7 +165,7 @@ impl RpcBlock { let inner = if !custody_columns.is_empty() { RpcBlockInner::BlockAndCustodyColumns( block, - RuntimeVariableList::new(custody_columns, spec.number_of_columns)?, + RuntimeVariableList::new(custody_columns, spec.number_of_columns as usize)?, ) } else { RpcBlockInner::Block(block) @@ -176,23 +176,6 @@ impl RpcBlock { }) } - pub fn new_from_fixed( - block_root: Hash256, - block: Arc>, - blobs: FixedBlobSidecarList, - ) -> Result { - let filtered = blobs - .into_iter() - .filter_map(|b| b.clone()) - .collect::>(); - let blobs = if filtered.is_empty() { - None - } else { - Some(VariableList::from(filtered)) - }; - Self::new(Some(block_root), block, blobs) - } - #[allow(clippy::type_complexity)] pub fn deconstruct( self, @@ -355,7 +338,8 @@ impl AvailabilityPendingExecutedBlock { } } -#[derive(Debug, PartialEq)] +#[derive(Debug, Derivative)] +#[derivative(PartialEq)] pub struct BlockImportData { pub block_root: Hash256, pub state: BeaconState, @@ -363,6 +347,12 @@ pub struct BlockImportData { pub parent_eth1_finalization_data: Eth1FinalizationData, pub confirmed_state_roots: Vec, pub consensus_context: ConsensusContext, + #[derivative(PartialEq = "ignore")] + /// An optional receiver for `DataColumnSidecarList`. + /// + /// This field is `Some` when data columns are being computed asynchronously. + /// The resulting `DataColumnSidecarList` will be sent through this receiver. + pub data_column_recv: Option>>, } impl BlockImportData { @@ -381,6 +371,7 @@ impl BlockImportData { }, confirmed_state_roots: vec![], consensus_context: ConsensusContext::new(Slot::new(0)), + data_column_recv: None, } } } diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index d1088b849c..2a201b1aeb 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -9,6 +9,7 @@ use crate::fork_choice_signal::ForkChoiceSignalTx; use crate::fork_revert::{reset_fork_choice_to_finalization, revert_to_fork_boundary}; use crate::graffiti_calculator::{GraffitiCalculator, GraffitiOrigin}; use crate::head_tracker::HeadTracker; +use crate::kzg_utils::blobs_to_data_column_sidecars; use crate::light_client_server_cache::LightClientServerCache; use crate::migrate::{BackgroundMigrator, MigratorConfig}; use crate::observed_data_sidecars::ObservedDataSidecars; @@ -562,9 +563,30 @@ where .put_block(&weak_subj_block_root, weak_subj_block.clone()) .map_err(|e| format!("Failed to store weak subjectivity block: {e:?}"))?; if let Some(blobs) = weak_subj_blobs { - store - .put_blobs(&weak_subj_block_root, blobs) - .map_err(|e| format!("Failed to store weak subjectivity blobs: {e:?}"))?; + if self + .spec + .is_peer_das_enabled_for_epoch(weak_subj_block.epoch()) + { + // After PeerDAS recompute columns from blobs to not force the checkpointz server + // into exposing another route. + let blobs = blobs + .iter() + .map(|blob_sidecar| &blob_sidecar.blob) + .collect::>(); + let data_columns = + blobs_to_data_column_sidecars(&blobs, &weak_subj_block, &self.kzg, &self.spec) + .map_err(|e| { + format!("Failed to compute weak subjectivity data_columns: {e:?}") + })?; + // TODO(das): only persist the columns under custody + store + .put_data_columns(&weak_subj_block_root, data_columns) + .map_err(|e| format!("Failed to store weak subjectivity data_column: {e:?}"))?; + } else { + store + .put_blobs(&weak_subj_block_root, blobs) + .map_err(|e| format!("Failed to store weak subjectivity blobs: {e:?}"))?; + } } // Stage the database's metadata fields for atomic storage when `build` is called. diff --git a/beacon_node/beacon_chain/src/canonical_head.rs b/beacon_node/beacon_chain/src/canonical_head.rs index 9cbeb03246..9c0067e3b4 100644 --- a/beacon_node/beacon_chain/src/canonical_head.rs +++ b/beacon_node/beacon_chain/src/canonical_head.rs @@ -1259,11 +1259,7 @@ pub fn find_reorg_slot( ($state: ident, $block_root: ident) => { std::iter::once(Ok(($state.slot(), $block_root))) .chain($state.rev_iter_block_roots(spec)) - .skip_while(|result| { - result - .as_ref() - .map_or(false, |(slot, _)| *slot > lowest_slot) - }) + .skip_while(|result| result.as_ref().is_ok_and(|(slot, _)| *slot > lowest_slot)) }; } diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 72806a74d2..aa4689121c 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -15,6 +15,7 @@ use std::num::NonZeroUsize; use std::sync::Arc; use std::time::Duration; use task_executor::TaskExecutor; +use tokio::sync::oneshot; use types::blob_sidecar::{BlobIdentifier, BlobSidecar, FixedBlobSidecarList}; use types::{ BlobSidecarList, ChainSpec, DataColumnIdentifier, DataColumnSidecar, DataColumnSidecarList, @@ -116,21 +117,16 @@ impl DataAvailabilityChecker { spec: Arc, log: Logger, ) -> Result { - let custody_subnet_count = if import_all_data_columns { - spec.data_column_sidecar_subnet_count as usize - } else { - spec.custody_requirement as usize - }; - - let subnet_sampling_size = - std::cmp::max(custody_subnet_count, spec.samples_per_slot as usize); - let sampling_column_count = - subnet_sampling_size.saturating_mul(spec.data_columns_per_subnet()); + let custody_group_count = spec.custody_group_count(import_all_data_columns); + // This should only panic if the chain spec contains invalid values. + let sampling_size = spec + .sampling_size(custody_group_count) + .expect("should compute node sampling size from valid chain spec"); let inner = DataAvailabilityCheckerInner::new( OVERFLOW_LRU_CAPACITY, store, - sampling_column_count, + sampling_size as usize, spec.clone(), )?; Ok(Self { @@ -147,7 +143,7 @@ impl DataAvailabilityChecker { } pub(crate) fn is_supernode(&self) -> bool { - self.get_sampling_column_count() == self.spec.number_of_columns + self.get_sampling_column_count() == self.spec.number_of_columns as usize } /// Checks if the block root is currenlty in the availability cache awaiting import because @@ -215,12 +211,15 @@ impl DataAvailabilityChecker { // Note: currently not reporting which specific blob is invalid because we fetch all blobs // from the same peer for both lookup and range sync. - let verified_blobs = - KzgVerifiedBlobList::new(blobs.iter().flatten().cloned(), &self.kzg, seen_timestamp) - .map_err(AvailabilityCheckError::InvalidBlobs)?; + let verified_blobs = KzgVerifiedBlobList::new( + blobs.into_vec().into_iter().flatten(), + &self.kzg, + seen_timestamp, + ) + .map_err(AvailabilityCheckError::InvalidBlobs)?; self.availability_cache - .put_kzg_verified_blobs(block_root, verified_blobs, &self.log) + .put_kzg_verified_blobs(block_root, verified_blobs, None, &self.log) } /// Put a list of custody columns received via RPC into the availability cache. This performs KZG @@ -260,6 +259,7 @@ impl DataAvailabilityChecker { &self, block_root: Hash256, blobs: FixedBlobSidecarList, + data_column_recv: Option>>, ) -> Result, AvailabilityCheckError> { let seen_timestamp = self .slot_clock @@ -269,8 +269,12 @@ impl DataAvailabilityChecker { let verified_blobs = KzgVerifiedBlobList::from_verified(blobs.iter().flatten().cloned(), seen_timestamp); - self.availability_cache - .put_kzg_verified_blobs(block_root, verified_blobs, &self.log) + self.availability_cache.put_kzg_verified_blobs( + block_root, + verified_blobs, + data_column_recv, + &self.log, + ) } /// Check if we've cached other blobs for this block. If it completes a set and we also @@ -285,6 +289,7 @@ impl DataAvailabilityChecker { self.availability_cache.put_kzg_verified_blobs( gossip_blob.block_root(), vec![gossip_blob.into_inner()], + None, &self.log, ) } @@ -400,14 +405,13 @@ impl DataAvailabilityChecker { blocks: Vec>, ) -> Result>, AvailabilityCheckError> { let mut results = Vec::with_capacity(blocks.len()); - let all_blobs: BlobSidecarList = blocks + let all_blobs = blocks .iter() .filter(|block| self.blobs_required_for_block(block.as_block())) // this clone is cheap as it's cloning an Arc .filter_map(|block| block.blobs().cloned()) .flatten() - .collect::>() - .into(); + .collect::>(); // verify kzg for all blobs at once if !all_blobs.is_empty() { @@ -424,7 +428,7 @@ impl DataAvailabilityChecker { .map(CustodyDataColumn::into_inner) .collect::>(); let all_data_columns = - RuntimeVariableList::from_vec(all_data_columns, self.spec.number_of_columns); + RuntimeVariableList::from_vec(all_data_columns, self.spec.number_of_columns as usize); // verify kzg for all data columns at once if !all_data_columns.is_empty() { @@ -519,13 +523,13 @@ impl DataAvailabilityChecker { /// Returns true if the given epoch lies within the da boundary and false otherwise. pub fn da_check_required_for_epoch(&self, block_epoch: Epoch) -> bool { self.data_availability_boundary() - .map_or(false, |da_epoch| block_epoch >= da_epoch) + .is_some_and(|da_epoch| block_epoch >= da_epoch) } /// Returns `true` if the current epoch is greater than or equal to the `Deneb` epoch. pub fn is_deneb(&self) -> bool { - self.slot_clock.now().map_or(false, |slot| { - self.spec.deneb_fork_epoch.map_or(false, |deneb_epoch| { + self.slot_clock.now().is_some_and(|slot| { + self.spec.deneb_fork_epoch.is_some_and(|deneb_epoch| { let now_epoch = slot.epoch(T::EthSpec::slots_per_epoch()); now_epoch >= deneb_epoch }) @@ -801,7 +805,6 @@ impl AvailableBlock { block, blobs, data_columns, - blobs_available_timestamp: _, .. } = self; (block_root, block, blobs, data_columns) diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index 40361574af..cd793c8394 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -10,40 +10,55 @@ use crate::BeaconChainTypes; use lru::LruCache; use parking_lot::RwLock; use slog::{debug, Logger}; -use ssz_types::FixedVector; use std::num::NonZeroUsize; use std::sync::Arc; +use tokio::sync::oneshot; use types::blob_sidecar::BlobIdentifier; use types::{ - BlobSidecar, ChainSpec, ColumnIndex, DataColumnIdentifier, DataColumnSidecar, Epoch, EthSpec, - Hash256, SignedBeaconBlock, + BlobSidecar, ChainSpec, ColumnIndex, DataColumnIdentifier, DataColumnSidecar, + DataColumnSidecarList, Epoch, EthSpec, Hash256, RuntimeFixedVector, RuntimeVariableList, + SignedBeaconBlock, }; /// This represents the components of a partially available block /// /// The blobs are all gossip and kzg verified. /// The block has completed all verifications except the availability check. -/// TODO(das): this struct can potentially be reafactored as blobs and data columns are mutually -/// exclusive and this could simplify `is_importable`. -#[derive(Clone)] pub struct PendingComponents { pub block_root: Hash256, - pub verified_blobs: FixedVector>, E::MaxBlobsPerBlock>, + pub verified_blobs: RuntimeFixedVector>>, pub verified_data_columns: Vec>, pub executed_block: Option>, pub reconstruction_started: bool, + /// Receiver for data columns that are computed asynchronously; + /// + /// If `data_column_recv` is `Some`, it means data column computation or reconstruction has been + /// started. This can happen either via engine blobs fetching or data column reconstruction + /// (triggered when >= 50% columns are received via gossip). + pub data_column_recv: Option>>, } impl PendingComponents { + /// Clones the `PendingComponent` without cloning `data_column_recv`, as `Receiver` is not cloneable. + /// This should only be used when the receiver is no longer needed. + pub fn clone_without_column_recv(&self) -> Self { + PendingComponents { + block_root: self.block_root, + verified_blobs: self.verified_blobs.clone(), + verified_data_columns: self.verified_data_columns.clone(), + executed_block: self.executed_block.clone(), + reconstruction_started: self.reconstruction_started, + data_column_recv: None, + } + } + /// Returns an immutable reference to the cached block. pub fn get_cached_block(&self) -> &Option> { &self.executed_block } /// Returns an immutable reference to the fixed vector of cached blobs. - pub fn get_cached_blobs( - &self, - ) -> &FixedVector>, E::MaxBlobsPerBlock> { + pub fn get_cached_blobs(&self) -> &RuntimeFixedVector>> { &self.verified_blobs } @@ -64,9 +79,7 @@ impl PendingComponents { } /// Returns a mutable reference to the fixed vector of cached blobs. - pub fn get_cached_blobs_mut( - &mut self, - ) -> &mut FixedVector>, E::MaxBlobsPerBlock> { + pub fn get_cached_blobs_mut(&mut self) -> &mut RuntimeFixedVector>> { &mut self.verified_blobs } @@ -138,10 +151,7 @@ impl PendingComponents { /// Blobs are only inserted if: /// 1. The blob entry at the index is empty and no block exists. /// 2. The block exists and its commitment matches the blob's commitment. - pub fn merge_blobs( - &mut self, - blobs: FixedVector>, E::MaxBlobsPerBlock>, - ) { + pub fn merge_blobs(&mut self, blobs: RuntimeFixedVector>>) { for (index, blob) in blobs.iter().cloned().enumerate() { let Some(blob) = blob else { continue }; self.merge_single_blob(index, blob); @@ -185,7 +195,7 @@ impl PendingComponents { /// Blobs that don't match the new block's commitments are evicted. pub fn merge_block(&mut self, block: DietAvailabilityPendingExecutedBlock) { self.insert_block(block); - let reinsert = std::mem::take(self.get_cached_blobs_mut()); + let reinsert = self.get_cached_blobs_mut().take(); self.merge_blobs(reinsert); } @@ -228,25 +238,23 @@ impl PendingComponents { ); let all_blobs_received = block_kzg_commitments_count_opt - .map_or(false, |num_expected_blobs| { - num_expected_blobs == num_received_blobs - }); + .is_some_and(|num_expected_blobs| num_expected_blobs == num_received_blobs); - let all_columns_received = expected_columns_opt.map_or(false, |num_expected_columns| { - num_expected_columns == num_received_columns - }); + let all_columns_received = expected_columns_opt + .is_some_and(|num_expected_columns| num_expected_columns == num_received_columns); all_blobs_received || all_columns_received } /// Returns an empty `PendingComponents` object with the given block root. - pub fn empty(block_root: Hash256) -> Self { + pub fn empty(block_root: Hash256, max_len: usize) -> Self { Self { block_root, - verified_blobs: FixedVector::default(), + verified_blobs: RuntimeFixedVector::new(vec![None; max_len]), verified_data_columns: vec![], executed_block: None, reconstruction_started: false, + data_column_recv: None, } } @@ -271,6 +279,7 @@ impl PendingComponents { verified_blobs, verified_data_columns, executed_block, + data_column_recv, .. } = self; @@ -302,17 +311,22 @@ impl PendingComponents { else { return Err(AvailabilityCheckError::Unexpected); }; - (Some(verified_blobs), None) + let max_len = spec.max_blobs_per_block(diet_executed_block.as_block().epoch()) as usize; + ( + Some(RuntimeVariableList::new(verified_blobs, max_len)?), + None, + ) }; - let executed_block = recover(diet_executed_block)?; let AvailabilityPendingExecutedBlock { block, - import_data, + mut import_data, payload_verification_outcome, } = executed_block; + import_data.data_column_recv = data_column_recv; + let available_block = AvailableBlock { block_root, block, @@ -344,10 +358,7 @@ impl PendingComponents { } if let Some(kzg_verified_data_column) = self.verified_data_columns.first() { - let epoch = kzg_verified_data_column - .as_data_column() - .slot() - .epoch(E::slots_per_epoch()); + let epoch = kzg_verified_data_column.as_data_column().epoch(); return Some(epoch); } @@ -454,13 +465,31 @@ impl DataAvailabilityCheckerInner { f(self.critical.read().peek(block_root)) } + /// Puts the KZG verified blobs into the availability cache as pending components. + /// + /// The `data_column_recv` parameter is an optional `Receiver` for data columns that are + /// computed asynchronously. This method remains **used** after PeerDAS activation, because + /// blocks can be made available if the EL already has the blobs and returns them via the + /// `getBlobsV1` engine method. More details in [fetch_blobs.rs](https://github.com/sigp/lighthouse/blob/44f8add41ea2252769bb967864af95b3c13af8ca/beacon_node/beacon_chain/src/fetch_blobs.rs). pub fn put_kzg_verified_blobs>>( &self, block_root: Hash256, kzg_verified_blobs: I, + data_column_recv: Option>>, log: &Logger, ) -> Result, AvailabilityCheckError> { - let mut fixed_blobs = FixedVector::default(); + let mut kzg_verified_blobs = kzg_verified_blobs.into_iter().peekable(); + + let Some(epoch) = kzg_verified_blobs + .peek() + .map(|verified_blob| verified_blob.as_blob().epoch()) + else { + // Verified blobs list should be non-empty. + return Err(AvailabilityCheckError::Unexpected); + }; + + let mut fixed_blobs = + RuntimeFixedVector::new(vec![None; self.spec.max_blobs_per_block(epoch) as usize]); for blob in kzg_verified_blobs { if let Some(blob_opt) = fixed_blobs.get_mut(blob.blob_index() as usize) { @@ -474,14 +503,24 @@ impl DataAvailabilityCheckerInner { let mut pending_components = write_lock .pop_entry(&block_root) .map(|(_, v)| v) - .unwrap_or_else(|| PendingComponents::empty(block_root)); + .unwrap_or_else(|| { + PendingComponents::empty(block_root, self.spec.max_blobs_per_block(epoch) as usize) + }); // Merge in the blobs. pending_components.merge_blobs(fixed_blobs); + if data_column_recv.is_some() { + // If `data_column_recv` is `Some`, it means we have all the blobs from engine, and have + // started computing data columns. We store the receiver in `PendingComponents` for + // later use when importing the block. + pending_components.data_column_recv = data_column_recv; + } + if pending_components.is_available(self.sampling_column_count, log) { - write_lock.put(block_root, pending_components.clone()); - // No need to hold the write lock anymore + // We keep the pending components in the availability cache during block import (#5845). + // `data_column_recv` is returned as part of the available block and is no longer needed here. + write_lock.put(block_root, pending_components.clone_without_column_recv()); drop(write_lock); pending_components.make_available(&self.spec, |diet_block| { self.state_cache.recover_pending_executed_block(diet_block) @@ -501,20 +540,32 @@ impl DataAvailabilityCheckerInner { kzg_verified_data_columns: I, log: &Logger, ) -> Result, AvailabilityCheckError> { + let mut kzg_verified_data_columns = kzg_verified_data_columns.into_iter().peekable(); + let Some(epoch) = kzg_verified_data_columns + .peek() + .map(|verified_blob| verified_blob.as_data_column().epoch()) + else { + // Verified data_columns list should be non-empty. + return Err(AvailabilityCheckError::Unexpected); + }; + let mut write_lock = self.critical.write(); // Grab existing entry or create a new entry. let mut pending_components = write_lock .pop_entry(&block_root) .map(|(_, v)| v) - .unwrap_or_else(|| PendingComponents::empty(block_root)); + .unwrap_or_else(|| { + PendingComponents::empty(block_root, self.spec.max_blobs_per_block(epoch) as usize) + }); // Merge in the data columns. pending_components.merge_data_columns(kzg_verified_data_columns)?; if pending_components.is_available(self.sampling_column_count, log) { - write_lock.put(block_root, pending_components.clone()); - // No need to hold the write lock anymore + // We keep the pending components in the availability cache during block import (#5845). + // `data_column_recv` is returned as part of the available block and is no longer needed here. + write_lock.put(block_root, pending_components.clone_without_column_recv()); drop(write_lock); pending_components.make_available(&self.spec, |diet_block| { self.state_cache.recover_pending_executed_block(diet_block) @@ -546,7 +597,7 @@ impl DataAvailabilityCheckerInner { // If we're sampling all columns, it means we must be custodying all columns. let custody_column_count = self.sampling_column_count(); - let total_column_count = self.spec.number_of_columns; + let total_column_count = self.spec.number_of_columns as usize; let received_column_count = pending_components.verified_data_columns.len(); if pending_components.reconstruction_started { @@ -555,7 +606,7 @@ impl DataAvailabilityCheckerInner { if custody_column_count != total_column_count { return ReconstructColumnsDecision::No("not required for full node"); } - if received_column_count == self.spec.number_of_columns { + if received_column_count >= total_column_count { return ReconstructColumnsDecision::No("all columns received"); } if received_column_count < total_column_count / 2 { @@ -563,7 +614,7 @@ impl DataAvailabilityCheckerInner { } pending_components.reconstruction_started = true; - ReconstructColumnsDecision::Yes(pending_components.clone()) + ReconstructColumnsDecision::Yes(pending_components.clone_without_column_recv()) } /// This could mean some invalid data columns made it through to the `DataAvailabilityChecker`. @@ -584,6 +635,7 @@ impl DataAvailabilityCheckerInner { log: &Logger, ) -> Result, AvailabilityCheckError> { let mut write_lock = self.critical.write(); + let epoch = executed_block.as_block().epoch(); let block_root = executed_block.import_data.block_root; // register the block to get the diet block @@ -595,15 +647,18 @@ impl DataAvailabilityCheckerInner { let mut pending_components = write_lock .pop_entry(&block_root) .map(|(_, v)| v) - .unwrap_or_else(|| PendingComponents::empty(block_root)); + .unwrap_or_else(|| { + PendingComponents::empty(block_root, self.spec.max_blobs_per_block(epoch) as usize) + }); // Merge in the block. pending_components.merge_block(diet_executed_block); // Check if we have all components and entire set is consistent. if pending_components.is_available(self.sampling_column_count, log) { - write_lock.put(block_root, pending_components.clone()); - // No need to hold the write lock anymore + // We keep the pending components in the availability cache during block import (#5845). + // `data_column_recv` is returned as part of the available block and is no longer needed here. + write_lock.put(block_root, pending_components.clone_without_column_recv()); drop(write_lock); pending_components.make_available(&self.spec, |diet_block| { self.state_cache.recover_pending_executed_block(diet_block) @@ -676,7 +731,7 @@ mod test { use slog::{info, Logger}; use state_processing::ConsensusContext; use std::collections::VecDeque; - use store::{HotColdDB, ItemStore, LevelDB, StoreConfig}; + use store::{database::interface::BeaconNodeBackend, HotColdDB, ItemStore, StoreConfig}; use tempfile::{tempdir, TempDir}; use types::non_zero_usize::new_non_zero_usize; use types::{ExecPayload, MinimalEthSpec}; @@ -688,7 +743,7 @@ mod test { db_path: &TempDir, spec: Arc, log: Logger, - ) -> Arc, LevelDB>> { + ) -> Arc, BeaconNodeBackend>> { let hot_path = db_path.path().join("hot_db"); let cold_path = db_path.path().join("cold_db"); let blobs_path = db_path.path().join("blobs_db"); @@ -815,7 +870,8 @@ mod test { info!(log, "done printing kzg commitments"); let gossip_verified_blobs = if let Some((kzg_proofs, blobs)) = maybe_blobs { - let sidecars = BlobSidecar::build_sidecars(blobs, &block, kzg_proofs).unwrap(); + let sidecars = + BlobSidecar::build_sidecars(blobs, &block, kzg_proofs, &chain.spec).unwrap(); Vec::from(sidecars) .into_iter() .map(|sidecar| { @@ -837,6 +893,7 @@ mod test { parent_eth1_finalization_data, confirmed_state_roots: vec![], consensus_context, + data_column_recv: None, }; let payload_verification_outcome = PayloadVerificationOutcome { @@ -862,7 +919,11 @@ mod test { ) where E: EthSpec, - T: BeaconChainTypes, ColdStore = LevelDB, EthSpec = E>, + T: BeaconChainTypes< + HotStore = BeaconNodeBackend, + ColdStore = BeaconNodeBackend, + EthSpec = E, + >, { let log = test_logger(); let chain_db_path = tempdir().expect("should get temp dir"); @@ -939,7 +1000,7 @@ mod test { for (blob_index, gossip_blob) in blobs.into_iter().enumerate() { kzg_verified_blobs.push(gossip_blob.into_inner()); let availability = cache - .put_kzg_verified_blobs(root, kzg_verified_blobs.clone(), harness.logger()) + .put_kzg_verified_blobs(root, kzg_verified_blobs.clone(), None, harness.logger()) .expect("should put blob"); if blob_index == blobs_expected - 1 { assert!(matches!(availability, Availability::Available(_))); @@ -948,6 +1009,8 @@ mod test { assert_eq!(cache.critical.read().len(), 1); } } + // remove the blob to simulate successful import + cache.remove_pending_components(root); assert!( cache.critical.read().is_empty(), "cache should be empty now that all components available" @@ -965,7 +1028,7 @@ mod test { for gossip_blob in blobs { kzg_verified_blobs.push(gossip_blob.into_inner()); let availability = cache - .put_kzg_verified_blobs(root, kzg_verified_blobs.clone(), harness.logger()) + .put_kzg_verified_blobs(root, kzg_verified_blobs.clone(), None, harness.logger()) .expect("should put blob"); assert_eq!( availability, @@ -1128,7 +1191,7 @@ mod pending_components_tests { use super::*; use crate::block_verification_types::BlockImportData; use crate::eth1_finalization_cache::Eth1FinalizationData; - use crate::test_utils::{generate_rand_block_and_blobs, NumBlobs}; + use crate::test_utils::{generate_rand_block_and_blobs, test_spec, NumBlobs}; use crate::PayloadVerificationOutcome; use fork_choice::PayloadVerificationStatus; use kzg::KzgCommitment; @@ -1144,15 +1207,19 @@ mod pending_components_tests { type Setup = ( SignedBeaconBlock, - FixedVector>>, ::MaxBlobsPerBlock>, - FixedVector>>, ::MaxBlobsPerBlock>, + RuntimeFixedVector>>>, + RuntimeFixedVector>>>, + usize, ); pub fn pre_setup() -> Setup { let mut rng = StdRng::seed_from_u64(0xDEADBEEF0BAD5EEDu64); + let spec = test_spec::(); let (block, blobs_vec) = - generate_rand_block_and_blobs::(ForkName::Deneb, NumBlobs::Random, &mut rng); - let mut blobs: FixedVector<_, ::MaxBlobsPerBlock> = FixedVector::default(); + generate_rand_block_and_blobs::(ForkName::Deneb, NumBlobs::Random, &mut rng, &spec); + let max_len = spec.max_blobs_per_block(block.epoch()) as usize; + let mut blobs: RuntimeFixedVector>>> = + RuntimeFixedVector::default(max_len); for blob in blobs_vec { if let Some(b) = blobs.get_mut(blob.index as usize) { @@ -1160,10 +1227,8 @@ mod pending_components_tests { } } - let mut invalid_blobs: FixedVector< - Option>>, - ::MaxBlobsPerBlock, - > = FixedVector::default(); + let mut invalid_blobs: RuntimeFixedVector>>> = + RuntimeFixedVector::default(max_len); for (index, blob) in blobs.iter().enumerate() { if let Some(invalid_blob) = blob { let mut blob_copy = invalid_blob.as_ref().clone(); @@ -1172,21 +1237,21 @@ mod pending_components_tests { } } - (block, blobs, invalid_blobs) + (block, blobs, invalid_blobs, max_len) } type PendingComponentsSetup = ( DietAvailabilityPendingExecutedBlock, - FixedVector>, ::MaxBlobsPerBlock>, - FixedVector>, ::MaxBlobsPerBlock>, + RuntimeFixedVector>>, + RuntimeFixedVector>>, ); pub fn setup_pending_components( block: SignedBeaconBlock, - valid_blobs: FixedVector>>, ::MaxBlobsPerBlock>, - invalid_blobs: FixedVector>>, ::MaxBlobsPerBlock>, + valid_blobs: RuntimeFixedVector>>>, + invalid_blobs: RuntimeFixedVector>>>, ) -> PendingComponentsSetup { - let blobs = FixedVector::from( + let blobs = RuntimeFixedVector::new( valid_blobs .iter() .map(|blob_opt| { @@ -1196,7 +1261,7 @@ mod pending_components_tests { }) .collect::>(), ); - let invalid_blobs = FixedVector::from( + let invalid_blobs = RuntimeFixedVector::new( invalid_blobs .iter() .map(|blob_opt| { @@ -1219,6 +1284,7 @@ mod pending_components_tests { }, confirmed_state_roots: vec![], consensus_context: ConsensusContext::new(Slot::new(0)), + data_column_recv: None, }, payload_verification_outcome: PayloadVerificationOutcome { payload_verification_status: PayloadVerificationStatus::Verified, @@ -1228,10 +1294,10 @@ mod pending_components_tests { (block.into(), blobs, invalid_blobs) } - pub fn assert_cache_consistent(cache: PendingComponents) { + pub fn assert_cache_consistent(cache: PendingComponents, max_len: usize) { if let Some(cached_block) = cache.get_cached_block() { let cached_block_commitments = cached_block.get_commitments(); - for index in 0..E::max_blobs_per_block() { + for index in 0..max_len { let block_commitment = cached_block_commitments.get(index).copied(); let blob_commitment_opt = cache.get_cached_blobs().get(index).unwrap(); let blob_commitment = blob_commitment_opt.as_ref().map(|b| *b.get_commitment()); @@ -1250,40 +1316,40 @@ mod pending_components_tests { #[test] fn valid_block_invalid_blobs_valid_blobs() { - let (block_commitments, blobs, random_blobs) = pre_setup(); + let (block_commitments, blobs, random_blobs, max_len) = pre_setup(); let (block_commitments, blobs, random_blobs) = setup_pending_components(block_commitments, blobs, random_blobs); let block_root = Hash256::zero(); - let mut cache = >::empty(block_root); + let mut cache = >::empty(block_root, max_len); cache.merge_block(block_commitments); cache.merge_blobs(random_blobs); cache.merge_blobs(blobs); - assert_cache_consistent(cache); + assert_cache_consistent(cache, max_len); } #[test] fn invalid_blobs_block_valid_blobs() { - let (block_commitments, blobs, random_blobs) = pre_setup(); + let (block_commitments, blobs, random_blobs, max_len) = pre_setup(); let (block_commitments, blobs, random_blobs) = setup_pending_components(block_commitments, blobs, random_blobs); let block_root = Hash256::zero(); - let mut cache = >::empty(block_root); + let mut cache = >::empty(block_root, max_len); cache.merge_blobs(random_blobs); cache.merge_block(block_commitments); cache.merge_blobs(blobs); - assert_cache_consistent(cache); + assert_cache_consistent(cache, max_len); } #[test] fn invalid_blobs_valid_blobs_block() { - let (block_commitments, blobs, random_blobs) = pre_setup(); + let (block_commitments, blobs, random_blobs, max_len) = pre_setup(); let (block_commitments, blobs, random_blobs) = setup_pending_components(block_commitments, blobs, random_blobs); let block_root = Hash256::zero(); - let mut cache = >::empty(block_root); + let mut cache = >::empty(block_root, max_len); cache.merge_blobs(random_blobs); cache.merge_blobs(blobs); cache.merge_block(block_commitments); @@ -1293,46 +1359,46 @@ mod pending_components_tests { #[test] fn block_valid_blobs_invalid_blobs() { - let (block_commitments, blobs, random_blobs) = pre_setup(); + let (block_commitments, blobs, random_blobs, max_len) = pre_setup(); let (block_commitments, blobs, random_blobs) = setup_pending_components(block_commitments, blobs, random_blobs); let block_root = Hash256::zero(); - let mut cache = >::empty(block_root); + let mut cache = >::empty(block_root, max_len); cache.merge_block(block_commitments); cache.merge_blobs(blobs); cache.merge_blobs(random_blobs); - assert_cache_consistent(cache); + assert_cache_consistent(cache, max_len); } #[test] fn valid_blobs_block_invalid_blobs() { - let (block_commitments, blobs, random_blobs) = pre_setup(); + let (block_commitments, blobs, random_blobs, max_len) = pre_setup(); let (block_commitments, blobs, random_blobs) = setup_pending_components(block_commitments, blobs, random_blobs); let block_root = Hash256::zero(); - let mut cache = >::empty(block_root); + let mut cache = >::empty(block_root, max_len); cache.merge_blobs(blobs); cache.merge_block(block_commitments); cache.merge_blobs(random_blobs); - assert_cache_consistent(cache); + assert_cache_consistent(cache, max_len); } #[test] fn valid_blobs_invalid_blobs_block() { - let (block_commitments, blobs, random_blobs) = pre_setup(); + let (block_commitments, blobs, random_blobs, max_len) = pre_setup(); let (block_commitments, blobs, random_blobs) = setup_pending_components(block_commitments, blobs, random_blobs); let block_root = Hash256::zero(); - let mut cache = >::empty(block_root); + let mut cache = >::empty(block_root, max_len); cache.merge_blobs(blobs); cache.merge_blobs(random_blobs); cache.merge_block(block_commitments); - assert_cache_consistent(cache); + assert_cache_consistent(cache, max_len); } } diff --git a/beacon_node/beacon_chain/src/data_availability_checker/state_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/state_lru_cache.rs index 5b9b7c7023..2a2a0431cc 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/state_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/state_lru_cache.rs @@ -136,6 +136,7 @@ impl StateLRUCache { consensus_context: diet_executed_block .consensus_context .into_consensus_context(), + data_column_recv: None, }, payload_verification_outcome: diet_executed_block.payload_verification_outcome, }) diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index 6cfd26786a..565e76704e 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -423,7 +423,7 @@ fn verify_data_column_sidecar( data_column: &DataColumnSidecar, spec: &ChainSpec, ) -> Result<(), GossipDataColumnError> { - if data_column.index >= spec.number_of_columns as u64 { + if data_column.index >= spec.number_of_columns { return Err(GossipDataColumnError::InvalidColumnIndex(data_column.index)); } if data_column.kzg_commitments.is_empty() { @@ -611,7 +611,7 @@ fn verify_index_matches_subnet( spec: &ChainSpec, ) -> Result<(), GossipDataColumnError> { let expected_subnet: u64 = - DataColumnSubnetId::from_column_index::(data_column.index as usize, spec).into(); + DataColumnSubnetId::from_column_index(data_column.index, spec).into(); if expected_subnet != subnet { return Err(GossipDataColumnError::InvalidSubnetId { received: subnet, @@ -699,7 +699,7 @@ mod test { #[tokio::test] async fn empty_data_column_sidecars_fails_validation() { - let spec = ForkName::latest().make_genesis_spec(E::default_spec()); + let spec = ForkName::Fulu.make_genesis_spec(E::default_spec()); let harness = BeaconChainHarness::builder(E::default()) .spec(spec.into()) .deterministic_keypairs(64) diff --git a/beacon_node/beacon_chain/src/early_attester_cache.rs b/beacon_node/beacon_chain/src/early_attester_cache.rs index 606610a748..c94ea0e941 100644 --- a/beacon_node/beacon_chain/src/early_attester_cache.rs +++ b/beacon_node/beacon_chain/src/early_attester_cache.rs @@ -145,7 +145,7 @@ impl EarlyAttesterCache { self.item .read() .as_ref() - .map_or(false, |item| item.beacon_block_root == block_root) + .is_some_and(|item| item.beacon_block_root == block_root) } /// Returns the block, if `block_root` matches the cached item. diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index 0fc6a32435..f59fe55571 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -228,6 +228,10 @@ pub enum BeaconChainError { EmptyRpcCustodyColumns, AttestationError(AttestationError), AttestationCommitteeIndexNotSet, + InsufficientColumnsToReconstructBlobs { + columns_found: usize, + }, + FailedToReconstructBlobs(String), } easy_from_to!(SlotProcessingError, BeaconChainError); diff --git a/beacon_node/beacon_chain/src/eth1_chain.rs b/beacon_node/beacon_chain/src/eth1_chain.rs index cb6e4c34f3..ad4f106517 100644 --- a/beacon_node/beacon_chain/src/eth1_chain.rs +++ b/beacon_node/beacon_chain/src/eth1_chain.rs @@ -153,7 +153,7 @@ fn get_sync_status( // Lighthouse is "cached and ready" when it has cached enough blocks to cover the start of the // current voting period. let lighthouse_is_cached_and_ready = - latest_cached_block_timestamp.map_or(false, |t| t >= voting_target_timestamp); + latest_cached_block_timestamp.is_some_and(|t| t >= voting_target_timestamp); Some(Eth1SyncStatusData { head_block_number, diff --git a/beacon_node/beacon_chain/src/events.rs b/beacon_node/beacon_chain/src/events.rs index 267d56220c..8c342893ae 100644 --- a/beacon_node/beacon_chain/src/events.rs +++ b/beacon_node/beacon_chain/src/events.rs @@ -8,6 +8,7 @@ const DEFAULT_CHANNEL_CAPACITY: usize = 16; pub struct ServerSentEventHandler { attestation_tx: Sender>, + single_attestation_tx: Sender>, block_tx: Sender>, blob_sidecar_tx: Sender>, finalized_tx: Sender>, @@ -37,6 +38,7 @@ impl ServerSentEventHandler { pub fn new_with_capacity(log: Logger, capacity: usize) -> Self { let (attestation_tx, _) = broadcast::channel(capacity); + let (single_attestation_tx, _) = broadcast::channel(capacity); let (block_tx, _) = broadcast::channel(capacity); let (blob_sidecar_tx, _) = broadcast::channel(capacity); let (finalized_tx, _) = broadcast::channel(capacity); @@ -56,6 +58,7 @@ impl ServerSentEventHandler { Self { attestation_tx, + single_attestation_tx, block_tx, blob_sidecar_tx, finalized_tx, @@ -90,6 +93,10 @@ impl ServerSentEventHandler { .attestation_tx .send(kind) .map(|count| log_count("attestation", count)), + EventKind::SingleAttestation(_) => self + .single_attestation_tx + .send(kind) + .map(|count| log_count("single_attestation", count)), EventKind::Block(_) => self .block_tx .send(kind) @@ -164,6 +171,10 @@ impl ServerSentEventHandler { self.attestation_tx.subscribe() } + pub fn subscribe_single_attestation(&self) -> Receiver> { + self.single_attestation_tx.subscribe() + } + pub fn subscribe_block(&self) -> Receiver> { self.block_tx.subscribe() } @@ -232,6 +243,10 @@ impl ServerSentEventHandler { self.attestation_tx.receiver_count() > 0 } + pub fn has_single_attestation_subscribers(&self) -> bool { + self.single_attestation_tx.receiver_count() > 0 + } + pub fn has_block_subscribers(&self) -> bool { self.block_tx.receiver_count() > 0 } diff --git a/beacon_node/beacon_chain/src/execution_payload.rs b/beacon_node/beacon_chain/src/execution_payload.rs index ef4b81e9ff..9b44c47eeb 100644 --- a/beacon_node/beacon_chain/src/execution_payload.rs +++ b/beacon_node/beacon_chain/src/execution_payload.rs @@ -140,9 +140,9 @@ impl PayloadNotifier { /// contains a few extra checks by running `partially_verify_execution_payload` first: /// /// https://github.com/ethereum/consensus-specs/blob/v1.1.9/specs/bellatrix/beacon-chain.md#notify_new_payload -async fn notify_new_payload<'a, T: BeaconChainTypes>( +async fn notify_new_payload( chain: &Arc>, - block: BeaconBlockRef<'a, T::EthSpec>, + block: BeaconBlockRef<'_, T::EthSpec>, il_transactions: InclusionListTransactions, ) -> Result { let execution_layer = chain @@ -258,9 +258,9 @@ async fn notify_new_payload<'a, T: BeaconChainTypes>( /// Equivalent to the `validate_merge_block` function in the merge Fork Choice Changes: /// /// https://github.com/ethereum/consensus-specs/blob/v1.1.5/specs/merge/fork-choice.md#validate_merge_block -pub async fn validate_merge_block<'a, T: BeaconChainTypes>( +pub async fn validate_merge_block( chain: &Arc>, - block: BeaconBlockRef<'a, T::EthSpec>, + block: BeaconBlockRef<'_, T::EthSpec>, allow_optimistic_import: AllowOptimisticImport, ) -> Result<(), BlockError> { let spec = &chain.spec; @@ -402,19 +402,15 @@ pub fn get_execution_payload( let latest_execution_payload_header = state.latest_execution_payload_header()?; let latest_execution_payload_header_block_hash = latest_execution_payload_header.block_hash(); let latest_execution_payload_header_gas_limit = latest_execution_payload_header.gas_limit(); - let withdrawals = match state { - &BeaconState::Capella(_) | &BeaconState::Deneb(_) | &BeaconState::Electra(_) => { - Some(get_expected_withdrawals(state, spec)?.0.into()) - } - &BeaconState::Bellatrix(_) => None, - // These shouldn't happen but they're here to make the pattern irrefutable - &BeaconState::Base(_) | &BeaconState::Altair(_) => None, + let withdrawals = if state.fork_name_unchecked().capella_enabled() { + Some(get_expected_withdrawals(state, spec)?.0.into()) + } else { + None }; - let parent_beacon_block_root = match state { - BeaconState::Deneb(_) | BeaconState::Electra(_) => Some(parent_block_root), - BeaconState::Bellatrix(_) | BeaconState::Capella(_) => None, - // These shouldn't happen but they're here to make the pattern irrefutable - BeaconState::Base(_) | BeaconState::Altair(_) => None, + let parent_beacon_block_root = if state.fork_name_unchecked().deneb_enabled() { + Some(parent_block_root) + } else { + None }; // Spawn a task to obtain the execution payload from the EL via a series of async calls. The diff --git a/beacon_node/beacon_chain/src/fetch_blobs.rs b/beacon_node/beacon_chain/src/fetch_blobs.rs index f740b693fb..5bc2b92ec3 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs.rs @@ -18,11 +18,11 @@ use slog::{debug, error, o, Logger}; use ssz_types::FixedVector; use state_processing::per_block_processing::deneb::kzg_commitment_to_versioned_hash; use std::sync::Arc; -use tokio::sync::mpsc::Receiver; +use tokio::sync::oneshot; use types::blob_sidecar::{BlobSidecarError, FixedBlobSidecarList}; use types::{ - BeaconStateError, BlobSidecar, DataColumnSidecar, DataColumnSidecarList, EthSpec, FullPayload, - Hash256, SignedBeaconBlock, SignedBeaconBlockHeader, + BeaconStateError, BlobSidecar, ChainSpec, DataColumnSidecar, DataColumnSidecarList, EthSpec, + FullPayload, Hash256, SignedBeaconBlock, SignedBeaconBlockHeader, }; pub enum BlobsOrDataColumns { @@ -112,6 +112,7 @@ pub async fn fetch_and_process_engine_blobs( response, signed_block_header, &kzg_commitments_proof, + &chain.spec, )?; let num_fetched_blobs = fixed_blob_sidecar_list @@ -162,6 +163,20 @@ pub async fn fetch_and_process_engine_blobs( return Ok(None); } + if chain + .canonical_head + .fork_choice_read_lock() + .contains_block(&block_root) + { + // Avoid computing columns if block has already been imported. + debug!( + log, + "Ignoring EL blobs response"; + "info" => "block has already been imported", + ); + return Ok(None); + } + let data_columns_receiver = spawn_compute_and_publish_data_columns_task( &chain, block.clone(), @@ -212,9 +227,9 @@ fn spawn_compute_and_publish_data_columns_task( blobs: FixedBlobSidecarList, publish_fn: impl Fn(BlobsOrDataColumns) + Send + 'static, log: Logger, -) -> Receiver>>> { +) -> oneshot::Receiver>>> { let chain_cloned = chain.clone(); - let (data_columns_sender, data_columns_receiver) = tokio::sync::mpsc::channel(1); + let (data_columns_sender, data_columns_receiver) = oneshot::channel(); chain.task_executor.spawn_blocking( move || { @@ -247,18 +262,21 @@ fn spawn_compute_and_publish_data_columns_task( } }; - if let Err(e) = data_columns_sender.try_send(all_data_columns.clone()) { - error!(log, "Failed to send computed data columns"; "error" => ?e); + if data_columns_sender.send(all_data_columns.clone()).is_err() { + // Data column receiver have been dropped - block may have already been imported. + // This race condition exists because gossip columns may arrive and trigger block + // import during the computation. Here we just drop the computed columns. + debug!( + log, + "Failed to send computed data columns"; + ); + return; }; - // Check indices from cache before sending the columns, to make sure we don't - // publish components already seen on gossip. - let is_supernode = chain_cloned.data_availability_checker.is_supernode(); - // At the moment non supernodes are not required to publish any columns. // TODO(das): we could experiment with having full nodes publish their custodied // columns here. - if !is_supernode { + if !chain_cloned.data_availability_checker.is_supernode() { return; } @@ -275,8 +293,11 @@ fn build_blob_sidecars( response: Vec>>, signed_block_header: SignedBeaconBlockHeader, kzg_commitments_inclusion_proof: &FixedVector, + spec: &ChainSpec, ) -> Result, FetchEngineBlobError> { - let mut fixed_blob_sidecar_list = FixedBlobSidecarList::default(); + let epoch = block.epoch(); + let mut fixed_blob_sidecar_list = + FixedBlobSidecarList::default(spec.max_blobs_per_block(epoch) as usize); for (index, blob_and_proof) in response .into_iter() .enumerate() diff --git a/beacon_node/beacon_chain/src/fulu_readiness.rs b/beacon_node/beacon_chain/src/fulu_readiness.rs new file mode 100644 index 0000000000..872fe58f2b --- /dev/null +++ b/beacon_node/beacon_chain/src/fulu_readiness.rs @@ -0,0 +1,115 @@ +//! Provides tools for checking if a node is ready for the Fulu upgrade. + +use crate::{BeaconChain, BeaconChainTypes}; +use execution_layer::http::{ENGINE_GET_PAYLOAD_V4, ENGINE_NEW_PAYLOAD_V4}; +use serde::{Deserialize, Serialize}; +use std::fmt; +use std::time::Duration; +use types::*; + +/// The time before the Fulu fork when we will start issuing warnings about preparation. +use super::bellatrix_readiness::SECONDS_IN_A_WEEK; +pub const FULU_READINESS_PREPARATION_SECONDS: u64 = SECONDS_IN_A_WEEK * 2; +pub const ENGINE_CAPABILITIES_REFRESH_INTERVAL: u64 = 300; + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +#[serde(tag = "type")] +pub enum FuluReadiness { + /// The execution engine is fulu-enabled (as far as we can tell) + Ready, + /// We are connected to an execution engine which doesn't support the V5 engine api methods + V5MethodsNotSupported { error: String }, + /// The transition configuration with the EL failed, there might be a problem with + /// connectivity, authentication or a difference in configuration. + ExchangeCapabilitiesFailed { error: String }, + /// The user has not configured an execution endpoint + NoExecutionEndpoint, +} + +impl fmt::Display for FuluReadiness { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + FuluReadiness::Ready => { + write!(f, "This node appears ready for Fulu.") + } + FuluReadiness::ExchangeCapabilitiesFailed { error } => write!( + f, + "Could not exchange capabilities with the \ + execution endpoint: {}", + error + ), + FuluReadiness::NoExecutionEndpoint => write!( + f, + "The --execution-endpoint flag is not specified, this is a \ + requirement post-merge" + ), + FuluReadiness::V5MethodsNotSupported { error } => write!( + f, + "Execution endpoint does not support Fulu methods: {}", + error + ), + } + } +} + +impl BeaconChain { + /// Returns `true` if fulu epoch is set and Fulu fork has occurred or will + /// occur within `FULU_READINESS_PREPARATION_SECONDS` + pub fn is_time_to_prepare_for_fulu(&self, current_slot: Slot) -> bool { + if let Some(fulu_epoch) = self.spec.fulu_fork_epoch { + let fulu_slot = fulu_epoch.start_slot(T::EthSpec::slots_per_epoch()); + let fulu_readiness_preparation_slots = + FULU_READINESS_PREPARATION_SECONDS / self.spec.seconds_per_slot; + // Return `true` if Fulu has happened or is within the preparation time. + current_slot + fulu_readiness_preparation_slots > fulu_slot + } else { + // The Fulu fork epoch has not been defined yet, no need to prepare. + false + } + } + + /// Attempts to connect to the EL and confirm that it is ready for fulu. + pub async fn check_fulu_readiness(&self) -> FuluReadiness { + if let Some(el) = self.execution_layer.as_ref() { + match el + .get_engine_capabilities(Some(Duration::from_secs( + ENGINE_CAPABILITIES_REFRESH_INTERVAL, + ))) + .await + { + Err(e) => { + // The EL was either unreachable or responded with an error + FuluReadiness::ExchangeCapabilitiesFailed { + error: format!("{:?}", e), + } + } + Ok(capabilities) => { + let mut missing_methods = String::from("Required Methods Unsupported:"); + let mut all_good = true; + // TODO(fulu) switch to v5 when the EL is ready + if !capabilities.get_payload_v4 { + missing_methods.push(' '); + missing_methods.push_str(ENGINE_GET_PAYLOAD_V4); + all_good = false; + } + if !capabilities.new_payload_v4 { + missing_methods.push(' '); + missing_methods.push_str(ENGINE_NEW_PAYLOAD_V4); + all_good = false; + } + + if all_good { + FuluReadiness::Ready + } else { + FuluReadiness::V5MethodsNotSupported { + error: missing_methods, + } + } + } + } + } else { + FuluReadiness::NoExecutionEndpoint + } + } +} diff --git a/beacon_node/beacon_chain/src/graffiti_calculator.rs b/beacon_node/beacon_chain/src/graffiti_calculator.rs index 4373164d62..8692d374ed 100644 --- a/beacon_node/beacon_chain/src/graffiti_calculator.rs +++ b/beacon_node/beacon_chain/src/graffiti_calculator.rs @@ -293,10 +293,7 @@ mod tests { .await .unwrap(); - let version_bytes = std::cmp::min( - lighthouse_version::VERSION.as_bytes().len(), - GRAFFITI_BYTES_LEN, - ); + let version_bytes = std::cmp::min(lighthouse_version::VERSION.len(), GRAFFITI_BYTES_LEN); // grab the slice of the graffiti that corresponds to the lighthouse version let graffiti_slice = &harness.chain.graffiti_calculator.get_graffiti(None).await.0[..version_bytes]; @@ -361,7 +358,7 @@ mod tests { let graffiti_str = "nice graffiti bro"; let mut graffiti_bytes = [0u8; GRAFFITI_BYTES_LEN]; - graffiti_bytes[..graffiti_str.as_bytes().len()].copy_from_slice(graffiti_str.as_bytes()); + graffiti_bytes[..graffiti_str.len()].copy_from_slice(graffiti_str.as_bytes()); let found_graffiti = harness .chain diff --git a/beacon_node/beacon_chain/src/historical_blocks.rs b/beacon_node/beacon_chain/src/historical_blocks.rs index ddae54f464..e22ec95a79 100644 --- a/beacon_node/beacon_chain/src/historical_blocks.rs +++ b/beacon_node/beacon_chain/src/historical_blocks.rs @@ -10,10 +10,7 @@ use std::borrow::Cow; use std::iter; use std::time::Duration; use store::metadata::DataColumnInfo; -use store::{ - get_key_for_col, AnchorInfo, BlobInfo, DBColumn, Error as StoreError, KeyValueStore, - KeyValueStoreOp, -}; +use store::{AnchorInfo, BlobInfo, DBColumn, Error as StoreError, KeyValueStore, KeyValueStoreOp}; use strum::IntoStaticStr; use types::{FixedBytesExtended, Hash256, Slot}; @@ -153,7 +150,8 @@ impl BeaconChain { // Store block roots, including at all skip slots in the freezer DB. for slot in (block.slot().as_u64()..prev_block_slot.as_u64()).rev() { cold_batch.push(KeyValueStoreOp::PutKeyValue( - get_key_for_col(DBColumn::BeaconBlockRoots.into(), &slot.to_be_bytes()), + DBColumn::BeaconBlockRoots, + slot.to_be_bytes().to_vec(), block_root.as_slice().to_vec(), )); } @@ -169,7 +167,8 @@ impl BeaconChain { let genesis_slot = self.spec.genesis_slot; for slot in genesis_slot.as_u64()..prev_block_slot.as_u64() { cold_batch.push(KeyValueStoreOp::PutKeyValue( - get_key_for_col(DBColumn::BeaconBlockRoots.into(), &slot.to_be_bytes()), + DBColumn::BeaconBlockRoots, + slot.to_be_bytes().to_vec(), self.genesis_block_root.as_slice().to_vec(), )); } diff --git a/beacon_node/beacon_chain/src/kzg_utils.rs b/beacon_node/beacon_chain/src/kzg_utils.rs index 1680c0298d..06cce14144 100644 --- a/beacon_node/beacon_chain/src/kzg_utils.rs +++ b/beacon_node/beacon_chain/src/kzg_utils.rs @@ -7,8 +7,9 @@ use std::sync::Arc; use types::beacon_block_body::KzgCommitments; use types::data_column_sidecar::{Cell, DataColumn, DataColumnSidecarError}; use types::{ - Blob, ChainSpec, ColumnIndex, DataColumnSidecar, DataColumnSidecarList, EthSpec, Hash256, - KzgCommitment, KzgProof, KzgProofs, SignedBeaconBlock, SignedBeaconBlockHeader, + Blob, BlobSidecar, BlobSidecarList, ChainSpec, ColumnIndex, DataColumnSidecar, + DataColumnSidecarList, EthSpec, Hash256, KzgCommitment, KzgProof, KzgProofs, SignedBeaconBlock, + SignedBeaconBlockHeader, SignedBlindedBeaconBlock, }; /// Converts a blob ssz List object to an array to be used with the kzg @@ -185,17 +186,19 @@ pub fn blobs_to_data_column_sidecars( .map_err(DataColumnSidecarError::BuildSidecarFailed) } -fn build_data_column_sidecars( +pub(crate) fn build_data_column_sidecars( kzg_commitments: KzgCommitments, kzg_commitments_inclusion_proof: FixedVector, signed_block_header: SignedBeaconBlockHeader, blob_cells_and_proofs_vec: Vec, spec: &ChainSpec, ) -> Result, String> { - let number_of_columns = spec.number_of_columns; - let mut columns = vec![Vec::with_capacity(E::max_blobs_per_block()); number_of_columns]; - let mut column_kzg_proofs = - vec![Vec::with_capacity(E::max_blobs_per_block()); number_of_columns]; + let number_of_columns = spec.number_of_columns as usize; + let max_blobs_per_block = spec + .max_blobs_per_block(signed_block_header.message.slot.epoch(E::slots_per_epoch())) + as usize; + let mut columns = vec![Vec::with_capacity(max_blobs_per_block); number_of_columns]; + let mut column_kzg_proofs = vec![Vec::with_capacity(max_blobs_per_block); number_of_columns]; for (blob_cells, blob_cell_proofs) in blob_cells_and_proofs_vec { // we iterate over each column, and we construct the column from "top to bottom", @@ -243,6 +246,85 @@ fn build_data_column_sidecars( Ok(sidecars) } +/// Reconstruct blobs from a subset of data column sidecars (requires at least 50%). +/// +/// If `blob_indices_opt` is `None`, this function attempts to reconstruct all blobs associated +/// with the block. +pub fn reconstruct_blobs( + kzg: &Kzg, + data_columns: &[Arc>], + blob_indices_opt: Option>, + signed_block: &SignedBlindedBeaconBlock, + spec: &ChainSpec, +) -> Result, String> { + // The data columns are from the database, so we assume their correctness. + let first_data_column = data_columns + .first() + .ok_or("data_columns should have at least one element".to_string())?; + + let blob_indices: Vec = match blob_indices_opt { + Some(indices) => indices.into_iter().map(|i| i as usize).collect(), + None => { + let num_of_blobs = first_data_column.kzg_commitments.len(); + (0..num_of_blobs).collect() + } + }; + + let blob_sidecars = blob_indices + .into_par_iter() + .map(|row_index| { + let mut cells: Vec = vec![]; + let mut cell_ids: Vec = vec![]; + for data_column in data_columns { + let cell = data_column + .column + .get(row_index) + .ok_or(format!("Missing data column at row index {row_index}")) + .and_then(|cell| { + ssz_cell_to_crypto_cell::(cell).map_err(|e| format!("{e:?}")) + })?; + + cells.push(cell); + cell_ids.push(data_column.index); + } + + let (cells, _kzg_proofs) = kzg + .recover_cells_and_compute_kzg_proofs(&cell_ids, &cells) + .map_err(|e| format!("Failed to recover cells and compute KZG proofs: {e:?}"))?; + + let num_cells_original_blob = cells.len() / 2; + let blob_bytes = cells + .into_iter() + .take(num_cells_original_blob) + .flat_map(|cell| cell.into_iter()) + .collect(); + + let blob = Blob::::new(blob_bytes).map_err(|e| format!("{e:?}"))?; + let kzg_commitment = first_data_column + .kzg_commitments + .get(row_index) + .ok_or(format!("Missing KZG commitment for blob {row_index}"))?; + let kzg_proof = compute_blob_kzg_proof::(kzg, &blob, *kzg_commitment) + .map_err(|e| format!("{e:?}"))?; + + BlobSidecar::::new_with_existing_proof( + row_index, + blob, + signed_block, + first_data_column.signed_block_header.clone(), + &first_data_column.kzg_commitments_inclusion_proof, + kzg_proof, + ) + .map(Arc::new) + .map_err(|e| format!("{e:?}")) + }) + .collect::, _>>()?; + + let max_blobs = spec.max_blobs_per_block(signed_block.epoch()) as usize; + + BlobSidecarList::new(blob_sidecars, max_blobs).map_err(|e| format!("{e:?}")) +} + /// Reconstruct all data columns from a subset of data column sidecars (requires at least 50%). pub fn reconstruct_data_columns( kzg: &Kzg, @@ -265,7 +347,7 @@ pub fn reconstruct_data_columns( for data_column in data_columns { let cell = data_column.column.get(row_index).ok_or( KzgError::InconsistentArrayLength(format!( - "Missing data column at index {row_index}" + "Missing data column at row index {row_index}" )), )?; @@ -289,12 +371,16 @@ pub fn reconstruct_data_columns( #[cfg(test)] mod test { - use crate::kzg_utils::{blobs_to_data_column_sidecars, reconstruct_data_columns}; + use crate::kzg_utils::{ + blobs_to_data_column_sidecars, reconstruct_blobs, reconstruct_data_columns, + }; use bls::Signature; + use eth2::types::BlobsBundle; + use execution_layer::test_utils::generate_blobs; use kzg::{trusted_setup::get_trusted_setup, Kzg, KzgCommitment, TrustedSetup}; use types::{ - beacon_block_body::KzgCommitments, BeaconBlock, BeaconBlockDeneb, Blob, BlobsList, - ChainSpec, EmptyBlock, EthSpec, MainnetEthSpec, SignedBeaconBlock, + beacon_block_body::KzgCommitments, BeaconBlock, BeaconBlockDeneb, BlobsList, ChainSpec, + EmptyBlock, EthSpec, MainnetEthSpec, SignedBeaconBlock, }; type E = MainnetEthSpec; @@ -308,6 +394,7 @@ mod test { test_build_data_columns_empty(&kzg, &spec); test_build_data_columns(&kzg, &spec); test_reconstruct_data_columns(&kzg, &spec); + test_reconstruct_blobs_from_data_columns(&kzg, &spec); } #[track_caller] @@ -341,7 +428,7 @@ mod test { .kzg_commitments_merkle_proof() .unwrap(); - assert_eq!(column_sidecars.len(), spec.number_of_columns); + assert_eq!(column_sidecars.len(), spec.number_of_columns as usize); for (idx, col_sidecar) in column_sidecars.iter().enumerate() { assert_eq!(col_sidecar.index, idx as u64); @@ -374,11 +461,42 @@ mod test { ) .unwrap(); - for i in 0..spec.number_of_columns { + for i in 0..spec.number_of_columns as usize { assert_eq!(reconstructed_columns.get(i), column_sidecars.get(i), "{i}"); } } + #[track_caller] + fn test_reconstruct_blobs_from_data_columns(kzg: &Kzg, spec: &ChainSpec) { + let num_of_blobs = 6; + let (signed_block, blobs) = create_test_block_and_blobs::(num_of_blobs, spec); + let blob_refs = blobs.iter().collect::>(); + let column_sidecars = + blobs_to_data_column_sidecars(&blob_refs, &signed_block, kzg, spec).unwrap(); + + // Now reconstruct + let signed_blinded_block = signed_block.into(); + let blob_indices = vec![3, 4, 5]; + let reconstructed_blobs = reconstruct_blobs( + kzg, + &column_sidecars.iter().as_slice()[0..column_sidecars.len() / 2], + Some(blob_indices.clone()), + &signed_blinded_block, + spec, + ) + .unwrap(); + + for i in blob_indices { + let reconstructed_blob = &reconstructed_blobs + .iter() + .find(|sidecar| sidecar.index == i) + .map(|sidecar| sidecar.blob.clone()) + .expect("reconstructed blob should exist"); + let original_blob = blobs.get(i as usize).unwrap(); + assert_eq!(reconstructed_blob, original_blob, "{i}"); + } + } + fn get_kzg() -> Kzg { let trusted_setup: TrustedSetup = serde_json::from_reader(get_trusted_setup().as_slice()) .map_err(|e| format!("Unable to read trusted setup file: {}", e)) @@ -397,12 +515,20 @@ mod test { KzgCommitments::::new(vec![KzgCommitment::empty_for_testing(); num_of_blobs]) .unwrap(); - let signed_block = SignedBeaconBlock::from_block(block, Signature::empty()); + let mut signed_block = SignedBeaconBlock::from_block(block, Signature::empty()); - let blobs = (0..num_of_blobs) - .map(|_| Blob::::default()) - .collect::>() - .into(); + let (blobs_bundle, _) = generate_blobs::(num_of_blobs).unwrap(); + let BlobsBundle { + blobs, + commitments, + proofs: _, + } = blobs_bundle; + + *signed_block + .message_mut() + .body_mut() + .blob_kzg_commitments_mut() + .unwrap() = commitments; (signed_block, blobs) } diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 1facde02c4..fe03420736 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -31,6 +31,7 @@ pub mod execution_payload; pub mod fetch_blobs; pub mod fork_choice_signal; pub mod fork_revert; +pub mod fulu_readiness; pub mod graffiti_calculator; mod head_tracker; pub mod historical_blocks; @@ -78,7 +79,7 @@ pub use beacon_fork_choice_store::{BeaconForkChoiceStore, Error as ForkChoiceSto pub use block_verification::{ build_blob_data_column_sidecars, get_block_root, BlockError, ExecutionPayloadError, ExecutionPendingBlock, GossipVerifiedBlock, IntoExecutionPendingBlock, IntoGossipVerifiedBlock, - PayloadVerificationOutcome, PayloadVerificationStatus, + InvalidSignature, PayloadVerificationOutcome, PayloadVerificationStatus, }; pub use block_verification_types::AvailabilityPendingExecutedBlock; pub use block_verification_types::ExecutedBlock; diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index c6aa9fbcac..ae3add7f03 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -1656,7 +1656,7 @@ pub static BLOB_SIDECAR_INCLUSION_PROOF_COMPUTATION: LazyLock> }); pub static DATA_COLUMN_SIDECAR_COMPUTATION: LazyLock> = LazyLock::new(|| { try_create_histogram_vec_with_buckets( - "data_column_sidecar_computation_seconds", + "beacon_data_column_sidecar_computation_seconds", "Time taken to compute data column sidecar, including cells, proofs and inclusion proof", Ok(vec![0.1, 0.15, 0.25, 0.35, 0.5, 0.7, 1.0, 2.5, 5.0, 10.0]), &["blob_count"], @@ -1665,7 +1665,7 @@ pub static DATA_COLUMN_SIDECAR_COMPUTATION: LazyLock> = Laz pub static DATA_COLUMN_SIDECAR_INCLUSION_PROOF_VERIFICATION: LazyLock> = LazyLock::new(|| { try_create_histogram( - "data_column_sidecar_inclusion_proof_verification_seconds", + "beacon_data_column_sidecar_inclusion_proof_verification_seconds", "Time taken to verify data_column sidecar inclusion proof", ) }); @@ -1693,7 +1693,7 @@ pub static DATA_COLUMN_SIDECAR_GOSSIP_VERIFICATION_TIMES: LazyLock> = LazyLock::new(|| { try_create_int_counter( - "beacon_blobs_column_sidecar_processing_successes_total", + "beacon_data_column_sidecar_processing_successes_total", "Number of data column sidecars verified for gossip", ) }); @@ -1847,7 +1847,7 @@ pub static KZG_VERIFICATION_BATCH_TIMES: LazyLock> = LazyLock: pub static KZG_VERIFICATION_DATA_COLUMN_SINGLE_TIMES: LazyLock> = LazyLock::new(|| { try_create_histogram_with_buckets( - "kzg_verification_data_column_single_seconds", + "beacon_kzg_verification_data_column_single_seconds", "Runtime of single data column kzg verification", Ok(vec![ 0.0005, 0.001, 0.0015, 0.002, 0.003, 0.004, 0.005, 0.007, 0.01, 0.02, 0.05, @@ -1857,7 +1857,7 @@ pub static KZG_VERIFICATION_DATA_COLUMN_SINGLE_TIMES: LazyLock pub static KZG_VERIFICATION_DATA_COLUMN_BATCH_TIMES: LazyLock> = LazyLock::new(|| { try_create_histogram_with_buckets( - "kzg_verification_data_column_batch_seconds", + "beacon_kzg_verification_data_column_batch_seconds", "Runtime of batched data column kzg verification", Ok(vec![ 0.002, 0.004, 0.006, 0.008, 0.01, 0.012, 0.015, 0.02, 0.03, 0.05, 0.07, @@ -1910,14 +1910,14 @@ pub static DATA_AVAILABILITY_OVERFLOW_STORE_CACHE_SIZE: LazyLock> = LazyLock::new(|| { try_create_histogram( - "data_availability_reconstruction_time_seconds", + "beacon_data_availability_reconstruction_time_seconds", "Time taken to reconstruct columns", ) }); pub static DATA_AVAILABILITY_RECONSTRUCTED_COLUMNS: LazyLock> = LazyLock::new(|| { try_create_int_counter( - "data_availability_reconstructed_columns_total", + "beacon_data_availability_reconstructed_columns_total", "Total count of reconstructed columns", ) }); @@ -1925,7 +1925,7 @@ pub static DATA_AVAILABILITY_RECONSTRUCTED_COLUMNS: LazyLock> pub static KZG_DATA_COLUMN_RECONSTRUCTION_ATTEMPTS: LazyLock> = LazyLock::new(|| { try_create_int_counter( - "kzg_data_column_reconstruction_attempts", + "beacon_kzg_data_column_reconstruction_attempts", "Count of times data column reconstruction has been attempted", ) }); @@ -1933,7 +1933,7 @@ pub static KZG_DATA_COLUMN_RECONSTRUCTION_ATTEMPTS: LazyLock> pub static KZG_DATA_COLUMN_RECONSTRUCTION_FAILURES: LazyLock> = LazyLock::new(|| { try_create_int_counter( - "kzg_data_column_reconstruction_failures", + "beacon_kzg_data_column_reconstruction_failures", "Count of times data column reconstruction has failed", ) }); @@ -1941,7 +1941,7 @@ pub static KZG_DATA_COLUMN_RECONSTRUCTION_FAILURES: LazyLock> pub static KZG_DATA_COLUMN_RECONSTRUCTION_INCOMPLETE_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter_vec( - "kzg_data_column_reconstruction_incomplete_total", + "beacon_kzg_data_column_reconstruction_incomplete_total", "Count of times data column reconstruction attempts did not result in an import", &["reason"], ) diff --git a/beacon_node/beacon_chain/src/observed_aggregates.rs b/beacon_node/beacon_chain/src/observed_aggregates.rs index dec012fb92..20ed36ace7 100644 --- a/beacon_node/beacon_chain/src/observed_aggregates.rs +++ b/beacon_node/beacon_chain/src/observed_aggregates.rs @@ -293,7 +293,7 @@ impl SlotHashSet { Ok(self .map .get(&root) - .map_or(false, |agg| agg.iter().any(|val| item.is_subset(val)))) + .is_some_and(|agg| agg.iter().any(|val| item.is_subset(val)))) } /// The number of observed items in `self`. diff --git a/beacon_node/beacon_chain/src/observed_attesters.rs b/beacon_node/beacon_chain/src/observed_attesters.rs index efb95f57a9..5bba8e4d8e 100644 --- a/beacon_node/beacon_chain/src/observed_attesters.rs +++ b/beacon_node/beacon_chain/src/observed_attesters.rs @@ -130,7 +130,7 @@ impl Item<()> for EpochBitfield { fn get(&self, validator_index: usize) -> Option<()> { self.bitfield .get(validator_index) - .map_or(false, |bit| *bit) + .is_some_and(|bit| *bit) .then_some(()) } } @@ -336,7 +336,7 @@ impl, E: EthSpec> AutoPruningEpochContainer { let exists = self .items .get(&epoch) - .map_or(false, |item| item.get(validator_index).is_some()); + .is_some_and(|item| item.get(validator_index).is_some()); Ok(exists) } diff --git a/beacon_node/beacon_chain/src/observed_data_sidecars.rs b/beacon_node/beacon_chain/src/observed_data_sidecars.rs index 53f8c71f54..1ca6c03f00 100644 --- a/beacon_node/beacon_chain/src/observed_data_sidecars.rs +++ b/beacon_node/beacon_chain/src/observed_data_sidecars.rs @@ -24,7 +24,7 @@ pub trait ObservableDataSidecar { fn slot(&self) -> Slot; fn block_proposer_index(&self) -> u64; fn index(&self) -> u64; - fn max_num_of_items(spec: &ChainSpec) -> usize; + fn max_num_of_items(spec: &ChainSpec, slot: Slot) -> usize; } impl ObservableDataSidecar for BlobSidecar { @@ -40,8 +40,8 @@ impl ObservableDataSidecar for BlobSidecar { self.index } - fn max_num_of_items(_spec: &ChainSpec) -> usize { - E::max_blobs_per_block() + fn max_num_of_items(spec: &ChainSpec, slot: Slot) -> usize { + spec.max_blobs_per_block(slot.epoch(E::slots_per_epoch())) as usize } } @@ -58,8 +58,8 @@ impl ObservableDataSidecar for DataColumnSidecar { self.index } - fn max_num_of_items(spec: &ChainSpec) -> usize { - spec.number_of_columns + fn max_num_of_items(spec: &ChainSpec, _slot: Slot) -> usize { + spec.number_of_columns as usize } } @@ -103,7 +103,9 @@ impl ObservedDataSidecars { slot: data_sidecar.slot(), proposer: data_sidecar.block_proposer_index(), }) - .or_insert_with(|| HashSet::with_capacity(T::max_num_of_items(&self.spec))); + .or_insert_with(|| { + HashSet::with_capacity(T::max_num_of_items(&self.spec, data_sidecar.slot())) + }); let did_not_exist = data_indices.insert(data_sidecar.index()); Ok(!did_not_exist) @@ -118,12 +120,12 @@ impl ObservedDataSidecars { slot: data_sidecar.slot(), proposer: data_sidecar.block_proposer_index(), }) - .map_or(false, |indices| indices.contains(&data_sidecar.index())); + .is_some_and(|indices| indices.contains(&data_sidecar.index())); Ok(is_known) } fn sanitize_data_sidecar(&self, data_sidecar: &T) -> Result<(), Error> { - if data_sidecar.index() >= T::max_num_of_items(&self.spec) as u64 { + if data_sidecar.index() >= T::max_num_of_items(&self.spec, data_sidecar.slot()) as u64 { return Err(Error::InvalidDataIndex(data_sidecar.index())); } let finalized_slot = self.finalized_slot; @@ -179,7 +181,7 @@ mod tests { use crate::test_utils::test_spec; use bls::Hash256; use std::sync::Arc; - use types::MainnetEthSpec; + use types::{Epoch, MainnetEthSpec}; type E = MainnetEthSpec; @@ -333,7 +335,7 @@ mod tests { #[test] fn simple_observations() { let spec = Arc::new(test_spec::()); - let mut cache = ObservedDataSidecars::>::new(spec); + let mut cache = ObservedDataSidecars::>::new(spec.clone()); // Slot 0, index 0 let proposer_index_a = 420; @@ -489,7 +491,7 @@ mod tests { ); // Try adding an out of bounds index - let invalid_index = E::max_blobs_per_block() as u64; + let invalid_index = spec.max_blobs_per_block(Epoch::new(0)); let sidecar_d = get_blob_sidecar(0, proposer_index_a, invalid_index); assert_eq!( cache.observe_sidecar(&sidecar_d), diff --git a/beacon_node/beacon_chain/src/schema_change/migration_schema_v21.rs b/beacon_node/beacon_chain/src/schema_change/migration_schema_v21.rs index fcc8b9884a..f02f5ee6f3 100644 --- a/beacon_node/beacon_chain/src/schema_change/migration_schema_v21.rs +++ b/beacon_node/beacon_chain/src/schema_change/migration_schema_v21.rs @@ -3,9 +3,7 @@ use crate::validator_pubkey_cache::DatabasePubkey; use slog::{info, Logger}; use ssz::{Decode, Encode}; use std::sync::Arc; -use store::{ - get_key_for_col, DBColumn, Error, HotColdDB, KeyValueStore, KeyValueStoreOp, StoreItem, -}; +use store::{DBColumn, Error, HotColdDB, KeyValueStore, KeyValueStoreOp, StoreItem}; use types::{Hash256, PublicKey}; const LOG_EVERY: usize = 200_000; @@ -62,9 +60,9 @@ pub fn downgrade_from_v21( message: format!("{e:?}"), })?; - let db_key = get_key_for_col(DBColumn::PubkeyCache.into(), key.as_slice()); ops.push(KeyValueStoreOp::PutKeyValue( - db_key, + DBColumn::PubkeyCache, + key.as_slice().to_vec(), pubkey_bytes.as_ssz_bytes(), )); diff --git a/beacon_node/beacon_chain/src/schema_change/migration_schema_v22.rs b/beacon_node/beacon_chain/src/schema_change/migration_schema_v22.rs index c34512eded..982c3ded46 100644 --- a/beacon_node/beacon_chain/src/schema_change/migration_schema_v22.rs +++ b/beacon_node/beacon_chain/src/schema_change/migration_schema_v22.rs @@ -4,7 +4,6 @@ use std::sync::Arc; use store::chunked_iter::ChunkedVectorIter; use store::{ chunked_vector::BlockRootsChunked, - get_key_for_col, metadata::{ SchemaVersion, ANCHOR_FOR_ARCHIVE_NODE, ANCHOR_UNINITIALIZED, STATE_UPPER_LIMIT_NO_RETAIN, }, @@ -21,7 +20,7 @@ fn load_old_schema_frozen_state( ) -> Result>, Error> { let Some(partial_state_bytes) = db .cold_db - .get_bytes(DBColumn::BeaconState.into(), state_root.as_slice())? + .get_bytes(DBColumn::BeaconState, state_root.as_slice())? else { return Ok(None); }; @@ -136,10 +135,7 @@ pub fn delete_old_schema_freezer_data( for column in columns { for res in db.cold_db.iter_column_keys::>(column) { let key = res?; - cold_ops.push(KeyValueStoreOp::DeleteKey(get_key_for_col( - column.as_str(), - &key, - ))); + cold_ops.push(KeyValueStoreOp::DeleteKey(column, key)); } } let delete_ops = cold_ops.len(); @@ -175,7 +171,8 @@ pub fn write_new_schema_block_roots( // Store the genesis block root if it would otherwise not be stored. if oldest_block_slot != 0 { cold_ops.push(KeyValueStoreOp::PutKeyValue( - get_key_for_col(DBColumn::BeaconBlockRoots.into(), &0u64.to_be_bytes()), + DBColumn::BeaconBlockRoots, + 0u64.to_be_bytes().to_vec(), genesis_block_root.as_slice().to_vec(), )); } @@ -192,10 +189,8 @@ pub fn write_new_schema_block_roots( // OK to hold these in memory (10M slots * 43 bytes per KV ~= 430 MB). for (i, (slot, block_root)) in block_root_iter.enumerate() { cold_ops.push(KeyValueStoreOp::PutKeyValue( - get_key_for_col( - DBColumn::BeaconBlockRoots.into(), - &(slot as u64).to_be_bytes(), - ), + DBColumn::BeaconBlockRoots, + slot.to_be_bytes().to_vec(), block_root.as_slice().to_vec(), )); diff --git a/beacon_node/beacon_chain/src/shuffling_cache.rs b/beacon_node/beacon_chain/src/shuffling_cache.rs index da1d60db17..67ca72254b 100644 --- a/beacon_node/beacon_chain/src/shuffling_cache.rs +++ b/beacon_node/beacon_chain/src/shuffling_cache.rs @@ -253,7 +253,7 @@ impl BlockShufflingIds { } else if self .previous .as_ref() - .map_or(false, |id| id.shuffling_epoch == epoch) + .is_some_and(|id| id.shuffling_epoch == epoch) { self.previous.clone() } else if epoch == self.next.shuffling_epoch { diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 093ee0c44b..4526b2b360 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -1,8 +1,9 @@ +use crate::blob_verification::GossipVerifiedBlob; use crate::block_verification_types::{AsBlock, RpcBlock}; -use crate::kzg_utils::blobs_to_data_column_sidecars; +use crate::data_column_verification::CustodyDataColumn; +use crate::kzg_utils::build_data_column_sidecars; use crate::observed_operations::ObservationOutcome; pub use crate::persisted_beacon_chain::PersistedBeaconChain; -use crate::BeaconBlockResponseWrapper; pub use crate::{ beacon_chain::{BEACON_CHAIN_DB_KEY, ETH1_CACHE_DB_KEY, FORK_CHOICE_DB_KEY, OP_POOL_DB_KEY}, migrate::MigratorConfig, @@ -16,6 +17,7 @@ use crate::{ BeaconChain, BeaconChainTypes, BlockError, ChainConfig, ServerSentEventHandler, StateSkipConfig, }; +use crate::{get_block_root, BeaconBlockResponseWrapper}; use bls::get_withdrawal_credentials; use eth2::types::SignedBlockContentsTuple; use execution_layer::test_utils::generate_genesis_header; @@ -56,7 +58,8 @@ use std::str::FromStr; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, LazyLock}; use std::time::Duration; -use store::{config::StoreConfig, HotColdDB, ItemStore, LevelDB, MemoryStore}; +use store::database::interface::BeaconNodeBackend; +use store::{config::StoreConfig, HotColdDB, ItemStore, MemoryStore}; use task_executor::TaskExecutor; use task_executor::{test_utils::TestRuntime, ShutdownReason}; use tree_hash::TreeHash; @@ -73,6 +76,11 @@ pub const FORK_NAME_ENV_VAR: &str = "FORK_NAME"; // Environment variable to read if `ci_logger` feature is enabled. pub const CI_LOGGER_DIR_ENV_VAR: &str = "CI_LOGGER_DIR"; +// Pre-computed data column sidecar using a single static blob from: +// `beacon_node/execution_layer/src/test_utils/fixtures/mainnet/test_blobs_bundle.ssz` +const TEST_DATA_COLUMN_SIDECARS_SSZ: &[u8] = + include_bytes!("test_utils/fixtures/test_data_column_sidecars.ssz"); + // Default target aggregators to set during testing, this ensures an aggregator at each slot. // // You should mutate the `ChainSpec` prior to initialising the harness if you would like to use @@ -104,7 +112,7 @@ static KZG_NO_PRECOMP: LazyLock> = LazyLock::new(|| { }); pub fn get_kzg(spec: &ChainSpec) -> Arc { - if spec.eip7594_fork_epoch.is_some() { + if spec.fulu_fork_epoch.is_some() { KZG_PEERDAS.clone() } else if spec.deneb_fork_epoch.is_some() { KZG.clone() @@ -116,7 +124,7 @@ pub fn get_kzg(spec: &ChainSpec) -> Arc { pub type BaseHarnessType = Witness, E, THotStore, TColdStore>; -pub type DiskHarnessType = BaseHarnessType, LevelDB>; +pub type DiskHarnessType = BaseHarnessType, BeaconNodeBackend>; pub type EphemeralHarnessType = BaseHarnessType, MemoryStore>; pub type BoxedMutator = Box< @@ -223,6 +231,7 @@ pub struct Builder { mock_execution_layer: Option>, testing_slot_clock: Option, validator_monitor_config: Option, + import_all_data_columns: bool, runtime: TestRuntime, log: Logger, } @@ -299,7 +308,10 @@ impl Builder> { impl Builder> { /// Disk store, start from genesis. - pub fn fresh_disk_store(mut self, store: Arc, LevelDB>>) -> Self { + pub fn fresh_disk_store( + mut self, + store: Arc, BeaconNodeBackend>>, + ) -> Self { let validator_keypairs = self .validator_keypairs .clone() @@ -324,7 +336,10 @@ impl Builder> { } /// Disk store, resume. - pub fn resumed_disk_store(mut self, store: Arc, LevelDB>>) -> Self { + pub fn resumed_disk_store( + mut self, + store: Arc, BeaconNodeBackend>>, + ) -> Self { let mutator = move |builder: BeaconChainBuilder<_>| { builder .resume_from_db() @@ -359,6 +374,7 @@ where mock_execution_layer: None, testing_slot_clock: None, validator_monitor_config: None, + import_all_data_columns: false, runtime, log, } @@ -451,6 +467,11 @@ where self } + pub fn import_all_data_columns(mut self, import_all_data_columns: bool) -> Self { + self.import_all_data_columns = import_all_data_columns; + self + } + pub fn execution_layer_from_url(mut self, url: &str) -> Self { assert!( self.execution_layer.is_none(), @@ -501,6 +522,9 @@ where spec.electra_fork_epoch.map(|epoch| { genesis_time + spec.seconds_per_slot * E::slots_per_epoch() * epoch.as_u64() }); + mock.server.execution_block_generator().osaka_time = spec.fulu_fork_epoch.map(|epoch| { + genesis_time + spec.seconds_per_slot * E::slots_per_epoch() * epoch.as_u64() + }); self } @@ -511,7 +535,7 @@ where pub fn mock_execution_layer_with_config(mut self) -> Self { let mock = mock_execution_layer_from_parts::( - self.spec.as_ref().expect("cannot build without spec"), + self.spec.clone().expect("cannot build without spec"), self.runtime.task_executor.clone(), ); self.execution_layer = Some(mock.el.clone()); @@ -565,6 +589,7 @@ where .expect("should build dummy backend") .shutdown_sender(shutdown_tx) .chain_config(chain_config) + .import_all_data_columns(self.import_all_data_columns) .event_handler(Some(ServerSentEventHandler::new_with_capacity( log.clone(), 5, @@ -611,7 +636,7 @@ where } pub fn mock_execution_layer_from_parts( - spec: &ChainSpec, + spec: Arc, task_executor: TaskExecutor, ) -> MockExecutionLayer { let shanghai_time = spec.capella_fork_epoch.map(|epoch| { @@ -623,8 +648,11 @@ pub fn mock_execution_layer_from_parts( let prague_time = spec.electra_fork_epoch.map(|epoch| { HARNESS_GENESIS_TIME + spec.seconds_per_slot * E::slots_per_epoch() * epoch.as_u64() }); + let osaka_time = spec.fulu_fork_epoch.map(|epoch| { + HARNESS_GENESIS_TIME + spec.seconds_per_slot * E::slots_per_epoch() * epoch.as_u64() + }); - let kzg = get_kzg(spec); + let kzg = get_kzg(&spec); MockExecutionLayer::new( task_executor, @@ -632,8 +660,9 @@ pub fn mock_execution_layer_from_parts( shanghai_time, cancun_time, prague_time, + osaka_time, Some(JwtKey::from_slice(&DEFAULT_JWT_SECRET).unwrap()), - spec.clone(), + spec, Some(kzg), ) } @@ -662,10 +691,16 @@ pub struct BeaconChainHarness { pub rng: Mutex, } +pub type CommitteeSingleAttestations = Vec<(SingleAttestation, SubnetId)>; pub type CommitteeAttestations = Vec<(Attestation, SubnetId)>; pub type HarnessAttestations = Vec<(CommitteeAttestations, Option>)>; +pub type HarnessSingleAttestations = Vec<( + CommitteeSingleAttestations, + Option>, +)>; + pub type HarnessSyncContributions = Vec<( Vec<(SyncCommitteeMessage, usize)>, Option>, @@ -742,15 +777,13 @@ where pub fn get_head_block(&self) -> RpcBlock { let block = self.chain.head_beacon_block(); let block_root = block.canonical_root(); - let blobs = self.chain.get_blobs(&block_root).unwrap(); - RpcBlock::new(Some(block_root), block, Some(blobs)).unwrap() + self.build_rpc_block_from_store_blobs(Some(block_root), block) } pub fn get_full_block(&self, block_root: &Hash256) -> RpcBlock { let block = self.chain.get_blinded_block(block_root).unwrap().unwrap(); let full_block = self.chain.store.make_full_block(block_root, block).unwrap(); - let blobs = self.chain.get_blobs(block_root).unwrap(); - RpcBlock::new(Some(*block_root), Arc::new(full_block), Some(blobs)).unwrap() + self.build_rpc_block_from_store_blobs(Some(*block_root), Arc::new(full_block)) } pub fn get_all_validators(&self) -> Vec { @@ -913,15 +946,12 @@ where &self.spec, )); - let block_contents: SignedBlockContentsTuple = match *signed_block { - SignedBeaconBlock::Base(_) - | SignedBeaconBlock::Altair(_) - | SignedBeaconBlock::Bellatrix(_) - | SignedBeaconBlock::Capella(_) => (signed_block, None), - SignedBeaconBlock::Deneb(_) | SignedBeaconBlock::Electra(_) => { + let block_contents: SignedBlockContentsTuple = + if signed_block.fork_name_unchecked().deneb_enabled() { (signed_block, block_response.blob_items) - } - }; + } else { + (signed_block, None) + }; (block_contents, block_response.state) } @@ -977,15 +1007,12 @@ where &self.spec, )); - let block_contents: SignedBlockContentsTuple = match *signed_block { - SignedBeaconBlock::Base(_) - | SignedBeaconBlock::Altair(_) - | SignedBeaconBlock::Bellatrix(_) - | SignedBeaconBlock::Capella(_) => (signed_block, None), - SignedBeaconBlock::Deneb(_) | SignedBeaconBlock::Electra(_) => { + let block_contents: SignedBlockContentsTuple = + if signed_block.fork_name_unchecked().deneb_enabled() { (signed_block, block_response.blob_items) - } - }; + } else { + (signed_block, None) + }; (block_contents, pre_state) } @@ -1023,6 +1050,99 @@ where ) } + #[allow(clippy::too_many_arguments)] + pub fn produce_single_attestation_for_block( + &self, + slot: Slot, + index: CommitteeIndex, + beacon_block_root: Hash256, + mut state: Cow>, + state_root: Hash256, + aggregation_bit_index: usize, + validator_index: usize, + ) -> Result { + let epoch = slot.epoch(E::slots_per_epoch()); + + if state.slot() > slot { + return Err(BeaconChainError::CannotAttestToFutureState); + } else if state.current_epoch() < epoch { + let mut_state = state.to_mut(); + complete_state_advance( + mut_state, + Some(state_root), + epoch.start_slot(E::slots_per_epoch()), + &self.spec, + )?; + mut_state.build_committee_cache(RelativeEpoch::Current, &self.spec)?; + } + + let committee_len = state.get_beacon_committee(slot, index)?.committee.len(); + + let target_slot = epoch.start_slot(E::slots_per_epoch()); + let target_root = if state.slot() <= target_slot { + beacon_block_root + } else { + *state.get_block_root(target_slot)? + }; + + let attestation: Attestation = Attestation::empty_for_signing( + index, + committee_len, + slot, + beacon_block_root, + state.current_justified_checkpoint(), + Checkpoint { + epoch, + root: target_root, + }, + &self.spec, + )?; + + let attestation = match attestation { + Attestation::Electra(mut attn) => { + attn.aggregation_bits + .set(aggregation_bit_index, true) + .unwrap(); + attn + } + Attestation::Base(_) => panic!("Must be an Electra attestation"), + }; + + let aggregation_bits = attestation.get_aggregation_bits(); + + if aggregation_bits.len() != 1 { + panic!("Must be an unaggregated attestation") + } + + let aggregation_bit = *aggregation_bits.first().unwrap(); + + let committee = state.get_beacon_committee(slot, index).unwrap(); + + let attester_index = committee + .committee + .iter() + .enumerate() + .find_map(|(i, &index)| { + if aggregation_bit as usize == i { + return Some(index); + } + None + }) + .unwrap(); + + let single_attestation = + attestation.to_single_attestation_with_attester_index(attester_index as u64)?; + + let attestation: Attestation = single_attestation.to_attestation(committee.committee)?; + + assert_eq!( + single_attestation.committee_index, + attestation.committee_index().unwrap() + ); + assert_eq!(single_attestation.attester_index, validator_index as u64); + Ok(single_attestation) + } + /// Produces an "unaggregated" attestation for the given `slot` and `index` that attests to /// `beacon_block_root`. The provided `state` should match the `block.state_root` for the /// `block` identified by `beacon_block_root`. @@ -1080,6 +1200,33 @@ where )?) } + /// A list of attestations for each committee for the given slot. + /// + /// The first layer of the Vec is organised per committee. For example, if the return value is + /// called `all_attestations`, then all attestations in `all_attestations[0]` will be for + /// committee 0, whilst all in `all_attestations[1]` will be for committee 1. + pub fn make_single_attestations( + &self, + attesting_validators: &[usize], + state: &BeaconState, + state_root: Hash256, + head_block_root: SignedBeaconBlockHash, + attestation_slot: Slot, + ) -> Vec { + let fork = self + .spec + .fork_at_epoch(attestation_slot.epoch(E::slots_per_epoch())); + self.make_single_attestations_with_opts( + attesting_validators, + state, + state_root, + head_block_root, + attestation_slot, + MakeAttestationOptions { limit: None, fork }, + ) + .0 + } + /// A list of attestations for each committee for the given slot. /// /// The first layer of the Vec is organised per committee. For example, if the return value is @@ -1107,6 +1254,99 @@ where .0 } + pub fn make_single_attestations_with_opts( + &self, + attesting_validators: &[usize], + state: &BeaconState, + state_root: Hash256, + head_block_root: SignedBeaconBlockHash, + attestation_slot: Slot, + opts: MakeAttestationOptions, + ) -> (Vec, Vec) { + let MakeAttestationOptions { limit, fork } = opts; + let committee_count = state.get_committee_count_at_slot(state.slot()).unwrap(); + let num_attesters = AtomicUsize::new(0); + + let (attestations, split_attesters) = state + .get_beacon_committees_at_slot(attestation_slot) + .expect("should get committees") + .iter() + .map(|bc| { + bc.committee + .par_iter() + .enumerate() + .filter_map(|(i, validator_index)| { + if !attesting_validators.contains(validator_index) { + return None; + } + + if let Some(limit) = limit { + // This atomics stuff is necessary because we're under a par_iter, + // and Rayon will deadlock if we use a mutex. + if num_attesters.fetch_add(1, Ordering::Relaxed) >= limit { + num_attesters.fetch_sub(1, Ordering::Relaxed); + return None; + } + } + + let mut attestation = self + .produce_single_attestation_for_block( + attestation_slot, + bc.index, + head_block_root.into(), + Cow::Borrowed(state), + state_root, + i, + *validator_index, + ) + .unwrap(); + + attestation.signature = { + let domain = self.spec.get_domain( + attestation.data.target.epoch, + Domain::BeaconAttester, + &fork, + state.genesis_validators_root(), + ); + + let message = attestation.data.signing_root(domain); + + let mut agg_sig = AggregateSignature::infinity(); + + agg_sig.add_assign( + &self.validator_keypairs[*validator_index].sk.sign(message), + ); + + agg_sig + }; + + let subnet_id = SubnetId::compute_subnet_for_single_attestation::( + &attestation, + committee_count, + &self.chain.spec, + ) + .unwrap(); + + Some(((attestation, subnet_id), validator_index)) + }) + .unzip::<_, _, Vec<_>, Vec<_>>() + }) + .unzip::<_, _, Vec<_>, Vec<_>>(); + + // Flatten attesters. + let attesters = split_attesters.into_iter().flatten().collect::>(); + + if let Some(limit) = limit { + assert_eq!(limit, num_attesters.load(Ordering::Relaxed)); + assert_eq!( + limit, + attesters.len(), + "failed to generate `limit` attestations" + ); + } + (attestations, attesters) + } + pub fn make_unaggregated_attestations_with_opts( &self, attesting_validators: &[usize], @@ -1287,6 +1527,32 @@ where ) } + /// A list of attestations for each committee for the given slot. + /// + /// The first layer of the Vec is organised per committee. For example, if the return value is + /// called `all_attestations`, then all attestations in `all_attestations[0]` will be for + /// committee 0, whilst all in `all_attestations[1]` will be for committee 1. + pub fn get_single_attestations( + &self, + attestation_strategy: &AttestationStrategy, + state: &BeaconState, + state_root: Hash256, + head_block_root: Hash256, + attestation_slot: Slot, + ) -> Vec> { + let validators: Vec = match attestation_strategy { + AttestationStrategy::AllValidators => self.get_all_validators(), + AttestationStrategy::SomeValidators(vals) => vals.clone(), + }; + self.make_single_attestations( + &validators, + state, + state_root, + head_block_root.into(), + attestation_slot, + ) + } + pub fn make_attestations( &self, attesting_validators: &[usize], @@ -2018,22 +2284,19 @@ where self.set_current_slot(slot); let (block, blob_items) = block_contents; - let sidecars = blob_items - .map(|(proofs, blobs)| BlobSidecar::build_sidecars(blobs, &block, proofs)) - .transpose() - .unwrap(); + let rpc_block = self.build_rpc_block_from_blobs(block_root, block, blob_items)?; let block_hash: SignedBeaconBlockHash = self .chain .process_block( block_root, - RpcBlock::new(Some(block_root), block, sidecars).unwrap(), + rpc_block, NotifyExecutionLayer::Yes, BlockImportSource::RangeSync, || Ok(()), ) .await? .try_into() - .unwrap(); + .expect("block blobs are available"); self.chain.recompute_head_at_current_slot().await; Ok(block_hash) } @@ -2044,16 +2307,13 @@ where ) -> Result { let (block, blob_items) = block_contents; - let sidecars = blob_items - .map(|(proofs, blobs)| BlobSidecar::build_sidecars(blobs, &block, proofs)) - .transpose() - .unwrap(); let block_root = block.canonical_root(); + let rpc_block = self.build_rpc_block_from_blobs(block_root, block, blob_items)?; let block_hash: SignedBeaconBlockHash = self .chain .process_block( block_root, - RpcBlock::new(Some(block_root), block, sidecars).unwrap(), + rpc_block, NotifyExecutionLayer::Yes, BlockImportSource::RangeSync, || Ok(()), @@ -2065,6 +2325,75 @@ where Ok(block_hash) } + /// Builds an `Rpc` block from a `SignedBeaconBlock` and blobs or data columns retrieved from + /// the database. + pub fn build_rpc_block_from_store_blobs( + &self, + block_root: Option, + block: Arc>, + ) -> RpcBlock { + let block_root = block_root.unwrap_or_else(|| get_block_root(&block)); + let has_blobs = block + .message() + .body() + .blob_kzg_commitments() + .is_ok_and(|c| !c.is_empty()); + if !has_blobs { + return RpcBlock::new_without_blobs(Some(block_root), block); + } + + // Blobs are stored as data columns from Fulu (PeerDAS) + if self.spec.is_peer_das_enabled_for_epoch(block.epoch()) { + let columns = self.chain.get_data_columns(&block_root).unwrap().unwrap(); + let custody_columns = columns + .into_iter() + .map(CustodyDataColumn::from_asserted_custody) + .collect::>(); + RpcBlock::new_with_custody_columns(Some(block_root), block, custody_columns, &self.spec) + .unwrap() + } else { + let blobs = self.chain.get_blobs(&block_root).unwrap().blobs(); + RpcBlock::new(Some(block_root), block, blobs).unwrap() + } + } + + /// Builds an `RpcBlock` from a `SignedBeaconBlock` and `BlobsList`. + fn build_rpc_block_from_blobs( + &self, + block_root: Hash256, + block: Arc>>, + blob_items: Option<(KzgProofs, BlobsList)>, + ) -> Result, BlockError> { + Ok(if self.spec.is_peer_das_enabled_for_epoch(block.epoch()) { + let sampling_column_count = self + .chain + .data_availability_checker + .get_sampling_column_count(); + + if blob_items.is_some_and(|(_, blobs)| !blobs.is_empty()) { + // Note: this method ignores the actual custody columns and just take the first + // `sampling_column_count` for testing purpose only, because the chain does not + // currently have any knowledge of the columns being custodied. + let columns = generate_data_column_sidecars_from_block(&block, &self.spec) + .into_iter() + .take(sampling_column_count) + .map(CustodyDataColumn::from_asserted_custody) + .collect::>(); + RpcBlock::new_with_custody_columns(Some(block_root), block, columns, &self.spec)? + } else { + RpcBlock::new_without_blobs(Some(block_root), block) + } + } else { + let blobs = blob_items + .map(|(proofs, blobs)| { + BlobSidecar::build_sidecars(blobs, &block, proofs, &self.spec) + }) + .transpose() + .unwrap(); + RpcBlock::new(Some(block_root), block, blobs)? + }) + } + pub fn process_attestations(&self, attestations: HarnessAttestations) { let num_validators = self.validator_keypairs.len(); let mut unaggregated = Vec::with_capacity(num_validators); @@ -2738,6 +3067,56 @@ where Ok(()) } + + /// Simulate some of the blobs / data columns being seen on gossip. + /// Converts the blobs to data columns if the slot is Fulu or later. + pub async fn process_gossip_blobs_or_columns<'a>( + &self, + block: &SignedBeaconBlock, + blobs: impl Iterator>, + proofs: impl Iterator, + custody_columns_opt: Option>, + ) { + let is_peerdas_enabled = self.chain.spec.is_peer_das_enabled_for_epoch(block.epoch()); + if is_peerdas_enabled { + let custody_columns = custody_columns_opt.unwrap_or_else(|| { + let sampling_column_count = self + .chain + .data_availability_checker + .get_sampling_column_count() as u64; + (0..sampling_column_count).collect() + }); + + let verified_columns = generate_data_column_sidecars_from_block(block, &self.spec) + .into_iter() + .filter(|c| custody_columns.contains(&c.index)) + .map(|sidecar| { + let column_index = sidecar.index; + self.chain + .verify_data_column_sidecar_for_gossip(sidecar, column_index) + }) + .collect::, _>>() + .unwrap(); + + if !verified_columns.is_empty() { + self.chain + .process_gossip_data_columns(verified_columns, || Ok(())) + .await + .unwrap(); + } + } else { + for (i, (kzg_proof, blob)) in proofs.into_iter().zip(blobs).enumerate() { + let sidecar = + Arc::new(BlobSidecar::new(i, blob.clone(), block, *kzg_proof).unwrap()); + let gossip_blob = GossipVerifiedBlob::new(sidecar, i as u64, &self.chain) + .expect("should obtain gossip verified blob"); + self.chain + .process_gossip_blob(gossip_blob) + .await + .expect("should import valid gossip verified blob"); + } + } + } } // Junk `Debug` impl to satistfy certain trait bounds during testing. @@ -2816,11 +3195,12 @@ pub fn generate_rand_block_and_blobs( fork_name: ForkName, num_blobs: NumBlobs, rng: &mut impl Rng, + spec: &ChainSpec, ) -> (SignedBeaconBlock>, Vec>) { let inner = map_fork_name!(fork_name, BeaconBlock, <_>::random_for_test(rng)); let mut block = SignedBeaconBlock::from_block(inner, types::Signature::random_for_test(rng)); - + let max_blobs = spec.max_blobs_per_block(block.epoch()) as usize; let mut blob_sidecars = vec![]; let bundle = match block { @@ -2830,7 +3210,7 @@ pub fn generate_rand_block_and_blobs( // Get either zero blobs or a random number of blobs between 1 and Max Blobs. let payload: &mut FullPayloadDeneb = &mut message.body.execution_payload; let num_blobs = match num_blobs { - NumBlobs::Random => rng.gen_range(1..=E::max_blobs_per_block()), + NumBlobs::Random => rng.gen_range(1..=max_blobs), NumBlobs::Number(n) => n, NumBlobs::None => 0, }; @@ -2850,7 +3230,26 @@ pub fn generate_rand_block_and_blobs( // Get either zero blobs or a random number of blobs between 1 and Max Blobs. let payload: &mut FullPayloadElectra = &mut message.body.execution_payload; let num_blobs = match num_blobs { - NumBlobs::Random => rng.gen_range(1..=E::max_blobs_per_block()), + NumBlobs::Random => rng.gen_range(1..=max_blobs), + NumBlobs::Number(n) => n, + NumBlobs::None => 0, + }; + let (bundle, transactions) = + execution_layer::test_utils::generate_blobs::(num_blobs).unwrap(); + payload.execution_payload.transactions = <_>::default(); + for tx in Vec::from(transactions) { + payload.execution_payload.transactions.push(tx).unwrap(); + } + message.body.blob_kzg_commitments = bundle.commitments.clone(); + bundle + } + SignedBeaconBlock::Fulu(SignedBeaconBlockFulu { + ref mut message, .. + }) => { + // Get either zero blobs or a random number of blobs between 1 and Max Blobs. + let payload: &mut FullPayloadFulu = &mut message.body.execution_payload; + let num_blobs = match num_blobs { + NumBlobs::Random => rng.gen_range(1..=max_blobs), NumBlobs::Number(n) => n, NumBlobs::None => 0, }; @@ -2903,10 +3302,59 @@ pub fn generate_rand_block_and_data_columns( SignedBeaconBlock>, DataColumnSidecarList, ) { - let kzg = get_kzg(spec); - let (block, blobs) = generate_rand_block_and_blobs(fork_name, num_blobs, rng); - let blob_refs = blobs.iter().map(|b| &b.blob).collect::>(); - let data_columns = blobs_to_data_column_sidecars(&blob_refs, &block, &kzg, spec).unwrap(); - + let (block, _blobs) = generate_rand_block_and_blobs(fork_name, num_blobs, rng, spec); + let data_columns = generate_data_column_sidecars_from_block(&block, spec); (block, data_columns) } + +/// Generate data column sidecars from pre-computed cells and proofs. +fn generate_data_column_sidecars_from_block( + block: &SignedBeaconBlock, + spec: &ChainSpec, +) -> DataColumnSidecarList { + let kzg_commitments = block.message().body().blob_kzg_commitments().unwrap(); + if kzg_commitments.is_empty() { + return vec![]; + } + + let kzg_commitments_inclusion_proof = block + .message() + .body() + .kzg_commitments_merkle_proof() + .unwrap(); + let signed_block_header = block.signed_block_header(); + + // load the precomputed column sidecar to avoid computing them for every block in the tests. + let template_data_columns = RuntimeVariableList::>::from_ssz_bytes( + TEST_DATA_COLUMN_SIDECARS_SSZ, + spec.number_of_columns as usize, + ) + .unwrap(); + + let (cells, proofs) = template_data_columns + .into_iter() + .map(|sidecar| { + let DataColumnSidecar { + column, kzg_proofs, .. + } = sidecar; + // There's only one cell per column for a single blob + let cell_bytes: Vec = column.into_iter().next().unwrap().into(); + let kzg_cell = cell_bytes.try_into().unwrap(); + let kzg_proof = kzg_proofs.into_iter().next().unwrap(); + (kzg_cell, kzg_proof) + }) + .collect::<(Vec<_>, Vec<_>)>(); + + // Repeat the cells and proofs for every blob + let blob_cells_and_proofs_vec = + vec![(cells.try_into().unwrap(), proofs.try_into().unwrap()); kzg_commitments.len()]; + + build_data_column_sidecars( + kzg_commitments.clone(), + kzg_commitments_inclusion_proof, + signed_block_header, + blob_cells_and_proofs_vec, + spec, + ) + .unwrap() +} diff --git a/beacon_node/beacon_chain/src/test_utils/fixtures/test_data_column_sidecars.ssz b/beacon_node/beacon_chain/src/test_utils/fixtures/test_data_column_sidecars.ssz new file mode 100644 index 0000000000..112dd43b04 Binary files /dev/null and b/beacon_node/beacon_chain/src/test_utils/fixtures/test_data_column_sidecars.ssz differ diff --git a/beacon_node/beacon_chain/tests/attestation_production.rs b/beacon_node/beacon_chain/tests/attestation_production.rs index 87fefe7114..621475a3ec 100644 --- a/beacon_node/beacon_chain/tests/attestation_production.rs +++ b/beacon_node/beacon_chain/tests/attestation_production.rs @@ -1,7 +1,6 @@ #![cfg(not(debug_assertions))] use beacon_chain::attestation_simulator::produce_unaggregated_attestation; -use beacon_chain::block_verification_types::RpcBlock; use beacon_chain::test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy}; use beacon_chain::validator_monitor::UNAGGREGATED_ATTESTATION_LAG_SLOTS; use beacon_chain::{metrics, StateSkipConfig, WhenSlotSkipped}; @@ -155,7 +154,6 @@ async fn produces_attestations() { .store .make_full_block(&block_root, blinded_block) .unwrap(); - let blobs = chain.get_blobs(&block_root).unwrap(); let epoch_boundary_slot = state .current_epoch() @@ -223,8 +221,7 @@ async fn produces_attestations() { assert_eq!(data.target.root, target_root, "bad target root"); let rpc_block = - RpcBlock::::new(None, Arc::new(block.clone()), Some(blobs.clone())) - .unwrap(); + harness.build_rpc_block_from_store_blobs(Some(block_root), Arc::new(block.clone())); let beacon_chain::data_availability_checker::MaybeAvailableBlock::Available( available_block, ) = chain @@ -296,13 +293,8 @@ async fn early_attester_cache_old_request() { .get_block(&head.beacon_block_root) .unwrap(); - let head_blobs = harness - .chain - .get_blobs(&head.beacon_block_root) - .expect("should get blobs"); - - let rpc_block = - RpcBlock::::new(None, head.beacon_block.clone(), Some(head_blobs)).unwrap(); + let rpc_block = harness + .build_rpc_block_from_store_blobs(Some(head.beacon_block_root), head.beacon_block.clone()); let beacon_chain::data_availability_checker::MaybeAvailableBlock::Available(available_block) = harness .chain diff --git a/beacon_node/beacon_chain/tests/block_verification.rs b/beacon_node/beacon_chain/tests/block_verification.rs index f094a173ee..2a881b5b0f 100644 --- a/beacon_node/beacon_chain/tests/block_verification.rs +++ b/beacon_node/beacon_chain/tests/block_verification.rs @@ -1,6 +1,7 @@ #![cfg(not(debug_assertions))] use beacon_chain::block_verification_types::{AsBlock, ExecutedBlock, RpcBlock}; +use beacon_chain::data_column_verification::CustodyDataColumn; use beacon_chain::{ test_utils::{ test_spec, AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType, @@ -9,7 +10,7 @@ use beacon_chain::{ }; use beacon_chain::{ BeaconSnapshot, BlockError, ChainConfig, ChainSegmentResult, IntoExecutionPendingBlock, - NotifyExecutionLayer, + InvalidSignature, NotifyExecutionLayer, }; use logging::test_logger; use slasher::{Config as SlasherConfig, Slasher}; @@ -34,7 +35,12 @@ const BLOCK_INDICES: &[usize] = &[0, 1, 32, 64, 68 + 1, 129, CHAIN_SEGMENT_LENGT static KEYPAIRS: LazyLock> = LazyLock::new(|| types::test_utils::generate_deterministic_keypairs(VALIDATOR_COUNT)); -async fn get_chain_segment() -> (Vec>, Vec>>) { +enum DataSidecars { + Blobs(BlobSidecarList), + DataColumns(Vec>), +} + +async fn get_chain_segment() -> (Vec>, Vec>>) { let harness = get_harness(VALIDATOR_COUNT); harness @@ -46,7 +52,7 @@ async fn get_chain_segment() -> (Vec>, Vec (Vec>, Vec (Vec>, Vec>>) { - let harness = get_harness(VALIDATOR_COUNT); - - harness - .extend_chain( - CHAIN_SEGMENT_LENGTH, - BlockStrategy::OnCanonicalHead, - AttestationStrategy::AllValidators, - ) - .await; - - let mut segment = Vec::with_capacity(CHAIN_SEGMENT_LENGTH); - let mut segment_blobs = Vec::with_capacity(CHAIN_SEGMENT_LENGTH); - for snapshot in harness - .chain - .chain_dump() - .expect("should dump chain") - .into_iter() - .skip(1) - { - let full_block = harness - .chain - .get_block(&snapshot.beacon_block_root) - .await - .unwrap() - .unwrap(); - segment.push(BeaconSnapshot { - beacon_block_root: snapshot.beacon_block_root, - beacon_block: Arc::new(full_block), - beacon_state: snapshot.beacon_state, - }); - let blob_sidecars = harness - .chain - .get_blobs(&snapshot.beacon_block_root) - .unwrap(); - segment_blobs.push(Some(blob_sidecars)) - } - (segment, segment_blobs) + (segment, segment_sidecars) } fn get_harness(validator_count: usize) -> BeaconChainHarness> { @@ -135,17 +119,35 @@ fn get_harness(validator_count: usize) -> BeaconChainHarness], - blobs: &[Option>], + chain_segment_sidecars: &[Option>], + spec: &ChainSpec, ) -> Vec> { chain_segment .iter() - .zip(blobs.iter()) - .map(|(snapshot, blobs)| { - RpcBlock::new(None, snapshot.beacon_block.clone(), blobs.clone()).unwrap() + .zip(chain_segment_sidecars.iter()) + .map(|(snapshot, data_sidecars)| { + let block = snapshot.beacon_block.clone(); + build_rpc_block(block, data_sidecars, spec) }) .collect() } +fn build_rpc_block( + block: Arc>, + data_sidecars: &Option>, + spec: &ChainSpec, +) -> RpcBlock { + match data_sidecars { + Some(DataSidecars::Blobs(blobs)) => { + RpcBlock::new(None, block, Some(blobs.clone())).unwrap() + } + Some(DataSidecars::DataColumns(columns)) => { + RpcBlock::new_with_custody_columns(None, block, columns.clone(), spec).unwrap() + } + None => RpcBlock::new_without_blobs(None, block), + } +} + fn junk_signature() -> Signature { let kp = generate_deterministic_keypair(VALIDATOR_COUNT); let message = Hash256::from_slice(&[42; 32]); @@ -184,18 +186,22 @@ fn update_proposal_signatures( } } -fn update_parent_roots( - snapshots: &mut [BeaconSnapshot], - blobs: &mut [Option>], -) { +fn update_parent_roots(snapshots: &mut [BeaconSnapshot], blobs: &mut [Option>]) { for i in 0..snapshots.len() { let root = snapshots[i].beacon_block.canonical_root(); if let (Some(child), Some(child_blobs)) = (snapshots.get_mut(i + 1), blobs.get_mut(i + 1)) { let (mut block, signature) = child.beacon_block.as_ref().clone().deconstruct(); *block.parent_root_mut() = root; let new_child = Arc::new(SignedBeaconBlock::from_block(block, signature)); - if let Some(blobs) = child_blobs { - update_blob_signed_header(&new_child, blobs); + if let Some(data_sidecars) = child_blobs { + match data_sidecars { + DataSidecars::Blobs(blobs) => { + update_blob_signed_header(&new_child, blobs); + } + DataSidecars::DataColumns(columns) => { + update_data_column_signed_header(&new_child, columns); + } + } } child.beacon_block = new_child; } @@ -206,7 +212,7 @@ fn update_blob_signed_header( signed_block: &SignedBeaconBlock, blobs: &mut BlobSidecarList, ) { - for old_blob_sidecar in blobs.iter_mut() { + for old_blob_sidecar in blobs.as_mut_slice() { let new_blob = Arc::new(BlobSidecar:: { index: old_blob_sidecar.index, blob: old_blob_sidecar.blob.clone(), @@ -223,13 +229,36 @@ fn update_blob_signed_header( } } +fn update_data_column_signed_header( + signed_block: &SignedBeaconBlock, + data_columns: &mut Vec>, +) { + for old_custody_column_sidecar in data_columns.as_mut_slice() { + let old_column_sidecar = old_custody_column_sidecar.as_data_column(); + let new_column_sidecar = Arc::new(DataColumnSidecar:: { + index: old_column_sidecar.index, + column: old_column_sidecar.column.clone(), + kzg_commitments: old_column_sidecar.kzg_commitments.clone(), + kzg_proofs: old_column_sidecar.kzg_proofs.clone(), + signed_block_header: signed_block.signed_block_header(), + kzg_commitments_inclusion_proof: signed_block + .message() + .body() + .kzg_commitments_merkle_proof() + .unwrap(), + }); + *old_custody_column_sidecar = CustodyDataColumn::from_asserted_custody(new_column_sidecar); + } +} + #[tokio::test] async fn chain_segment_full_segment() { let harness = get_harness(VALIDATOR_COUNT); let (chain_segment, chain_segment_blobs) = get_chain_segment().await; - let blocks: Vec> = chain_segment_blocks(&chain_segment, &chain_segment_blobs) - .into_iter() - .collect(); + let blocks: Vec> = + chain_segment_blocks(&chain_segment, &chain_segment_blobs, &harness.spec) + .into_iter() + .collect(); harness .chain @@ -265,9 +294,10 @@ async fn chain_segment_varying_chunk_size() { for chunk_size in &[1, 2, 3, 5, 31, 32, 33, 42] { let harness = get_harness(VALIDATOR_COUNT); let (chain_segment, chain_segment_blobs) = get_chain_segment().await; - let blocks: Vec> = chain_segment_blocks(&chain_segment, &chain_segment_blobs) - .into_iter() - .collect(); + let blocks: Vec> = + chain_segment_blocks(&chain_segment, &chain_segment_blobs, &harness.spec) + .into_iter() + .collect(); harness .chain @@ -306,9 +336,10 @@ async fn chain_segment_non_linear_parent_roots() { /* * Test with a block removed. */ - let mut blocks: Vec> = chain_segment_blocks(&chain_segment, &chain_segment_blobs) - .into_iter() - .collect(); + let mut blocks: Vec> = + chain_segment_blocks(&chain_segment, &chain_segment_blobs, &harness.spec) + .into_iter() + .collect(); blocks.remove(2); assert!( @@ -326,9 +357,10 @@ async fn chain_segment_non_linear_parent_roots() { /* * Test with a modified parent root. */ - let mut blocks: Vec> = chain_segment_blocks(&chain_segment, &chain_segment_blobs) - .into_iter() - .collect(); + let mut blocks: Vec> = + chain_segment_blocks(&chain_segment, &chain_segment_blobs, &harness.spec) + .into_iter() + .collect(); let (mut block, signature) = blocks[3].as_block().clone().deconstruct(); *block.parent_root_mut() = Hash256::zero(); @@ -363,9 +395,10 @@ async fn chain_segment_non_linear_slots() { * Test where a child is lower than the parent. */ - let mut blocks: Vec> = chain_segment_blocks(&chain_segment, &chain_segment_blobs) - .into_iter() - .collect(); + let mut blocks: Vec> = + chain_segment_blocks(&chain_segment, &chain_segment_blobs, &harness.spec) + .into_iter() + .collect(); let (mut block, signature) = blocks[3].as_block().clone().deconstruct(); *block.slot_mut() = Slot::new(0); blocks[3] = RpcBlock::new_without_blobs( @@ -389,9 +422,10 @@ async fn chain_segment_non_linear_slots() { * Test where a child is equal to the parent. */ - let mut blocks: Vec> = chain_segment_blocks(&chain_segment, &chain_segment_blobs) - .into_iter() - .collect(); + let mut blocks: Vec> = + chain_segment_blocks(&chain_segment, &chain_segment_blobs, &harness.spec) + .into_iter() + .collect(); let (mut block, signature) = blocks[3].as_block().clone().deconstruct(); *block.slot_mut() = blocks[2].slot(); blocks[3] = RpcBlock::new_without_blobs( @@ -414,7 +448,7 @@ async fn chain_segment_non_linear_slots() { async fn assert_invalid_signature( chain_segment: &[BeaconSnapshot], - chain_segment_blobs: &[Option>], + chain_segment_blobs: &[Option>], harness: &BeaconChainHarness>, block_index: usize, snapshots: &[BeaconSnapshot], @@ -424,7 +458,7 @@ async fn assert_invalid_signature( .iter() .zip(chain_segment_blobs.iter()) .map(|(snapshot, blobs)| { - RpcBlock::new(None, snapshot.beacon_block.clone(), blobs.clone()).unwrap() + build_rpc_block(snapshot.beacon_block.clone(), blobs, &harness.spec) }) .collect(); @@ -436,7 +470,7 @@ async fn assert_invalid_signature( .process_chain_segment(blocks, NotifyExecutionLayer::Yes) .await .into_block_error(), - Err(BlockError::InvalidSignature) + Err(BlockError::InvalidSignature(InvalidSignature::Unknown)) ), "should not import chain segment with an invalid {} signature", item @@ -451,7 +485,7 @@ async fn assert_invalid_signature( .take(block_index) .zip(chain_segment_blobs.iter()) .map(|(snapshot, blobs)| { - RpcBlock::new(None, snapshot.beacon_block.clone(), blobs.clone()).unwrap() + build_rpc_block(snapshot.beacon_block.clone(), blobs, &harness.spec) }) .collect(); // We don't care if this fails, we just call this to ensure that all prior blocks have been @@ -466,19 +500,23 @@ async fn assert_invalid_signature( .chain .process_block( snapshots[block_index].beacon_block.canonical_root(), - RpcBlock::new( - None, + build_rpc_block( snapshots[block_index].beacon_block.clone(), - chain_segment_blobs[block_index].clone(), - ) - .unwrap(), + &chain_segment_blobs[block_index], + &harness.spec, + ), NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), ) .await; assert!( - matches!(process_res, Err(BlockError::InvalidSignature)), + matches!( + process_res, + Err(BlockError::InvalidSignature( + InvalidSignature::BlockBodySignatures + )) + ), "should not import individual block with an invalid {} signature, got: {:?}", item, process_res @@ -524,7 +562,7 @@ async fn invalid_signature_gossip_block() { .take(block_index) .zip(chain_segment_blobs.iter()) .map(|(snapshot, blobs)| { - RpcBlock::new(None, snapshot.beacon_block.clone(), blobs.clone()).unwrap() + build_rpc_block(snapshot.beacon_block.clone(), blobs, &harness.spec) }) .collect(); harness @@ -534,21 +572,25 @@ async fn invalid_signature_gossip_block() { .into_block_error() .expect("should import all blocks prior to the one being tested"); let signed_block = SignedBeaconBlock::from_block(block, junk_signature()); + let process_res = harness + .chain + .process_block( + signed_block.canonical_root(), + Arc::new(signed_block), + NotifyExecutionLayer::Yes, + BlockImportSource::Lookup, + || Ok(()), + ) + .await; assert!( matches!( - harness - .chain - .process_block( - signed_block.canonical_root(), - Arc::new(signed_block), - NotifyExecutionLayer::Yes, - BlockImportSource::Lookup, - || Ok(()), - ) - .await, - Err(BlockError::InvalidSignature) + process_res, + Err(BlockError::InvalidSignature( + InvalidSignature::ProposerSignature + )) ), - "should not import individual block with an invalid gossip signature", + "should not import individual block with an invalid gossip signature, got: {:?}", + process_res ); } } @@ -572,20 +614,22 @@ async fn invalid_signature_block_proposal() { .iter() .zip(chain_segment_blobs.iter()) .map(|(snapshot, blobs)| { - RpcBlock::new(None, snapshot.beacon_block.clone(), blobs.clone()).unwrap() + build_rpc_block(snapshot.beacon_block.clone(), blobs, &harness.spec) }) .collect::>(); // Ensure the block will be rejected if imported in a chain segment. + let process_res = harness + .chain + .process_chain_segment(blocks, NotifyExecutionLayer::Yes) + .await + .into_block_error(); assert!( matches!( - harness - .chain - .process_chain_segment(blocks, NotifyExecutionLayer::Yes) - .await - .into_block_error(), - Err(BlockError::InvalidSignature) + process_res, + Err(BlockError::InvalidSignature(InvalidSignature::Unknown)) ), - "should not import chain segment with an invalid block signature", + "should not import chain segment with an invalid block signature, got: {:?}", + process_res ); } } @@ -754,6 +798,11 @@ async fn invalid_signature_attester_slashing() { .push(attester_slashing.as_electra().unwrap().clone()) .expect("should update attester slashing"); } + BeaconBlockBodyRefMut::Fulu(ref mut blk) => { + blk.attester_slashings + .push(attester_slashing.as_electra().unwrap().clone()) + .expect("should update attester slashing"); + } } snapshots[block_index].beacon_block = Arc::new(SignedBeaconBlock::from_block(block, signature)); @@ -809,6 +858,10 @@ async fn invalid_signature_attestation() { .attestations .get_mut(0) .map(|att| att.signature = junk_aggregate_signature()), + BeaconBlockBodyRefMut::Fulu(ref mut blk) => blk + .attestations + .get_mut(0) + .map(|att| att.signature = junk_aggregate_signature()), }; if block.body().attestations_len() > 0 { @@ -869,7 +922,7 @@ async fn invalid_signature_deposit() { .iter() .zip(chain_segment_blobs.iter()) .map(|(snapshot, blobs)| { - RpcBlock::new(None, snapshot.beacon_block.clone(), blobs.clone()).unwrap() + build_rpc_block(snapshot.beacon_block.clone(), blobs, &harness.spec) }) .collect(); assert!( @@ -879,7 +932,7 @@ async fn invalid_signature_deposit() { .process_chain_segment(blocks, NotifyExecutionLayer::Yes) .await .into_block_error(), - Err(BlockError::InvalidSignature) + Err(BlockError::InvalidSignature(InvalidSignature::Unknown)) ), "should not throw an invalid signature error for a bad deposit signature" ); @@ -935,7 +988,7 @@ fn unwrap_err(result: Result) -> U { #[tokio::test] async fn block_gossip_verification() { let harness = get_harness(VALIDATOR_COUNT); - let (chain_segment, chain_segment_blobs) = get_chain_segment_with_blob_sidecars().await; + let (chain_segment, chain_segment_blobs) = get_chain_segment().await; let block_index = CHAIN_SEGMENT_LENGTH - 2; @@ -947,7 +1000,7 @@ async fn block_gossip_verification() { // Import the ancestors prior to the block we're testing. for (snapshot, blobs_opt) in chain_segment[0..block_index] .iter() - .zip(chain_segment_blobs.iter()) + .zip(chain_segment_blobs.into_iter()) { let gossip_verified = harness .chain @@ -966,20 +1019,8 @@ async fn block_gossip_verification() { ) .await .expect("should import valid gossip verified block"); - if let Some(blob_sidecars) = blobs_opt { - for blob_sidecar in blob_sidecars { - let blob_index = blob_sidecar.index; - let gossip_verified = harness - .chain - .verify_blob_sidecar_for_gossip(blob_sidecar.clone(), blob_index) - .expect("should obtain gossip verified blob"); - - harness - .chain - .process_gossip_blob(gossip_verified) - .await - .expect("should import valid gossip verified blob"); - } + if let Some(data_sidecars) = blobs_opt { + verify_and_process_gossip_data_sidecars(&harness, data_sidecars).await; } } @@ -1075,7 +1116,7 @@ async fn block_gossip_verification() { ))) .await ), - BlockError::ProposalSignatureInvalid + BlockError::InvalidSignature(InvalidSignature::ProposerSignature) ), "should not import a block with an invalid proposal signature" ); @@ -1207,6 +1248,51 @@ async fn block_gossip_verification() { ); } +async fn verify_and_process_gossip_data_sidecars( + harness: &BeaconChainHarness>, + data_sidecars: DataSidecars, +) { + match data_sidecars { + DataSidecars::Blobs(blob_sidecars) => { + for blob_sidecar in blob_sidecars { + let blob_index = blob_sidecar.index; + let gossip_verified = harness + .chain + .verify_blob_sidecar_for_gossip(blob_sidecar.clone(), blob_index) + .expect("should obtain gossip verified blob"); + + harness + .chain + .process_gossip_blob(gossip_verified) + .await + .expect("should import valid gossip verified blob"); + } + } + DataSidecars::DataColumns(column_sidecars) => { + let gossip_verified = column_sidecars + .into_iter() + .map(|column_sidecar| { + let subnet_id = DataColumnSubnetId::from_column_index( + column_sidecar.index(), + &harness.spec, + ); + harness.chain.verify_data_column_sidecar_for_gossip( + column_sidecar.into_inner(), + *subnet_id, + ) + }) + .collect::, _>>() + .expect("should obtain gossip verified columns"); + + harness + .chain + .process_gossip_data_columns(gossip_verified, || Ok(())) + .await + .expect("should import valid gossip verified columns"); + } + } +} + #[tokio::test] async fn verify_block_for_gossip_slashing_detection() { let slasher_dir = tempdir().unwrap(); @@ -1214,7 +1300,7 @@ async fn verify_block_for_gossip_slashing_detection() { let slasher = Arc::new( Slasher::open( SlasherConfig::new(slasher_dir.path().into()), - spec, + spec.clone(), test_logger(), ) .unwrap(), @@ -1237,20 +1323,14 @@ async fn verify_block_for_gossip_slashing_detection() { let verified_block = harness.chain.verify_block_for_gossip(block1).await.unwrap(); if let Some((kzg_proofs, blobs)) = blobs1 { - let sidecars = - BlobSidecar::build_sidecars(blobs, verified_block.block(), kzg_proofs).unwrap(); - for sidecar in sidecars { - let blob_index = sidecar.index; - let verified_blob = harness - .chain - .verify_blob_sidecar_for_gossip(sidecar, blob_index) - .unwrap(); - harness - .chain - .process_gossip_blob(verified_blob) - .await - .unwrap(); - } + harness + .process_gossip_blobs_or_columns( + verified_block.block(), + blobs.iter(), + kzg_proofs.iter(), + None, + ) + .await; } harness .chain @@ -1726,7 +1806,7 @@ async fn import_execution_pending_block( .unwrap() { ExecutedBlock::Available(block) => chain - .import_available_block(Box::from(block), None) + .import_available_block(Box::from(block)) .await .map_err(|e| format!("{e:?}")), ExecutedBlock::AvailabilityPending(_) => { diff --git a/beacon_node/beacon_chain/tests/events.rs b/beacon_node/beacon_chain/tests/events.rs index ab784d3be4..c9bd55e062 100644 --- a/beacon_node/beacon_chain/tests/events.rs +++ b/beacon_node/beacon_chain/tests/events.rs @@ -73,7 +73,7 @@ async fn blob_sidecar_event_on_process_rpc_blobs() { let blob_1 = Arc::new(blob_1); let blob_2 = Arc::new(blob_2); - let blobs = FixedBlobSidecarList::from(vec![Some(blob_1.clone()), Some(blob_2.clone())]); + let blobs = FixedBlobSidecarList::new(vec![Some(blob_1.clone()), Some(blob_2.clone())]); let expected_sse_blobs = vec![ SseBlobSidecar::from_blob_sidecar(blob_1.as_ref()), SseBlobSidecar::from_blob_sidecar(blob_2.as_ref()), diff --git a/beacon_node/beacon_chain/tests/op_verification.rs b/beacon_node/beacon_chain/tests/op_verification.rs index df0d561e1c..44fb298d6c 100644 --- a/beacon_node/beacon_chain/tests/op_verification.rs +++ b/beacon_node/beacon_chain/tests/op_verification.rs @@ -14,7 +14,8 @@ use state_processing::per_block_processing::errors::{ AttesterSlashingInvalid, BlockOperationError, ExitInvalid, ProposerSlashingInvalid, }; use std::sync::{Arc, LazyLock}; -use store::{LevelDB, StoreConfig}; +use store::database::interface::BeaconNodeBackend; +use store::StoreConfig; use tempfile::{tempdir, TempDir}; use types::*; @@ -26,7 +27,7 @@ static KEYPAIRS: LazyLock> = type E = MinimalEthSpec; type TestHarness = BeaconChainHarness>; -type HotColdDB = store::HotColdDB, LevelDB>; +type HotColdDB = store::HotColdDB, BeaconNodeBackend>; fn get_store(db_path: &TempDir) -> Arc { let spec = Arc::new(test_spec::()); diff --git a/beacon_node/beacon_chain/tests/store_tests.rs b/beacon_node/beacon_chain/tests/store_tests.rs index e1258ccdea..8654b33646 100644 --- a/beacon_node/beacon_chain/tests/store_tests.rs +++ b/beacon_node/beacon_chain/tests/store_tests.rs @@ -1,7 +1,6 @@ #![cfg(not(debug_assertions))] use beacon_chain::attestation_verification::Error as AttnError; -use beacon_chain::block_verification_types::RpcBlock; use beacon_chain::builder::BeaconChainBuilder; use beacon_chain::data_availability_checker::AvailableBlock; use beacon_chain::schema_change::migrate_schema; @@ -25,10 +24,11 @@ use std::collections::HashSet; use std::convert::TryInto; use std::sync::{Arc, LazyLock}; use std::time::Duration; +use store::database::interface::BeaconNodeBackend; use store::metadata::{SchemaVersion, CURRENT_SCHEMA_VERSION, STATE_UPPER_LIMIT_NO_RETAIN}; use store::{ iter::{BlockRootsIterator, StateRootsIterator}, - BlobInfo, DBColumn, HotColdDB, LevelDB, StoreConfig, + BlobInfo, DBColumn, HotColdDB, StoreConfig, }; use tempfile::{tempdir, TempDir}; use tokio::time::sleep; @@ -46,7 +46,7 @@ static KEYPAIRS: LazyLock> = type E = MinimalEthSpec; type TestHarness = BeaconChainHarness>; -fn get_store(db_path: &TempDir) -> Arc, LevelDB>> { +fn get_store(db_path: &TempDir) -> Arc, BeaconNodeBackend>> { get_store_generic(db_path, StoreConfig::default(), test_spec::()) } @@ -54,7 +54,7 @@ fn get_store_generic( db_path: &TempDir, config: StoreConfig, spec: ChainSpec, -) -> Arc, LevelDB>> { +) -> Arc, BeaconNodeBackend>> { let hot_path = db_path.path().join("chain_db"); let cold_path = db_path.path().join("freezer_db"); let blobs_path = db_path.path().join("blobs_db"); @@ -73,7 +73,7 @@ fn get_store_generic( } fn get_harness( - store: Arc, LevelDB>>, + store: Arc, BeaconNodeBackend>>, validator_count: usize, ) -> TestHarness { // Most tests expect to retain historic states, so we use this as the default. @@ -81,13 +81,26 @@ fn get_harness( reconstruct_historic_states: true, ..ChainConfig::default() }; - get_harness_generic(store, validator_count, chain_config) + get_harness_generic(store, validator_count, chain_config, false) +} + +fn get_harness_import_all_data_columns( + store: Arc, BeaconNodeBackend>>, + validator_count: usize, +) -> TestHarness { + // Most tests expect to retain historic states, so we use this as the default. + let chain_config = ChainConfig { + reconstruct_historic_states: true, + ..ChainConfig::default() + }; + get_harness_generic(store, validator_count, chain_config, true) } fn get_harness_generic( - store: Arc, LevelDB>>, + store: Arc, BeaconNodeBackend>>, validator_count: usize, chain_config: ChainConfig, + import_all_data_columns: bool, ) -> TestHarness { let harness = TestHarness::builder(MinimalEthSpec) .spec(store.get_chain_spec().clone()) @@ -96,6 +109,7 @@ fn get_harness_generic( .fresh_disk_store(store) .mock_execution_layer() .chain_config(chain_config) + .import_all_data_columns(import_all_data_columns) .build(); harness.advance_slot(); harness @@ -147,6 +161,7 @@ async fn light_client_bootstrap_test() { LightClientBootstrap::Capella(lc_bootstrap) => lc_bootstrap.header.beacon.slot, LightClientBootstrap::Deneb(lc_bootstrap) => lc_bootstrap.header.beacon.slot, LightClientBootstrap::Electra(lc_bootstrap) => lc_bootstrap.header.beacon.slot, + LightClientBootstrap::Fulu(lc_bootstrap) => lc_bootstrap.header.beacon.slot, }; assert_eq!( @@ -243,7 +258,6 @@ async fn full_participation_no_skips() { AttestationStrategy::AllValidators, ) .await; - check_finalization(&harness, num_blocks_produced); check_split_slot(&harness, store); check_chain_dump(&harness, num_blocks_produced + 1); @@ -2285,7 +2299,12 @@ async fn weak_subjectivity_sync_test(slots: Vec, checkpoint_slot: Slot) { let temp1 = tempdir().unwrap(); let full_store = get_store(&temp1); - let harness = get_harness(full_store.clone(), LOW_VALIDATOR_COUNT); + + // TODO(das): Run a supernode so the node has full blobs stored. + // This may not be required in the future if we end up implementing downloading checkpoint + // blobs from p2p peers: + // https://github.com/sigp/lighthouse/issues/6837 + let harness = get_harness_import_all_data_columns(full_store.clone(), LOW_VALIDATOR_COUNT); let all_validators = (0..LOW_VALIDATOR_COUNT).collect::>(); @@ -2316,7 +2335,10 @@ async fn weak_subjectivity_sync_test(slots: Vec, checkpoint_slot: Slot) { .get_full_block(&wss_block_root) .unwrap() .unwrap(); - let wss_blobs_opt = harness.chain.store.get_blobs(&wss_block_root).unwrap(); + let wss_blobs_opt = harness + .chain + .get_or_reconstruct_blobs(&wss_block_root) + .unwrap(); let wss_state = full_store .get_state(&wss_state_root, Some(checkpoint_slot)) .unwrap() @@ -2341,8 +2363,10 @@ async fn weak_subjectivity_sync_test(slots: Vec, checkpoint_slot: Slot) { let kzg = get_kzg(&spec); - let mock = - mock_execution_layer_from_parts(&harness.spec, harness.runtime.task_executor.clone()); + let mock = mock_execution_layer_from_parts( + harness.spec.clone(), + harness.runtime.task_executor.clone(), + ); // Initialise a new beacon chain from the finalized checkpoint. // The slot clock must be set to a time ahead of the checkpoint state. @@ -2387,10 +2411,16 @@ async fn weak_subjectivity_sync_test(slots: Vec, checkpoint_slot: Slot) { .await .unwrap() .unwrap(); - let store_wss_blobs_opt = beacon_chain.store.get_blobs(&wss_block_root).unwrap(); + // This test may break in the future if we no longer store the full checkpoint data columns. + let store_wss_blobs_opt = beacon_chain + .get_or_reconstruct_blobs(&wss_block_root) + .unwrap(); assert_eq!(store_wss_block, wss_block); - assert_eq!(store_wss_blobs_opt, wss_blobs_opt); + // TODO(fulu): Remove this condition once #6760 (PeerDAS checkpoint sync) is merged. + if !beacon_chain.spec.is_peer_das_scheduled() { + assert_eq!(store_wss_blobs_opt, wss_blobs_opt); + } // Apply blocks forward to reach head. let chain_dump = harness.chain.chain_dump().unwrap(); @@ -2406,7 +2436,7 @@ async fn weak_subjectivity_sync_test(slots: Vec, checkpoint_slot: Slot) { .await .unwrap() .unwrap(); - let blobs = harness.chain.get_blobs(&block_root).expect("blobs"); + let slot = full_block.slot(); let state_root = full_block.state_root(); @@ -2414,7 +2444,7 @@ async fn weak_subjectivity_sync_test(slots: Vec, checkpoint_slot: Slot) { beacon_chain .process_block( full_block.canonical_root(), - RpcBlock::new(Some(block_root), Arc::new(full_block), Some(blobs)).unwrap(), + harness.build_rpc_block_from_store_blobs(Some(block_root), Arc::new(full_block)), NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), @@ -2468,13 +2498,12 @@ async fn weak_subjectivity_sync_test(slots: Vec, checkpoint_slot: Slot) { .await .expect("should get block") .expect("should get block"); - let blobs = harness.chain.get_blobs(&block_root).expect("blobs"); if let MaybeAvailableBlock::Available(block) = harness .chain .data_availability_checker .verify_kzg_for_rpc_block( - RpcBlock::new(Some(block_root), Arc::new(full_block), Some(blobs)).unwrap(), + harness.build_rpc_block_from_store_blobs(Some(block_root), Arc::new(full_block)), ) .expect("should verify kzg") { @@ -2575,7 +2604,7 @@ async fn process_blocks_and_attestations_for_unaligned_checkpoint() { reconstruct_historic_states: false, ..ChainConfig::default() }; - let harness = get_harness_generic(store.clone(), LOW_VALIDATOR_COUNT, chain_config); + let harness = get_harness_generic(store.clone(), LOW_VALIDATOR_COUNT, chain_config, false); let all_validators = (0..LOW_VALIDATOR_COUNT).collect::>(); @@ -3063,6 +3092,10 @@ async fn deneb_prune_blobs_happy_case() { let db_path = tempdir().unwrap(); let store = get_store(&db_path); + if store.get_chain_spec().is_peer_das_scheduled() { + // TODO(fulu): add prune tests for Fulu / PeerDAS data columns. + return; + } let Some(deneb_fork_epoch) = store.get_chain_spec().deneb_fork_epoch else { // No-op prior to Deneb. return; @@ -3110,6 +3143,10 @@ async fn deneb_prune_blobs_no_finalization() { let db_path = tempdir().unwrap(); let store = get_store(&db_path); + if store.get_chain_spec().is_peer_das_scheduled() { + // TODO(fulu): add prune tests for Fulu / PeerDAS data columns. + return; + } let Some(deneb_fork_epoch) = store.get_chain_spec().deneb_fork_epoch else { // No-op prior to Deneb. return; @@ -3254,6 +3291,10 @@ async fn deneb_prune_blobs_margin_test(margin: u64) { let db_path = tempdir().unwrap(); let store = get_store_generic(&db_path, config, test_spec::()); + if store.get_chain_spec().is_peer_das_scheduled() { + // TODO(fulu): add prune tests for Fulu / PeerDAS data columns. + return; + } let Some(deneb_fork_epoch) = store.get_chain_spec().deneb_fork_epoch else { // No-op prior to Deneb. return; @@ -3350,7 +3391,7 @@ fn check_blob_existence( .unwrap() .map(Result::unwrap) { - if let Some(blobs) = harness.chain.store.get_blobs(&block_root).unwrap() { + if let Some(blobs) = harness.chain.store.get_blobs(&block_root).unwrap().blobs() { assert!(should_exist, "blobs at slot {slot} exist but should not"); blobs_seen += blobs.len(); } else { @@ -3496,7 +3537,10 @@ fn check_finalization(harness: &TestHarness, expected_slot: u64) { } /// Check that the HotColdDB's split_slot is equal to the start slot of the last finalized epoch. -fn check_split_slot(harness: &TestHarness, store: Arc, LevelDB>>) { +fn check_split_slot( + harness: &TestHarness, + store: Arc, BeaconNodeBackend>>, +) { let split_slot = store.get_split_slot(); assert_eq!( harness diff --git a/beacon_node/beacon_chain/tests/validator_monitor.rs b/beacon_node/beacon_chain/tests/validator_monitor.rs index b4a54d2667..180db6d76d 100644 --- a/beacon_node/beacon_chain/tests/validator_monitor.rs +++ b/beacon_node/beacon_chain/tests/validator_monitor.rs @@ -4,7 +4,7 @@ use beacon_chain::test_utils::{ use beacon_chain::validator_monitor::{ValidatorMonitorConfig, MISSED_BLOCK_LAG_SLOTS}; use logging::test_logger; use std::sync::LazyLock; -use types::{Epoch, EthSpec, ForkName, Keypair, MainnetEthSpec, PublicKeyBytes, Slot}; +use types::{Epoch, EthSpec, Keypair, MainnetEthSpec, PublicKeyBytes, Slot}; // Should ideally be divisible by 3. pub const VALIDATOR_COUNT: usize = 48; @@ -117,7 +117,7 @@ async fn missed_blocks_across_epochs() { } #[tokio::test] -async fn produces_missed_blocks() { +async fn missed_blocks_basic() { let validator_count = 16; let slots_per_epoch = E::slots_per_epoch(); @@ -127,13 +127,10 @@ async fn produces_missed_blocks() { // Generate 63 slots (2 epochs * 32 slots per epoch - 1) let initial_blocks = slots_per_epoch * nb_epoch_to_simulate.as_u64() - 1; - // The validator index of the validator that is 'supposed' to miss a block - let validator_index_to_monitor = 1; - // 1st scenario // // // Missed block happens when slot and prev_slot are in the same epoch - let harness1 = get_harness(validator_count, vec![validator_index_to_monitor]); + let harness1 = get_harness(validator_count, vec![]); harness1 .extend_chain( initial_blocks as usize, @@ -153,7 +150,7 @@ async fn produces_missed_blocks() { let mut prev_slot = Slot::new(idx - 1); let mut duplicate_block_root = *_state.block_roots().get(idx as usize).unwrap(); let mut validator_indexes = _state.get_beacon_proposer_indices(&harness1.spec).unwrap(); - let mut validator_index = validator_indexes[slot_in_epoch.as_usize()]; + let mut missed_block_proposer = validator_indexes[slot_in_epoch.as_usize()]; let mut proposer_shuffling_decision_root = _state .proposer_shuffling_decision_root(duplicate_block_root) .unwrap(); @@ -170,7 +167,7 @@ async fn produces_missed_blocks() { beacon_proposer_cache.lock().insert( epoch, proposer_shuffling_decision_root, - validator_indexes.into_iter().collect::>(), + validator_indexes, _state.fork() ), Ok(()) @@ -187,12 +184,15 @@ async fn produces_missed_blocks() { // Let's validate the state which will call the function responsible for // adding the missed blocks to the validator monitor let mut validator_monitor = harness1.chain.validator_monitor.write(); + + validator_monitor.add_validator_pubkey(KEYPAIRS[missed_block_proposer].pk.compress()); validator_monitor.process_valid_state(nb_epoch_to_simulate, _state, &harness1.chain.spec); // We should have one entry in the missed blocks map assert_eq!( - validator_monitor.get_monitored_validator_missed_block_count(validator_index as u64), - 1 + validator_monitor + .get_monitored_validator_missed_block_count(missed_block_proposer as u64), + 1, ); } @@ -201,22 +201,7 @@ async fn produces_missed_blocks() { // Missed block happens when slot and prev_slot are not in the same epoch // making sure that the cache reloads when the epoch changes // in that scenario the slot that missed a block is the first slot of the epoch - // We are adding other validators to monitor as these ones will miss a block depending on - // the fork name specified when running the test as the proposer cache differs depending on - // the fork name (cf. seed) - // - // If you are adding a new fork and seeing errors, print - // `validator_indexes[slot_in_epoch.as_usize()]` and add it below. - let validator_index_to_monitor = match harness1.spec.fork_name_at_slot::(Slot::new(0)) { - ForkName::Base => 7, - ForkName::Altair => 2, - ForkName::Bellatrix => 4, - ForkName::Capella => 11, - ForkName::Deneb => 3, - ForkName::Electra => 1, - }; - - let harness2 = get_harness(validator_count, vec![validator_index_to_monitor]); + let harness2 = get_harness(validator_count, vec![]); let advance_slot_by = 9; harness2 .extend_chain( @@ -237,11 +222,7 @@ async fn produces_missed_blocks() { slot_in_epoch = slot % slots_per_epoch; duplicate_block_root = *_state2.block_roots().get(idx as usize).unwrap(); validator_indexes = _state2.get_beacon_proposer_indices(&harness2.spec).unwrap(); - validator_index = validator_indexes[slot_in_epoch.as_usize()]; - // If you are adding a new fork and seeing errors, it means the fork seed has changed the - // validator_index. Uncomment this line, run the test again and add the resulting index to the - // list above. - //eprintln!("new index which needs to be added => {:?}", validator_index); + missed_block_proposer = validator_indexes[slot_in_epoch.as_usize()]; let beacon_proposer_cache = harness2 .chain @@ -255,7 +236,7 @@ async fn produces_missed_blocks() { beacon_proposer_cache.lock().insert( epoch, duplicate_block_root, - validator_indexes.into_iter().collect::>(), + validator_indexes.clone(), _state2.fork() ), Ok(()) @@ -270,10 +251,12 @@ async fn produces_missed_blocks() { // Let's validate the state which will call the function responsible for // adding the missed blocks to the validator monitor let mut validator_monitor2 = harness2.chain.validator_monitor.write(); + validator_monitor2.add_validator_pubkey(KEYPAIRS[missed_block_proposer].pk.compress()); validator_monitor2.process_valid_state(epoch, _state2, &harness2.chain.spec); // We should have one entry in the missed blocks map assert_eq!( - validator_monitor2.get_monitored_validator_missed_block_count(validator_index as u64), + validator_monitor2 + .get_monitored_validator_missed_block_count(missed_block_proposer as u64), 1 ); @@ -281,19 +264,20 @@ async fn produces_missed_blocks() { // // A missed block happens but the validator is not monitored // it should not be flagged as a missed block - idx = initial_blocks + (advance_slot_by) - 7; + while validator_indexes[(idx % slots_per_epoch) as usize] == missed_block_proposer + && idx / slots_per_epoch == epoch.as_u64() + { + idx += 1; + } slot = Slot::new(idx); prev_slot = Slot::new(idx - 1); slot_in_epoch = slot % slots_per_epoch; duplicate_block_root = *_state2.block_roots().get(idx as usize).unwrap(); - validator_indexes = _state2.get_beacon_proposer_indices(&harness2.spec).unwrap(); - let not_monitored_validator_index = validator_indexes[slot_in_epoch.as_usize()]; - // This could do with a refactor: https://github.com/sigp/lighthouse/issues/6293 - assert_ne!( - not_monitored_validator_index, - validator_index_to_monitor, - "this test has a fragile dependency on hardcoded indices. you need to tweak some settings or rewrite this" - ); + let second_missed_block_proposer = validator_indexes[slot_in_epoch.as_usize()]; + + // This test may fail if we can't find another distinct proposer in the same epoch. + // However, this should be vanishingly unlikely: P ~= (1/16)^32 = 2e-39. + assert_ne!(missed_block_proposer, second_missed_block_proposer); assert_eq!( _state2.set_block_root(prev_slot, duplicate_block_root), @@ -305,10 +289,9 @@ async fn produces_missed_blocks() { validator_monitor2.process_valid_state(epoch, _state2, &harness2.chain.spec); // We shouldn't have any entry in the missed blocks map - assert_ne!(validator_index, not_monitored_validator_index); assert_eq!( validator_monitor2 - .get_monitored_validator_missed_block_count(not_monitored_validator_index as u64), + .get_monitored_validator_missed_block_count(second_missed_block_proposer as u64), 0 ); } @@ -317,7 +300,7 @@ async fn produces_missed_blocks() { // // A missed block happens at state.slot - LOG_SLOTS_PER_EPOCH // it shouldn't be flagged as a missed block - let harness3 = get_harness(validator_count, vec![validator_index_to_monitor]); + let harness3 = get_harness(validator_count, vec![]); harness3 .extend_chain( slots_per_epoch as usize, @@ -337,7 +320,7 @@ async fn produces_missed_blocks() { prev_slot = Slot::new(idx - 1); duplicate_block_root = *_state3.block_roots().get(idx as usize).unwrap(); validator_indexes = _state3.get_beacon_proposer_indices(&harness3.spec).unwrap(); - validator_index = validator_indexes[slot_in_epoch.as_usize()]; + missed_block_proposer = validator_indexes[slot_in_epoch.as_usize()]; proposer_shuffling_decision_root = _state3 .proposer_shuffling_decision_root_at_epoch(epoch, duplicate_block_root) .unwrap(); @@ -354,7 +337,7 @@ async fn produces_missed_blocks() { beacon_proposer_cache.lock().insert( epoch, proposer_shuffling_decision_root, - validator_indexes.into_iter().collect::>(), + validator_indexes, _state3.fork() ), Ok(()) @@ -371,11 +354,13 @@ async fn produces_missed_blocks() { // Let's validate the state which will call the function responsible for // adding the missed blocks to the validator monitor let mut validator_monitor3 = harness3.chain.validator_monitor.write(); + validator_monitor3.add_validator_pubkey(KEYPAIRS[missed_block_proposer].pk.compress()); validator_monitor3.process_valid_state(epoch, _state3, &harness3.chain.spec); // We shouldn't have one entry in the missed blocks map assert_eq!( - validator_monitor3.get_monitored_validator_missed_block_count(validator_index as u64), + validator_monitor3 + .get_monitored_validator_missed_block_count(missed_block_proposer as u64), 0 ); } diff --git a/beacon_node/beacon_processor/src/lib.rs b/beacon_node/beacon_processor/src/lib.rs index d83426a274..cbd9e0b872 100644 --- a/beacon_node/beacon_processor/src/lib.rs +++ b/beacon_node/beacon_processor/src/lib.rs @@ -1030,7 +1030,7 @@ impl BeaconProcessor { let can_spawn = self.current_workers < self.config.max_workers; let drop_during_sync = work_event .as_ref() - .map_or(false, |event| event.drop_during_sync); + .is_some_and(|event| event.drop_during_sync); let idle_tx = idle_tx.clone(); let modified_queue_id = match work_event { @@ -1731,7 +1731,7 @@ mod tests { #[test] fn min_queue_len() { // State with no validators. - let spec = ForkName::latest().make_genesis_spec(ChainSpec::mainnet()); + let spec = ForkName::latest_stable().make_genesis_spec(ChainSpec::mainnet()); let genesis_time = 0; let state = BeaconState::::new(genesis_time, Eth1Data::default(), &spec); assert_eq!(state.validators().len(), 0); diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index 7c6a253aca..e3bfd60a48 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -14,7 +14,7 @@ use beacon_chain::{ eth1_chain::{CachingEth1Backend, Eth1Chain}, slot_clock::{SlotClock, SystemTimeSlotClock}, state_advance_timer::spawn_state_advance_timer, - store::{HotColdDB, ItemStore, LevelDB, StoreConfig}, + store::{HotColdDB, ItemStore, StoreConfig}, BeaconChain, BeaconChainTypes, Eth1ChainBackend, MigratorConfig, ServerSentEventHandler, }; use beacon_chain::{Kzg, LightClientProducerEvent}; @@ -36,12 +36,12 @@ use network::{NetworkConfig, NetworkSenders, NetworkService}; use slasher::Slasher; use slasher_service::SlasherService; use slog::{debug, info, warn, Logger}; -use ssz::Decode; use std::net::TcpListener; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; use std::time::{SystemTime, UNIX_EPOCH}; +use store::database::interface::BeaconNodeBackend; use timer::spawn_timer; use tokio::sync::oneshot; use types::{ @@ -361,10 +361,11 @@ where let anchor_block = SignedBeaconBlock::from_ssz_bytes(&anchor_block_bytes, &spec) .map_err(|e| format!("Unable to parse weak subj block SSZ: {:?}", e))?; let anchor_blobs = if anchor_block.message().body().has_blobs() { + let max_blobs_len = spec.max_blobs_per_block(anchor_block.epoch()) as usize; let anchor_blobs_bytes = anchor_blobs_bytes .ok_or("Blobs for checkpoint must be provided using --checkpoint-blobs")?; Some( - BlobSidecarList::from_ssz_bytes(&anchor_blobs_bytes) + BlobSidecarList::from_ssz_bytes(&anchor_blobs_bytes, max_blobs_len) .map_err(|e| format!("Unable to parse weak subj blobs SSZ: {e:?}"))?, ) } else { @@ -910,7 +911,7 @@ where .forkchoice_update_parameters(); if params .head_hash - .map_or(false, |hash| hash != ExecutionBlockHash::zero()) + .is_some_and(|hash| hash != ExecutionBlockHash::zero()) { // Spawn a new task to update the EE without waiting for it to complete. let inner_chain = beacon_chain.clone(); @@ -1030,7 +1031,7 @@ where } impl - ClientBuilder, LevelDB>> + ClientBuilder, BeaconNodeBackend>> where TSlotClock: SlotClock + 'static, TEth1Backend: Eth1ChainBackend + 'static, diff --git a/beacon_node/client/src/notifier.rs b/beacon_node/client/src/notifier.rs index f686c2c650..0c3b1578d6 100644 --- a/beacon_node/client/src/notifier.rs +++ b/beacon_node/client/src/notifier.rs @@ -4,6 +4,7 @@ use beacon_chain::{ capella_readiness::CapellaReadiness, deneb_readiness::DenebReadiness, electra_readiness::ElectraReadiness, + fulu_readiness::FuluReadiness, BeaconChain, BeaconChainTypes, ExecutionStatus, }; use lighthouse_network::{types::SyncState, NetworkGlobals}; @@ -197,7 +198,7 @@ pub fn spawn_notifier( ); let speed = speedo.slots_per_second(); - let display_speed = speed.map_or(false, |speed| speed != 0.0); + let display_speed = speed.is_some_and(|speed| speed != 0.0); if display_speed { info!( @@ -233,7 +234,7 @@ pub fn spawn_notifier( ); let speed = speedo.slots_per_second(); - let display_speed = speed.map_or(false, |speed| speed != 0.0); + let display_speed = speed.is_some_and(|speed| speed != 0.0); if display_speed { info!( @@ -315,6 +316,7 @@ pub fn spawn_notifier( capella_readiness_logging(current_slot, &beacon_chain, &log).await; deneb_readiness_logging(current_slot, &beacon_chain, &log).await; electra_readiness_logging(current_slot, &beacon_chain, &log).await; + fulu_readiness_logging(current_slot, &beacon_chain, &log).await; } }; @@ -339,9 +341,7 @@ async fn bellatrix_readiness_logging( .message() .body() .execution_payload() - .map_or(false, |payload| { - payload.parent_hash() != ExecutionBlockHash::zero() - }); + .is_ok_and(|payload| payload.parent_hash() != ExecutionBlockHash::zero()); let has_execution_layer = beacon_chain.execution_layer.is_some(); @@ -588,6 +588,62 @@ async fn electra_readiness_logging( } } +/// Provides some helpful logging to users to indicate if their node is ready for Fulu. +async fn fulu_readiness_logging( + current_slot: Slot, + beacon_chain: &BeaconChain, + log: &Logger, +) { + let fulu_completed = beacon_chain + .canonical_head + .cached_head() + .snapshot + .beacon_state + .fork_name_unchecked() + .fulu_enabled(); + + let has_execution_layer = beacon_chain.execution_layer.is_some(); + + if fulu_completed && has_execution_layer + || !beacon_chain.is_time_to_prepare_for_fulu(current_slot) + { + return; + } + + if fulu_completed && !has_execution_layer { + error!( + log, + "Execution endpoint required"; + "info" => "you need a Fulu enabled execution engine to validate blocks." + ); + return; + } + + match beacon_chain.check_fulu_readiness().await { + FuluReadiness::Ready => { + info!( + log, + "Ready for Fulu"; + "info" => "ensure the execution endpoint is updated to the latest Fulu release" + ) + } + readiness @ FuluReadiness::ExchangeCapabilitiesFailed { error: _ } => { + error!( + log, + "Not ready for Fulu"; + "hint" => "the execution endpoint may be offline", + "info" => %readiness, + ) + } + readiness => warn!( + log, + "Not ready for Fulu"; + "hint" => "try updating the execution endpoint", + "info" => %readiness, + ), + } +} + async fn genesis_execution_payload_logging( beacon_chain: &BeaconChain, log: &Logger, diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index df16a90305..8b8bc257c7 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -1,10 +1,10 @@ use crate::engines::ForkchoiceState; use crate::http::{ ENGINE_FORKCHOICE_UPDATED_V1, ENGINE_FORKCHOICE_UPDATED_V2, ENGINE_FORKCHOICE_UPDATED_V3, - ENGINE_GET_BLOBS_V1, ENGINE_GET_CLIENT_VERSION_V1, ENGINE_GET_INCLUSION_LIST_V1, - ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V1, ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V1, - ENGINE_GET_PAYLOAD_V1, ENGINE_GET_PAYLOAD_V2, ENGINE_GET_PAYLOAD_V3, ENGINE_GET_PAYLOAD_V4, - ENGINE_NEW_PAYLOAD_V1, ENGINE_NEW_PAYLOAD_V2, ENGINE_NEW_PAYLOAD_V3, ENGINE_NEW_PAYLOAD_V4, + ENGINE_GET_BLOBS_V1, ENGINE_GET_CLIENT_VERSION_V1, ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V1, + ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V1, ENGINE_GET_PAYLOAD_V1, ENGINE_GET_PAYLOAD_V2, + ENGINE_GET_PAYLOAD_V3, ENGINE_GET_PAYLOAD_V4, ENGINE_GET_PAYLOAD_V5, ENGINE_NEW_PAYLOAD_V1, + ENGINE_NEW_PAYLOAD_V2, ENGINE_NEW_PAYLOAD_V3, ENGINE_NEW_PAYLOAD_V4, ENGINE_NEW_PAYLOAD_V5, ENGINE_GET_INCLUSION_LIST_V1 }; use eth2::types::{ BlobsBundle, SsePayloadAttributes, SsePayloadAttributesV1, SsePayloadAttributesV2, @@ -24,7 +24,7 @@ pub use types::{ }; use types::{ ExecutionPayloadBellatrix, ExecutionPayloadCapella, ExecutionPayloadDeneb, - ExecutionPayloadElectra, ExecutionRequests, KzgProofs, + ExecutionPayloadElectra, ExecutionPayloadFulu, ExecutionRequests, KzgProofs, }; use types::{Graffiti, GRAFFITI_BYTES_LEN}; @@ -35,7 +35,7 @@ mod new_payload_request; pub use new_payload_request::{ NewPayloadRequest, NewPayloadRequestBellatrix, NewPayloadRequestCapella, - NewPayloadRequestDeneb, NewPayloadRequestElectra, + NewPayloadRequestDeneb, NewPayloadRequestElectra, NewPayloadRequestFulu, }; pub const LATEST_TAG: &str = "latest"; @@ -261,7 +261,7 @@ pub struct ProposeBlindedBlockResponse { } #[superstruct( - variants(Bellatrix, Capella, Deneb, Electra), + variants(Bellatrix, Capella, Deneb, Electra, Fulu), variant_attributes(derive(Clone, Debug, PartialEq),), map_into(ExecutionPayload), map_ref_into(ExecutionPayloadRef), @@ -281,12 +281,14 @@ pub struct GetPayloadResponse { pub execution_payload: ExecutionPayloadDeneb, #[superstruct(only(Electra), partial_getter(rename = "execution_payload_electra"))] pub execution_payload: ExecutionPayloadElectra, + #[superstruct(only(Fulu), partial_getter(rename = "execution_payload_fulu"))] + pub execution_payload: ExecutionPayloadFulu, pub block_value: Uint256, - #[superstruct(only(Deneb, Electra))] + #[superstruct(only(Deneb, Electra, Fulu))] pub blobs_bundle: BlobsBundle, - #[superstruct(only(Deneb, Electra), partial_getter(copy))] + #[superstruct(only(Deneb, Electra, Fulu), partial_getter(copy))] pub should_override_builder: bool, - #[superstruct(only(Electra))] + #[superstruct(only(Electra, Fulu))] pub requests: ExecutionRequests, } @@ -354,6 +356,12 @@ impl From> Some(inner.blobs_bundle), Some(inner.requests), ), + GetPayloadResponse::Fulu(inner) => ( + ExecutionPayload::Fulu(inner.execution_payload), + inner.block_value, + Some(inner.blobs_bundle), + Some(inner.requests), + ), } } } @@ -487,6 +495,34 @@ impl ExecutionPayloadBodyV1 { )) } } + ExecutionPayloadHeader::Fulu(header) => { + if let Some(withdrawals) = self.withdrawals { + Ok(ExecutionPayload::Fulu(ExecutionPayloadFulu { + parent_hash: header.parent_hash, + fee_recipient: header.fee_recipient, + state_root: header.state_root, + receipts_root: header.receipts_root, + logs_bloom: header.logs_bloom, + prev_randao: header.prev_randao, + block_number: header.block_number, + gas_limit: header.gas_limit, + gas_used: header.gas_used, + timestamp: header.timestamp, + extra_data: header.extra_data, + base_fee_per_gas: header.base_fee_per_gas, + block_hash: header.block_hash, + transactions: self.transactions, + withdrawals, + blob_gas_used: header.blob_gas_used, + excess_blob_gas: header.excess_blob_gas, + })) + } else { + Err(format!( + "block {} is post capella but payload body doesn't have withdrawals", + header.block_hash + )) + } + } } } } @@ -497,6 +533,7 @@ pub struct EngineCapabilities { pub new_payload_v2: bool, pub new_payload_v3: bool, pub new_payload_v4: bool, + pub new_payload_v5: bool, pub forkchoice_updated_v1: bool, pub forkchoice_updated_v2: bool, pub forkchoice_updated_v3: bool, @@ -506,6 +543,7 @@ pub struct EngineCapabilities { pub get_payload_v2: bool, pub get_payload_v3: bool, pub get_payload_v4: bool, + pub get_payload_v5: bool, pub get_client_version_v1: bool, pub get_blobs_v1: bool, pub get_inclusion_list_v1: bool, @@ -526,6 +564,9 @@ impl EngineCapabilities { if self.new_payload_v4 { response.push(ENGINE_NEW_PAYLOAD_V4); } + if self.new_payload_v5 { + response.push(ENGINE_NEW_PAYLOAD_V5); + } if self.forkchoice_updated_v1 { response.push(ENGINE_FORKCHOICE_UPDATED_V1); } @@ -553,6 +594,9 @@ impl EngineCapabilities { if self.get_payload_v4 { response.push(ENGINE_GET_PAYLOAD_V4); } + if self.get_payload_v5 { + response.push(ENGINE_GET_PAYLOAD_V5); + } if self.get_client_version_v1 { response.push(ENGINE_GET_CLIENT_VERSION_V1); } diff --git a/beacon_node/execution_layer/src/engine_api/http.rs b/beacon_node/execution_layer/src/engine_api/http.rs index 5d22be9afd..2d5716ea56 100644 --- a/beacon_node/execution_layer/src/engine_api/http.rs +++ b/beacon_node/execution_layer/src/engine_api/http.rs @@ -35,12 +35,14 @@ pub const ENGINE_NEW_PAYLOAD_V1: &str = "engine_newPayloadV1"; pub const ENGINE_NEW_PAYLOAD_V2: &str = "engine_newPayloadV2"; pub const ENGINE_NEW_PAYLOAD_V3: &str = "engine_newPayloadV3"; pub const ENGINE_NEW_PAYLOAD_V4: &str = "engine_newPayloadV4"; +pub const ENGINE_NEW_PAYLOAD_V5: &str = "engine_newPayloadV5"; pub const ENGINE_NEW_PAYLOAD_TIMEOUT: Duration = Duration::from_secs(8); pub const ENGINE_GET_PAYLOAD_V1: &str = "engine_getPayloadV1"; pub const ENGINE_GET_PAYLOAD_V2: &str = "engine_getPayloadV2"; pub const ENGINE_GET_PAYLOAD_V3: &str = "engine_getPayloadV3"; pub const ENGINE_GET_PAYLOAD_V4: &str = "engine_getPayloadV4"; +pub const ENGINE_GET_PAYLOAD_V5: &str = "engine_getPayloadV5"; pub const ENGINE_GET_PAYLOAD_TIMEOUT: Duration = Duration::from_secs(2); pub const ENGINE_FORKCHOICE_UPDATED_V1: &str = "engine_forkchoiceUpdatedV1"; @@ -75,10 +77,12 @@ pub static LIGHTHOUSE_CAPABILITIES: &[&str] = &[ ENGINE_NEW_PAYLOAD_V2, ENGINE_NEW_PAYLOAD_V3, ENGINE_NEW_PAYLOAD_V4, + ENGINE_NEW_PAYLOAD_V5, ENGINE_GET_PAYLOAD_V1, ENGINE_GET_PAYLOAD_V2, ENGINE_GET_PAYLOAD_V3, ENGINE_GET_PAYLOAD_V4, + ENGINE_GET_PAYLOAD_V5, ENGINE_FORKCHOICE_UPDATED_V1, ENGINE_FORKCHOICE_UPDATED_V2, ENGINE_FORKCHOICE_UPDATED_V3, @@ -162,9 +166,7 @@ pub mod deposit_log { }; let signature_is_valid = deposit_pubkey_signature_message(&deposit_data, spec) - .map_or(false, |(public_key, signature, msg)| { - signature.verify(&public_key, msg) - }); + .is_some_and(|(public_key, signature, msg)| signature.verify(&public_key, msg)); Ok(DepositLog { deposit_data, @@ -596,7 +598,7 @@ impl CachedResponse { /// returns `true` if the entry's age is >= age_limit pub fn older_than(&self, age_limit: Option) -> bool { - age_limit.map_or(false, |limit| self.age() >= limit) + age_limit.is_some_and(|limit| self.age() >= limit) } } @@ -743,9 +745,9 @@ impl HttpJsonRpc { .await } - pub async fn get_block_by_number<'a>( + pub async fn get_block_by_number( &self, - query: BlockByNumberQuery<'a>, + query: BlockByNumberQuery<'_>, ) -> Result, Error> { let params = json!([query, RETURN_FULL_TRANSACTION_OBJECTS]); @@ -850,6 +852,31 @@ impl HttpJsonRpc { Ok(response.into()) } + // TODO(fulu): switch to v5 endpoint when the EL is ready for Fulu + pub async fn new_payload_v4_fulu( + &self, + new_payload_request_fulu: NewPayloadRequestFulu<'_, E>, + ) -> Result { + let params = json!([ + JsonExecutionPayload::V5(new_payload_request_fulu.execution_payload.clone().into()), + new_payload_request_fulu.versioned_hashes, + new_payload_request_fulu.parent_beacon_block_root, + new_payload_request_fulu + .execution_requests + .get_execution_requests_list(), + ]); + + let response: JsonPayloadStatusV1 = self + .rpc_request( + ENGINE_NEW_PAYLOAD_V4, + params, + ENGINE_NEW_PAYLOAD_TIMEOUT * self.execution_timeout_multiplier, + ) + .await?; + + Ok(response.into()) + } + pub async fn get_payload_v1( &self, payload_id: PayloadId, @@ -905,9 +932,10 @@ impl HttpJsonRpc { .try_into() .map_err(Error::BadResponse) } - ForkName::Base | ForkName::Altair | ForkName::Deneb | ForkName::Electra => Err( - Error::UnsupportedForkVariant(format!("called get_payload_v2 with {}", fork_name)), - ), + _ => Err(Error::UnsupportedForkVariant(format!( + "called get_payload_v2 with {}", + fork_name + ))), } } @@ -931,11 +959,7 @@ impl HttpJsonRpc { .try_into() .map_err(Error::BadResponse) } - ForkName::Base - | ForkName::Altair - | ForkName::Bellatrix - | ForkName::Capella - | ForkName::Electra => Err(Error::UnsupportedForkVariant(format!( + _ => Err(Error::UnsupportedForkVariant(format!( "called get_payload_v3 with {}", fork_name ))), @@ -962,17 +986,53 @@ impl HttpJsonRpc { .try_into() .map_err(Error::BadResponse) } - ForkName::Base - | ForkName::Altair - | ForkName::Bellatrix - | ForkName::Capella - | ForkName::Deneb => Err(Error::UnsupportedForkVariant(format!( + // TODO(fulu): remove when v5 method is ready. + ForkName::Fulu => { + let response: JsonGetPayloadResponseV5 = self + .rpc_request( + ENGINE_GET_PAYLOAD_V4, + params, + ENGINE_GET_PAYLOAD_TIMEOUT * self.execution_timeout_multiplier, + ) + .await?; + JsonGetPayloadResponse::V5(response) + .try_into() + .map_err(Error::BadResponse) + } + _ => Err(Error::UnsupportedForkVariant(format!( "called get_payload_v4 with {}", fork_name ))), } } + pub async fn get_payload_v5( + &self, + fork_name: ForkName, + payload_id: PayloadId, + ) -> Result, Error> { + let params = json!([JsonPayloadIdRequest::from(payload_id)]); + + match fork_name { + ForkName::Fulu => { + let response: JsonGetPayloadResponseV5 = self + .rpc_request( + ENGINE_GET_PAYLOAD_V5, + params, + ENGINE_GET_PAYLOAD_TIMEOUT * self.execution_timeout_multiplier, + ) + .await?; + JsonGetPayloadResponse::V5(response) + .try_into() + .map_err(Error::BadResponse) + } + _ => Err(Error::UnsupportedForkVariant(format!( + "called get_payload_v5 with {}", + fork_name + ))), + } + } + pub async fn forkchoice_updated_v1( &self, forkchoice_state: ForkchoiceState, @@ -1096,6 +1156,7 @@ impl HttpJsonRpc { new_payload_v2: capabilities.contains(ENGINE_NEW_PAYLOAD_V2), new_payload_v3: capabilities.contains(ENGINE_NEW_PAYLOAD_V3), new_payload_v4: capabilities.contains(ENGINE_NEW_PAYLOAD_V4), + new_payload_v5: capabilities.contains(ENGINE_NEW_PAYLOAD_V5), forkchoice_updated_v1: capabilities.contains(ENGINE_FORKCHOICE_UPDATED_V1), forkchoice_updated_v2: capabilities.contains(ENGINE_FORKCHOICE_UPDATED_V2), forkchoice_updated_v3: capabilities.contains(ENGINE_FORKCHOICE_UPDATED_V3), @@ -1107,6 +1168,7 @@ impl HttpJsonRpc { get_payload_v2: capabilities.contains(ENGINE_GET_PAYLOAD_V2), get_payload_v3: capabilities.contains(ENGINE_GET_PAYLOAD_V3), get_payload_v4: capabilities.contains(ENGINE_GET_PAYLOAD_V4), + get_payload_v5: capabilities.contains(ENGINE_GET_PAYLOAD_V5), get_client_version_v1: capabilities.contains(ENGINE_GET_CLIENT_VERSION_V1), get_blobs_v1: capabilities.contains(ENGINE_GET_BLOBS_V1), get_inclusion_list_v1: capabilities.contains(ENGINE_GET_INCLUSION_LIST_V1), @@ -1238,6 +1300,14 @@ impl HttpJsonRpc { Err(Error::RequiredMethodUnsupported("engine_newPayloadV4")) } } + NewPayloadRequest::Fulu(new_payload_request_fulu) => { + // TODO(fulu): switch to v5 endpoint when the EL is ready for Fulu + if engine_capabilities.new_payload_v4 { + self.new_payload_v4_fulu(new_payload_request_fulu).await + } else { + Err(Error::RequiredMethodUnsupported("engine_newPayloadV4")) + } + } } } @@ -1273,6 +1343,14 @@ impl HttpJsonRpc { Err(Error::RequiredMethodUnsupported("engine_getPayloadv4")) } } + ForkName::Fulu => { + // TODO(fulu): switch to v5 when the EL is ready + if engine_capabilities.get_payload_v4 { + self.get_payload_v4(fork_name, payload_id).await + } else { + Err(Error::RequiredMethodUnsupported("engine_getPayloadv5")) + } + } ForkName::Base | ForkName::Altair => Err(Error::UnsupportedForkVariant(format!( "called get_payload with {}", fork_name @@ -1345,7 +1423,8 @@ mod test { impl Tester { pub fn new(with_auth: bool) -> Self { - let server = MockServer::unit_testing(); + let spec = Arc::new(MainnetEthSpec::default_spec()); + let server = MockServer::unit_testing(spec); let rpc_url = SensitiveUrl::parse(&server.url()).unwrap(); let echo_url = SensitiveUrl::parse(&format!("{}/echo", server.url())).unwrap(); diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index 1c6639804e..96615297d8 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -7,7 +7,7 @@ use superstruct::superstruct; use types::beacon_block_body::KzgCommitments; use types::blob_sidecar::BlobsList; use types::execution_requests::{ - ConsolidationRequests, DepositRequests, RequestPrefix, WithdrawalRequests, + ConsolidationRequests, DepositRequests, RequestType, WithdrawalRequests, }; use types::{Blob, FixedVector, KzgProof, Unsigned}; @@ -65,7 +65,7 @@ pub struct JsonPayloadIdResponse { } #[superstruct( - variants(V1, V2, V3, V4), + variants(V1, V2, V3, V4, V5), variant_attributes( derive(Debug, PartialEq, Default, Serialize, Deserialize,), serde(bound = "E: EthSpec", rename_all = "camelCase"), @@ -100,12 +100,12 @@ pub struct JsonExecutionPayload { pub block_hash: ExecutionBlockHash, #[serde(with = "ssz_types::serde_utils::list_of_hex_var_list")] pub transactions: Transactions, - #[superstruct(only(V2, V3, V4))] + #[superstruct(only(V2, V3, V4, V5))] pub withdrawals: VariableList, - #[superstruct(only(V3, V4))] + #[superstruct(only(V3, V4, V5))] #[serde(with = "serde_utils::u64_hex_be")] pub blob_gas_used: u64, - #[superstruct(only(V3, V4))] + #[superstruct(only(V3, V4, V5))] #[serde(with = "serde_utils::u64_hex_be")] pub excess_blob_gas: u64, } @@ -214,6 +214,35 @@ impl From> for JsonExecutionPayloadV4 } } +impl From> for JsonExecutionPayloadV5 { + fn from(payload: ExecutionPayloadFulu) -> Self { + JsonExecutionPayloadV5 { + parent_hash: payload.parent_hash, + fee_recipient: payload.fee_recipient, + state_root: payload.state_root, + receipts_root: payload.receipts_root, + logs_bloom: payload.logs_bloom, + prev_randao: payload.prev_randao, + block_number: payload.block_number, + gas_limit: payload.gas_limit, + gas_used: payload.gas_used, + timestamp: payload.timestamp, + extra_data: payload.extra_data, + base_fee_per_gas: payload.base_fee_per_gas, + block_hash: payload.block_hash, + transactions: payload.transactions, + withdrawals: payload + .withdrawals + .into_iter() + .map(Into::into) + .collect::>() + .into(), + blob_gas_used: payload.blob_gas_used, + excess_blob_gas: payload.excess_blob_gas, + } + } +} + impl From> for JsonExecutionPayload { fn from(execution_payload: ExecutionPayload) -> Self { match execution_payload { @@ -221,6 +250,7 @@ impl From> for JsonExecutionPayload { ExecutionPayload::Capella(payload) => JsonExecutionPayload::V2(payload.into()), ExecutionPayload::Deneb(payload) => JsonExecutionPayload::V3(payload.into()), ExecutionPayload::Electra(payload) => JsonExecutionPayload::V4(payload.into()), + ExecutionPayload::Fulu(payload) => JsonExecutionPayload::V5(payload.into()), } } } @@ -330,6 +360,35 @@ impl From> for ExecutionPayloadElectra } } +impl From> for ExecutionPayloadFulu { + fn from(payload: JsonExecutionPayloadV5) -> Self { + ExecutionPayloadFulu { + parent_hash: payload.parent_hash, + fee_recipient: payload.fee_recipient, + state_root: payload.state_root, + receipts_root: payload.receipts_root, + logs_bloom: payload.logs_bloom, + prev_randao: payload.prev_randao, + block_number: payload.block_number, + gas_limit: payload.gas_limit, + gas_used: payload.gas_used, + timestamp: payload.timestamp, + extra_data: payload.extra_data, + base_fee_per_gas: payload.base_fee_per_gas, + block_hash: payload.block_hash, + transactions: payload.transactions, + withdrawals: payload + .withdrawals + .into_iter() + .map(Into::into) + .collect::>() + .into(), + blob_gas_used: payload.blob_gas_used, + excess_blob_gas: payload.excess_blob_gas, + } + } +} + impl From> for ExecutionPayload { fn from(json_execution_payload: JsonExecutionPayload) -> Self { match json_execution_payload { @@ -337,51 +396,85 @@ impl From> for ExecutionPayload { JsonExecutionPayload::V2(payload) => ExecutionPayload::Capella(payload.into()), JsonExecutionPayload::V3(payload) => ExecutionPayload::Deneb(payload.into()), JsonExecutionPayload::V4(payload) => ExecutionPayload::Electra(payload.into()), + JsonExecutionPayload::V5(payload) => ExecutionPayload::Fulu(payload.into()), } } } +#[derive(Debug, Clone)] +pub enum RequestsError { + InvalidHex(hex::FromHexError), + EmptyRequest(usize), + InvalidOrdering, + InvalidPrefix(u8), + DecodeError(String), +} + /// Format of `ExecutionRequests` received over the engine api. /// -/// Array of ssz-encoded requests list encoded as hex bytes. -/// The prefix of the request type is used to index into the array. -/// -/// For e.g. [0xab, 0xcd, 0xef] -/// Here, 0xab are the deposits bytes (prefix and index == 0) -/// 0xcd are the withdrawals bytes (prefix and index == 1) -/// 0xef are the consolidations bytes (prefix and index == 2) +/// Array of ssz-encoded requests list encoded as hex bytes prefixed +/// with a `RequestType` #[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)] #[serde(transparent)] pub struct JsonExecutionRequests(pub Vec); impl TryFrom for ExecutionRequests { - type Error = String; + type Error = RequestsError; fn try_from(value: JsonExecutionRequests) -> Result { let mut requests = ExecutionRequests::default(); - + let mut prev_prefix: Option = None; for (i, request) in value.0.into_iter().enumerate() { // hex string let decoded_bytes = hex::decode(request.strip_prefix("0x").unwrap_or(&request)) - .map_err(|e| format!("Invalid hex {:?}", e))?; - match RequestPrefix::from_prefix(i as u8) { - Some(RequestPrefix::Deposit) => { - requests.deposits = DepositRequests::::from_ssz_bytes(&decoded_bytes) - .map_err(|e| format!("Failed to decode DepositRequest from EL: {:?}", e))?; + .map_err(RequestsError::InvalidHex)?; + + // The first byte of each element is the `request_type` and the remaining bytes are the `request_data`. + // Elements with empty `request_data` **MUST** be excluded from the list. + let Some((prefix_byte, request_bytes)) = decoded_bytes.split_first() else { + return Err(RequestsError::EmptyRequest(i)); + }; + if request_bytes.is_empty() { + return Err(RequestsError::EmptyRequest(i)); + } + // Elements of the list **MUST** be ordered by `request_type` in ascending order + let current_prefix = RequestType::from_u8(*prefix_byte) + .ok_or(RequestsError::InvalidPrefix(*prefix_byte))?; + if let Some(prev) = prev_prefix { + if prev.to_u8() >= current_prefix.to_u8() { + return Err(RequestsError::InvalidOrdering); } - Some(RequestPrefix::Withdrawal) => { - requests.withdrawals = WithdrawalRequests::::from_ssz_bytes(&decoded_bytes) + } + prev_prefix = Some(current_prefix); + + match current_prefix { + RequestType::Deposit => { + requests.deposits = DepositRequests::::from_ssz_bytes(request_bytes) .map_err(|e| { - format!("Failed to decode WithdrawalRequest from EL: {:?}", e) + RequestsError::DecodeError(format!( + "Failed to decode DepositRequest from EL: {:?}", + e + )) })?; } - Some(RequestPrefix::Consolidation) => { - requests.consolidations = - ConsolidationRequests::::from_ssz_bytes(&decoded_bytes).map_err( - |e| format!("Failed to decode ConsolidationRequest from EL: {:?}", e), - )?; + RequestType::Withdrawal => { + requests.withdrawals = WithdrawalRequests::::from_ssz_bytes(request_bytes) + .map_err(|e| { + RequestsError::DecodeError(format!( + "Failed to decode WithdrawalRequest from EL: {:?}", + e + )) + })?; + } + RequestType::Consolidation => { + requests.consolidations = + ConsolidationRequests::::from_ssz_bytes(request_bytes).map_err(|e| { + RequestsError::DecodeError(format!( + "Failed to decode ConsolidationRequest from EL: {:?}", + e + )) + })?; } - None => return Err("Empty requests string".to_string()), } } Ok(requests) @@ -389,7 +482,7 @@ impl TryFrom for ExecutionRequests { } #[superstruct( - variants(V1, V2, V3, V4), + variants(V1, V2, V3, V4, V5), variant_attributes( derive(Debug, PartialEq, Serialize, Deserialize), serde(bound = "E: EthSpec", rename_all = "camelCase") @@ -408,13 +501,15 @@ pub struct JsonGetPayloadResponse { pub execution_payload: JsonExecutionPayloadV3, #[superstruct(only(V4), partial_getter(rename = "execution_payload_v4"))] pub execution_payload: JsonExecutionPayloadV4, + #[superstruct(only(V5), partial_getter(rename = "execution_payload_v5"))] + pub execution_payload: JsonExecutionPayloadV5, #[serde(with = "serde_utils::u256_hex_be")] pub block_value: Uint256, - #[superstruct(only(V3, V4))] + #[superstruct(only(V3, V4, V5))] pub blobs_bundle: JsonBlobsBundleV1, - #[superstruct(only(V3, V4))] + #[superstruct(only(V3, V4, V5))] pub should_override_builder: bool, - #[superstruct(only(V4))] + #[superstruct(only(V4, V5))] pub execution_requests: JsonExecutionRequests, } @@ -448,7 +543,20 @@ impl TryFrom> for GetPayloadResponse { block_value: response.block_value, blobs_bundle: response.blobs_bundle.into(), should_override_builder: response.should_override_builder, - requests: response.execution_requests.try_into()?, + requests: response.execution_requests.try_into().map_err(|e| { + format!("Failed to convert json to execution requests : {:?}", e) + })?, + })) + } + JsonGetPayloadResponse::V5(response) => { + Ok(GetPayloadResponse::Fulu(GetPayloadResponseFulu { + execution_payload: response.execution_payload.into(), + block_value: response.block_value, + blobs_bundle: response.blobs_bundle.into(), + should_override_builder: response.should_override_builder, + requests: response.execution_requests.try_into().map_err(|e| { + format!("Failed to convert json to execution requests {:?}", e) + })?, })) } } @@ -883,3 +991,154 @@ impl TryFrom for ClientVersionV1 { }) } } + +#[cfg(test)] +mod tests { + use ssz::Encode; + use types::{ + ConsolidationRequest, DepositRequest, MainnetEthSpec, PublicKeyBytes, RequestType, + SignatureBytes, WithdrawalRequest, + }; + + use super::*; + + fn create_request_string(prefix: u8, request_bytes: &T) -> String { + format!( + "0x{:02x}{}", + prefix, + hex::encode(request_bytes.as_ssz_bytes()) + ) + } + + /// Tests all error conditions except ssz decoding errors + /// + /// *** + /// Elements of the list MUST be ordered by request_type in ascending order. + /// Elements with empty request_data MUST be excluded from the list. + /// If any element is out of order, has a length of 1-byte or shorter, + /// or more than one element has the same type byte, client software MUST return -32602: Invalid params error. + /// *** + #[test] + fn test_invalid_execution_requests() { + let deposit_request = DepositRequest { + pubkey: PublicKeyBytes::empty(), + withdrawal_credentials: Hash256::random(), + amount: 32, + signature: SignatureBytes::empty(), + index: 0, + }; + + let consolidation_request = ConsolidationRequest { + source_address: Address::random(), + source_pubkey: PublicKeyBytes::empty(), + target_pubkey: PublicKeyBytes::empty(), + }; + + let withdrawal_request = WithdrawalRequest { + amount: 32, + source_address: Address::random(), + validator_pubkey: PublicKeyBytes::empty(), + }; + + // First check a valid request with all requests + assert!( + ExecutionRequests::::try_from(JsonExecutionRequests(vec![ + create_request_string(RequestType::Deposit.to_u8(), &deposit_request), + create_request_string(RequestType::Withdrawal.to_u8(), &withdrawal_request), + create_request_string(RequestType::Consolidation.to_u8(), &consolidation_request), + ])) + .is_ok() + ); + + // Single requests + assert!( + ExecutionRequests::::try_from(JsonExecutionRequests(vec![ + create_request_string(RequestType::Deposit.to_u8(), &deposit_request), + ])) + .is_ok() + ); + + assert!( + ExecutionRequests::::try_from(JsonExecutionRequests(vec![ + create_request_string(RequestType::Withdrawal.to_u8(), &withdrawal_request), + ])) + .is_ok() + ); + + assert!( + ExecutionRequests::::try_from(JsonExecutionRequests(vec![ + create_request_string(RequestType::Consolidation.to_u8(), &consolidation_request), + ])) + .is_ok() + ); + + // Out of order + assert!(matches!( + ExecutionRequests::::try_from(JsonExecutionRequests(vec![ + create_request_string(RequestType::Withdrawal.to_u8(), &withdrawal_request), + create_request_string(RequestType::Deposit.to_u8(), &deposit_request), + ])) + .unwrap_err(), + RequestsError::InvalidOrdering + )); + + assert!(matches!( + ExecutionRequests::::try_from(JsonExecutionRequests(vec![ + create_request_string(RequestType::Consolidation.to_u8(), &consolidation_request), + create_request_string(RequestType::Withdrawal.to_u8(), &withdrawal_request), + ])) + .unwrap_err(), + RequestsError::InvalidOrdering + )); + + assert!(matches!( + ExecutionRequests::::try_from(JsonExecutionRequests(vec![ + create_request_string(RequestType::Consolidation.to_u8(), &consolidation_request), + create_request_string(RequestType::Deposit.to_u8(), &deposit_request), + ])) + .unwrap_err(), + RequestsError::InvalidOrdering + )); + + // Multiple requests of same type + assert!(matches!( + ExecutionRequests::::try_from(JsonExecutionRequests(vec![ + create_request_string(RequestType::Deposit.to_u8(), &deposit_request), + create_request_string(RequestType::Deposit.to_u8(), &deposit_request), + ])) + .unwrap_err(), + RequestsError::InvalidOrdering + )); + + // Invalid prefix + assert!(matches!( + ExecutionRequests::::try_from(JsonExecutionRequests(vec![ + create_request_string(42, &deposit_request), + ])) + .unwrap_err(), + RequestsError::InvalidPrefix(42) + )); + + // Prefix followed by no data + assert!(matches!( + ExecutionRequests::::try_from(JsonExecutionRequests(vec![ + create_request_string(RequestType::Deposit.to_u8(), &deposit_request), + create_request_string( + RequestType::Consolidation.to_u8(), + &Vec::::new() + ), + ])) + .unwrap_err(), + RequestsError::EmptyRequest(1) + )); + // Empty request + assert!(matches!( + ExecutionRequests::::try_from(JsonExecutionRequests(vec![ + create_request_string(RequestType::Deposit.to_u8(), &deposit_request), + "0x".to_string() + ])) + .unwrap_err(), + RequestsError::EmptyRequest(1) + )); + } +} diff --git a/beacon_node/execution_layer/src/engine_api/new_payload_request.rs b/beacon_node/execution_layer/src/engine_api/new_payload_request.rs index b3dc84d253..e3c2215463 100644 --- a/beacon_node/execution_layer/src/engine_api/new_payload_request.rs +++ b/beacon_node/execution_layer/src/engine_api/new_payload_request.rs @@ -9,11 +9,11 @@ use types::{ }; use types::{ ExecutionPayloadBellatrix, ExecutionPayloadCapella, ExecutionPayloadDeneb, - ExecutionPayloadElectra, ExecutionRequests, + ExecutionPayloadElectra, ExecutionPayloadFulu, ExecutionRequests, }; #[superstruct( - variants(Bellatrix, Capella, Deneb, Electra), + variants(Bellatrix, Capella, Deneb, Electra, Fulu), variant_attributes(derive(Clone, Debug, PartialEq),), map_into(ExecutionPayload), map_ref_into(ExecutionPayloadRef), @@ -39,13 +39,15 @@ pub struct NewPayloadRequest<'block, E: EthSpec> { pub execution_payload: &'block ExecutionPayloadDeneb, #[superstruct(only(Electra), partial_getter(rename = "execution_payload_electra"))] pub execution_payload: &'block ExecutionPayloadElectra, - #[superstruct(only(Deneb, Electra))] + #[superstruct(only(Fulu), partial_getter(rename = "execution_payload_fulu"))] + pub execution_payload: &'block ExecutionPayloadFulu, + #[superstruct(only(Deneb, Electra, Fulu))] pub versioned_hashes: Vec, - #[superstruct(only(Deneb, Electra))] + #[superstruct(only(Deneb, Electra, Fulu))] pub parent_beacon_block_root: Hash256, - #[superstruct(only(Electra))] + #[superstruct(only(Electra, Fulu))] pub execution_requests: &'block ExecutionRequests, - #[superstruct(only(Electra))] + #[superstruct(only(Electra, Fulu))] pub il_transactions: InclusionListTransactions, } @@ -56,6 +58,7 @@ impl<'block, E: EthSpec> NewPayloadRequest<'block, E> { Self::Capella(payload) => payload.execution_payload.parent_hash, Self::Deneb(payload) => payload.execution_payload.parent_hash, Self::Electra(payload) => payload.execution_payload.parent_hash, + Self::Fulu(payload) => payload.execution_payload.parent_hash, } } @@ -65,6 +68,7 @@ impl<'block, E: EthSpec> NewPayloadRequest<'block, E> { Self::Capella(payload) => payload.execution_payload.block_hash, Self::Deneb(payload) => payload.execution_payload.block_hash, Self::Electra(payload) => payload.execution_payload.block_hash, + Self::Fulu(payload) => payload.execution_payload.block_hash, } } @@ -74,6 +78,7 @@ impl<'block, E: EthSpec> NewPayloadRequest<'block, E> { Self::Capella(payload) => payload.execution_payload.block_number, Self::Deneb(payload) => payload.execution_payload.block_number, Self::Electra(payload) => payload.execution_payload.block_number, + Self::Fulu(payload) => payload.execution_payload.block_number, } } @@ -83,6 +88,7 @@ impl<'block, E: EthSpec> NewPayloadRequest<'block, E> { Self::Capella(request) => ExecutionPayloadRef::Capella(request.execution_payload), Self::Deneb(request) => ExecutionPayloadRef::Deneb(request.execution_payload), Self::Electra(request) => ExecutionPayloadRef::Electra(request.execution_payload), + Self::Fulu(request) => ExecutionPayloadRef::Fulu(request.execution_payload), } } @@ -94,6 +100,7 @@ impl<'block, E: EthSpec> NewPayloadRequest<'block, E> { Self::Capella(request) => ExecutionPayload::Capella(request.execution_payload.clone()), Self::Deneb(request) => ExecutionPayload::Deneb(request.execution_payload.clone()), Self::Electra(request) => ExecutionPayload::Electra(request.execution_payload.clone()), + Self::Fulu(request) => ExecutionPayload::Fulu(request.execution_payload.clone()), } } @@ -123,6 +130,11 @@ impl<'block, E: EthSpec> NewPayloadRequest<'block, E> { let _timer = metrics::start_timer(&metrics::EXECUTION_LAYER_VERIFY_BLOCK_HASH); + // Check that no transactions in the payload are zero length + if payload.transactions().iter().any(|slice| slice.is_empty()) { + return Err(Error::ZeroLengthTransaction); + } + let (header_hash, rlp_transactions_root) = calculate_execution_block_hash( payload, parent_beacon_block_root, @@ -194,6 +206,18 @@ impl<'a, E: EthSpec> NewPayloadRequest<'a, E> { execution_requests: &block_ref.body.execution_requests, il_transactions, })), + BeaconBlockRef::Fulu(block_ref) => Ok(Self::Fulu(NewPayloadRequestFulu { + execution_payload: &block_ref.body.execution_payload.execution_payload, + versioned_hashes: block_ref + .body + .blob_kzg_commitments + .iter() + .map(kzg_commitment_to_versioned_hash) + .collect(), + parent_beacon_block_root: block_ref.parent_root, + execution_requests: &block_ref.body.execution_requests, + il_transactions, + })), } } } @@ -237,6 +261,18 @@ impl<'a, E: EthSpec> TryFrom> for NewPayloadRequest<'a, E> execution_requests: &block_ref.body.execution_requests, il_transactions: vec![].into(), })), + BeaconBlockRef::Fulu(block_ref) => Ok(Self::Fulu(NewPayloadRequestFulu { + execution_payload: &block_ref.body.execution_payload.execution_payload, + versioned_hashes: block_ref + .body + .blob_kzg_commitments + .iter() + .map(kzg_commitment_to_versioned_hash) + .collect(), + parent_beacon_block_root: block_ref.parent_root, + execution_requests: &block_ref.body.execution_requests, + il_transactions: vec![].into() + })), } } } @@ -256,6 +292,7 @@ impl<'a, E: EthSpec> TryFrom> for NewPayloadRequest<' })), ExecutionPayloadRef::Deneb(_) => Err(Self::Error::IncorrectStateVariant), ExecutionPayloadRef::Electra(_) => Err(Self::Error::IncorrectStateVariant), + ExecutionPayloadRef::Fulu(_) => Err(Self::Error::IncorrectStateVariant), } } } diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 41f5baa67b..9ea172689d 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -54,8 +54,8 @@ use types::{ }; use types::{ BeaconStateError, BlindedPayload, ChainSpec, Epoch, ExecPayload, ExecutionPayloadBellatrix, - ExecutionPayloadCapella, ExecutionPayloadElectra, FullPayload, InclusionListTransactions, - ProposerPreparationData, PublicKeyBytes, Signature, Slot, + ExecutionPayloadCapella, ExecutionPayloadElectra, ExecutionPayloadFulu, FullPayload, + ProposerPreparationData, PublicKeyBytes, Signature, Slot, InclusionListTransactions }; mod block_hash; @@ -121,7 +121,14 @@ impl TryFrom> for ProvenancedPayload BlockProposalContents::PayloadAndBlobs { + payload: ExecutionPayloadHeader::Fulu(builder_bid.header).into(), + block_value: builder_bid.value, + kzg_commitments: builder_bid.blob_kzg_commitments, + blobs_and_proofs: None, + // TODO(fulu): update this with builder api returning the requests requests: None, }, }; @@ -149,6 +156,7 @@ pub enum Error { payload: ExecutionBlockHash, transactions_root: Hash256, }, + ZeroLengthTransaction, PayloadBodiesByRangeNotSupported, InvalidJWTSecret(String), InvalidForkForPayload, @@ -321,7 +329,7 @@ impl> BlockProposalContents { pub parent_hash: ExecutionBlockHash, pub parent_gas_limit: u64, @@ -1822,6 +1830,7 @@ impl ExecutionLayer { ForkName::Capella => ExecutionPayloadCapella::default().into(), ForkName::Deneb => ExecutionPayloadDeneb::default().into(), ForkName::Electra => ExecutionPayloadElectra::default().into(), + ForkName::Fulu => ExecutionPayloadFulu::default().into(), ForkName::Base | ForkName::Altair => { return Err(Error::InvalidForkForPayload); } @@ -2109,7 +2118,7 @@ fn verify_builder_bid( payload: header.timestamp(), expected: payload_attributes.timestamp(), })) - } else if block_number.map_or(false, |n| n != header.block_number()) { + } else if block_number.is_some_and(|n| n != header.block_number()) { Err(Box::new(InvalidBuilderPayload::BlockNumber { payload: header.block_number(), expected: block_number, diff --git a/beacon_node/execution_layer/src/payload_status.rs b/beacon_node/execution_layer/src/payload_status.rs index 5405fd7009..cf0be8ed0d 100644 --- a/beacon_node/execution_layer/src/payload_status.rs +++ b/beacon_node/execution_layer/src/payload_status.rs @@ -41,7 +41,7 @@ pub fn process_payload_status( PayloadStatusV1Status::Valid => { if response .latest_valid_hash - .map_or(false, |h| h == head_block_hash) + .is_some_and(|h| h == head_block_hash) { // The response is only valid if `latest_valid_hash` is not `null` and // equal to the provided `block_hash`. diff --git a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs index 4fab7150ce..9fa375b375 100644 --- a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs +++ b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs @@ -19,7 +19,7 @@ use tree_hash::TreeHash; use tree_hash_derive::TreeHash; use types::{ Blob, ChainSpec, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadBellatrix, - ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadElectra, + ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadElectra, ExecutionPayloadFulu, ExecutionPayloadHeader, FixedBytesExtended, ForkName, Hash256, Transaction, Transactions, Uint256, }; @@ -147,12 +147,14 @@ pub struct ExecutionBlockGenerator { pub shanghai_time: Option, // capella pub cancun_time: Option, // deneb pub prague_time: Option, // electra + pub osaka_time: Option, // fulu /* * deneb stuff */ pub blobs_bundles: HashMap>, pub kzg: Option>, rng: Arc>, + spec: Arc, } fn make_rng() -> Arc> { @@ -162,6 +164,7 @@ fn make_rng() -> Arc> { } impl ExecutionBlockGenerator { + #[allow(clippy::too_many_arguments)] pub fn new( terminal_total_difficulty: Uint256, terminal_block_number: u64, @@ -169,6 +172,8 @@ impl ExecutionBlockGenerator { shanghai_time: Option, cancun_time: Option, prague_time: Option, + osaka_time: Option, + spec: Arc, kzg: Option>, ) -> Self { let mut gen = Self { @@ -185,9 +190,11 @@ impl ExecutionBlockGenerator { shanghai_time, cancun_time, prague_time, + osaka_time, blobs_bundles: <_>::default(), kzg, rng: make_rng(), + spec, }; gen.insert_pow_block(0).unwrap(); @@ -233,13 +240,16 @@ impl ExecutionBlockGenerator { } pub fn get_fork_at_timestamp(&self, timestamp: u64) -> ForkName { - match self.prague_time { - Some(fork_time) if timestamp >= fork_time => ForkName::Electra, - _ => match self.cancun_time { - Some(fork_time) if timestamp >= fork_time => ForkName::Deneb, - _ => match self.shanghai_time { - Some(fork_time) if timestamp >= fork_time => ForkName::Capella, - _ => ForkName::Bellatrix, + match self.osaka_time { + Some(fork_time) if timestamp >= fork_time => ForkName::Fulu, + _ => match self.prague_time { + Some(fork_time) if timestamp >= fork_time => ForkName::Electra, + _ => match self.cancun_time { + Some(fork_time) if timestamp >= fork_time => ForkName::Deneb, + _ => match self.shanghai_time { + Some(fork_time) if timestamp >= fork_time => ForkName::Capella, + _ => ForkName::Bellatrix, + }, }, }, } @@ -664,6 +674,25 @@ impl ExecutionBlockGenerator { blob_gas_used: 0, excess_blob_gas: 0, }), + ForkName::Fulu => ExecutionPayload::Fulu(ExecutionPayloadFulu { + parent_hash: head_block_hash, + fee_recipient: pa.suggested_fee_recipient, + receipts_root: Hash256::repeat_byte(42), + state_root: Hash256::repeat_byte(43), + logs_bloom: vec![0; 256].into(), + prev_randao: pa.prev_randao, + block_number: parent.block_number() + 1, + gas_limit: DEFAULT_GAS_LIMIT, + gas_used: GAS_USED, + timestamp: pa.timestamp, + extra_data: "block gen was here".as_bytes().to_vec().into(), + base_fee_per_gas: Uint256::from(1u64), + block_hash: ExecutionBlockHash::zero(), + transactions: vec![].into(), + withdrawals: pa.withdrawals.clone().into(), + blob_gas_used: 0, + excess_blob_gas: 0, + }), _ => unreachable!(), }, }; @@ -671,7 +700,11 @@ impl ExecutionBlockGenerator { if execution_payload.fork_name().deneb_enabled() { // get random number between 0 and Max Blobs let mut rng = self.rng.lock(); - let num_blobs = rng.gen::() % (E::max_blobs_per_block() + 1); + let max_blobs = self + .spec + .max_blobs_per_block_by_fork(execution_payload.fork_name()) + as usize; + let num_blobs = rng.gen::() % (max_blobs + 1); let (bundle, transactions) = generate_blobs(num_blobs)?; for tx in Vec::from(transactions) { execution_payload @@ -811,6 +844,12 @@ pub fn generate_genesis_header( *header.transactions_root_mut() = empty_transactions_root; Some(header) } + ForkName::Fulu => { + let mut header = ExecutionPayloadHeader::Fulu(<_>::default()); + *header.block_hash_mut() = genesis_block_hash.unwrap_or_default(); + *header.transactions_root_mut() = empty_transactions_root; + Some(header) + } } } @@ -874,6 +913,7 @@ mod test { const TERMINAL_DIFFICULTY: u64 = 10; const TERMINAL_BLOCK: u64 = 10; const DIFFICULTY_INCREMENT: u64 = 1; + let spec = Arc::new(MainnetEthSpec::default_spec()); let mut generator: ExecutionBlockGenerator = ExecutionBlockGenerator::new( Uint256::from(TERMINAL_DIFFICULTY), @@ -883,6 +923,8 @@ mod test { None, None, None, + spec, + None, ); for i in 0..=TERMINAL_BLOCK { diff --git a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs index 9365024ffb..d727d2c159 100644 --- a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs +++ b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs @@ -99,7 +99,8 @@ pub async fn handle_rpc( ENGINE_NEW_PAYLOAD_V1 | ENGINE_NEW_PAYLOAD_V2 | ENGINE_NEW_PAYLOAD_V3 - | ENGINE_NEW_PAYLOAD_V4 => { + | ENGINE_NEW_PAYLOAD_V4 + | ENGINE_NEW_PAYLOAD_V5 => { let request = match method { ENGINE_NEW_PAYLOAD_V1 => JsonExecutionPayload::V1( get_param::>(params, 0) @@ -121,6 +122,9 @@ pub async fn handle_rpc( ENGINE_NEW_PAYLOAD_V4 => get_param::>(params, 0) .map(|jep| JsonExecutionPayload::V4(jep)) .map_err(|s| (s, BAD_PARAMS_ERROR_CODE))?, + ENGINE_NEW_PAYLOAD_V5 => get_param::>(params, 0) + .map(|jep| JsonExecutionPayload::V5(jep)) + .map_err(|s| (s, BAD_PARAMS_ERROR_CODE))?, _ => unreachable!(), }; @@ -222,6 +226,56 @@ pub async fn handle_rpc( )); } } + ForkName::Fulu => { + if method == ENGINE_NEW_PAYLOAD_V1 + || method == ENGINE_NEW_PAYLOAD_V2 + || method == ENGINE_NEW_PAYLOAD_V3 + // TODO(fulu): Uncomment this once v5 method is ready for Fulu + // || method == ENGINE_NEW_PAYLOAD_V4 + { + return Err(( + format!("{} called after Fulu fork!", method), + GENERIC_ERROR_CODE, + )); + } + if matches!(request, JsonExecutionPayload::V1(_)) { + return Err(( + format!( + "{} called with `ExecutionPayloadV1` after Fulu fork!", + method + ), + GENERIC_ERROR_CODE, + )); + } + if matches!(request, JsonExecutionPayload::V2(_)) { + return Err(( + format!( + "{} called with `ExecutionPayloadV2` after Fulu fork!", + method + ), + GENERIC_ERROR_CODE, + )); + } + if matches!(request, JsonExecutionPayload::V3(_)) { + return Err(( + format!( + "{} called with `ExecutionPayloadV3` after Fulu fork!", + method + ), + GENERIC_ERROR_CODE, + )); + } + // TODO(fulu): remove once we switch to v5 + // if matches!(request, JsonExecutionPayload::V4(_)) { + // return Err(( + // format!( + // "{} called with `ExecutionPayloadV4` after Fulu fork!", + // method + // ), + // GENERIC_ERROR_CODE, + // )); + // } + } _ => unreachable!(), }; @@ -260,7 +314,8 @@ pub async fn handle_rpc( ENGINE_GET_PAYLOAD_V1 | ENGINE_GET_PAYLOAD_V2 | ENGINE_GET_PAYLOAD_V3 - | ENGINE_GET_PAYLOAD_V4 => { + | ENGINE_GET_PAYLOAD_V4 + | ENGINE_GET_PAYLOAD_V5 => { let request: JsonPayloadIdRequest = get_param(params, 0).map_err(|s| (s, BAD_PARAMS_ERROR_CODE))?; let id = request.into(); @@ -320,6 +375,24 @@ pub async fn handle_rpc( )); } + // validate method called correctly according to fulu fork time + if ctx + .execution_block_generator + .read() + .get_fork_at_timestamp(response.timestamp()) + == ForkName::Fulu + && (method == ENGINE_GET_PAYLOAD_V1 + || method == ENGINE_GET_PAYLOAD_V2 + || method == ENGINE_GET_PAYLOAD_V3) + // TODO(fulu): Uncomment this once v5 method is ready for Fulu + // || method == ENGINE_GET_PAYLOAD_V4) + { + return Err(( + format!("{} called after Fulu fork!", method), + FORK_REQUEST_MISMATCH_ERROR_CODE, + )); + } + match method { ENGINE_GET_PAYLOAD_V1 => { Ok(serde_json::to_value(JsonExecutionPayload::from(response)).unwrap()) @@ -378,6 +451,40 @@ pub async fn handle_rpc( }) .unwrap() } + // TODO(fulu): remove this once we switch to v5 method + JsonExecutionPayload::V5(execution_payload) => { + serde_json::to_value(JsonGetPayloadResponseV5 { + execution_payload, + block_value: Uint256::from(DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI), + blobs_bundle: maybe_blobs + .ok_or(( + "No blobs returned despite V5 Payload".to_string(), + GENERIC_ERROR_CODE, + ))? + .into(), + should_override_builder: false, + execution_requests: Default::default(), + }) + .unwrap() + } + _ => unreachable!(), + }), + ENGINE_GET_PAYLOAD_V5 => Ok(match JsonExecutionPayload::from(response) { + JsonExecutionPayload::V5(execution_payload) => { + serde_json::to_value(JsonGetPayloadResponseV5 { + execution_payload, + block_value: Uint256::from(DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI), + blobs_bundle: maybe_blobs + .ok_or(( + "No blobs returned despite V5 Payload".to_string(), + GENERIC_ERROR_CODE, + ))? + .into(), + should_override_builder: false, + execution_requests: Default::default(), + }) + .unwrap() + } _ => unreachable!(), }), _ => unreachable!(), @@ -411,7 +518,10 @@ pub async fn handle_rpc( .map(|opt| opt.map(JsonPayloadAttributes::V1)) .transpose() } - ForkName::Capella | ForkName::Deneb | ForkName::Electra => { + ForkName::Capella + | ForkName::Deneb + | ForkName::Electra + | ForkName::Fulu => { get_param::>(params, 1) .map(|opt| opt.map(JsonPayloadAttributes::V2)) .transpose() @@ -475,7 +585,7 @@ pub async fn handle_rpc( )); } } - ForkName::Deneb | ForkName::Electra => { + ForkName::Deneb | ForkName::Electra | ForkName::Fulu => { if method == ENGINE_FORKCHOICE_UPDATED_V1 { return Err(( format!("{} called after Deneb fork!", method), diff --git a/beacon_node/execution_layer/src/test_utils/mock_builder.rs b/beacon_node/execution_layer/src/test_utils/mock_builder.rs index 879b54eb07..3540909fe4 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_builder.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_builder.rs @@ -1,10 +1,15 @@ use crate::test_utils::{DEFAULT_BUILDER_PAYLOAD_VALUE_WEI, DEFAULT_JWT_SECRET}; use crate::{Config, ExecutionLayer, PayloadAttributes, PayloadParameters}; -use eth2::types::{BlobsBundle, BlockId, StateId, ValidatorId}; +use eth2::types::PublishBlockRequest; +use eth2::types::{ + BlobsBundle, BlockId, BroadcastValidation, EventKind, EventTopic, FullPayloadContents, + ProposerData, StateId, ValidatorId, +}; use eth2::{BeaconNodeHttpClient, Timeouts, CONSENSUS_VERSION_HEADER}; use fork_choice::ForkchoiceUpdateParameters; use parking_lot::RwLock; use sensitive_url::SensitiveUrl; +use slog::{debug, error, info, warn, Logger}; use std::collections::HashMap; use std::fmt::Debug; use std::future::Future; @@ -13,20 +18,26 @@ use std::sync::Arc; use std::time::Duration; use task_executor::TaskExecutor; use tempfile::NamedTempFile; +use tokio_stream::StreamExt; use tree_hash::TreeHash; use types::builder_bid::{ BuilderBid, BuilderBidBellatrix, BuilderBidCapella, BuilderBidDeneb, BuilderBidElectra, - SignedBuilderBid, + BuilderBidFulu, SignedBuilderBid, }; use types::{ - Address, BeaconState, ChainSpec, EthSpec, ExecPayload, ExecutionPayload, - ExecutionPayloadHeaderRefMut, ExecutionRequests, FixedBytesExtended, ForkName, - ForkVersionedResponse, Hash256, PublicKeyBytes, Signature, SignedBlindedBeaconBlock, - SignedRoot, SignedValidatorRegistrationData, Slot, Uint256, + Address, BeaconState, ChainSpec, Epoch, EthSpec, ExecPayload, ExecutionPayload, + ExecutionPayloadHeaderRefMut, ExecutionRequests, ForkName, ForkVersionedResponse, Hash256, + PublicKeyBytes, Signature, SignedBlindedBeaconBlock, SignedRoot, + SignedValidatorRegistrationData, Slot, Uint256, }; use types::{ExecutionBlockHash, SecretKey}; use warp::{Filter, Rejection}; +pub const DEFAULT_FEE_RECIPIENT: Address = Address::repeat_byte(42); +pub const DEFAULT_GAS_LIMIT: u64 = 30_000_000; +pub const DEFAULT_BUILDER_PRIVATE_KEY: &str = + "607a11b45a7219cc61a3d9c5fd08c7eebd602a6a19a977f8d3771d5711a550f2"; + #[derive(Clone)] pub enum Operation { FeeRecipient(Address), @@ -95,6 +106,9 @@ impl BidStuff for BuilderBid { ExecutionPayloadHeaderRefMut::Electra(header) => { header.fee_recipient = fee_recipient; } + ExecutionPayloadHeaderRefMut::Fulu(header) => { + header.fee_recipient = fee_recipient; + } } } @@ -112,6 +126,9 @@ impl BidStuff for BuilderBid { ExecutionPayloadHeaderRefMut::Electra(header) => { header.gas_limit = gas_limit; } + ExecutionPayloadHeaderRefMut::Fulu(header) => { + header.gas_limit = gas_limit; + } } } @@ -133,6 +150,9 @@ impl BidStuff for BuilderBid { ExecutionPayloadHeaderRefMut::Electra(header) => { header.parent_hash = ExecutionBlockHash::from_root(parent_hash); } + ExecutionPayloadHeaderRefMut::Fulu(header) => { + header.parent_hash = ExecutionBlockHash::from_root(parent_hash); + } } } @@ -150,6 +170,9 @@ impl BidStuff for BuilderBid { ExecutionPayloadHeaderRefMut::Electra(header) => { header.prev_randao = prev_randao; } + ExecutionPayloadHeaderRefMut::Fulu(header) => { + header.prev_randao = prev_randao; + } } } @@ -167,6 +190,9 @@ impl BidStuff for BuilderBid { ExecutionPayloadHeaderRefMut::Electra(header) => { header.block_number = block_number; } + ExecutionPayloadHeaderRefMut::Fulu(header) => { + header.block_number = block_number; + } } } @@ -184,6 +210,9 @@ impl BidStuff for BuilderBid { ExecutionPayloadHeaderRefMut::Electra(header) => { header.timestamp = timestamp; } + ExecutionPayloadHeaderRefMut::Fulu(header) => { + header.timestamp = timestamp; + } } } @@ -201,6 +230,9 @@ impl BidStuff for BuilderBid { ExecutionPayloadHeaderRefMut::Electra(header) => { header.withdrawals_root = withdrawals_root; } + ExecutionPayloadHeaderRefMut::Fulu(header) => { + header.withdrawals_root = withdrawals_root; + } } } @@ -230,10 +262,25 @@ impl BidStuff for BuilderBid { header.extra_data = extra_data; header.block_hash = ExecutionBlockHash::from_root(header.tree_hash_root()); } + ExecutionPayloadHeaderRefMut::Fulu(header) => { + header.extra_data = extra_data; + header.block_hash = ExecutionBlockHash::from_root(header.tree_hash_root()); + } } } } +// Non referenced version of `PayloadParameters` +#[derive(Clone)] +pub struct PayloadParametersCloned { + pub parent_hash: ExecutionBlockHash, + pub parent_gas_limit: u64, + pub proposer_gas_limit: Option, + pub payload_attributes: PayloadAttributes, + pub forkchoice_update_params: ForkchoiceUpdateParameters, + pub current_fork: ForkName, +} + #[derive(Clone)] pub struct MockBuilder { el: ExecutionLayer, @@ -243,6 +290,20 @@ pub struct MockBuilder { builder_sk: SecretKey, operations: Arc>>, invalidate_signatures: Arc>, + genesis_time: Option, + /// Only returns bids for registered validators if set to true. `true` by default. + validate_pubkey: bool, + /// Do not apply any operations if set to `false`. + /// Applying operations might modify the cached header in the execution layer. + /// Use this if you want get_header to return a valid bid that can be eventually submitted as + /// a valid block. + apply_operations: bool, + payload_id_cache: Arc>>, + /// If set to `true`, sets the bid returned by `get_header` to Uint256::MAX + max_bid: bool, + /// A cache that stores the proposers index for a given epoch + proposers_cache: Arc>>>, + log: Logger, } impl MockBuilder { @@ -270,7 +331,12 @@ impl MockBuilder { let builder = MockBuilder::new( el, BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(Duration::from_secs(1))), + true, + true, + false, spec, + None, + executor.log().clone(), ); let host: Ipv4Addr = Ipv4Addr::LOCALHOST; let port = 0; @@ -278,21 +344,47 @@ impl MockBuilder { (builder, server) } + #[allow(clippy::too_many_arguments)] pub fn new( el: ExecutionLayer, beacon_client: BeaconNodeHttpClient, + validate_pubkey: bool, + apply_operations: bool, + max_bid: bool, spec: Arc, + sk: Option<&[u8]>, + log: Logger, ) -> Self { - let sk = SecretKey::random(); + let builder_sk = if let Some(sk_bytes) = sk { + match SecretKey::deserialize(sk_bytes) { + Ok(sk) => sk, + Err(_) => { + error!( + log, + "Invalid sk_bytes provided, generating random secret key" + ); + SecretKey::random() + } + } + } else { + SecretKey::deserialize(&hex::decode(DEFAULT_BUILDER_PRIVATE_KEY).unwrap()).unwrap() + }; Self { el, beacon_client, // Should keep spec and context consistent somehow spec, val_registration_cache: Arc::new(RwLock::new(HashMap::new())), - builder_sk: sk, + builder_sk, + validate_pubkey, operations: Arc::new(RwLock::new(vec![])), invalidate_signatures: Arc::new(RwLock::new(false)), + payload_id_cache: Arc::new(RwLock::new(HashMap::new())), + proposers_cache: Arc::new(RwLock::new(HashMap::new())), + apply_operations, + max_bid, + genesis_time: None, + log, } } @@ -317,8 +409,523 @@ impl MockBuilder { } bid.stamp_payload(); } + + /// Return the public key of the builder + pub fn public_key(&self) -> PublicKeyBytes { + self.builder_sk.public_key().compress() + } + + pub async fn register_validators( + &self, + registrations: Vec, + ) -> Result<(), String> { + info!( + self.log, + "Registering validators"; + "count" => registrations.len(), + ); + for registration in registrations { + if !registration.verify_signature(&self.spec) { + error!( + self.log, + "Failed to register validator"; + "error" => "invalid signature", + "validator" => %registration.message.pubkey + ); + return Err("invalid signature".to_string()); + } + self.val_registration_cache + .write() + .insert(registration.message.pubkey, registration); + } + Ok(()) + } + + pub async fn submit_blinded_block( + &self, + block: SignedBlindedBeaconBlock, + ) -> Result, String> { + let root = match &block { + SignedBlindedBeaconBlock::Base(_) | types::SignedBeaconBlock::Altair(_) => { + return Err("invalid fork".to_string()); + } + SignedBlindedBeaconBlock::Bellatrix(block) => { + block.message.body.execution_payload.tree_hash_root() + } + SignedBlindedBeaconBlock::Capella(block) => { + block.message.body.execution_payload.tree_hash_root() + } + SignedBlindedBeaconBlock::Deneb(block) => { + block.message.body.execution_payload.tree_hash_root() + } + SignedBlindedBeaconBlock::Electra(block) => { + block.message.body.execution_payload.tree_hash_root() + } + SignedBlindedBeaconBlock::Fulu(block) => { + block.message.body.execution_payload.tree_hash_root() + } + }; + info!( + self.log, + "Submitting blinded beacon block to builder"; + "block_hash" => %root + ); + let payload = self + .el + .get_payload_by_root(&root) + .ok_or_else(|| "missing payload for tx root".to_string())?; + + let (payload, blobs) = payload.deconstruct(); + let full_block = block + .try_into_full_block(Some(payload.clone())) + .ok_or("Internal error, just provided a payload")?; + debug!( + self.log, + "Got full payload, sending to local beacon node for propagation"; + "txs_count" => payload.transactions().len(), + "blob_count" => blobs.as_ref().map(|b| b.commitments.len()) + ); + let publish_block_request = PublishBlockRequest::new( + Arc::new(full_block), + blobs.clone().map(|b| (b.proofs, b.blobs)), + ); + self.beacon_client + .post_beacon_blocks_v2(&publish_block_request, Some(BroadcastValidation::Gossip)) + .await + .map_err(|e| format!("Failed to post blinded block {:?}", e))?; + Ok(FullPayloadContents::new(payload, blobs)) + } + + pub async fn get_header( + &self, + slot: Slot, + parent_hash: ExecutionBlockHash, + pubkey: PublicKeyBytes, + ) -> Result, String> { + info!(self.log, "In get_header"); + // Check if the pubkey has registered with the builder if required + if self.validate_pubkey && !self.val_registration_cache.read().contains_key(&pubkey) { + return Err("validator not registered with builder".to_string()); + } + let payload_parameters = { + let mut guard = self.payload_id_cache.write(); + guard.remove(&parent_hash) + }; + + let payload_parameters = match payload_parameters { + Some(params) => params, + None => { + warn!( + self.log, + "Payload params not cached for parent_hash {}", parent_hash + ); + self.get_payload_params(slot, None, pubkey, None).await? + } + }; + + info!(self.log, "Got payload params"); + + let fork = self.fork_name_at_slot(slot); + let payload_response_type = self + .el + .get_full_payload_caching(PayloadParameters { + parent_hash: payload_parameters.parent_hash, + parent_gas_limit: payload_parameters.parent_gas_limit, + proposer_gas_limit: payload_parameters.proposer_gas_limit, + payload_attributes: &payload_parameters.payload_attributes, + forkchoice_update_params: &payload_parameters.forkchoice_update_params, + current_fork: payload_parameters.current_fork, + }) + .await + .map_err(|e| format!("couldn't get payload {:?}", e))?; + + info!(self.log, "Got payload message, fork {}", fork); + + let mut message = match payload_response_type { + crate::GetPayloadResponseType::Full(payload_response) => { + #[allow(clippy::type_complexity)] + let (payload, value, maybe_blobs_bundle, maybe_requests): ( + ExecutionPayload, + Uint256, + Option>, + Option>, + ) = payload_response.into(); + + match fork { + ForkName::Fulu => BuilderBid::Fulu(BuilderBidFulu { + header: payload + .as_fulu() + .map_err(|_| "incorrect payload variant".to_string())? + .into(), + blob_kzg_commitments: maybe_blobs_bundle + .map(|b| b.commitments) + .unwrap_or_default(), + value: self.get_bid_value(value), + pubkey: self.builder_sk.public_key().compress(), + execution_requests: maybe_requests.unwrap_or_default(), + }), + ForkName::Electra => BuilderBid::Electra(BuilderBidElectra { + header: payload + .as_electra() + .map_err(|_| "incorrect payload variant".to_string())? + .into(), + blob_kzg_commitments: maybe_blobs_bundle + .map(|b| b.commitments) + .unwrap_or_default(), + value: self.get_bid_value(value), + pubkey: self.builder_sk.public_key().compress(), + execution_requests: maybe_requests.unwrap_or_default(), + }), + ForkName::Deneb => BuilderBid::Deneb(BuilderBidDeneb { + header: payload + .as_deneb() + .map_err(|_| "incorrect payload variant".to_string())? + .into(), + blob_kzg_commitments: maybe_blobs_bundle + .map(|b| b.commitments) + .unwrap_or_default(), + value: self.get_bid_value(value), + pubkey: self.builder_sk.public_key().compress(), + }), + ForkName::Capella => BuilderBid::Capella(BuilderBidCapella { + header: payload + .as_capella() + .map_err(|_| "incorrect payload variant".to_string())? + .into(), + value: self.get_bid_value(value), + pubkey: self.builder_sk.public_key().compress(), + }), + ForkName::Bellatrix => BuilderBid::Bellatrix(BuilderBidBellatrix { + header: payload + .as_bellatrix() + .map_err(|_| "incorrect payload variant".to_string())? + .into(), + value: self.get_bid_value(value), + pubkey: self.builder_sk.public_key().compress(), + }), + ForkName::Base | ForkName::Altair => return Err("invalid fork".to_string()), + } + } + _ => panic!("just requested full payload, cannot get blinded"), + }; + + if self.apply_operations { + info!(self.log, "Applying operations"); + self.apply_operations(&mut message); + } + info!(self.log, "Signing builder message"); + + let mut signature = message.sign_builder_message(&self.builder_sk, &self.spec); + + if *self.invalidate_signatures.read() { + signature = Signature::empty(); + }; + let signed_bid = SignedBuilderBid { message, signature }; + info!(self.log, "Builder bid {:?}", &signed_bid.message.value()); + Ok(signed_bid) + } + + fn fork_name_at_slot(&self, slot: Slot) -> ForkName { + self.spec.fork_name_at_slot::(slot) + } + + fn get_bid_value(&self, value: Uint256) -> Uint256 { + if self.max_bid { + Uint256::MAX + } else if !self.apply_operations { + value + } else { + Uint256::from(DEFAULT_BUILDER_PAYLOAD_VALUE_WEI) + } + } + + /// Prepare the execution layer for payload creation every slot for the correct + /// proposer index + pub async fn prepare_execution_layer(&self) -> Result<(), String> { + info!( + self.log, + "Starting a task to prepare the execution layer"; + ); + let mut head_event_stream = self + .beacon_client + .get_events::(&[EventTopic::Head]) + .await + .map_err(|e| format!("Failed to get head event {:?}", e))?; + + while let Some(Ok(event)) = head_event_stream.next().await { + match event { + EventKind::Head(head) => { + debug!( + self.log, + "Got a new head event"; + "block_hash" => %head.block + ); + let next_slot = head.slot + 1; + // Find the next proposer index from the cached data or through a beacon api call + let epoch = next_slot.epoch(E::slots_per_epoch()); + let position_in_slot = next_slot.as_u64() % E::slots_per_epoch(); + let proposer_data = { + let proposers_opt = { + let proposers_cache = self.proposers_cache.read(); + proposers_cache.get(&epoch).cloned() + }; + match proposers_opt { + Some(proposers) => proposers + .get(position_in_slot as usize) + .expect("position in slot is max epoch size") + .clone(), + None => { + // make a call to the beacon api and populate the cache + let duties: Vec<_> = self + .beacon_client + .get_validator_duties_proposer(epoch) + .await + .map_err(|e| { + format!( + "Failed to get proposer duties for epoch: {}, {:?}", + epoch, e + ) + })? + .data; + let proposer_data = duties + .get(position_in_slot as usize) + .expect("position in slot is max epoch size") + .clone(); + self.proposers_cache.write().insert(epoch, duties); + proposer_data + } + } + }; + self.prepare_execution_layer_internal( + head.slot, + head.block, + proposer_data.validator_index, + proposer_data.pubkey, + ) + .await?; + } + e => { + warn!( + self.log, + "Got an unexpected event"; + "event" => %e.topic_name() + ); + } + } + } + Ok(()) + } + + async fn prepare_execution_layer_internal( + &self, + current_slot: Slot, + head_block_root: Hash256, + validator_index: u64, + pubkey: PublicKeyBytes, + ) -> Result<(), String> { + let next_slot = current_slot + 1; + let payload_parameters = self + .get_payload_params( + next_slot, + Some(head_block_root), + pubkey, + Some(validator_index), + ) + .await?; + + self.payload_id_cache + .write() + .insert(payload_parameters.parent_hash, payload_parameters); + Ok(()) + } + + /// Get the `PayloadParameters` for requesting an ExecutionPayload for `slot` + /// for the given `validator_index` and `pubkey`. + async fn get_payload_params( + &self, + slot: Slot, + head_block_root: Option, + pubkey: PublicKeyBytes, + validator_index: Option, + ) -> Result { + let fork = self.fork_name_at_slot(slot); + + let block_id = match head_block_root { + Some(block_root) => BlockId::Root(block_root), + None => BlockId::Head, + }; + let head = self + .beacon_client + .get_beacon_blocks::(block_id) + .await + .map_err(|_| "couldn't get head".to_string())? + .ok_or_else(|| "missing head block".to_string())? + .data; + + let head_block_root = head_block_root.unwrap_or(head.canonical_root()); + + let head_execution_payload = head + .message() + .body() + .execution_payload() + .map_err(|_| "pre-merge block".to_string())?; + let head_execution_hash = head_execution_payload.block_hash(); + let head_gas_limit = head_execution_payload.gas_limit(); + + let finalized_execution_hash = self + .beacon_client + .get_beacon_blocks::(BlockId::Finalized) + .await + .map_err(|_| "couldn't get finalized block".to_string())? + .ok_or_else(|| "missing finalized block".to_string())? + .data + .message() + .body() + .execution_payload() + .map_err(|_| "pre-merge block".to_string())? + .block_hash(); + + let justified_execution_hash = self + .beacon_client + .get_beacon_blocks::(BlockId::Justified) + .await + .map_err(|_| "couldn't get justified block".to_string())? + .ok_or_else(|| "missing justified block".to_string())? + .data + .message() + .body() + .execution_payload() + .map_err(|_| "pre-merge block".to_string())? + .block_hash(); + + let (fee_recipient, proposer_gas_limit) = + match self.val_registration_cache.read().get(&pubkey) { + Some(cached_data) => ( + cached_data.message.fee_recipient, + cached_data.message.gas_limit, + ), + None => { + warn!( + self.log, + "Validator not registered {}, using default fee recipient and gas limits", + pubkey + ); + (DEFAULT_FEE_RECIPIENT, DEFAULT_GAS_LIMIT) + } + }; + let slots_since_genesis = slot.as_u64() - self.spec.genesis_slot.as_u64(); + + let genesis_time = if let Some(genesis_time) = self.genesis_time { + genesis_time + } else { + self.beacon_client + .get_beacon_genesis() + .await + .map_err(|_| "couldn't get beacon genesis".to_string())? + .data + .genesis_time + }; + let timestamp = (slots_since_genesis * self.spec.seconds_per_slot) + genesis_time; + + let head_state: BeaconState = self + .beacon_client + .get_debug_beacon_states(StateId::Head) + .await + .map_err(|_| "couldn't get state".to_string())? + .ok_or_else(|| "missing state".to_string())? + .data; + + let prev_randao = head_state + .get_randao_mix(head_state.current_epoch()) + .map_err(|_| "couldn't get prev randao".to_string())?; + + let expected_withdrawals = if fork.capella_enabled() { + Some( + self.beacon_client + .get_expected_withdrawals(&StateId::Head) + .await + .map_err(|e| format!("Failed to get expected withdrawals: {:?}", e))? + .data, + ) + } else { + None + }; + + let payload_attributes = match fork { + // the withdrawals root is filled in by operations, but we supply the valid withdrawals + // first to avoid polluting the execution block generator with invalid payload attributes + // NOTE: this was part of an effort to add payload attribute uniqueness checks, + // which was abandoned because it broke too many tests in subtle ways. + ForkName::Bellatrix | ForkName::Capella => PayloadAttributes::new( + timestamp, + *prev_randao, + fee_recipient, + expected_withdrawals, + None, + ), + ForkName::Deneb | ForkName::Electra | ForkName::Fulu => PayloadAttributes::new( + timestamp, + *prev_randao, + fee_recipient, + expected_withdrawals, + Some(head_block_root), + ), + ForkName::Base | ForkName::Altair => { + return Err("invalid fork".to_string()); + } + }; + + // Tells the execution layer that the `validator_index` is expected to propose + // a block on top of `head_block_root` for the given slot + let val_index = validator_index.unwrap_or( + self.beacon_client + .get_beacon_states_validator_id(StateId::Head, &ValidatorId::PublicKey(pubkey)) + .await + .map_err(|_| "couldn't get validator".to_string())? + .ok_or_else(|| "missing validator".to_string())? + .data + .index, + ); + + self.el + .insert_proposer(slot, head_block_root, val_index, payload_attributes.clone()) + .await; + + let forkchoice_update_params = ForkchoiceUpdateParameters { + head_hash: Some(head_execution_hash), + finalized_hash: Some(finalized_execution_hash), + justified_hash: Some(justified_execution_hash), + head_root: head_block_root, + }; + + let _status = self + .el + .notify_forkchoice_updated( + head_execution_hash, + justified_execution_hash, + finalized_execution_hash, + slot - 1, + head_block_root, + ) + .await + .map_err(|e| format!("fcu call failed : {:?}", e))?; + + let payload_parameters = PayloadParametersCloned { + parent_hash: head_execution_hash, + parent_gas_limit: head_gas_limit, + proposer_gas_limit: Some(proposer_gas_limit), + payload_attributes, + forkchoice_update_params, + current_fork: fork, + }; + Ok(payload_parameters) + } } +/// Serve the builder api using warp. Uses the functions defined in `MockBuilder` to serve +/// the requests. +/// +/// We should eventually move this to axum when we move everything else. pub fn serve( listen_addr: Ipv4Addr, listen_port: u16, @@ -337,19 +944,16 @@ pub fn serve( .and(warp::path::end()) .and(ctx_filter.clone()) .and_then( - |registrations: Vec, builder: MockBuilder| async move { - for registration in registrations { - if !registration.verify_signature(&builder.spec) { - return Err(reject("invalid signature")); - } - builder - .val_registration_cache - .write() - .insert(registration.message.pubkey, registration); - } - Ok(warp::reply()) + |registrations: Vec, + builder: MockBuilder| async move { + builder + .register_validators(registrations) + .await + .map_err(|e| warp::reject::custom(Custom(e)))?; + Ok::<_, Rejection>(warp::reply()) }, - ); + ) + .boxed(); let blinded_block = prefix @@ -362,27 +966,10 @@ pub fn serve( |block: SignedBlindedBeaconBlock, fork_name: ForkName, builder: MockBuilder| async move { - let root = match block { - SignedBlindedBeaconBlock::Base(_) | types::SignedBeaconBlock::Altair(_) => { - return Err(reject("invalid fork")); - } - SignedBlindedBeaconBlock::Bellatrix(block) => { - block.message.body.execution_payload.tree_hash_root() - } - SignedBlindedBeaconBlock::Capella(block) => { - block.message.body.execution_payload.tree_hash_root() - } - SignedBlindedBeaconBlock::Deneb(block) => { - block.message.body.execution_payload.tree_hash_root() - } - SignedBlindedBeaconBlock::Electra(block) => { - block.message.body.execution_payload.tree_hash_root() - } - }; let payload = builder - .el - .get_payload_by_root(&root) - .ok_or_else(|| reject("missing payload for tx root"))?; + .submit_blinded_block(block) + .await + .map_err(|e| warp::reject::custom(Custom(e)))?; let resp: ForkVersionedResponse<_> = ForkVersionedResponse { version: Some(fork_name), metadata: Default::default(), @@ -425,283 +1012,12 @@ pub fn serve( parent_hash: ExecutionBlockHash, pubkey: PublicKeyBytes, builder: MockBuilder| async move { - let fork = builder.spec.fork_name_at_slot::(slot); - let signed_cached_data = builder - .val_registration_cache - .read() - .get(&pubkey) - .ok_or_else(|| reject("missing registration"))? - .clone(); - let cached_data = signed_cached_data.message; - - let head = builder - .beacon_client - .get_beacon_blocks::(BlockId::Head) + let fork_name = builder.fork_name_at_slot(slot); + let signed_bid = builder + .get_header(slot, parent_hash, pubkey) .await - .map_err(|_| reject("couldn't get head"))? - .ok_or_else(|| reject("missing head block"))?; + .map_err(|e| warp::reject::custom(Custom(e)))?; - let block = head.data.message(); - let head_block_root = block.tree_hash_root(); - let head_execution_payload = block - .body() - .execution_payload() - .map_err(|_| reject("pre-merge block"))?; - let head_execution_hash = head_execution_payload.block_hash(); - let head_gas_limit = head_execution_payload.gas_limit(); - if head_execution_hash != parent_hash { - return Err(reject("head mismatch")); - } - - let finalized_execution_hash = builder - .beacon_client - .get_beacon_blocks::(BlockId::Finalized) - .await - .map_err(|_| reject("couldn't get finalized block"))? - .ok_or_else(|| reject("missing finalized block"))? - .data - .message() - .body() - .execution_payload() - .map_err(|_| reject("pre-merge block"))? - .block_hash(); - - let justified_execution_hash = builder - .beacon_client - .get_beacon_blocks::(BlockId::Justified) - .await - .map_err(|_| reject("couldn't get justified block"))? - .ok_or_else(|| reject("missing justified block"))? - .data - .message() - .body() - .execution_payload() - .map_err(|_| reject("pre-merge block"))? - .block_hash(); - - let val_index = builder - .beacon_client - .get_beacon_states_validator_id(StateId::Head, &ValidatorId::PublicKey(pubkey)) - .await - .map_err(|_| reject("couldn't get validator"))? - .ok_or_else(|| reject("missing validator"))? - .data - .index; - let fee_recipient = cached_data.fee_recipient; - let slots_since_genesis = slot.as_u64() - builder.spec.genesis_slot.as_u64(); - - let genesis_data = builder - .beacon_client - .get_beacon_genesis() - .await - .map_err(|_| reject("couldn't get beacon genesis"))? - .data; - let genesis_time = genesis_data.genesis_time; - let timestamp = - (slots_since_genesis * builder.spec.seconds_per_slot) + genesis_time; - - let head_state: BeaconState = builder - .beacon_client - .get_debug_beacon_states(StateId::Head) - .await - .map_err(|_| reject("couldn't get state"))? - .ok_or_else(|| reject("missing state"))? - .data; - let prev_randao = head_state - .get_randao_mix(head_state.current_epoch()) - .map_err(|_| reject("couldn't get prev randao"))?; - - let expected_withdrawals = if fork.capella_enabled() { - Some( - builder - .beacon_client - .get_expected_withdrawals(&StateId::Head) - .await - .unwrap() - .data, - ) - } else { - None - }; - - let payload_attributes = match fork { - // the withdrawals root is filled in by operations, but we supply the valid withdrawals - // first to avoid polluting the execution block generator with invalid payload attributes - // NOTE: this was part of an effort to add payload attribute uniqueness checks, - // which was abandoned because it broke too many tests in subtle ways. - ForkName::Bellatrix | ForkName::Capella => PayloadAttributes::new( - timestamp, - *prev_randao, - fee_recipient, - expected_withdrawals, - None, - ), - ForkName::Deneb | ForkName::Electra => PayloadAttributes::new( - timestamp, - *prev_randao, - fee_recipient, - expected_withdrawals, - Some(head_block_root), - ), - ForkName::Base | ForkName::Altair => { - return Err(reject("invalid fork")); - } - }; - - builder - .el - .insert_proposer(slot, head_block_root, val_index, payload_attributes.clone()) - .await; - - let forkchoice_update_params = ForkchoiceUpdateParameters { - head_root: Hash256::zero(), - head_hash: None, - justified_hash: Some(justified_execution_hash), - finalized_hash: Some(finalized_execution_hash), - }; - - let proposer_gas_limit = builder - .val_registration_cache - .read() - .get(&pubkey) - .map(|v| v.message.gas_limit); - - let payload_parameters = PayloadParameters { - parent_hash: head_execution_hash, - parent_gas_limit: head_gas_limit, - proposer_gas_limit, - payload_attributes: &payload_attributes, - forkchoice_update_params: &forkchoice_update_params, - current_fork: fork, - }; - - let payload_response_type = builder - .el - .get_full_payload_caching(payload_parameters) - .await - .map_err(|_| reject("couldn't get payload"))?; - - let mut message = match payload_response_type { - crate::GetPayloadResponseType::Full(payload_response) => { - #[allow(clippy::type_complexity)] - let (payload, _block_value, maybe_blobs_bundle, _maybe_requests): ( - ExecutionPayload, - Uint256, - Option>, - Option>, - ) = payload_response.into(); - - match fork { - ForkName::Electra => BuilderBid::Electra(BuilderBidElectra { - header: payload - .as_electra() - .map_err(|_| reject("incorrect payload variant"))? - .into(), - blob_kzg_commitments: maybe_blobs_bundle - .map(|b| b.commitments) - .unwrap_or_default(), - value: Uint256::from(DEFAULT_BUILDER_PAYLOAD_VALUE_WEI), - pubkey: builder.builder_sk.public_key().compress(), - }), - ForkName::Deneb => BuilderBid::Deneb(BuilderBidDeneb { - header: payload - .as_deneb() - .map_err(|_| reject("incorrect payload variant"))? - .into(), - blob_kzg_commitments: maybe_blobs_bundle - .map(|b| b.commitments) - .unwrap_or_default(), - value: Uint256::from(DEFAULT_BUILDER_PAYLOAD_VALUE_WEI), - pubkey: builder.builder_sk.public_key().compress(), - }), - ForkName::Capella => BuilderBid::Capella(BuilderBidCapella { - header: payload - .as_capella() - .map_err(|_| reject("incorrect payload variant"))? - .into(), - value: Uint256::from(DEFAULT_BUILDER_PAYLOAD_VALUE_WEI), - pubkey: builder.builder_sk.public_key().compress(), - }), - ForkName::Bellatrix => BuilderBid::Bellatrix(BuilderBidBellatrix { - header: payload - .as_bellatrix() - .map_err(|_| reject("incorrect payload variant"))? - .into(), - value: Uint256::from(DEFAULT_BUILDER_PAYLOAD_VALUE_WEI), - pubkey: builder.builder_sk.public_key().compress(), - }), - ForkName::Base | ForkName::Altair => { - return Err(reject("invalid fork")) - } - } - } - crate::GetPayloadResponseType::Blinded(payload_response) => { - #[allow(clippy::type_complexity)] - let (payload, _block_value, maybe_blobs_bundle, _maybe_requests): ( - ExecutionPayload, - Uint256, - Option>, - Option>, - ) = payload_response.into(); - match fork { - ForkName::Electra => BuilderBid::Electra(BuilderBidElectra { - header: payload - .as_electra() - .map_err(|_| reject("incorrect payload variant"))? - .into(), - blob_kzg_commitments: maybe_blobs_bundle - .map(|b| b.commitments) - .unwrap_or_default(), - value: Uint256::from(DEFAULT_BUILDER_PAYLOAD_VALUE_WEI), - pubkey: builder.builder_sk.public_key().compress(), - }), - ForkName::Deneb => BuilderBid::Deneb(BuilderBidDeneb { - header: payload - .as_deneb() - .map_err(|_| reject("incorrect payload variant"))? - .into(), - blob_kzg_commitments: maybe_blobs_bundle - .map(|b| b.commitments) - .unwrap_or_default(), - value: Uint256::from(DEFAULT_BUILDER_PAYLOAD_VALUE_WEI), - pubkey: builder.builder_sk.public_key().compress(), - }), - ForkName::Capella => BuilderBid::Capella(BuilderBidCapella { - header: payload - .as_capella() - .map_err(|_| reject("incorrect payload variant"))? - .into(), - value: Uint256::from(DEFAULT_BUILDER_PAYLOAD_VALUE_WEI), - pubkey: builder.builder_sk.public_key().compress(), - }), - ForkName::Bellatrix => BuilderBid::Bellatrix(BuilderBidBellatrix { - header: payload - .as_bellatrix() - .map_err(|_| reject("incorrect payload variant"))? - .into(), - value: Uint256::from(DEFAULT_BUILDER_PAYLOAD_VALUE_WEI), - pubkey: builder.builder_sk.public_key().compress(), - }), - ForkName::Base | ForkName::Altair => { - return Err(reject("invalid fork")) - } - } - } - }; - - builder.apply_operations(&mut message); - - let mut signature = - message.sign_builder_message(&builder.builder_sk, &builder.spec); - - if *builder.invalidate_signatures.read() { - signature = Signature::empty(); - } - - let fork_name = builder - .spec - .fork_name_at_epoch(slot.epoch(E::slots_per_epoch())); - let signed_bid = SignedBuilderBid { message, signature }; let resp: ForkVersionedResponse<_> = ForkVersionedResponse { version: Some(fork_name), metadata: Default::default(), diff --git a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs index 48372a39be..f45bfda9ff 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs @@ -13,7 +13,7 @@ pub struct MockExecutionLayer { pub server: MockServer, pub el: ExecutionLayer, pub executor: TaskExecutor, - pub spec: ChainSpec, + pub spec: Arc, } impl MockExecutionLayer { @@ -28,8 +28,9 @@ impl MockExecutionLayer { None, None, None, + None, Some(JwtKey::from_slice(&DEFAULT_JWT_SECRET).unwrap()), - spec, + Arc::new(spec), None, ) } @@ -41,8 +42,9 @@ impl MockExecutionLayer { shanghai_time: Option, cancun_time: Option, prague_time: Option, + osaka_time: Option, jwt_key: Option, - spec: ChainSpec, + spec: Arc, kzg: Option>, ) -> Self { let handle = executor.handle().unwrap(); @@ -57,6 +59,8 @@ impl MockExecutionLayer { shanghai_time, cancun_time, prague_time, + osaka_time, + spec.clone(), kzg, ); @@ -318,9 +322,9 @@ impl MockExecutionLayer { (self, block_hash) } - pub async fn with_terminal_block<'a, U, V>(self, func: U) -> Self + pub async fn with_terminal_block(self, func: U) -> Self where - U: Fn(ChainSpec, ExecutionLayer, Option) -> V, + U: Fn(Arc, ExecutionLayer, Option) -> V, V: Future, { let terminal_block_number = self diff --git a/beacon_node/execution_layer/src/test_utils/mod.rs b/beacon_node/execution_layer/src/test_utils/mod.rs index a390e33d76..c89c1d0533 100644 --- a/beacon_node/execution_layer/src/test_utils/mod.rs +++ b/beacon_node/execution_layer/src/test_utils/mod.rs @@ -21,7 +21,7 @@ use std::marker::PhantomData; use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; use std::sync::{Arc, LazyLock}; use tokio::{runtime, sync::oneshot}; -use types::{EthSpec, ExecutionBlockHash, Uint256}; +use types::{ChainSpec, EthSpec, ExecutionBlockHash, Uint256}; use warp::{http::StatusCode, Filter, Rejection}; use crate::EngineCapabilities; @@ -44,6 +44,7 @@ pub const DEFAULT_ENGINE_CAPABILITIES: EngineCapabilities = EngineCapabilities { new_payload_v2: true, new_payload_v3: true, new_payload_v4: true, + new_payload_v5: true, forkchoice_updated_v1: true, forkchoice_updated_v2: true, forkchoice_updated_v3: true, @@ -53,6 +54,7 @@ pub const DEFAULT_ENGINE_CAPABILITIES: EngineCapabilities = EngineCapabilities { get_payload_v2: true, get_payload_v3: true, get_payload_v4: true, + get_payload_v5: true, get_client_version_v1: true, get_blobs_v1: true, get_inclusion_list_v1: true, @@ -83,6 +85,7 @@ pub struct MockExecutionConfig { pub shanghai_time: Option, pub cancun_time: Option, pub prague_time: Option, + pub osaka_time: Option, } impl Default for MockExecutionConfig { @@ -96,6 +99,7 @@ impl Default for MockExecutionConfig { shanghai_time: None, cancun_time: None, prague_time: None, + osaka_time: None, } } } @@ -108,7 +112,7 @@ pub struct MockServer { } impl MockServer { - pub fn unit_testing() -> Self { + pub fn unit_testing(chain_spec: Arc) -> Self { Self::new( &runtime::Handle::current(), JwtKey::from_slice(&DEFAULT_JWT_SECRET).unwrap(), @@ -118,6 +122,8 @@ impl MockServer { None, // FIXME(capella): should this be the default? None, // FIXME(deneb): should this be the default? None, // FIXME(electra): should this be the default? + None, // FIXME(fulu): should this be the default? + chain_spec, None, ) } @@ -125,6 +131,7 @@ impl MockServer { pub fn new_with_config( handle: &runtime::Handle, config: MockExecutionConfig, + spec: Arc, kzg: Option>, ) -> Self { let MockExecutionConfig { @@ -136,6 +143,7 @@ impl MockServer { shanghai_time, cancun_time, prague_time, + osaka_time, } = config; let last_echo_request = Arc::new(RwLock::new(None)); let preloaded_responses = Arc::new(Mutex::new(vec![])); @@ -146,6 +154,8 @@ impl MockServer { shanghai_time, cancun_time, prague_time, + osaka_time, + spec, kzg, ); @@ -209,6 +219,8 @@ impl MockServer { shanghai_time: Option, cancun_time: Option, prague_time: Option, + osaka_time: Option, + spec: Arc, kzg: Option>, ) -> Self { Self::new_with_config( @@ -222,7 +234,9 @@ impl MockServer { shanghai_time, cancun_time, prague_time, + osaka_time, }, + spec, kzg, ) } diff --git a/beacon_node/genesis/Cargo.toml b/beacon_node/genesis/Cargo.toml index b01e6a6aea..eeca393947 100644 --- a/beacon_node/genesis/Cargo.toml +++ b/beacon_node/genesis/Cargo.toml @@ -9,16 +9,16 @@ eth1_test_rig = { workspace = true } sensitive_url = { workspace = true } [dependencies] -futures = { workspace = true } -types = { workspace = true } environment = { workspace = true } eth1 = { workspace = true } -rayon = { workspace = true } -state_processing = { workspace = true } -merkle_proof = { workspace = true } -ethereum_ssz = { workspace = true } ethereum_hashing = { workspace = true } -tree_hash = { workspace = true } -tokio = { workspace = true } -slog = { workspace = true } +ethereum_ssz = { workspace = true } +futures = { workspace = true } int_to_bytes = { workspace = true } +merkle_proof = { workspace = true } +rayon = { workspace = true } +slog = { workspace = true } +state_processing = { workspace = true } +tokio = { workspace = true } +tree_hash = { workspace = true } +types = { workspace = true } diff --git a/beacon_node/genesis/src/eth1_genesis_service.rs b/beacon_node/genesis/src/eth1_genesis_service.rs index 3981833a5c..b5f4bd50ee 100644 --- a/beacon_node/genesis/src/eth1_genesis_service.rs +++ b/beacon_node/genesis/src/eth1_genesis_service.rs @@ -270,7 +270,7 @@ impl Eth1GenesisService { // Ignore any block that has already been processed or update the highest processed // block. - if highest_processed_block.map_or(false, |highest| highest >= block.number) { + if highest_processed_block.is_some_and(|highest| highest >= block.number) { continue; } else { self.stats diff --git a/beacon_node/http_api/Cargo.toml b/beacon_node/http_api/Cargo.toml index 5d601008bc..2fb3ec06bf 100644 --- a/beacon_node/http_api/Cargo.toml +++ b/beacon_node/http_api/Cargo.toml @@ -11,12 +11,14 @@ beacon_processor = { workspace = true } bs58 = "0.4.0" bytes = { workspace = true } directory = { workspace = true } +either = { workspace = true } eth1 = { workspace = true } eth2 = { workspace = true } ethereum_serde_utils = { workspace = true } ethereum_ssz = { workspace = true } execution_layer = { workspace = true } futures = { workspace = true } +health_metrics = { workspace = true } hex = { workspace = true } lighthouse_network = { workspace = true } lighthouse_version = { workspace = true } @@ -30,6 +32,7 @@ rand = { workspace = true } safe_arith = { workspace = true } sensitive_url = { workspace = true } serde = { workspace = true } +serde_json = { workspace = true } slog = { workspace = true } slot_clock = { workspace = true } state_processing = { workspace = true } diff --git a/beacon_node/http_api/src/attestation_performance.rs b/beacon_node/http_api/src/attestation_performance.rs index d4f9916814..2f3f340445 100644 --- a/beacon_node/http_api/src/attestation_performance.rs +++ b/beacon_node/http_api/src/attestation_performance.rs @@ -7,7 +7,7 @@ use state_processing::{ }; use std::sync::Arc; use types::{BeaconState, BeaconStateError, EthSpec, Hash256}; -use warp_utils::reject::{beacon_chain_error, custom_bad_request, custom_server_error}; +use warp_utils::reject::{custom_bad_request, custom_server_error, unhandled_error}; const MAX_REQUEST_RANGE_EPOCHS: usize = 100; const BLOCK_ROOT_CHUNK_SIZE: usize = 100; @@ -50,7 +50,7 @@ pub fn get_attestation_performance( let end_slot = end_epoch.end_slot(T::EthSpec::slots_per_epoch()); // Ensure end_epoch is smaller than the current epoch - 1. - let current_epoch = chain.epoch().map_err(beacon_chain_error)?; + let current_epoch = chain.epoch().map_err(unhandled_error)?; if query.end_epoch >= current_epoch - 1 { return Err(custom_bad_request(format!( "end_epoch must be less than the current epoch - 1. current: {}, end: {}", @@ -83,7 +83,7 @@ pub fn get_attestation_performance( let index_range = if target.to_lowercase() == "global" { chain .with_head(|head| Ok((0..head.beacon_state.validators().len() as u64).collect())) - .map_err(beacon_chain_error)? + .map_err(unhandled_error::)? } else { vec![target.parse::().map_err(|_| { custom_bad_request(format!( @@ -96,10 +96,10 @@ pub fn get_attestation_performance( // Load block roots. let mut block_roots: Vec = chain .forwards_iter_block_roots_until(start_slot, end_slot) - .map_err(beacon_chain_error)? + .map_err(unhandled_error)? .map(|res| res.map(|(root, _)| root)) .collect::, _>>() - .map_err(beacon_chain_error)?; + .map_err(unhandled_error)?; block_roots.dedup(); // Load first block so we can get its parent. @@ -113,7 +113,7 @@ pub fn get_attestation_performance( .and_then(|maybe_block| { maybe_block.ok_or(BeaconChainError::MissingBeaconBlock(*first_block_root)) }) - .map_err(beacon_chain_error)?; + .map_err(unhandled_error)?; // Load the block of the prior slot which will be used to build the starting state. let prior_block = chain @@ -122,14 +122,14 @@ pub fn get_attestation_performance( maybe_block .ok_or_else(|| BeaconChainError::MissingBeaconBlock(first_block.parent_root())) }) - .map_err(beacon_chain_error)?; + .map_err(unhandled_error)?; // Load state for block replay. let state_root = prior_block.state_root(); let state = chain .get_state(&state_root, Some(prior_slot)) .and_then(|maybe_state| maybe_state.ok_or(BeaconChainError::MissingBeaconState(state_root))) - .map_err(beacon_chain_error)?; + .map_err(unhandled_error)?; // Allocate an AttestationPerformance vector for each validator in the range. let mut perfs: Vec = @@ -198,7 +198,7 @@ pub fn get_attestation_performance( .and_then(|maybe_block| { maybe_block.ok_or(BeaconChainError::MissingBeaconBlock(*root)) }) - .map_err(beacon_chain_error) + .map_err(unhandled_error) }) .collect::, _>>()?; diff --git a/beacon_node/http_api/src/attester_duties.rs b/beacon_node/http_api/src/attester_duties.rs index 6c7dc3348c..8905b24cde 100644 --- a/beacon_node/http_api/src/attester_duties.rs +++ b/beacon_node/http_api/src/attester_duties.rs @@ -16,9 +16,7 @@ pub fn attester_duties( request_indices: &[u64], chain: &BeaconChain, ) -> Result { - let current_epoch = chain - .epoch() - .map_err(warp_utils::reject::beacon_chain_error)?; + let current_epoch = chain.epoch().map_err(warp_utils::reject::unhandled_error)?; // Determine what the current epoch would be if we fast-forward our system clock by // `MAXIMUM_GOSSIP_CLOCK_DISPARITY`. @@ -57,7 +55,7 @@ fn cached_attestation_duties( let (duties, dependent_root, execution_status) = chain .validator_attestation_duties(request_indices, request_epoch, head_block_root) - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; convert_to_api_response( duties, @@ -82,7 +80,7 @@ fn compute_historic_attester_duties( let (cached_head, execution_status) = chain .canonical_head .head_and_execution_status() - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; let head = &cached_head.snapshot; if head.beacon_state.current_epoch() <= request_epoch { @@ -131,13 +129,13 @@ fn compute_historic_attester_duties( state .build_committee_cache(relative_epoch, &chain.spec) .map_err(BeaconChainError::from) - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; let dependent_root = state // The only block which decides its own shuffling is the genesis block. .attester_shuffling_decision_root(chain.genesis_block_root, relative_epoch) .map_err(BeaconChainError::from) - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; let duties = request_indices .iter() @@ -147,7 +145,7 @@ fn compute_historic_attester_duties( .map_err(BeaconChainError::from) }) .collect::>() - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; convert_to_api_response( duties, @@ -181,7 +179,7 @@ fn ensure_state_knows_attester_duties_for_epoch( // A "partial" state advance is adequate since attester duties don't rely on state roots. partial_state_advance(state, Some(state_root), target_slot, spec) .map_err(BeaconChainError::from) - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; } Ok(()) @@ -208,7 +206,7 @@ fn convert_to_api_response( let usize_indices = indices.iter().map(|i| *i as usize).collect::>(); let index_to_pubkey_map = chain .validator_pubkey_bytes_many(&usize_indices) - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; let data = duties .into_iter() diff --git a/beacon_node/http_api/src/block_id.rs b/beacon_node/http_api/src/block_id.rs index dba8eb1ef3..cdef1521ec 100644 --- a/beacon_node/http_api/src/block_id.rs +++ b/beacon_node/http_api/src/block_id.rs @@ -1,4 +1,5 @@ use crate::{state_id::checkpoint_slot_and_execution_optimistic, ExecutionOptimistic}; +use beacon_chain::kzg_utils::reconstruct_blobs; use beacon_chain::{BeaconChain, BeaconChainError, BeaconChainTypes, WhenSlotSkipped}; use eth2::types::BlobIndicesQuery; use eth2::types::BlockId as CoreBlockId; @@ -9,6 +10,7 @@ use types::{ BlobSidecarList, EthSpec, FixedBytesExtended, Hash256, SignedBeaconBlock, SignedBlindedBeaconBlock, Slot, }; +use warp::Rejection; /// Wraps `eth2::types::BlockId` and provides a simple way to obtain a block or root for a given /// `BlockId`. @@ -36,7 +38,7 @@ impl BlockId { let (cached_head, execution_status) = chain .canonical_head .head_and_execution_status() - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; Ok(( cached_head.head_block_root(), execution_status.is_optimistic_or_invalid(), @@ -61,10 +63,10 @@ impl BlockId { CoreBlockId::Slot(slot) => { let execution_optimistic = chain .is_optimistic_or_invalid_head() - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; let root = chain .block_root_at_slot(*slot, WhenSlotSkipped::None) - .map_err(warp_utils::reject::beacon_chain_error) + .map_err(warp_utils::reject::unhandled_error) .and_then(|root_opt| { root_opt.ok_or_else(|| { warp_utils::reject::custom_not_found(format!( @@ -94,17 +96,17 @@ impl BlockId { .store .block_exists(root) .map_err(BeaconChainError::DBError) - .map_err(warp_utils::reject::beacon_chain_error)? + .map_err(warp_utils::reject::unhandled_error)? { let execution_optimistic = chain .canonical_head .fork_choice_read_lock() .is_optimistic_or_invalid_block(root) .map_err(BeaconChainError::ForkChoiceError) - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; let blinded_block = chain .get_blinded_block(root) - .map_err(warp_utils::reject::beacon_chain_error)? + .map_err(warp_utils::reject::unhandled_error)? .ok_or_else(|| { warp_utils::reject::custom_not_found(format!( "beacon block with root {}", @@ -114,7 +116,7 @@ impl BlockId { let block_slot = blinded_block.slot(); let finalized = chain .is_finalized_block(root, block_slot) - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; Ok((*root, execution_optimistic, finalized)) } else { Err(warp_utils::reject::custom_not_found(format!( @@ -132,7 +134,7 @@ impl BlockId { ) -> Result>, warp::Rejection> { chain .get_blinded_block(root) - .map_err(warp_utils::reject::beacon_chain_error) + .map_err(warp_utils::reject::unhandled_error) } /// Return the `SignedBeaconBlock` identified by `self`. @@ -152,7 +154,7 @@ impl BlockId { let (cached_head, execution_status) = chain .canonical_head .head_and_execution_status() - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; Ok(( cached_head.snapshot.beacon_block.clone_as_blinded(), execution_status.is_optimistic_or_invalid(), @@ -209,7 +211,7 @@ impl BlockId { let (cached_head, execution_status) = chain .canonical_head .head_and_execution_status() - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; Ok(( cached_head.snapshot.beacon_block.clone(), execution_status.is_optimistic_or_invalid(), @@ -221,7 +223,7 @@ impl BlockId { chain .get_block(&root) .await - .map_err(warp_utils::reject::beacon_chain_error) + .map_err(warp_utils::reject::unhandled_error) .and_then(|block_opt| match block_opt { Some(block) => { if block.slot() != *slot { @@ -243,7 +245,7 @@ impl BlockId { chain .get_block(&root) .await - .map_err(warp_utils::reject::beacon_chain_error) + .map_err(warp_utils::reject::unhandled_error) .and_then(|block_opt| { block_opt .map(|block| (Arc::new(block), execution_optimistic, finalized)) @@ -261,7 +263,7 @@ impl BlockId { #[allow(clippy::type_complexity)] pub fn get_blinded_block_and_blob_list_filtered( &self, - indices: BlobIndicesQuery, + query: BlobIndicesQuery, chain: &BeaconChain, ) -> Result< ( @@ -285,37 +287,92 @@ impl BlockId { })?; // Return the `BlobSidecarList` identified by `self`. + let max_blobs_per_block = chain.spec.max_blobs_per_block(block.epoch()) as usize; let blob_sidecar_list = if !blob_kzg_commitments.is_empty() { - chain - .store - .get_blobs(&root) - .map_err(|e| warp_utils::reject::beacon_chain_error(e.into()))? - .ok_or_else(|| { - warp_utils::reject::custom_not_found(format!( - "no blobs stored for block {root}" - )) - })? + if chain.spec.is_peer_das_enabled_for_epoch(block.epoch()) { + Self::get_blobs_from_data_columns(chain, root, query.indices, &block)? + } else { + Self::get_blobs(chain, root, query.indices, max_blobs_per_block)? + } } else { - BlobSidecarList::default() + BlobSidecarList::new(vec![], max_blobs_per_block) + .map_err(|e| warp_utils::reject::custom_server_error(format!("{:?}", e)))? }; - let blob_sidecar_list_filtered = match indices.indices { + Ok((block, blob_sidecar_list, execution_optimistic, finalized)) + } + + fn get_blobs( + chain: &BeaconChain, + root: Hash256, + indices: Option>, + max_blobs_per_block: usize, + ) -> Result, Rejection> { + let blob_sidecar_list = chain + .store + .get_blobs(&root) + .map_err(|e| warp_utils::reject::unhandled_error(BeaconChainError::from(e)))? + .blobs() + .ok_or_else(|| { + warp_utils::reject::custom_not_found(format!("no blobs stored for block {root}")) + })?; + + let blob_sidecar_list_filtered = match indices { Some(vec) => { - let list = blob_sidecar_list + let list: Vec<_> = blob_sidecar_list .into_iter() .filter(|blob_sidecar| vec.contains(&blob_sidecar.index)) .collect(); - BlobSidecarList::new(list) + + BlobSidecarList::new(list, max_blobs_per_block) .map_err(|e| warp_utils::reject::custom_server_error(format!("{:?}", e)))? } None => blob_sidecar_list, }; - Ok(( - block, - blob_sidecar_list_filtered, - execution_optimistic, - finalized, - )) + + Ok(blob_sidecar_list_filtered) + } + + fn get_blobs_from_data_columns( + chain: &BeaconChain, + root: Hash256, + blob_indices: Option>, + block: &SignedBlindedBeaconBlock<::EthSpec>, + ) -> Result, Rejection> { + let column_indices = chain.store.get_data_column_keys(root).map_err(|e| { + warp_utils::reject::custom_server_error(format!( + "Error fetching data columns keys: {e:?}" + )) + })?; + + let num_found_column_keys = column_indices.len(); + let num_required_columns = chain.spec.number_of_columns / 2; + let is_blob_available = num_found_column_keys >= num_required_columns as usize; + + if is_blob_available { + let data_columns = column_indices + .into_iter() + .filter_map( + |column_index| match chain.get_data_column(&root, &column_index) { + Ok(Some(data_column)) => Some(Ok(data_column)), + Ok(None) => None, + Err(e) => Some(Err(warp_utils::reject::unhandled_error(e))), + }, + ) + .collect::, _>>()?; + + reconstruct_blobs(&chain.kzg, &data_columns, blob_indices, block, &chain.spec).map_err( + |e| { + warp_utils::reject::custom_server_error(format!( + "Error reconstructing data columns: {e:?}" + )) + }, + ) + } else { + Err(warp_utils::reject::custom_server_error( + format!("Insufficient data columns to reconstruct blobs: required {num_required_columns}, but only {num_found_column_keys} were found.") + )) + } } } diff --git a/beacon_node/http_api/src/block_packing_efficiency.rs b/beacon_node/http_api/src/block_packing_efficiency.rs index 66c7187278..431547f10b 100644 --- a/beacon_node/http_api/src/block_packing_efficiency.rs +++ b/beacon_node/http_api/src/block_packing_efficiency.rs @@ -13,7 +13,7 @@ use types::{ AttestationRef, BeaconCommittee, BeaconState, BeaconStateError, BlindedPayload, ChainSpec, Epoch, EthSpec, Hash256, OwnedBeaconCommittee, RelativeEpoch, SignedBeaconBlock, Slot, }; -use warp_utils::reject::{beacon_chain_error, custom_bad_request, custom_server_error}; +use warp_utils::reject::{custom_bad_request, custom_server_error, unhandled_error}; /// Load blocks from block roots in chunks to reduce load on memory. const BLOCK_ROOT_CHUNK_SIZE: usize = 100; @@ -263,9 +263,9 @@ pub fn get_block_packing_efficiency( // Load block roots. let mut block_roots: Vec = chain .forwards_iter_block_roots_until(start_slot_of_prior_epoch, end_slot) - .map_err(beacon_chain_error)? + .map_err(unhandled_error)? .collect::, _>>() - .map_err(beacon_chain_error)? + .map_err(unhandled_error)? .iter() .map(|(root, _)| *root) .collect(); @@ -280,7 +280,7 @@ pub fn get_block_packing_efficiency( .and_then(|maybe_block| { maybe_block.ok_or(BeaconChainError::MissingBeaconBlock(*first_block_root)) }) - .map_err(beacon_chain_error)?; + .map_err(unhandled_error)?; // Load state for block replay. let starting_state_root = first_block.state_root(); @@ -290,7 +290,7 @@ pub fn get_block_packing_efficiency( .and_then(|maybe_state| { maybe_state.ok_or(BeaconChainError::MissingBeaconState(starting_state_root)) }) - .map_err(beacon_chain_error)?; + .map_err(unhandled_error)?; // Initialize response vector. let mut response = Vec::new(); @@ -392,7 +392,7 @@ pub fn get_block_packing_efficiency( .and_then(|maybe_block| { maybe_block.ok_or(BeaconChainError::MissingBeaconBlock(*root)) }) - .map_err(beacon_chain_error) + .map_err(unhandled_error) }) .collect::, _>>()?; diff --git a/beacon_node/http_api/src/block_rewards.rs b/beacon_node/http_api/src/block_rewards.rs index ad71e9e9d0..0cc878bb48 100644 --- a/beacon_node/http_api/src/block_rewards.rs +++ b/beacon_node/http_api/src/block_rewards.rs @@ -7,7 +7,7 @@ use std::num::NonZeroUsize; use std::sync::Arc; use types::beacon_block::BlindedBeaconBlock; use types::non_zero_usize::new_non_zero_usize; -use warp_utils::reject::{beacon_chain_error, beacon_state_error, custom_bad_request}; +use warp_utils::reject::{beacon_state_error, custom_bad_request, unhandled_error}; const STATE_CACHE_SIZE: NonZeroUsize = new_non_zero_usize(2); @@ -30,23 +30,23 @@ pub fn get_block_rewards( let end_block_root = chain .block_root_at_slot(end_slot, WhenSlotSkipped::Prev) - .map_err(beacon_chain_error)? + .map_err(unhandled_error)? .ok_or_else(|| custom_bad_request(format!("block at end slot {} unknown", end_slot)))?; let blocks = chain .store .load_blocks_to_replay(start_slot, end_slot, end_block_root) - .map_err(|e| beacon_chain_error(e.into()))?; + .map_err(|e| unhandled_error(BeaconChainError::from(e)))?; let state_root = chain .state_root_at_slot(prior_slot) - .map_err(beacon_chain_error)? + .map_err(unhandled_error)? .ok_or_else(|| custom_bad_request(format!("prior state at slot {} unknown", prior_slot)))?; let mut state = chain .get_state(&state_root, Some(prior_slot)) .and_then(|maybe_state| maybe_state.ok_or(BeaconChainError::MissingBeaconState(state_root))) - .map_err(beacon_chain_error)?; + .map_err(unhandled_error)?; state .build_caches(&chain.spec) @@ -73,12 +73,12 @@ pub fn get_block_rewards( .state_root_iter( chain .forwards_iter_state_roots_until(prior_slot, end_slot) - .map_err(beacon_chain_error)?, + .map_err(unhandled_error)?, ) .no_signature_verification() .minimal_block_root_verification() .apply_blocks(blocks, None) - .map_err(beacon_chain_error)?; + .map_err(unhandled_error)?; if block_replayer.state_root_miss() { warn!( @@ -125,7 +125,7 @@ pub fn compute_block_rewards( ); let parent_block = chain .get_blinded_block(&parent_root) - .map_err(beacon_chain_error)? + .map_err(unhandled_error)? .ok_or_else(|| { custom_bad_request(format!( "parent block not known or not canonical: {:?}", @@ -135,7 +135,7 @@ pub fn compute_block_rewards( let parent_state = chain .get_state(&parent_block.state_root(), Some(parent_block.slot())) - .map_err(beacon_chain_error)? + .map_err(unhandled_error)? .ok_or_else(|| { custom_bad_request(format!( "no state known for parent block: {:?}", @@ -148,7 +148,7 @@ pub fn compute_block_rewards( .state_root_iter([Ok((parent_block.state_root(), parent_block.slot()))].into_iter()) .minimal_block_root_verification() .apply_blocks(vec![], Some(block.slot())) - .map_err(beacon_chain_error)?; + .map_err(unhandled_error::)?; if block_replayer.state_root_miss() { warn!( @@ -176,7 +176,7 @@ pub fn compute_block_rewards( &mut reward_cache, true, ) - .map_err(beacon_chain_error)?; + .map_err(unhandled_error)?; block_rewards.push(block_reward); } diff --git a/beacon_node/http_api/src/build_block_contents.rs b/beacon_node/http_api/src/build_block_contents.rs index c2ccb6695e..fb8fba0731 100644 --- a/beacon_node/http_api/src/build_block_contents.rs +++ b/beacon_node/http_api/src/build_block_contents.rs @@ -23,7 +23,7 @@ pub fn build_block_contents( } = block; let Some((kzg_proofs, blobs)) = blob_items else { - return Err(warp_utils::reject::block_production_error( + return Err(warp_utils::reject::unhandled_error( BlockProductionError::MissingBlobs, )); }; diff --git a/beacon_node/http_api/src/inclusion_list_duties.rs b/beacon_node/http_api/src/inclusion_list_duties.rs index 745f6b78cc..45e1f782e6 100644 --- a/beacon_node/http_api/src/inclusion_list_duties.rs +++ b/beacon_node/http_api/src/inclusion_list_duties.rs @@ -14,7 +14,9 @@ pub fn inclusion_list_duties( ) -> Result { let current_epoch = chain .epoch() - .map_err(warp_utils::reject::beacon_chain_error)?; + // TODO(focil) unwrap + .unwrap(); + // .map_err(warp_utils::reject::beacon_chain_error)?; // Determine what the current epoch would be if we fast-forward our system clock by // `MAXIMUM_GOSSIP_CLOCK_DISPARITY`. @@ -35,7 +37,9 @@ pub fn inclusion_list_duties( let head_block_root = chain.canonical_head.cached_head().head_block_root(); let (duties, dependent_root) = chain .validator_inclusion_list_duties(request_indices, request_epoch, head_block_root) - .map_err(warp_utils::reject::beacon_chain_error)?; + // TODO(focil) unwrap + .unwrap(); + //.map_err(warp_utils::reject::beacon_chain_error)?; convert_to_api_response(duties, request_indices, dependent_root, chain) } else if request_epoch > current_epoch + 1 { Err(warp_utils::reject::custom_bad_request(format!( @@ -73,7 +77,9 @@ fn convert_to_api_response( let usize_indices = indices.iter().map(|i| *i as usize).collect::>(); let index_to_pubkey_map = chain .validator_pubkey_bytes_many(&usize_indices) - .map_err(warp_utils::reject::beacon_chain_error)?; + // TODO(focil) unwrap + .unwrap(); + // .map_err(warp_utils::reject::beacon_chain_error)?; let data = duties .into_iter() diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 1a024135aa..9bd27133b9 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -31,7 +31,6 @@ mod validator; mod validator_inclusion; mod validators; mod version; - use crate::light_client::{get_light_client_bootstrap, get_light_client_updates}; use crate::produce_block::{produce_blinded_block_v2, produce_block_v2, produce_block_v3}; use crate::version::fork_versioned_response; @@ -45,12 +44,15 @@ pub use block_id::BlockId; use builder_states::get_next_withdrawals; use bytes::Bytes; use directory::DEFAULT_ROOT_DIR; +use either::Either; use eth2::types::{ self as api_types, BroadcastValidation, EndpointVersion, ForkChoice, ForkChoiceNode, LightClientUpdatesQuery, PublishBlockRequest, ValidatorBalancesRequestBody, ValidatorId, ValidatorStatus, ValidatorsRequestBody, }; use eth2::{CONSENSUS_VERSION_HEADER, CONTENT_TYPE_HEADER, SSZ_CONTENT_TYPE_HEADER}; +use health_metrics::observe::Observe; +use lighthouse_network::rpc::methods::MetaData; use lighthouse_network::{types::SyncState, EnrExt, NetworkGlobals, PeerId, PubsubMessage}; use lighthouse_version::version_with_platform; use logging::SSELoggingComponents; @@ -61,6 +63,7 @@ pub use publish_blocks::{ publish_blinded_block, publish_block, reconstruct_block, ProvenancedBlock, }; use serde::{Deserialize, Serialize}; +use serde_json::Value; use slog::{crit, debug, error, info, warn, Logger}; use slot_clock::SlotClock; use ssz::Encode; @@ -83,11 +86,11 @@ use tokio_stream::{ }; use types::{ fork_versioned_response::EmptyMetadata, Attestation, AttestationData, AttestationShufflingId, - AttesterSlashing, BeaconStateError, CommitteeCache, ConfigAndPreset, Epoch, EthSpec, ForkName, - ForkVersionedResponse, Hash256, ProposerPreparationData, ProposerSlashing, RelativeEpoch, - SignedAggregateAndProof, SignedBlindedBeaconBlock, SignedBlsToExecutionChange, - SignedContributionAndProof, SignedInclusionList, SignedValidatorRegistrationData, - SignedVoluntaryExit, Slot, SyncCommitteeMessage, SyncContributionData, + AttesterSlashing, BeaconStateError, ChainSpec, CommitteeCache, ConfigAndPreset, Epoch, EthSpec, + ForkName, ForkVersionedResponse, Hash256, ProposerPreparationData, ProposerSlashing, + RelativeEpoch, SignedAggregateAndProof, SignedBlindedBeaconBlock, SignedBlsToExecutionChange, + SignedContributionAndProof, SignedValidatorRegistrationData, SignedVoluntaryExit, Slot, + SyncCommitteeMessage, SyncContributionData, SignedInclusionList }; use validator::pubkey_to_validator_index; use version::{ @@ -941,9 +944,9 @@ pub fn serve( ) } } - _ => { - warp_utils::reject::beacon_chain_error(e.into()) - } + _ => warp_utils::reject::unhandled_error( + BeaconChainError::from(e), + ), } })?; @@ -1070,7 +1073,7 @@ pub fn serve( let validators = chain .validator_indices(sync_committee.pubkeys.iter()) - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; let validator_aggregates = validators .chunks_exact(T::EthSpec::sync_subcommittee_size()) @@ -1150,7 +1153,7 @@ pub fn serve( let (cached_head, execution_status) = chain .canonical_head .head_and_execution_status() - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; ( cached_head.head_block_root(), cached_head.snapshot.beacon_block.clone_as_blinded(), @@ -1164,13 +1167,13 @@ pub fn serve( BlockId::from_root(parent_root).blinded_block(&chain)?; let (root, _slot) = chain .forwards_iter_block_roots(parent.slot()) - .map_err(warp_utils::reject::beacon_chain_error)? + .map_err(warp_utils::reject::unhandled_error)? // Ignore any skip-slots immediately following the parent. .find(|res| { - res.as_ref().map_or(false, |(root, _)| *root != parent_root) + res.as_ref().is_ok_and(|(root, _)| *root != parent_root) }) .transpose() - .map_err(warp_utils::reject::beacon_chain_error)? + .map_err(warp_utils::reject::unhandled_error)? .ok_or_else(|| { warp_utils::reject::custom_not_found(format!( "child of block with root {}", @@ -1251,8 +1254,8 @@ pub fn serve( let canonical = chain .block_root_at_slot(block.slot(), WhenSlotSkipped::None) - .map_err(warp_utils::reject::beacon_chain_error)? - .map_or(false, |canonical| root == canonical); + .map_err(warp_utils::reject::unhandled_error)? + .is_some_and(|canonical| root == canonical); let data = api_types::BlockHeaderData { root, @@ -1278,6 +1281,9 @@ pub fn serve( let consensus_version_header_filter = warp::header::header::(CONSENSUS_VERSION_HEADER); + let optional_consensus_version_header_filter = + warp::header::optional::(CONSENSUS_VERSION_HEADER); + // POST beacon/blocks let post_beacon_blocks = eth_v1 .and(warp::path("beacon")) @@ -1828,32 +1834,86 @@ pub fn serve( .and(task_spawner_filter.clone()) .and(chain_filter.clone()); + let beacon_pool_path_v2 = eth_v2 + .and(warp::path("beacon")) + .and(warp::path("pool")) + .and(task_spawner_filter.clone()) + .and(chain_filter.clone()); + let beacon_pool_path_any = any_version .and(warp::path("beacon")) .and(warp::path("pool")) .and(task_spawner_filter.clone()) .and(chain_filter.clone()); - // POST beacon/pool/attestations - let post_beacon_pool_attestations = beacon_pool_path_any + let post_beacon_pool_attestations_v1 = beacon_pool_path .clone() .and(warp::path("attestations")) .and(warp::path::end()) .and(warp_utils::json::json()) .and(network_tx_filter.clone()) - .and(reprocess_send_filter) + .and(reprocess_send_filter.clone()) .and(log_filter.clone()) .then( - // V1 and V2 are identical except V2 has a consensus version header in the request. - // We only require this header for SSZ deserialization, which isn't supported for - // this endpoint presently. - |_endpoint_version: EndpointVersion, - task_spawner: TaskSpawner, + |task_spawner: TaskSpawner, chain: Arc>, attestations: Vec>, network_tx: UnboundedSender>, reprocess_tx: Option>, log: Logger| async move { + let attestations = attestations.into_iter().map(Either::Left).collect(); + let result = crate::publish_attestations::publish_attestations( + task_spawner, + chain, + attestations, + network_tx, + reprocess_tx, + log, + ) + .await + .map(|()| warp::reply::json(&())); + convert_rejection(result).await + }, + ); + + let post_beacon_pool_attestations_v2 = beacon_pool_path_v2 + .clone() + .and(warp::path("attestations")) + .and(warp::path::end()) + .and(warp_utils::json::json::()) + .and(optional_consensus_version_header_filter) + .and(network_tx_filter.clone()) + .and(reprocess_send_filter.clone()) + .and(log_filter.clone()) + .then( + |task_spawner: TaskSpawner, + chain: Arc>, + payload: Value, + fork_name: Option, + network_tx: UnboundedSender>, + reprocess_tx: Option>, + log: Logger| async move { + let attestations = + match crate::publish_attestations::deserialize_attestation_payload::( + payload, fork_name, &log, + ) { + Ok(attestations) => attestations, + Err(err) => { + warn!( + log, + "Unable to deserialize attestation POST request"; + "error" => ?err + ); + return warp::reply::with_status( + warp::reply::json( + &"Unable to deserialize request body".to_string(), + ), + eth2::StatusCode::BAD_REQUEST, + ) + .into_response(); + } + }; + let result = crate::publish_attestations::publish_attestations( task_spawner, chain, @@ -2891,36 +2951,24 @@ pub fn serve( .and(warp::path::end()) .and(task_spawner_filter.clone()) .and(network_globals.clone()) + .and(chain_filter.clone()) .then( |task_spawner: TaskSpawner, - network_globals: Arc>| { + network_globals: Arc>, + chain: Arc>| { task_spawner.blocking_json_task(Priority::P1, move || { let enr = network_globals.local_enr(); let p2p_addresses = enr.multiaddr_p2p_tcp(); let discovery_addresses = enr.multiaddr_p2p_udp(); - let meta_data = network_globals.local_metadata.read(); Ok(api_types::GenericResponse::from(api_types::IdentityData { peer_id: network_globals.local_peer_id().to_base58(), enr, p2p_addresses, discovery_addresses, - metadata: api_types::MetaData { - seq_number: *meta_data.seq_number(), - attnets: format!( - "0x{}", - hex::encode(meta_data.attnets().clone().into_bytes()), - ), - syncnets: format!( - "0x{}", - hex::encode( - meta_data - .syncnets() - .cloned() - .unwrap_or_default() - .into_bytes() - ) - ), - }, + metadata: from_meta_data::( + &network_globals.local_metadata, + &chain.spec, + ), })) }) }, @@ -2963,7 +3011,7 @@ pub fn serve( let (head, head_execution_status) = chain .canonical_head .head_and_execution_status() - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; let head_slot = head.head_slot(); let current_slot = chain.slot_clock.now_or_genesis().ok_or_else(|| { @@ -3023,7 +3071,7 @@ pub fn serve( .blocking_response_task(Priority::P0, move || { let is_optimistic = chain .is_optimistic_or_invalid_head() - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; let is_syncing = !network_globals.sync_state.read().is_synced(); @@ -3333,9 +3381,7 @@ pub fn serve( task_spawner.blocking_json_task(Priority::P0, move || { not_synced_filter?; - let current_slot = chain - .slot() - .map_err(warp_utils::reject::beacon_chain_error)?; + let current_slot = chain.slot().map_err(warp_utils::reject::unhandled_error)?; // allow a tolerance of one slot to account for clock skew if query.slot > current_slot + 1 { @@ -3349,7 +3395,7 @@ pub fn serve( .produce_unaggregated_attestation(query.slot, query.committee_index) .map(|attestation| attestation.data().clone()) .map(api_types::GenericResponse::from) - .map_err(warp_utils::reject::beacon_chain_error) + .map_err(warp_utils::reject::unhandled_error) }) }, ); @@ -3544,7 +3590,9 @@ pub fn serve( let current_slot = chain .slot() - .map_err(warp_utils::reject::beacon_chain_error)?; + .unwrap(); + // TODO(focil) unwrap + // .map_err(warp_utils::reject::beacon_chain_error)?; // allow a tolerance of one slot to account for clock skew // @@ -3560,7 +3608,9 @@ pub fn serve( .produce_inclusion_list(query.slot) .await .map(api_types::GenericResponse::from) - .map_err(warp_utils::reject::beacon_chain_error)?; + // TODO(focil) unwrap + .unwrap(); + // .map_err(warp_utils::reject::beacon_chain_error)?; Ok::<_, warp::reject::Rejection>(warp::reply::json(&data).into_response()) }) }, @@ -3789,11 +3839,9 @@ pub fn serve( .execution_layer .as_ref() .ok_or(BeaconChainError::ExecutionLayerMissing) - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; - let current_slot = chain - .slot() - .map_err(warp_utils::reject::beacon_chain_error)?; + let current_slot = chain.slot().map_err(warp_utils::reject::unhandled_error)?; let current_epoch = current_slot.epoch(T::EthSpec::slots_per_epoch()); debug!( @@ -3846,12 +3894,12 @@ pub fn serve( .execution_layer .as_ref() .ok_or(BeaconChainError::ExecutionLayerMissing) - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; let current_slot = chain .slot_clock .now_or_genesis() .ok_or(BeaconChainError::UnableToReadSlot) - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; let current_epoch = current_slot.epoch(T::EthSpec::slots_per_epoch()); debug!( @@ -3947,12 +3995,12 @@ pub fn serve( .execution_layer .as_ref() .ok_or(BeaconChainError::ExecutionLayerMissing) - .map_err(warp_utils::reject::beacon_chain_error)? + .map_err(warp_utils::reject::unhandled_error)? .builder(); let builder = arc_builder .as_ref() .ok_or(BeaconChainError::BuilderMissing) - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; builder .post_builder_validators(&filtered_registration_data) .await @@ -4068,9 +4116,8 @@ pub fn serve( chain: Arc>| { task_spawner.blocking_json_task(Priority::P0, move || { // Ensure the request is for either the current, previous or next epoch. - let current_epoch = chain - .epoch() - .map_err(warp_utils::reject::beacon_chain_error)?; + let current_epoch = + chain.epoch().map_err(warp_utils::reject::unhandled_error)?; let prev_epoch = current_epoch.saturating_sub(Epoch::new(1)); let next_epoch = current_epoch.saturating_add(Epoch::new(1)); @@ -4109,9 +4156,8 @@ pub fn serve( chain: Arc>| { task_spawner.blocking_json_task(Priority::P0, move || { // Ensure the request is for either the current, previous or next epoch. - let current_epoch = chain - .epoch() - .map_err(warp_utils::reject::beacon_chain_error)?; + let current_epoch = + chain.epoch().map_err(warp_utils::reject::unhandled_error)?; let prev_epoch = current_epoch.saturating_sub(Epoch::new(1)); let next_epoch = current_epoch.saturating_add(Epoch::new(1)); @@ -4613,6 +4659,9 @@ pub fn serve( api_types::EventTopic::Attestation => { event_handler.subscribe_attestation() } + api_types::EventTopic::SingleAttestation => { + event_handler.subscribe_single_attestation() + } api_types::EventTopic::VoluntaryExit => { event_handler.subscribe_exit() } @@ -4840,7 +4889,8 @@ pub fn serve( .uor(post_beacon_blinded_blocks) .uor(post_beacon_blocks_v2) .uor(post_beacon_blinded_blocks_v2) - .uor(post_beacon_pool_attestations) + .uor(post_beacon_pool_attestations_v1) + .uor(post_beacon_pool_attestations_v2) .uor(post_beacon_pool_attester_slashings) .uor(post_beacon_pool_proposer_slashings) .uor(post_beacon_pool_voluntary_exits) @@ -4909,6 +4959,39 @@ pub fn serve( Ok(http_server) } +fn from_meta_data( + meta_data: &RwLock>, + spec: &ChainSpec, +) -> api_types::MetaData { + let meta_data = meta_data.read(); + let format_hex = |bytes: &[u8]| format!("0x{}", hex::encode(bytes)); + + let seq_number = *meta_data.seq_number(); + let attnets = format_hex(&meta_data.attnets().clone().into_bytes()); + let syncnets = format_hex( + &meta_data + .syncnets() + .cloned() + .unwrap_or_default() + .into_bytes(), + ); + + if spec.is_peer_das_scheduled() { + api_types::MetaData::V3(api_types::MetaDataV3 { + seq_number, + attnets, + syncnets, + custody_group_count: meta_data.custody_group_count().cloned().unwrap_or_default(), + }) + } else { + api_types::MetaData::V2(api_types::MetaDataV2 { + seq_number, + attnets, + syncnets, + }) + } +} + /// Publish a message to the libp2p pubsub network. fn publish_pubsub_message( network_tx: &UnboundedSender>, diff --git a/beacon_node/http_api/src/produce_block.rs b/beacon_node/http_api/src/produce_block.rs index ed30da7362..0e24e8f175 100644 --- a/beacon_node/http_api/src/produce_block.rs +++ b/beacon_node/http_api/src/produce_block.rs @@ -153,7 +153,7 @@ pub async fn produce_blinded_block_v2( BlockProductionVersion::BlindedV2, ) .await - .map_err(warp_utils::reject::block_production_error)?; + .map_err(warp_utils::reject::unhandled_error)?; build_response_v2(chain, block_response_type, endpoint_version, accept_header) } @@ -184,7 +184,7 @@ pub async fn produce_block_v2( BlockProductionVersion::FullV2, ) .await - .map_err(warp_utils::reject::block_production_error)?; + .map_err(warp_utils::reject::unhandled_error)?; build_response_v2(chain, block_response_type, endpoint_version, accept_header) } diff --git a/beacon_node/http_api/src/proposer_duties.rs b/beacon_node/http_api/src/proposer_duties.rs index 515599ce88..c4945df9d7 100644 --- a/beacon_node/http_api/src/proposer_duties.rs +++ b/beacon_node/http_api/src/proposer_duties.rs @@ -26,7 +26,7 @@ pub fn proposer_duties( .now_or_genesis() .map(|slot| slot.epoch(T::EthSpec::slots_per_epoch())) .ok_or(BeaconChainError::UnableToReadSlot) - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; // Determine what the current epoch would be if we fast-forward our system clock by // `MAXIMUM_GOSSIP_CLOCK_DISPARITY`. @@ -66,7 +66,7 @@ pub fn proposer_duties( { let (proposers, dependent_root, execution_status, _fork) = compute_proposer_duties_from_head(request_epoch, chain) - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; convert_to_api_response( chain, request_epoch, @@ -114,7 +114,7 @@ fn try_proposer_duties_from_cache( .map_err(warp_utils::reject::beacon_state_error)?; let execution_optimistic = chain .is_optimistic_or_invalid_head_block(head_block) - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; let dependent_root = match head_epoch.cmp(&request_epoch) { // head_epoch == request_epoch @@ -163,7 +163,7 @@ fn compute_and_cache_proposer_duties( ) -> Result { let (indices, dependent_root, execution_status, fork) = compute_proposer_duties_from_head(current_epoch, chain) - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; // Prime the proposer shuffling cache with the newly-learned value. chain @@ -171,7 +171,7 @@ fn compute_and_cache_proposer_duties( .lock() .insert(current_epoch, dependent_root, indices.clone(), fork) .map_err(BeaconChainError::from) - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; convert_to_api_response( chain, @@ -195,7 +195,7 @@ fn compute_historic_proposer_duties( let (cached_head, execution_status) = chain .canonical_head .head_and_execution_status() - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; let head = &cached_head.snapshot; if head.beacon_state.current_epoch() <= epoch { @@ -214,7 +214,7 @@ fn compute_historic_proposer_duties( // If we've loaded the head state it might be from a previous epoch, ensure it's in a // suitable epoch. ensure_state_is_in_epoch(&mut state, state_root, epoch, &chain.spec) - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; (state, execution_optimistic) } else { let (state, execution_optimistic, _finalized) = @@ -234,14 +234,14 @@ fn compute_historic_proposer_duties( let indices = state .get_beacon_proposer_indices(&chain.spec) .map_err(BeaconChainError::from) - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; // We can supply the genesis block root as the block root since we know that the only block that // decides its own root is the genesis block. let dependent_root = state .proposer_shuffling_decision_root(chain.genesis_block_root) .map_err(BeaconChainError::from) - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; convert_to_api_response(chain, epoch, dependent_root, execution_optimistic, indices) } @@ -257,7 +257,7 @@ fn convert_to_api_response( ) -> Result { let index_to_pubkey_map = chain .validator_pubkey_bytes_many(&indices) - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; // Map our internal data structure into the API structure. let proposer_data = indices diff --git a/beacon_node/http_api/src/publish_attestations.rs b/beacon_node/http_api/src/publish_attestations.rs index 0065476532..1b9949d4d5 100644 --- a/beacon_node/http_api/src/publish_attestations.rs +++ b/beacon_node/http_api/src/publish_attestations.rs @@ -40,21 +40,24 @@ use beacon_chain::{ BeaconChainTypes, }; use beacon_processor::work_reprocessing_queue::{QueuedUnaggregate, ReprocessQueueMessage}; +use either::Either; use eth2::types::Failure; use lighthouse_network::PubsubMessage; use network::NetworkMessage; +use serde_json::Value; use slog::{debug, error, warn, Logger}; +use std::borrow::Cow; use std::sync::Arc; use std::time::Duration; use tokio::sync::{ mpsc::{Sender, UnboundedSender}, oneshot, }; -use types::Attestation; +use types::{Attestation, EthSpec, ForkName, SingleAttestation}; // Error variants are only used in `Debug` and considered `dead_code` by the compiler. #[derive(Debug)] -enum Error { +pub enum Error { Validation(AttestationError), Publication, ForkChoice(#[allow(dead_code)] BeaconChainError), @@ -62,6 +65,8 @@ enum Error { ReprocessDisabled, ReprocessFull, ReprocessTimeout, + InvalidJson(#[allow(dead_code)] serde_json::Error), + FailedConversion(#[allow(dead_code)] BeaconChainError), } enum PublishAttestationResult { @@ -71,26 +76,71 @@ enum PublishAttestationResult { Failure(Error), } +#[allow(clippy::type_complexity)] +pub fn deserialize_attestation_payload( + payload: Value, + fork_name: Option, + log: &Logger, +) -> Result, SingleAttestation>>, Error> { + if fork_name.is_some_and(|fork_name| fork_name.electra_enabled()) || fork_name.is_none() { + if fork_name.is_none() { + warn!( + log, + "No Consensus Version header specified."; + ); + } + + Ok(serde_json::from_value::>(payload) + .map_err(Error::InvalidJson)? + .into_iter() + .map(Either::Right) + .collect()) + } else { + Ok( + serde_json::from_value::>>(payload) + .map_err(Error::InvalidJson)? + .into_iter() + .map(Either::Left) + .collect(), + ) + } +} + fn verify_and_publish_attestation( chain: &Arc>, - attestation: &Attestation, + either_attestation: &Either, SingleAttestation>, seen_timestamp: Duration, network_tx: &UnboundedSender>, log: &Logger, ) -> Result<(), Error> { - let attestation = chain - .verify_unaggregated_attestation_for_gossip(attestation, None) + let attestation = convert_to_attestation(chain, either_attestation)?; + let verified_attestation = chain + .verify_unaggregated_attestation_for_gossip(&attestation, None) .map_err(Error::Validation)?; - // Publish. - network_tx - .send(NetworkMessage::Publish { - messages: vec![PubsubMessage::Attestation(Box::new(( - attestation.subnet_id(), - attestation.attestation().clone_as_attestation(), - )))], - }) - .map_err(|_| Error::Publication)?; + match either_attestation { + Either::Left(attestation) => { + // Publish. + network_tx + .send(NetworkMessage::Publish { + messages: vec![PubsubMessage::Attestation(Box::new(( + verified_attestation.subnet_id(), + attestation.clone(), + )))], + }) + .map_err(|_| Error::Publication)?; + } + Either::Right(single_attestation) => { + network_tx + .send(NetworkMessage::Publish { + messages: vec![PubsubMessage::SingleAttestation(Box::new(( + verified_attestation.subnet_id(), + single_attestation.clone(), + )))], + }) + .map_err(|_| Error::Publication)?; + } + } // Notify the validator monitor. chain @@ -98,12 +148,12 @@ fn verify_and_publish_attestation( .read() .register_api_unaggregated_attestation( seen_timestamp, - attestation.indexed_attestation(), + verified_attestation.indexed_attestation(), &chain.slot_clock, ); - let fc_result = chain.apply_attestation_to_fork_choice(&attestation); - let naive_aggregation_result = chain.add_to_naive_aggregation_pool(&attestation); + let fc_result = chain.apply_attestation_to_fork_choice(&verified_attestation); + let naive_aggregation_result = chain.add_to_naive_aggregation_pool(&verified_attestation); if let Err(e) = &fc_result { warn!( @@ -129,10 +179,48 @@ fn verify_and_publish_attestation( } } +fn convert_to_attestation<'a, T: BeaconChainTypes>( + chain: &Arc>, + attestation: &'a Either, SingleAttestation>, +) -> Result>, Error> { + let a = match attestation { + Either::Left(a) => Cow::Borrowed(a), + Either::Right(single_attestation) => chain + .with_committee_cache( + single_attestation.data.target.root, + single_attestation + .data + .slot + .epoch(T::EthSpec::slots_per_epoch()), + |committee_cache, _| { + let Some(committee) = committee_cache.get_beacon_committee( + single_attestation.data.slot, + single_attestation.committee_index, + ) else { + return Err(BeaconChainError::AttestationError( + types::AttestationError::NoCommitteeForSlotAndIndex { + slot: single_attestation.data.slot, + index: single_attestation.committee_index, + }, + )); + }; + + let attestation = + single_attestation.to_attestation::(committee.committee)?; + + Ok(Cow::Owned(attestation)) + }, + ) + .map_err(Error::FailedConversion)?, + }; + + Ok(a) +} + pub async fn publish_attestations( task_spawner: TaskSpawner, chain: Arc>, - attestations: Vec>, + attestations: Vec, SingleAttestation>>, network_tx: UnboundedSender>, reprocess_send: Option>, log: Logger, @@ -141,7 +229,10 @@ pub async fn publish_attestations( // move the `attestations` vec into the blocking task, so this small overhead is unavoidable. let attestation_metadata = attestations .iter() - .map(|att| (att.data().slot, att.committee_index())) + .map(|att| match att { + Either::Left(att) => (att.data().slot, att.committee_index()), + Either::Right(att) => (att.data.slot, Some(att.committee_index)), + }) .collect::>(); // Gossip validate and publish attestations that can be immediately processed. diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index b5aa23acf8..60d4b2f16e 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -395,9 +395,8 @@ fn build_gossip_verified_data_columns( let gossip_verified_data_columns = data_column_sidecars .into_iter() .map(|data_column_sidecar| { - let column_index = data_column_sidecar.index as usize; - let subnet = - DataColumnSubnetId::from_column_index::(column_index, &chain.spec); + let column_index = data_column_sidecar.index; + let subnet = DataColumnSubnetId::from_column_index(column_index, &chain.spec); let gossip_verified_column = GossipVerifiedDataColumn::new(data_column_sidecar, subnet.into(), chain); @@ -520,10 +519,7 @@ fn publish_column_sidecars( let pubsub_messages = data_column_sidecars .into_iter() .map(|data_col| { - let subnet = DataColumnSubnetId::from_column_index::( - data_col.index as usize, - &chain.spec, - ); + let subnet = DataColumnSubnetId::from_column_index(data_col.index, &chain.spec); PubsubMessage::DataColumnSidecar(Box::new((subnet, data_col))) }) .collect::>(); diff --git a/beacon_node/http_api/src/standard_block_rewards.rs b/beacon_node/http_api/src/standard_block_rewards.rs index 1ab75374ea..372a2765da 100644 --- a/beacon_node/http_api/src/standard_block_rewards.rs +++ b/beacon_node/http_api/src/standard_block_rewards.rs @@ -4,7 +4,7 @@ use crate::ExecutionOptimistic; use beacon_chain::{BeaconChain, BeaconChainTypes}; use eth2::lighthouse::StandardBlockReward; use std::sync::Arc; -use warp_utils::reject::beacon_chain_error; +use warp_utils::reject::unhandled_error; /// The difference between block_rewards and beacon_block_rewards is the later returns block /// reward format that satisfies beacon-api specs pub fn compute_beacon_block_rewards( @@ -19,7 +19,7 @@ pub fn compute_beacon_block_rewards( let rewards = chain .compute_beacon_block_reward(block_ref, &mut state) - .map_err(beacon_chain_error)?; + .map_err(unhandled_error)?; Ok((rewards, execution_optimistic, finalized)) } diff --git a/beacon_node/http_api/src/state_id.rs b/beacon_node/http_api/src/state_id.rs index ddacde9a3f..353390cdad 100644 --- a/beacon_node/http_api/src/state_id.rs +++ b/beacon_node/http_api/src/state_id.rs @@ -30,7 +30,7 @@ impl StateId { let (cached_head, execution_status) = chain .canonical_head .head_and_execution_status() - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; return Ok(( cached_head.head_state_root(), execution_status.is_optimistic_or_invalid(), @@ -56,7 +56,7 @@ impl StateId { *slot, chain .is_optimistic_or_invalid_head() - .map_err(warp_utils::reject::beacon_chain_error)?, + .map_err(warp_utils::reject::unhandled_error)?, *slot <= chain .canonical_head @@ -70,11 +70,11 @@ impl StateId { .store .load_hot_state_summary(root) .map_err(BeaconChainError::DBError) - .map_err(warp_utils::reject::beacon_chain_error)? + .map_err(warp_utils::reject::unhandled_error)? { let finalization_status = chain .state_finalization_and_canonicity(root, hot_summary.slot) - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; let finalized = finalization_status.is_finalized(); let fork_choice = chain.canonical_head.fork_choice_read_lock(); let execution_optimistic = if finalization_status.slot_is_finalized @@ -94,14 +94,14 @@ impl StateId { fork_choice .is_optimistic_or_invalid_block(&hot_summary.latest_block_root) .map_err(BeaconChainError::ForkChoiceError) - .map_err(warp_utils::reject::beacon_chain_error)? + .map_err(warp_utils::reject::unhandled_error)? }; return Ok((*root, execution_optimistic, finalized)); } else if let Some(_cold_state_slot) = chain .store .load_cold_state_slot(root) .map_err(BeaconChainError::DBError) - .map_err(warp_utils::reject::beacon_chain_error)? + .map_err(warp_utils::reject::unhandled_error)? { let fork_choice = chain.canonical_head.fork_choice_read_lock(); let finalized_root = fork_choice @@ -111,7 +111,7 @@ impl StateId { let execution_optimistic = fork_choice .is_optimistic_or_invalid_block_no_fallback(&finalized_root) .map_err(BeaconChainError::ForkChoiceError) - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; return Ok((*root, execution_optimistic, true)); } else { return Err(warp_utils::reject::custom_not_found(format!( @@ -124,7 +124,7 @@ impl StateId { let root = chain .state_root_at_slot(slot) - .map_err(warp_utils::reject::beacon_chain_error)? + .map_err(warp_utils::reject::unhandled_error)? .ok_or_else(|| { warp_utils::reject::custom_not_found(format!("beacon state at slot {}", slot)) })?; @@ -178,7 +178,7 @@ impl StateId { let (cached_head, execution_status) = chain .canonical_head .head_and_execution_status() - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; return Ok(( cached_head.snapshot.beacon_state.clone(), execution_status.is_optimistic_or_invalid(), @@ -191,7 +191,7 @@ impl StateId { let state = chain .get_state(&state_root, slot_opt) - .map_err(warp_utils::reject::beacon_chain_error) + .map_err(warp_utils::reject::unhandled_error) .and_then(|opt| { opt.ok_or_else(|| { warp_utils::reject::custom_not_found(format!( @@ -224,7 +224,7 @@ impl StateId { let (head, execution_status) = chain .canonical_head .head_and_execution_status() - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; return func( &head.snapshot.beacon_state, execution_status.is_optimistic_or_invalid(), @@ -273,7 +273,7 @@ pub fn checkpoint_slot_and_execution_optimistic( let execution_optimistic = fork_choice .is_optimistic_or_invalid_block_no_fallback(root) .map_err(BeaconChainError::ForkChoiceError) - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; Ok((slot, execution_optimistic)) } diff --git a/beacon_node/http_api/src/sync_committee_rewards.rs b/beacon_node/http_api/src/sync_committee_rewards.rs index 68a06b1ce8..ec63372406 100644 --- a/beacon_node/http_api/src/sync_committee_rewards.rs +++ b/beacon_node/http_api/src/sync_committee_rewards.rs @@ -6,7 +6,7 @@ use slog::{debug, Logger}; use state_processing::BlockReplayer; use std::sync::Arc; use types::{BeaconState, SignedBlindedBeaconBlock}; -use warp_utils::reject::{beacon_chain_error, custom_not_found}; +use warp_utils::reject::{custom_not_found, unhandled_error}; pub fn compute_sync_committee_rewards( chain: Arc>, @@ -20,7 +20,7 @@ pub fn compute_sync_committee_rewards( let reward_payload = chain .compute_sync_committee_rewards(block.message(), &mut state) - .map_err(beacon_chain_error)?; + .map_err(unhandled_error)?; let data = if reward_payload.is_empty() { debug!(log, "compute_sync_committee_rewards returned empty"); @@ -71,7 +71,7 @@ pub fn get_state_before_applying_block( .state_root_iter([Ok((parent_block.state_root(), parent_block.slot()))].into_iter()) .minimal_block_root_verification() .apply_blocks(vec![], Some(block.slot())) - .map_err(beacon_chain_error)?; + .map_err(unhandled_error::)?; Ok(replayer.into_state()) } diff --git a/beacon_node/http_api/src/sync_committees.rs b/beacon_node/http_api/src/sync_committees.rs index 3e5b1dc524..da9f9b7a06 100644 --- a/beacon_node/http_api/src/sync_committees.rs +++ b/beacon_node/http_api/src/sync_committees.rs @@ -39,7 +39,7 @@ pub fn sync_committee_duties( // still dependent on the head. So using `is_optimistic_head` is fine for both cases. let execution_optimistic = chain .is_optimistic_or_invalid_head() - .map_err(warp_utils::reject::beacon_chain_error)?; + .map_err(warp_utils::reject::unhandled_error)?; // Try using the head's sync committees to satisfy the request. This should be sufficient for // the vast majority of requests. Rather than checking if we think the request will succeed in a @@ -55,7 +55,7 @@ pub fn sync_committee_duties( .. })) | Err(BeaconChainError::SyncDutiesError(BeaconStateError::IncorrectStateVariant)) => (), - Err(e) => return Err(warp_utils::reject::beacon_chain_error(e)), + Err(e) => return Err(warp_utils::reject::unhandled_error(e)), } let duties = duties_from_state_load(request_epoch, request_indices, altair_fork_epoch, chain) @@ -67,7 +67,7 @@ pub fn sync_committee_duties( "invalid epoch: {}, current epoch: {}", request_epoch, current_epoch )), - e => warp_utils::reject::beacon_chain_error(e), + e => warp_utils::reject::unhandled_error(e), })?; Ok(convert_to_response( verify_unknown_validators(duties, request_epoch, chain)?, @@ -164,7 +164,7 @@ fn verify_unknown_validators( BeaconChainError::SyncDutiesError(BeaconStateError::UnknownValidator(idx)) => { warp_utils::reject::custom_bad_request(format!("invalid validator index: {idx}")) } - e => warp_utils::reject::beacon_chain_error(e), + e => warp_utils::reject::unhandled_error(e), }) } diff --git a/beacon_node/http_api/src/test_utils.rs b/beacon_node/http_api/src/test_utils.rs index 7b48d64e36..fbc92a45cc 100644 --- a/beacon_node/http_api/src/test_utils.rs +++ b/beacon_node/http_api/src/test_utils.rs @@ -8,6 +8,7 @@ use beacon_processor::{ }; use directory::DEFAULT_ROOT_DIR; use eth2::{BeaconNodeHttpClient, Timeouts}; +use lighthouse_network::rpc::methods::MetaDataV3; use lighthouse_network::{ discv5::enr::CombinedKey, libp2p::swarm::{ @@ -150,11 +151,21 @@ pub async fn create_api_server_with_config( let (network_senders, network_receivers) = NetworkSenders::new(); // Default metadata - let meta_data = MetaData::V2(MetaDataV2 { - seq_number: SEQ_NUMBER, - attnets: EnrAttestationBitfield::::default(), - syncnets: EnrSyncCommitteeBitfield::::default(), - }); + let meta_data = if chain.spec.is_peer_das_scheduled() { + MetaData::V3(MetaDataV3 { + seq_number: SEQ_NUMBER, + attnets: EnrAttestationBitfield::::default(), + syncnets: EnrSyncCommitteeBitfield::::default(), + custody_group_count: chain.spec.custody_requirement, + }) + } else { + MetaData::V2(MetaDataV2 { + seq_number: SEQ_NUMBER, + attnets: EnrAttestationBitfield::::default(), + syncnets: EnrSyncCommitteeBitfield::::default(), + }) + }; + let enr_key = CombinedKey::generate_secp256k1(); let enr = Enr::builder().build(&enr_key).unwrap(); let network_config = Arc::new(NetworkConfig::default()); diff --git a/beacon_node/http_api/src/ui.rs b/beacon_node/http_api/src/ui.rs index 616745dbef..80a9ed896d 100644 --- a/beacon_node/http_api/src/ui.rs +++ b/beacon_node/http_api/src/ui.rs @@ -5,7 +5,7 @@ use eth2::types::{Epoch, ValidatorStatus}; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::sync::Arc; -use warp_utils::reject::beacon_chain_error; +use warp_utils::reject::unhandled_error; #[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)] pub struct ValidatorCountResponse { @@ -58,7 +58,7 @@ pub fn get_validator_count( } Ok::<(), BeaconChainError>(()) }) - .map_err(beacon_chain_error)?; + .map_err(unhandled_error)?; Ok(ValidatorCountResponse { active_ongoing, @@ -101,7 +101,7 @@ pub fn get_validator_info( request_data: ValidatorInfoRequestData, chain: Arc>, ) -> Result { - let current_epoch = chain.epoch().map_err(beacon_chain_error)?; + let current_epoch = chain.epoch().map_err(unhandled_error)?; let epochs = current_epoch.saturating_sub(HISTORIC_EPOCHS).as_u64()..=current_epoch.as_u64(); diff --git a/beacon_node/http_api/src/validator.rs b/beacon_node/http_api/src/validator.rs index 7f11ddd8f4..baa41e33ed 100644 --- a/beacon_node/http_api/src/validator.rs +++ b/beacon_node/http_api/src/validator.rs @@ -14,7 +14,7 @@ pub fn pubkey_to_validator_index( state .validators() .get(index) - .map_or(false, |v| v.pubkey == *pubkey) + .is_some_and(|v| v.pubkey == *pubkey) }) .map(Result::Ok) .transpose() diff --git a/beacon_node/http_api/tests/broadcast_validation_tests.rs b/beacon_node/http_api/tests/broadcast_validation_tests.rs index e1ecf2d4fc..1baa71699c 100644 --- a/beacon_node/http_api/tests/broadcast_validation_tests.rs +++ b/beacon_node/http_api/tests/broadcast_validation_tests.rs @@ -1,4 +1,3 @@ -use beacon_chain::blob_verification::GossipVerifiedBlob; use beacon_chain::{ test_utils::{AttestationStrategy, BlockStrategy}, GossipVerifiedBlock, IntoGossipVerifiedBlock, @@ -7,9 +6,10 @@ use eth2::reqwest::StatusCode; use eth2::types::{BroadcastValidation, PublishBlockRequest}; use http_api::test_utils::InteractiveTester; use http_api::{publish_blinded_block, publish_block, reconstruct_block, Config, ProvenancedBlock}; +use std::collections::HashSet; use std::sync::Arc; use types::{ - BlobSidecar, Epoch, EthSpec, FixedBytesExtended, ForkName, Hash256, MainnetEthSpec, Slot, + ColumnIndex, Epoch, EthSpec, FixedBytesExtended, ForkName, Hash256, MainnetEthSpec, Slot, }; use warp::Rejection; use warp_utils::reject::CustomBadRequest; @@ -17,6 +17,8 @@ use warp_utils::reject::CustomBadRequest; type E = MainnetEthSpec; /* + * TODO(fulu): write PeerDAS equivalent tests for these. + * * We have the following test cases, which are duplicated for the blinded variant of the route: * * - `broadcast_validation=gossip` @@ -75,7 +77,7 @@ pub async fn gossip_invalid() { let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2(&PublishBlockRequest::new(block, blobs), validation_level) + .post_beacon_blocks_v2_ssz(&PublishBlockRequest::new(block, blobs), validation_level) .await; assert!(response.is_err()); @@ -124,7 +126,7 @@ pub async fn gossip_partial_pass() { let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2(&PublishBlockRequest::new(block, blobs), validation_level) + .post_beacon_blocks_v2_ssz(&PublishBlockRequest::new(block, blobs), validation_level) .await; assert!(response.is_err()); @@ -165,7 +167,7 @@ pub async fn gossip_full_pass() { let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2( + .post_beacon_blocks_v2_ssz( &PublishBlockRequest::new(block.clone(), blobs), validation_level, ) @@ -261,7 +263,7 @@ pub async fn consensus_invalid() { let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2(&PublishBlockRequest::new(block, blobs), validation_level) + .post_beacon_blocks_v2_ssz(&PublishBlockRequest::new(block, blobs), validation_level) .await; assert!(response.is_err()); @@ -307,7 +309,7 @@ pub async fn consensus_gossip() { let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2(&PublishBlockRequest::new(block, blobs), validation_level) + .post_beacon_blocks_v2_ssz(&PublishBlockRequest::new(block, blobs), validation_level) .await; assert!(response.is_err()); @@ -423,7 +425,7 @@ pub async fn consensus_full_pass() { let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2( + .post_beacon_blocks_v2_ssz( &PublishBlockRequest::new(block.clone(), blobs), validation_level, ) @@ -475,7 +477,7 @@ pub async fn equivocation_invalid() { let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2(&PublishBlockRequest::new(block, blobs), validation_level) + .post_beacon_blocks_v2_ssz(&PublishBlockRequest::new(block, blobs), validation_level) .await; assert!(response.is_err()); @@ -533,7 +535,7 @@ pub async fn equivocation_consensus_early_equivocation() { /* submit `block_a` as valid */ assert!(tester .client - .post_beacon_blocks_v2( + .post_beacon_blocks_v2_ssz( &PublishBlockRequest::new(block_a.clone(), blobs_a), validation_level ) @@ -547,7 +549,7 @@ pub async fn equivocation_consensus_early_equivocation() { /* submit `block_b` which should induce equivocation */ let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2( + .post_beacon_blocks_v2_ssz( &PublishBlockRequest::new(block_b.clone(), blobs_b), validation_level, ) @@ -596,7 +598,7 @@ pub async fn equivocation_gossip() { let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2(&PublishBlockRequest::new(block, blobs), validation_level) + .post_beacon_blocks_v2_ssz(&PublishBlockRequest::new(block, blobs), validation_level) .await; assert!(response.is_err()); @@ -721,7 +723,7 @@ pub async fn equivocation_full_pass() { let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2( + .post_beacon_blocks_v2_ssz( &PublishBlockRequest::new(block.clone(), blobs), validation_level, ) @@ -1375,7 +1377,7 @@ pub async fn block_seen_on_gossip_without_blobs() { // `validator_count // 32`. let validator_count = 64; let num_initial: u64 = 31; - let spec = ForkName::latest().make_genesis_spec(E::default_spec()); + let spec = ForkName::latest_stable().make_genesis_spec(E::default_spec()); let tester = InteractiveTester::::new(Some(spec), validator_count).await; // Create some chain depth. @@ -1413,7 +1415,7 @@ pub async fn block_seen_on_gossip_without_blobs() { // Post the block *and* blobs to the HTTP API. let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2( + .post_beacon_blocks_v2_ssz( &PublishBlockRequest::new(block.clone(), Some(blobs)), validation_level, ) @@ -1437,7 +1439,7 @@ pub async fn block_seen_on_gossip_with_some_blobs() { // `validator_count // 32`. let validator_count = 64; let num_initial: u64 = 31; - let spec = ForkName::latest().make_genesis_spec(E::default_spec()); + let spec = ForkName::latest_stable().make_genesis_spec(E::default_spec()); let tester = InteractiveTester::::new(Some(spec), validator_count).await; // Create some chain depth. @@ -1460,11 +1462,12 @@ pub async fn block_seen_on_gossip_with_some_blobs() { let blobs = blobs.expect("should have some blobs"); assert!( blobs.0.len() >= 2, - "need at least 2 blobs for partial reveal" + "need at least 2 blobs for partial reveal, got: {}", + blobs.0.len() ); - let partial_kzg_proofs = vec![*blobs.0.first().unwrap()]; - let partial_blobs = vec![blobs.1.first().unwrap().clone()]; + let partial_kzg_proofs = [*blobs.0.first().unwrap()]; + let partial_blobs = [blobs.1.first().unwrap().clone()]; // Simulate the block being seen on gossip. block @@ -1473,21 +1476,15 @@ pub async fn block_seen_on_gossip_with_some_blobs() { .unwrap(); // Simulate some of the blobs being seen on gossip. - for (i, (kzg_proof, blob)) in partial_kzg_proofs - .into_iter() - .zip(partial_blobs) - .enumerate() - { - let sidecar = Arc::new(BlobSidecar::new(i, blob, &block, kzg_proof).unwrap()); - let gossip_blob = - GossipVerifiedBlob::new(sidecar, i as u64, &tester.harness.chain).unwrap(); - tester - .harness - .chain - .process_gossip_blob(gossip_blob) - .await - .unwrap(); - } + tester + .harness + .process_gossip_blobs_or_columns( + &block, + partial_blobs.iter(), + partial_kzg_proofs.iter(), + Some(get_custody_columns(&tester)), + ) + .await; // It should not yet be added to fork choice because all blobs have not been seen. assert!(!tester @@ -1498,7 +1495,7 @@ pub async fn block_seen_on_gossip_with_some_blobs() { // Post the block *and* all blobs to the HTTP API. let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2( + .post_beacon_blocks_v2_ssz( &PublishBlockRequest::new(block.clone(), Some(blobs)), validation_level, ) @@ -1522,7 +1519,7 @@ pub async fn blobs_seen_on_gossip_without_block() { // `validator_count // 32`. let validator_count = 64; let num_initial: u64 = 31; - let spec = ForkName::latest().make_genesis_spec(E::default_spec()); + let spec = ForkName::latest_stable().make_genesis_spec(E::default_spec()); let tester = InteractiveTester::::new(Some(spec), validator_count).await; // Create some chain depth. @@ -1545,22 +1542,15 @@ pub async fn blobs_seen_on_gossip_without_block() { let (kzg_proofs, blobs) = blobs.expect("should have some blobs"); // Simulate the blobs being seen on gossip. - for (i, (kzg_proof, blob)) in kzg_proofs - .clone() - .into_iter() - .zip(blobs.clone()) - .enumerate() - { - let sidecar = Arc::new(BlobSidecar::new(i, blob, &block, kzg_proof).unwrap()); - let gossip_blob = - GossipVerifiedBlob::new(sidecar, i as u64, &tester.harness.chain).unwrap(); - tester - .harness - .chain - .process_gossip_blob(gossip_blob) - .await - .unwrap(); - } + tester + .harness + .process_gossip_blobs_or_columns( + &block, + blobs.iter(), + kzg_proofs.iter(), + Some(get_custody_columns(&tester)), + ) + .await; // It should not yet be added to fork choice because the block has not been seen. assert!(!tester @@ -1571,7 +1561,7 @@ pub async fn blobs_seen_on_gossip_without_block() { // Post the block *and* all blobs to the HTTP API. let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2( + .post_beacon_blocks_v2_ssz( &PublishBlockRequest::new(block.clone(), Some((kzg_proofs, blobs))), validation_level, ) @@ -1595,7 +1585,7 @@ pub async fn blobs_seen_on_gossip_without_block_and_no_http_blobs() { // `validator_count // 32`. let validator_count = 64; let num_initial: u64 = 31; - let spec = ForkName::latest().make_genesis_spec(E::default_spec()); + let spec = ForkName::latest_stable().make_genesis_spec(E::default_spec()); let tester = InteractiveTester::::new(Some(spec), validator_count).await; // Create some chain depth. @@ -1619,22 +1609,15 @@ pub async fn blobs_seen_on_gossip_without_block_and_no_http_blobs() { assert!(!blobs.is_empty()); // Simulate the blobs being seen on gossip. - for (i, (kzg_proof, blob)) in kzg_proofs - .clone() - .into_iter() - .zip(blobs.clone()) - .enumerate() - { - let sidecar = Arc::new(BlobSidecar::new(i, blob, &block, kzg_proof).unwrap()); - let gossip_blob = - GossipVerifiedBlob::new(sidecar, i as u64, &tester.harness.chain).unwrap(); - tester - .harness - .chain - .process_gossip_blob(gossip_blob) - .await - .unwrap(); - } + tester + .harness + .process_gossip_blobs_or_columns( + &block, + blobs.iter(), + kzg_proofs.iter(), + Some(get_custody_columns(&tester)), + ) + .await; // It should not yet be added to fork choice because the block has not been seen. assert!(!tester @@ -1645,7 +1628,7 @@ pub async fn blobs_seen_on_gossip_without_block_and_no_http_blobs() { // Post just the block to the HTTP API (blob lists are empty). let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2( + .post_beacon_blocks_v2_ssz( &PublishBlockRequest::new( block.clone(), Some((Default::default(), Default::default())), @@ -1671,7 +1654,7 @@ pub async fn slashable_blobs_seen_on_gossip_cause_failure() { // `validator_count // 32`. let validator_count = 64; let num_initial: u64 = 31; - let spec = ForkName::latest().make_genesis_spec(E::default_spec()); + let spec = ForkName::latest_stable().make_genesis_spec(E::default_spec()); let tester = InteractiveTester::::new(Some(spec), validator_count).await; // Create some chain depth. @@ -1696,17 +1679,15 @@ pub async fn slashable_blobs_seen_on_gossip_cause_failure() { let (kzg_proofs_b, blobs_b) = blobs_b.expect("should have some blobs"); // Simulate the blobs of block B being seen on gossip. - for (i, (kzg_proof, blob)) in kzg_proofs_b.into_iter().zip(blobs_b).enumerate() { - let sidecar = Arc::new(BlobSidecar::new(i, blob, &block_b, kzg_proof).unwrap()); - let gossip_blob = - GossipVerifiedBlob::new(sidecar, i as u64, &tester.harness.chain).unwrap(); - tester - .harness - .chain - .process_gossip_blob(gossip_blob) - .await - .unwrap(); - } + tester + .harness + .process_gossip_blobs_or_columns( + &block_b, + blobs_b.iter(), + kzg_proofs_b.iter(), + Some(get_custody_columns(&tester)), + ) + .await; // It should not yet be added to fork choice because block B has not been seen. assert!(!tester @@ -1717,7 +1698,7 @@ pub async fn slashable_blobs_seen_on_gossip_cause_failure() { // Post block A *and* all its blobs to the HTTP API. let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2( + .post_beacon_blocks_v2_ssz( &PublishBlockRequest::new(block_a.clone(), Some((kzg_proofs_a, blobs_a))), validation_level, ) @@ -1741,7 +1722,7 @@ pub async fn duplicate_block_status_code() { // `validator_count // 32`. let validator_count = 64; let num_initial: u64 = 31; - let spec = ForkName::latest().make_genesis_spec(E::default_spec()); + let spec = ForkName::latest_stable().make_genesis_spec(E::default_spec()); let duplicate_block_status_code = StatusCode::IM_A_TEAPOT; let tester = InteractiveTester::::new_with_initializer_and_mutator( Some(spec), @@ -1778,7 +1759,7 @@ pub async fn duplicate_block_status_code() { let block_request = PublishBlockRequest::new(block.clone(), Some((kzg_proofs, blobs))); let response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2(&block_request, validation_level) + .post_beacon_blocks_v2_ssz(&block_request, validation_level) .await; // This should result in the block being fully imported. @@ -1791,7 +1772,7 @@ pub async fn duplicate_block_status_code() { // Post again. let duplicate_response: Result<(), eth2::Error> = tester .client - .post_beacon_blocks_v2(&block_request, validation_level) + .post_beacon_blocks_v2_ssz(&block_request, validation_level) .await; let err = duplicate_response.unwrap_err(); assert_eq!(err.status().unwrap(), duplicate_block_status_code); @@ -1803,3 +1784,13 @@ fn assert_server_message_error(error_response: eth2::Error, expected_message: St }; assert_eq!(err.message, expected_message); } + +fn get_custody_columns(tester: &InteractiveTester) -> HashSet { + tester + .ctx + .network_globals + .as_ref() + .unwrap() + .sampling_columns + .clone() +} diff --git a/beacon_node/http_api/tests/fork_tests.rs b/beacon_node/http_api/tests/fork_tests.rs index 8cb6053e9f..d6b8df33b3 100644 --- a/beacon_node/http_api/tests/fork_tests.rs +++ b/beacon_node/http_api/tests/fork_tests.rs @@ -155,10 +155,6 @@ async fn attestations_across_fork_with_skip_slots() { .post_beacon_pool_attestations_v1(&unaggregated_attestations) .await .unwrap(); - client - .post_beacon_pool_attestations_v2(&unaggregated_attestations, fork_name) - .await - .unwrap(); let signed_aggregates = attestations .into_iter() diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index e45dcf221c..bb3086945b 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -5,6 +5,7 @@ use beacon_chain::{ ChainConfig, }; use beacon_processor::work_reprocessing_queue::ReprocessQueueMessage; +use either::Either; use eth2::types::ProduceBlockV3Response; use eth2::types::{DepositContractData, StateId}; use execution_layer::{ForkchoiceState, PayloadAttributes}; @@ -161,7 +162,7 @@ impl ForkChoiceUpdates { update .payload_attributes .as_ref() - .map_or(false, |payload_attributes| { + .is_some_and(|payload_attributes| { payload_attributes.timestamp() == proposal_timestamp }) }) @@ -890,27 +891,50 @@ async fn queue_attestations_from_http() { let pre_state = harness.get_current_state(); let (block, post_state) = harness.make_block(pre_state, attestation_slot).await; let block_root = block.0.canonical_root(); + let fork_name = tester.harness.spec.fork_name_at_slot::(attestation_slot); // Make attestations to the block and POST them to the beacon node on a background thread. - let attestations = harness - .make_unaggregated_attestations( - &all_validators, - &post_state, - block.0.state_root(), - block_root.into(), - attestation_slot, - ) - .into_iter() - .flat_map(|attestations| attestations.into_iter().map(|(att, _subnet)| att)) - .collect::>(); + let attestation_future = if fork_name.electra_enabled() { + let single_attestations = harness + .make_single_attestations( + &all_validators, + &post_state, + block.0.state_root(), + block_root.into(), + attestation_slot, + ) + .into_iter() + .flat_map(|attestations| attestations.into_iter().map(|(att, _subnet)| att)) + .collect::>(); - let fork_name = tester.harness.spec.fork_name_at_slot::(attestation_slot); - let attestation_future = tokio::spawn(async move { - client - .post_beacon_pool_attestations_v2(&attestations, fork_name) - .await - .expect("attestations should be processed successfully") - }); + let attestations = Either::Right(single_attestations); + + tokio::spawn(async move { + client + .post_beacon_pool_attestations_v2::(attestations, fork_name) + .await + .expect("attestations should be processed successfully") + }) + } else { + let attestations = harness + .make_unaggregated_attestations( + &all_validators, + &post_state, + block.0.state_root(), + block_root.into(), + attestation_slot, + ) + .into_iter() + .flat_map(|attestations| attestations.into_iter().map(|(att, _subnet)| att)) + .collect::>(); + + tokio::spawn(async move { + client + .post_beacon_pool_attestations_v1(&attestations) + .await + .expect("attestations should be processed successfully") + }) + }; // In parallel, apply the block. We need to manually notify the reprocess queue, because the // `beacon_chain` does not know about the queue and will not update it for us. diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 7007a14466..bc3159e074 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -3,6 +3,7 @@ use beacon_chain::{ test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType}, BeaconChain, ChainConfig, StateSkipConfig, WhenSlotSkipped, }; +use either::Either; use eth2::{ mixin::{RequestAccept, ResponseForkName, ResponseOptional}, reqwest::RequestBuilder, @@ -40,7 +41,8 @@ use tree_hash::TreeHash; use types::application_domain::ApplicationDomain; use types::{ attestation::AttestationBase, AggregateSignature, BitList, Domain, EthSpec, ExecutionBlockHash, - Hash256, Keypair, MainnetEthSpec, RelativeEpoch, SelectionProof, SignedRoot, Slot, + Hash256, Keypair, MainnetEthSpec, RelativeEpoch, SelectionProof, SignedRoot, SingleAttestation, + Slot, }; type E = MainnetEthSpec; @@ -71,6 +73,7 @@ struct ApiTester { next_block: PublishBlockRequest, reorg_block: PublishBlockRequest, attestations: Vec>, + single_attestations: Vec, contribution_and_proofs: Vec>, attester_slashing: AttesterSlashing, proposer_slashing: ProposerSlashing, @@ -203,6 +206,27 @@ impl ApiTester { "precondition: attestations for testing" ); + let fork_name = harness + .chain + .spec + .fork_name_at_slot::(harness.chain.slot().unwrap()); + + let single_attestations = if fork_name.electra_enabled() { + harness + .get_single_attestations( + &AttestationStrategy::AllValidators, + &head.beacon_state, + head_state_root, + head.beacon_block_root, + harness.chain.slot().unwrap(), + ) + .into_iter() + .flat_map(|vec| vec.into_iter().map(|(attestation, _subnet_id)| attestation)) + .collect::>() + } else { + vec![] + }; + let current_epoch = harness .chain .slot() @@ -294,6 +318,7 @@ impl ApiTester { next_block, reorg_block, attestations, + single_attestations, contribution_and_proofs, attester_slashing, proposer_slashing, @@ -381,6 +406,7 @@ impl ApiTester { next_block, reorg_block, attestations, + single_attestations: vec![], contribution_and_proofs: vec![], attester_slashing, proposer_slashing, @@ -1278,7 +1304,7 @@ impl ApiTester { .chain .block_root_at_slot(block.slot(), WhenSlotSkipped::None) .unwrap() - .map_or(false, |canonical| block_root == canonical); + .is_some_and(|canonical| block_root == canonical); assert_eq!(result.canonical, canonical, "{:?}", block_id); assert_eq!(result.root, block_root, "{:?}", block_id); @@ -1785,12 +1811,25 @@ impl ApiTester { self } - pub async fn test_post_beacon_pool_attestations_valid_v1(mut self) -> Self { + pub async fn test_post_beacon_pool_attestations_valid(mut self) -> Self { self.client .post_beacon_pool_attestations_v1(self.attestations.as_slice()) .await .unwrap(); + let fork_name = self + .attestations + .first() + .map(|att| self.chain.spec.fork_name_at_slot::(att.data().slot)) + .unwrap(); + + let attestations = Either::Left(self.attestations.clone()); + + self.client + .post_beacon_pool_attestations_v2::(attestations, fork_name) + .await + .unwrap(); + assert!( self.network_rx.network_recv.recv().await.is_some(), "valid attestation should be sent to network" @@ -1800,13 +1839,18 @@ impl ApiTester { } pub async fn test_post_beacon_pool_attestations_valid_v2(mut self) -> Self { + if self.single_attestations.is_empty() { + return self; + } let fork_name = self - .attestations + .single_attestations .first() - .map(|att| self.chain.spec.fork_name_at_slot::(att.data().slot)) + .map(|att| self.chain.spec.fork_name_at_slot::(att.data.slot)) .unwrap(); + + let attestations = Either::Right(self.single_attestations.clone()); self.client - .post_beacon_pool_attestations_v2(self.attestations.as_slice(), fork_name) + .post_beacon_pool_attestations_v2::(attestations, fork_name) .await .unwrap(); assert!( @@ -1854,10 +1898,13 @@ impl ApiTester { self } pub async fn test_post_beacon_pool_attestations_invalid_v2(mut self) -> Self { + if self.single_attestations.is_empty() { + return self; + } let mut attestations = Vec::new(); - for attestation in &self.attestations { + for attestation in &self.single_attestations { let mut invalid_attestation = attestation.clone(); - invalid_attestation.data_mut().slot += 1; + invalid_attestation.data.slot += 1; // add both to ensure we only fail on invalid attestations attestations.push(attestation.clone()); @@ -1869,10 +1916,10 @@ impl ApiTester { .first() .map(|att| self.chain.spec.fork_name_at_slot::(att.data().slot)) .unwrap(); - + let attestations = Either::Right(attestations); let err_v2 = self .client - .post_beacon_pool_attestations_v2(attestations.as_slice(), fork_name) + .post_beacon_pool_attestations_v2::(attestations, fork_name) .await .unwrap_err(); @@ -1902,7 +1949,7 @@ impl ApiTester { .sync_committee_period(&self.chain.spec) .unwrap(); - let result = match self + match self .client .get_beacon_light_client_updates::(current_sync_committee_period, 1) .await @@ -1923,7 +1970,6 @@ impl ApiTester { .unwrap(); assert_eq!(1, expected.len()); - assert_eq!(result.clone().unwrap().len(), expected.len()); self } @@ -1948,7 +1994,6 @@ impl ApiTester { .get_light_client_bootstrap(&self.chain.store, &block_root, 1u64, &self.chain.spec); assert!(expected.is_ok()); - assert_eq!(result.unwrap().data, expected.unwrap().unwrap().0); self @@ -2232,9 +2277,9 @@ impl ApiTester { pub async fn test_get_config_spec(self) -> Self { let result = self .client - .get_config_spec::() + .get_config_spec::() .await - .map(|res| ConfigAndPreset::Electra(res.data)) + .map(|res| ConfigAndPreset::Fulu(res.data)) .unwrap(); let expected = ConfigAndPreset::from_chain_spec::(&self.chain.spec, None); @@ -2331,11 +2376,11 @@ impl ApiTester { enr: self.local_enr.clone(), p2p_addresses: self.local_enr.multiaddr_p2p_tcp(), discovery_addresses: self.local_enr.multiaddr_p2p_udp(), - metadata: eth2::types::MetaData { + metadata: MetaData::V2(MetaDataV2 { seq_number: 0, attnets: "0x0000000000000000".to_string(), syncnets: "0x00".to_string(), - }, + }), }; assert_eq!(result, expected); @@ -6011,6 +6056,48 @@ impl ApiTester { self } + pub async fn test_get_events_electra(self) -> Self { + let topics = vec![EventTopic::SingleAttestation]; + let mut events_future = self + .client + .get_events::(topics.as_slice()) + .await + .unwrap(); + + let expected_attestation_len = self.single_attestations.len(); + + let fork_name = self + .chain + .spec + .fork_name_at_slot::(self.chain.slot().unwrap()); + let attestations = Either::Right(self.single_attestations.clone()); + self.client + .post_beacon_pool_attestations_v2::(attestations, fork_name) + .await + .unwrap(); + + let attestation_events = poll_events( + &mut events_future, + expected_attestation_len, + Duration::from_millis(10000), + ) + .await; + + assert_eq!( + attestation_events.as_slice(), + self.single_attestations + .clone() + .into_iter() + .map(|single_attestation| EventKind::SingleAttestation(Box::new( + single_attestation + ))) + .collect::>() + .as_slice() + ); + + self + } + pub async fn test_get_events_altair(self) -> Self { let topics = vec![EventTopic::ContributionAndProof]; let mut events_future = self @@ -6158,6 +6245,20 @@ async fn get_events_altair() { .await; } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn get_events_electra() { + let mut config = ApiTesterConfig::default(); + config.spec.altair_fork_epoch = Some(Epoch::new(0)); + config.spec.bellatrix_fork_epoch = Some(Epoch::new(0)); + config.spec.capella_fork_epoch = Some(Epoch::new(0)); + config.spec.deneb_fork_epoch = Some(Epoch::new(0)); + config.spec.electra_fork_epoch = Some(Epoch::new(0)); + ApiTester::new_from_config(config) + .await + .test_get_events_electra() + .await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn get_events_from_genesis() { ApiTester::new_from_genesis() @@ -6290,10 +6391,10 @@ async fn post_beacon_blocks_duplicate() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn beacon_pools_post_attestations_valid_v1() { +async fn beacon_pools_post_attestations_valid() { ApiTester::new() .await - .test_post_beacon_pool_attestations_valid_v1() + .test_post_beacon_pool_attestations_valid() .await; } diff --git a/beacon_node/http_metrics/Cargo.toml b/beacon_node/http_metrics/Cargo.toml index d92f986440..9ad073439d 100644 --- a/beacon_node/http_metrics/Cargo.toml +++ b/beacon_node/http_metrics/Cargo.toml @@ -7,6 +7,7 @@ edition = { workspace = true } [dependencies] beacon_chain = { workspace = true } +health_metrics = { workspace = true } lighthouse_network = { workspace = true } lighthouse_version = { workspace = true } malloc_utils = { workspace = true } diff --git a/beacon_node/http_metrics/src/metrics.rs b/beacon_node/http_metrics/src/metrics.rs index d751c51e4c..bcfb8e4c9c 100644 --- a/beacon_node/http_metrics/src/metrics.rs +++ b/beacon_node/http_metrics/src/metrics.rs @@ -39,7 +39,7 @@ pub fn gather_prometheus_metrics( lighthouse_network::scrape_discovery_metrics(); - warp_utils::metrics::scrape_health_metrics(); + health_metrics::metrics::scrape_health_metrics(); // It's important to ensure these metrics are explicitly enabled in the case that users aren't // using glibc and this function causes panics. diff --git a/beacon_node/lighthouse_network/Cargo.toml b/beacon_node/lighthouse_network/Cargo.toml index 485f32b37a..3c89ece442 100644 --- a/beacon_node/lighthouse_network/Cargo.toml +++ b/beacon_node/lighthouse_network/Cargo.toml @@ -20,7 +20,7 @@ futures = { workspace = true } gossipsub = { workspace = true } hex = { workspace = true } itertools = { workspace = true } -libp2p-mplex = "0.42" +libp2p-mplex = "0.43" lighthouse_version = { workspace = true } lru = { workspace = true } lru_cache = { workspace = true } @@ -50,7 +50,7 @@ unused_port = { workspace = true } void = "1.0.2" [dependencies.libp2p] -version = "0.54" +version = "0.55" default-features = false features = ["identify", "yamux", "noise", "dns", "tcp", "tokio", "plaintext", "secp256k1", "macros", "ecdsa", "metrics", "quic", "upnp"] diff --git a/beacon_node/lighthouse_network/gossipsub/Cargo.toml b/beacon_node/lighthouse_network/gossipsub/Cargo.toml index 61f5730c08..239caae47a 100644 --- a/beacon_node/lighthouse_network/gossipsub/Cargo.toml +++ b/beacon_node/lighthouse_network/gossipsub/Cargo.toml @@ -26,7 +26,7 @@ futures-timer = "3.0.2" getrandom = "0.2.12" hashlink = { workspace = true } hex_fmt = "0.3.0" -libp2p = { version = "0.54", default-features = false } +libp2p = { version = "0.55", default-features = false } prometheus-client = "0.22.0" quick-protobuf = "0.8" quick-protobuf-codec = "0.3" diff --git a/beacon_node/lighthouse_network/gossipsub/src/backoff.rs b/beacon_node/lighthouse_network/gossipsub/src/backoff.rs index 537d2319c2..0d77e2cd0f 100644 --- a/beacon_node/lighthouse_network/gossipsub/src/backoff.rs +++ b/beacon_node/lighthouse_network/gossipsub/src/backoff.rs @@ -124,7 +124,7 @@ impl BackoffStorage { pub(crate) fn is_backoff_with_slack(&self, topic: &TopicHash, peer: &PeerId) -> bool { self.backoffs .get(topic) - .map_or(false, |m| m.contains_key(peer)) + .is_some_and(|m| m.contains_key(peer)) } pub(crate) fn get_backoff_time(&self, topic: &TopicHash, peer: &PeerId) -> Option { diff --git a/beacon_node/lighthouse_network/gossipsub/src/behaviour.rs b/beacon_node/lighthouse_network/gossipsub/src/behaviour.rs index c4e20e4397..7eb35cc49b 100644 --- a/beacon_node/lighthouse_network/gossipsub/src/behaviour.rs +++ b/beacon_node/lighthouse_network/gossipsub/src/behaviour.rs @@ -1770,8 +1770,7 @@ where // reject messages claiming to be from ourselves but not locally published let self_published = !self.config.allow_self_origin() && if let Some(own_id) = self.publish_config.get_own_id() { - own_id != propagation_source - && raw_message.source.as_ref().map_or(false, |s| s == own_id) + own_id != propagation_source && raw_message.source.as_ref() == Some(own_id) } else { self.published_message_ids.contains(msg_id) }; @@ -1842,6 +1841,30 @@ where peer_score.duplicated_message(propagation_source, &msg_id, &message.topic); } self.mcache.observe_duplicate(&msg_id, propagation_source); + // track metrics for the source of the duplicates + if let Some(metrics) = self.metrics.as_mut() { + if self + .mesh + .get(&message.topic) + .is_some_and(|peers| peers.contains(propagation_source)) + { + // duplicate was received from a mesh peer + metrics.mesh_duplicates(&message.topic); + } else if self + .gossip_promises + .contains_peer(&msg_id, propagation_source) + { + // duplicate was received from an iwant request + metrics.iwant_duplicates(&message.topic); + } else { + tracing::warn!( + messsage=%msg_id, + peer=%propagation_source, + topic=%message.topic, + "Peer should not have sent message" + ); + } + } return; } diff --git a/beacon_node/lighthouse_network/gossipsub/src/gossip_promises.rs b/beacon_node/lighthouse_network/gossipsub/src/gossip_promises.rs index 3f72709245..ce1dee2a72 100644 --- a/beacon_node/lighthouse_network/gossipsub/src/gossip_promises.rs +++ b/beacon_node/lighthouse_network/gossipsub/src/gossip_promises.rs @@ -41,6 +41,13 @@ impl GossipPromises { self.promises.contains_key(message) } + /// Returns true if the message id exists in the promises and contains the given peer. + pub(crate) fn contains_peer(&self, message: &MessageId, peer: &PeerId) -> bool { + self.promises + .get(message) + .is_some_and(|peers| peers.contains_key(peer)) + } + ///Get the peers we sent IWANT the input message id. pub(crate) fn peers_for_message(&self, message_id: &MessageId) -> Vec { self.promises diff --git a/beacon_node/lighthouse_network/gossipsub/src/handler.rs b/beacon_node/lighthouse_network/gossipsub/src/handler.rs index d89013eb2f..0f25db6e3d 100644 --- a/beacon_node/lighthouse_network/gossipsub/src/handler.rs +++ b/beacon_node/lighthouse_network/gossipsub/src/handler.rs @@ -194,7 +194,6 @@ impl EnabledHandler { &mut self, FullyNegotiatedOutbound { protocol, .. }: FullyNegotiatedOutbound< ::OutboundProtocol, - ::OutboundOpenInfo, >, ) { let (substream, peer_kind) = protocol; @@ -217,7 +216,7 @@ impl EnabledHandler { ) -> Poll< ConnectionHandlerEvent< ::OutboundProtocol, - ::OutboundOpenInfo, + (), ::ToBehaviour, >, > { @@ -423,7 +422,7 @@ impl ConnectionHandler for Handler { type OutboundOpenInfo = (); type OutboundProtocol = ProtocolConfig; - fn listen_protocol(&self) -> SubstreamProtocol { + fn listen_protocol(&self) -> SubstreamProtocol { match self { Handler::Enabled(handler) => { SubstreamProtocol::new(either::Either::Left(handler.listen_protocol.clone()), ()) @@ -458,9 +457,7 @@ impl ConnectionHandler for Handler { fn poll( &mut self, cx: &mut Context<'_>, - ) -> Poll< - ConnectionHandlerEvent, - > { + ) -> Poll> { match self { Handler::Enabled(handler) => handler.poll(cx), Handler::Disabled(DisabledHandler::ProtocolUnsupported { peer_kind_sent }) => { @@ -479,12 +476,7 @@ impl ConnectionHandler for Handler { fn on_connection_event( &mut self, - event: ConnectionEvent< - Self::InboundProtocol, - Self::OutboundProtocol, - Self::InboundOpenInfo, - Self::OutboundOpenInfo, - >, + event: ConnectionEvent, ) { match self { Handler::Enabled(handler) => { @@ -521,7 +513,7 @@ impl ConnectionHandler for Handler { }) => match protocol { Either::Left(protocol) => handler.on_fully_negotiated_inbound(protocol), #[allow(unreachable_patterns)] - Either::Right(v) => void::unreachable(v), + Either::Right(v) => libp2p::core::util::unreachable(v), }, ConnectionEvent::FullyNegotiatedOutbound(fully_negotiated_outbound) => { handler.on_fully_negotiated_outbound(fully_negotiated_outbound) diff --git a/beacon_node/lighthouse_network/gossipsub/src/metrics.rs b/beacon_node/lighthouse_network/gossipsub/src/metrics.rs index d3ca6c299e..2989f95a26 100644 --- a/beacon_node/lighthouse_network/gossipsub/src/metrics.rs +++ b/beacon_node/lighthouse_network/gossipsub/src/metrics.rs @@ -194,6 +194,12 @@ pub(crate) struct Metrics { /// Number of full messages we received that we previously sent a IDONTWANT for. idontwant_messages_ignored_per_topic: Family, + /// Count of duplicate messages we have received from mesh peers for a given topic. + mesh_duplicates: Family, + + /// Count of duplicate messages we have received from by requesting them over iwant for a given topic. + iwant_duplicates: Family, + /// The size of the priority queue. priority_queue_size: Histogram, /// The size of the non-priority queue. @@ -359,6 +365,16 @@ impl Metrics { "IDONTWANT messages that were sent but we received the full message regardless" ); + let mesh_duplicates = register_family!( + "mesh_duplicates_per_topic", + "Count of duplicate messages received from mesh peers per topic" + ); + + let iwant_duplicates = register_family!( + "iwant_duplicates_per_topic", + "Count of duplicate messages received from non-mesh peers that we sent iwants for" + ); + let idontwant_bytes = { let metric = Counter::default(); registry.register( @@ -425,6 +441,8 @@ impl Metrics { idontwant_msgs_ids, idontwant_messages_sent_per_topic, idontwant_messages_ignored_per_topic, + mesh_duplicates, + iwant_duplicates, priority_queue_size, non_priority_queue_size, } @@ -597,6 +615,20 @@ impl Metrics { } } + /// Register a duplicate message received from a mesh peer. + pub(crate) fn mesh_duplicates(&mut self, topic: &TopicHash) { + if self.register_topic(topic).is_ok() { + self.mesh_duplicates.get_or_create(topic).inc(); + } + } + + /// Register a duplicate message received from a non-mesh peer on an iwant request. + pub(crate) fn iwant_duplicates(&mut self, topic: &TopicHash) { + if self.register_topic(topic).is_ok() { + self.iwant_duplicates.get_or_create(topic).inc(); + } + } + pub(crate) fn register_msg_validation( &mut self, topic: &TopicHash, diff --git a/beacon_node/lighthouse_network/src/config.rs b/beacon_node/lighthouse_network/src/config.rs index 21f3dc830f..55c1dbf491 100644 --- a/beacon_node/lighthouse_network/src/config.rs +++ b/beacon_node/lighthouse_network/src/config.rs @@ -116,7 +116,8 @@ pub struct Config { pub network_load: u8, /// Indicates if the user has set the network to be in private mode. Currently this - /// prevents sending client identifying information over identify. + /// prevents sending client identifying information over identify and prevents + /// EIP-7636 indentifiable information being provided in the ENR. pub private: bool, /// Shutdown beacon node after sync is completed. @@ -166,7 +167,7 @@ impl Config { tcp_port, }); self.discv5_config.listen_config = discv5::ListenConfig::from_ip(addr.into(), disc_port); - self.discv5_config.table_filter = |enr| enr.ip4().as_ref().map_or(false, is_global_ipv4) + self.discv5_config.table_filter = |enr| enr.ip4().as_ref().is_some_and(is_global_ipv4) } /// Sets the listening address to use an ipv6 address. The discv5 ip_mode and table filter is @@ -187,7 +188,7 @@ impl Config { }); self.discv5_config.listen_config = discv5::ListenConfig::from_ip(addr.into(), disc_port); - self.discv5_config.table_filter = |enr| enr.ip6().as_ref().map_or(false, is_global_ipv6) + self.discv5_config.table_filter = |enr| enr.ip6().as_ref().is_some_and(is_global_ipv6) } /// Sets the listening address to use both an ipv4 and ipv6 address. The discv5 ip_mode and @@ -317,7 +318,7 @@ impl Default for Config { .filter_rate_limiter(filter_rate_limiter) .filter_max_bans_per_ip(Some(5)) .filter_max_nodes_per_ip(Some(10)) - .table_filter(|enr| enr.ip4().map_or(false, |ip| is_global_ipv4(&ip))) // Filter non-global IPs + .table_filter(|enr| enr.ip4().is_some_and(|ip| is_global_ipv4(&ip))) // Filter non-global IPs .ban_duration(Some(Duration::from_secs(3600))) .ping_interval(Duration::from_secs(300)) .build(); diff --git a/beacon_node/lighthouse_network/src/discovery/enr.rs b/beacon_node/lighthouse_network/src/discovery/enr.rs index ce29480ffd..8067711954 100644 --- a/beacon_node/lighthouse_network/src/discovery/enr.rs +++ b/beacon_node/lighthouse_network/src/discovery/enr.rs @@ -8,6 +8,7 @@ use crate::types::{Enr, EnrAttestationBitfield, EnrSyncCommitteeBitfield}; use crate::NetworkConfig; use alloy_rlp::bytes::Bytes; use libp2p::identity::Keypair; +use lighthouse_version::{client_name, version}; use slog::{debug, warn}; use ssz::{Decode, Encode}; use ssz_types::BitVector; @@ -25,8 +26,8 @@ pub const ETH2_ENR_KEY: &str = "eth2"; pub const ATTESTATION_BITFIELD_ENR_KEY: &str = "attnets"; /// The ENR field specifying the sync committee subnet bitfield. pub const SYNC_COMMITTEE_BITFIELD_ENR_KEY: &str = "syncnets"; -/// The ENR field specifying the peerdas custody subnet count. -pub const PEERDAS_CUSTODY_SUBNET_COUNT_ENR_KEY: &str = "csc"; +/// The ENR field specifying the peerdas custody group count. +pub const PEERDAS_CUSTODY_GROUP_COUNT_ENR_KEY: &str = "cgc"; /// Extension trait for ENR's within Eth2. pub trait Eth2Enr { @@ -38,8 +39,8 @@ pub trait Eth2Enr { &self, ) -> Result, &'static str>; - /// The peerdas custody subnet count associated with the ENR. - fn custody_subnet_count(&self, spec: &ChainSpec) -> Result; + /// The peerdas custody group count associated with the ENR. + fn custody_group_count(&self, spec: &ChainSpec) -> Result; fn eth2(&self) -> Result; } @@ -67,16 +68,16 @@ impl Eth2Enr for Enr { .map_err(|_| "Could not decode the ENR syncnets bitfield") } - fn custody_subnet_count(&self, spec: &ChainSpec) -> Result { - let csc = self - .get_decodable::(PEERDAS_CUSTODY_SUBNET_COUNT_ENR_KEY) - .ok_or("ENR custody subnet count non-existent")? - .map_err(|_| "Could not decode the ENR custody subnet count")?; + fn custody_group_count(&self, spec: &ChainSpec) -> Result { + let cgc = self + .get_decodable::(PEERDAS_CUSTODY_GROUP_COUNT_ENR_KEY) + .ok_or("ENR custody group count non-existent")? + .map_err(|_| "Could not decode the ENR custody group count")?; - if csc >= spec.custody_requirement && csc <= spec.data_column_sidecar_subnet_count { - Ok(csc) + if (spec.custody_requirement..=spec.number_of_custody_groups).contains(&cgc) { + Ok(cgc) } else { - Err("Invalid custody subnet count in ENR") + Err("Invalid custody group count in ENR") } } @@ -188,6 +189,11 @@ pub fn build_enr( builder.udp6(udp6_port.get()); } + // Add EIP 7636 client information + if !config.private { + builder.client_info(client_name().to_string(), version().to_string(), None); + } + // Add QUIC fields to the ENR. // Since QUIC is used as an alternative transport for the libp2p protocols, // the related fields should only be added when both QUIC and libp2p are enabled @@ -253,14 +259,14 @@ pub fn build_enr( &bitfield.as_ssz_bytes().into(), ); - // only set `csc` if PeerDAS fork epoch has been scheduled + // only set `cgc` if PeerDAS fork epoch has been scheduled if spec.is_peer_das_scheduled() { - let custody_subnet_count = if config.subscribe_all_data_column_subnets { - spec.data_column_sidecar_subnet_count + let custody_group_count = if config.subscribe_all_data_column_subnets { + spec.number_of_custody_groups } else { spec.custody_requirement }; - builder.add_value(PEERDAS_CUSTODY_SUBNET_COUNT_ENR_KEY, &custody_subnet_count); + builder.add_value(PEERDAS_CUSTODY_GROUP_COUNT_ENR_KEY, &custody_group_count); } builder @@ -287,11 +293,11 @@ fn compare_enr(local_enr: &Enr, disk_enr: &Enr) -> bool { && (local_enr.udp4().is_none() || local_enr.udp4() == disk_enr.udp4()) && (local_enr.udp6().is_none() || local_enr.udp6() == disk_enr.udp6()) // we need the ATTESTATION_BITFIELD_ENR_KEY and SYNC_COMMITTEE_BITFIELD_ENR_KEY and - // PEERDAS_CUSTODY_SUBNET_COUNT_ENR_KEY key to match, otherwise we use a new ENR. This will + // PEERDAS_CUSTODY_GROUP_COUNT_ENR_KEY key to match, otherwise we use a new ENR. This will // likely only be true for non-validating nodes. && local_enr.get_decodable::(ATTESTATION_BITFIELD_ENR_KEY) == disk_enr.get_decodable(ATTESTATION_BITFIELD_ENR_KEY) && local_enr.get_decodable::(SYNC_COMMITTEE_BITFIELD_ENR_KEY) == disk_enr.get_decodable(SYNC_COMMITTEE_BITFIELD_ENR_KEY) - && local_enr.get_decodable::(PEERDAS_CUSTODY_SUBNET_COUNT_ENR_KEY) == disk_enr.get_decodable(PEERDAS_CUSTODY_SUBNET_COUNT_ENR_KEY) + && local_enr.get_decodable::(PEERDAS_CUSTODY_GROUP_COUNT_ENR_KEY) == disk_enr.get_decodable(PEERDAS_CUSTODY_GROUP_COUNT_ENR_KEY) } /// Loads enr from the given directory @@ -333,9 +339,9 @@ mod test { type E = MainnetEthSpec; - fn make_eip7594_spec() -> ChainSpec { + fn make_fulu_spec() -> ChainSpec { let mut spec = E::default_spec(); - spec.eip7594_fork_epoch = Some(Epoch::new(10)); + spec.fulu_fork_epoch = Some(Epoch::new(10)); spec } @@ -348,33 +354,33 @@ mod test { } #[test] - fn custody_subnet_count_default() { + fn custody_group_count_default() { let config = NetworkConfig { subscribe_all_data_column_subnets: false, ..NetworkConfig::default() }; - let spec = make_eip7594_spec(); + let spec = make_fulu_spec(); let enr = build_enr_with_config(config, &spec).0; assert_eq!( - enr.custody_subnet_count::(&spec).unwrap(), + enr.custody_group_count::(&spec).unwrap(), spec.custody_requirement, ); } #[test] - fn custody_subnet_count_all() { + fn custody_group_count_all() { let config = NetworkConfig { subscribe_all_data_column_subnets: true, ..NetworkConfig::default() }; - let spec = make_eip7594_spec(); + let spec = make_fulu_spec(); let enr = build_enr_with_config(config, &spec).0; assert_eq!( - enr.custody_subnet_count::(&spec).unwrap(), - spec.data_column_sidecar_subnet_count, + enr.custody_group_count::(&spec).unwrap(), + spec.number_of_custody_groups, ); } diff --git a/beacon_node/lighthouse_network/src/discovery/mod.rs b/beacon_node/lighthouse_network/src/discovery/mod.rs index 578bb52b51..33c7775ae2 100644 --- a/beacon_node/lighthouse_network/src/discovery/mod.rs +++ b/beacon_node/lighthouse_network/src/discovery/mod.rs @@ -994,7 +994,7 @@ impl NetworkBehaviour for Discovery { &mut self, _peer_id: PeerId, _connection_id: ConnectionId, - _event: void::Void, + _event: std::convert::Infallible, ) { } diff --git a/beacon_node/lighthouse_network/src/discovery/subnet_predicate.rs b/beacon_node/lighthouse_network/src/discovery/subnet_predicate.rs index 02ff0cc3ca..400a0c2d56 100644 --- a/beacon_node/lighthouse_network/src/discovery/subnet_predicate.rs +++ b/beacon_node/lighthouse_network/src/discovery/subnet_predicate.rs @@ -1,10 +1,10 @@ //! The subnet predicate used for searching for a particular subnet. use super::*; use crate::types::{EnrAttestationBitfield, EnrSyncCommitteeBitfield}; -use itertools::Itertools; use slog::trace; use std::ops::Deref; -use types::{ChainSpec, DataColumnSubnetId}; +use types::data_column_custody_group::compute_subnets_for_node; +use types::ChainSpec; /// Returns the predicate for a given subnet. pub fn subnet_predicate( @@ -35,15 +35,11 @@ where .unwrap_or(false), Subnet::SyncCommittee(s) => sync_committee_bitfield .as_ref() - .map_or(false, |b| b.get(*s.deref() as usize).unwrap_or(false)), + .is_ok_and(|b| b.get(*s.deref() as usize).unwrap_or(false)), Subnet::DataColumn(s) => { - if let Ok(custody_subnet_count) = enr.custody_subnet_count::(&spec) { - DataColumnSubnetId::compute_custody_subnets::( - enr.node_id().raw(), - custody_subnet_count, - &spec, - ) - .map_or(false, |mut subnets| subnets.contains(s)) + if let Ok(custody_group_count) = enr.custody_group_count::(&spec) { + compute_subnets_for_node(enr.node_id().raw(), custody_group_count, &spec) + .is_ok_and(|subnets| subnets.contains(s)) } else { false } diff --git a/beacon_node/lighthouse_network/src/metrics.rs b/beacon_node/lighthouse_network/src/metrics.rs index cb9c007b91..b36cb8075d 100644 --- a/beacon_node/lighthouse_network/src/metrics.rs +++ b/beacon_node/lighthouse_network/src/metrics.rs @@ -93,11 +93,11 @@ pub static PEERS_PER_CLIENT: LazyLock> = LazyLock::new(|| { ) }); -pub static PEERS_PER_CUSTODY_SUBNET_COUNT: LazyLock> = LazyLock::new(|| { +pub static PEERS_PER_CUSTODY_GROUP_COUNT: LazyLock> = LazyLock::new(|| { try_create_int_gauge_vec( - "peers_per_custody_subnet_count", - "The current count of peers by custody subnet count", - &["custody_subnet_count"], + "peers_per_custody_group_count", + "The current count of peers by custody group count", + &["custody_group_count"], ) }); diff --git a/beacon_node/lighthouse_network/src/peer_manager/mod.rs b/beacon_node/lighthouse_network/src/peer_manager/mod.rs index 4df2566dac..07c4be7959 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/mod.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/mod.rs @@ -34,6 +34,9 @@ pub use peerdb::sync_status::{SyncInfo, SyncStatus}; use std::collections::{hash_map::Entry, HashMap, HashSet}; use std::net::IpAddr; use strum::IntoEnumIterator; +use types::data_column_custody_group::{ + compute_subnets_from_custody_group, get_custody_groups, CustodyIndex, +}; pub mod config; mod network_behaviour; @@ -63,6 +66,8 @@ pub const MIN_OUTBOUND_ONLY_FACTOR: f32 = 0.2; /// limit is 55, and we are at 55 peers, the following parameter provisions a few more slots of /// dialing priority peers we need for validator duties. pub const PRIORITY_PEER_EXCESS: f32 = 0.2; +/// The numbre of inbound libp2p peers we have seen before we consider our NAT to be open. +pub const LIBP2P_NAT_OPEN_THRESHOLD: usize = 3; /// The main struct that handles peer's reputation and connection status. pub struct PeerManager { @@ -99,6 +104,8 @@ pub struct PeerManager { /// discovery queries for subnet peers if we disconnect from existing sync /// committee subnet peers. sync_committee_subnets: HashMap, + /// A mapping of all custody groups to column subnets to avoid re-computation. + subnets_by_custody_group: HashMap>, /// The heartbeat interval to perform routine maintenance. heartbeat: tokio::time::Interval, /// Keeps track of whether the discovery service is enabled or not. @@ -158,6 +165,21 @@ impl PeerManager { // Set up the peer manager heartbeat interval let heartbeat = tokio::time::interval(tokio::time::Duration::from_secs(HEARTBEAT_INTERVAL)); + // Compute subnets for all custody groups + let subnets_by_custody_group = if network_globals.spec.is_peer_das_scheduled() { + (0..network_globals.spec.number_of_custody_groups) + .map(|custody_index| { + let subnets = + compute_subnets_from_custody_group(custody_index, &network_globals.spec) + .expect("Should compute subnets for all custody groups") + .collect(); + (custody_index, subnets) + }) + .collect::>>() + } else { + HashMap::new() + }; + Ok(PeerManager { network_globals, events: SmallVec::new(), @@ -168,6 +190,7 @@ impl PeerManager { target_peers: target_peer_count, temporary_banned_peers: LRUTimeCache::new(PEER_RECONNECTION_TIMEOUT), sync_committee_subnets: Default::default(), + subnets_by_custody_group, heartbeat, discovery_enabled, metrics_enabled, @@ -709,22 +732,39 @@ impl PeerManager { "peer_id" => %peer_id, "new_seq_no" => meta_data.seq_number()); } - let custody_subnet_count_opt = meta_data.custody_subnet_count().copied().ok(); + let custody_group_count_opt = meta_data.custody_group_count().copied().ok(); peer_info.set_meta_data(meta_data); if self.network_globals.spec.is_peer_das_scheduled() { // Gracefully ignore metadata/v2 peers. Potentially downscore after PeerDAS to // prioritize PeerDAS peers. - if let Some(custody_subnet_count) = custody_subnet_count_opt { - match self.compute_peer_custody_subnets(peer_id, custody_subnet_count) { - Ok(custody_subnets) => { + if let Some(custody_group_count) = custody_group_count_opt { + match self.compute_peer_custody_groups(peer_id, custody_group_count) { + Ok(custody_groups) => { + let custody_subnets = custody_groups + .into_iter() + .flat_map(|custody_index| { + self.subnets_by_custody_group + .get(&custody_index) + .cloned() + .unwrap_or_else(|| { + warn!( + self.log, + "Custody group not found in subnet mapping"; + "custody_index" => custody_index, + "peer_id" => %peer_id + ); + vec![] + }) + }) + .collect(); peer_info.set_custody_subnets(custody_subnets); } Err(err) => { - debug!(self.log, "Unable to compute peer custody subnets from metadata"; + debug!(self.log, "Unable to compute peer custody groups from metadata"; "info" => "Sending goodbye to peer", "peer_id" => %peer_id, - "custody_subnet_count" => custody_subnet_count, + "custody_group_count" => custody_group_count, "error" => ?err, ); invalid_meta_data = true; @@ -1307,8 +1347,10 @@ impl PeerManager { fn update_peer_count_metrics(&self) { let mut peers_connected = 0; let mut clients_per_peer = HashMap::new(); - let mut peers_connected_mutli: HashMap<(&str, &str), i32> = HashMap::new(); - let mut peers_per_custody_subnet_count: HashMap = HashMap::new(); + let mut inbound_ipv4_peers_connected: usize = 0; + let mut inbound_ipv6_peers_connected: usize = 0; + let mut peers_connected_multi: HashMap<(&str, &str), i32> = HashMap::new(); + let mut peers_per_custody_group_count: HashMap = HashMap::new(); for (_, peer_info) in self.network_globals.peers.read().connected_peers() { peers_connected += 1; @@ -1336,25 +1378,48 @@ impl PeerManager { }) }) .unwrap_or("unknown"); - *peers_connected_mutli + *peers_connected_multi .entry((direction, transport)) .or_default() += 1; if let Some(MetaData::V3(meta_data)) = peer_info.meta_data() { - *peers_per_custody_subnet_count - .entry(meta_data.custody_subnet_count) + *peers_per_custody_group_count + .entry(meta_data.custody_group_count) .or_default() += 1; } + // Check if incoming peer is ipv4 + if peer_info.is_incoming_ipv4_connection() { + inbound_ipv4_peers_connected += 1; + } + + // Check if incoming peer is ipv6 + if peer_info.is_incoming_ipv6_connection() { + inbound_ipv6_peers_connected += 1; + } + } + + // Set ipv4 nat_open metric flag if threshold of peercount is met, unset if below threshold + if inbound_ipv4_peers_connected >= LIBP2P_NAT_OPEN_THRESHOLD { + metrics::set_gauge_vec(&metrics::NAT_OPEN, &["libp2p_ipv4"], 1); + } else { + metrics::set_gauge_vec(&metrics::NAT_OPEN, &["libp2p_ipv4"], 0); + } + + // Set ipv6 nat_open metric flag if threshold of peercount is met, unset if below threshold + if inbound_ipv6_peers_connected >= LIBP2P_NAT_OPEN_THRESHOLD { + metrics::set_gauge_vec(&metrics::NAT_OPEN, &["libp2p_ipv6"], 1); + } else { + metrics::set_gauge_vec(&metrics::NAT_OPEN, &["libp2p_ipv6"], 0); } // PEERS_CONNECTED metrics::set_gauge(&metrics::PEERS_CONNECTED, peers_connected); - // CUSTODY_SUBNET_COUNT - for (custody_subnet_count, peer_count) in peers_per_custody_subnet_count.into_iter() { + // CUSTODY_GROUP_COUNT + for (custody_group_count, peer_count) in peers_per_custody_group_count.into_iter() { metrics::set_gauge_vec( - &metrics::PEERS_PER_CUSTODY_SUBNET_COUNT, - &[&custody_subnet_count.to_string()], + &metrics::PEERS_PER_CUSTODY_GROUP_COUNT, + &[&custody_group_count.to_string()], peer_count, ) } @@ -1375,7 +1440,7 @@ impl PeerManager { metrics::set_gauge_vec( &metrics::PEERS_CONNECTED_MULTI, &[direction, transport], - *peers_connected_mutli + *peers_connected_multi .get(&(direction, transport)) .unwrap_or(&0) as i64, ); @@ -1383,43 +1448,27 @@ impl PeerManager { } } - fn compute_peer_custody_subnets( + fn compute_peer_custody_groups( &self, peer_id: &PeerId, - custody_subnet_count: u64, - ) -> Result, String> { + custody_group_count: u64, + ) -> Result, String> { // If we don't have a node id, we cannot compute the custody duties anyway let node_id = peer_id_to_node_id(peer_id)?; let spec = &self.network_globals.spec; - if !(spec.custody_requirement..=spec.data_column_sidecar_subnet_count) - .contains(&custody_subnet_count) + if !(spec.custody_requirement..=spec.number_of_custody_groups) + .contains(&custody_group_count) { - return Err("Invalid custody subnet count in metadata: out of range".to_string()); + return Err("Invalid custody group count in metadata: out of range".to_string()); } - let custody_subnets = DataColumnSubnetId::compute_custody_subnets::( - node_id.raw(), - custody_subnet_count, - spec, - ) - .map(|subnets| subnets.collect()) - .unwrap_or_else(|e| { - // This is an unreachable scenario unless there's a bug, as we've validated the csc - // just above. - error!( - self.log, - "Computing peer custody subnets failed unexpectedly"; - "info" => "Falling back to default custody requirement subnets", - "peer_id" => %peer_id, - "custody_subnet_count" => custody_subnet_count, - "error" => ?e - ); - DataColumnSubnetId::compute_custody_requirement_subnets::(node_id.raw(), spec) - .collect() - }); - - Ok(custody_subnets) + get_custody_groups(node_id.raw(), custody_group_count, spec).map_err(|e| { + format!( + "Error computing peer custody groups for node {} with cgc={}: {:?}", + node_id, custody_group_count, e + ) + }) } } diff --git a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs index 9fd059df85..abafb200be 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs @@ -37,7 +37,10 @@ impl NetworkBehaviour for PeerManager { // no events from the dummy handler } - fn poll(&mut self, cx: &mut Context<'_>) -> Poll> { + fn poll( + &mut self, + cx: &mut Context<'_>, + ) -> Poll> { // perform the heartbeat when necessary while self.heartbeat.poll_tick(cx).is_ready() { self.heartbeat(); diff --git a/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs b/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs index d2effd4d03..37cb5df6ea 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs @@ -1,4 +1,4 @@ -use crate::discovery::enr::PEERDAS_CUSTODY_SUBNET_COUNT_ENR_KEY; +use crate::discovery::enr::PEERDAS_CUSTODY_GROUP_COUNT_ENR_KEY; use crate::discovery::{peer_id_to_node_id, CombinedKey}; use crate::{metrics, multiaddr::Multiaddr, types::Subnet, Enr, EnrExt, Gossipsub, PeerId}; use itertools::Itertools; @@ -13,6 +13,7 @@ use std::{ fmt::Formatter, }; use sync_status::SyncStatus; +use types::data_column_custody_group::compute_subnets_for_node; use types::{ChainSpec, DataColumnSubnetId, EthSpec}; pub mod client; @@ -695,8 +696,8 @@ impl PeerDB { if supernode { enr.insert( - PEERDAS_CUSTODY_SUBNET_COUNT_ENR_KEY, - &spec.data_column_sidecar_subnet_count, + PEERDAS_CUSTODY_GROUP_COUNT_ENR_KEY, + &spec.number_of_custody_groups, &enr_key, ) .expect("u64 can be encoded"); @@ -714,19 +715,14 @@ impl PeerDB { if supernode { let peer_info = self.peers.get_mut(&peer_id).expect("peer exists"); let all_subnets = (0..spec.data_column_sidecar_subnet_count) - .map(|csc| csc.into()) + .map(|subnet_id| subnet_id.into()) .collect(); peer_info.set_custody_subnets(all_subnets); } else { let peer_info = self.peers.get_mut(&peer_id).expect("peer exists"); let node_id = peer_id_to_node_id(&peer_id).expect("convert peer_id to node_id"); - let subnets = DataColumnSubnetId::compute_custody_subnets::( - node_id.raw(), - spec.custody_requirement, - spec, - ) - .expect("should compute custody subnets") - .collect(); + let subnets = compute_subnets_for_node(node_id.raw(), spec.custody_requirement, spec) + .expect("should compute custody subnets"); peer_info.set_custody_subnets(subnets); } @@ -1305,7 +1301,7 @@ impl BannedPeersCount { pub fn ip_is_banned(&self, ip: &IpAddr) -> bool { self.banned_peers_per_ip .get(ip) - .map_or(false, |count| *count > BANNED_PEERS_PER_IP_THRESHOLD) + .is_some_and(|count| *count > BANNED_PEERS_PER_IP_THRESHOLD) } } diff --git a/beacon_node/lighthouse_network/src/peer_manager/peerdb/peer_info.rs b/beacon_node/lighthouse_network/src/peer_manager/peerdb/peer_info.rs index ee8c27f474..2e8f462565 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/peerdb/peer_info.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/peerdb/peer_info.rs @@ -89,7 +89,7 @@ impl PeerInfo { } /// Returns if the peer is subscribed to a given `Subnet` from the metadata attnets/syncnets field. - /// Also returns true if the peer is assigned to custody a given data column `Subnet` computed from the metadata `custody_column_count` field or ENR `csc` field. + /// Also returns true if the peer is assigned to custody a given data column `Subnet` computed from the metadata `custody_group_count` field or ENR `cgc` field. pub fn on_subnet_metadata(&self, subnet: &Subnet) -> bool { if let Some(meta_data) = &self.meta_data { match subnet { @@ -99,9 +99,11 @@ impl PeerInfo { Subnet::SyncCommittee(id) => { return meta_data .syncnets() - .map_or(false, |s| s.get(**id as usize).unwrap_or(false)) + .is_ok_and(|s| s.get(**id as usize).unwrap_or(false)) + } + Subnet::DataColumn(subnet_id) => { + return self.is_assigned_to_custody_subnet(subnet_id) } - Subnet::DataColumn(column) => return self.custody_subnets.contains(column), } } false @@ -122,6 +124,24 @@ impl PeerInfo { self.connection_direction.as_ref() } + /// Returns true if this is an incoming ipv4 connection. + pub fn is_incoming_ipv4_connection(&self) -> bool { + self.seen_multiaddrs.iter().any(|multiaddr| { + multiaddr + .iter() + .any(|protocol| matches!(protocol, libp2p::core::multiaddr::Protocol::Ip4(_))) + }) + } + + /// Returns true if this is an incoming ipv6 connection. + pub fn is_incoming_ipv6_connection(&self) -> bool { + self.seen_multiaddrs.iter().any(|multiaddr| { + multiaddr + .iter() + .any(|protocol| matches!(protocol, libp2p::core::multiaddr::Protocol::Ip6(_))) + }) + } + /// Returns the sync status of the peer. pub fn sync_status(&self) -> &SyncStatus { &self.sync_status @@ -264,7 +284,7 @@ impl PeerInfo { /// Reports if this peer has some future validator duty in which case it is valuable to keep it. pub fn has_future_duty(&self) -> bool { - self.min_ttl.map_or(false, |i| i >= Instant::now()) + self.min_ttl.is_some_and(|i| i >= Instant::now()) } /// Returns score of the peer. diff --git a/beacon_node/lighthouse_network/src/rpc/codec.rs b/beacon_node/lighthouse_network/src/rpc/codec.rs index 5d86936d41..2bf35b0e35 100644 --- a/beacon_node/lighthouse_network/src/rpc/codec.rs +++ b/beacon_node/lighthouse_network/src/rpc/codec.rs @@ -20,7 +20,7 @@ use types::{ LightClientBootstrap, LightClientFinalityUpdate, LightClientOptimisticUpdate, LightClientUpdate, RuntimeVariableList, SignedBeaconBlock, SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockBellatrix, SignedBeaconBlockCapella, - SignedBeaconBlockDeneb, SignedBeaconBlockElectra, + SignedBeaconBlockDeneb, SignedBeaconBlockElectra, SignedBeaconBlockFulu, }; use unsigned_varint::codec::Uvi; @@ -186,6 +186,7 @@ impl Decoder for SSZSnappyInboundCodec { handle_rpc_request( self.protocol.versioned_protocol, &decoded_buffer, + self.fork_context.current_fork(), &self.fork_context.spec, ) } @@ -458,6 +459,9 @@ fn context_bytes( return match **ref_box_block { // NOTE: If you are adding another fork type here, be sure to modify the // `fork_context.to_context_bytes()` function to support it as well! + SignedBeaconBlock::Fulu { .. } => { + fork_context.to_context_bytes(ForkName::Fulu) + } SignedBeaconBlock::Electra { .. } => { fork_context.to_context_bytes(ForkName::Electra) } @@ -481,17 +485,9 @@ fn context_bytes( RpcSuccessResponse::BlobsByRange(_) | RpcSuccessResponse::BlobsByRoot(_) => { return fork_context.to_context_bytes(ForkName::Deneb); } - RpcSuccessResponse::DataColumnsByRoot(d) - | RpcSuccessResponse::DataColumnsByRange(d) => { - // TODO(das): Remove deneb fork after `peerdas-devnet-2`. - return if matches!( - fork_context.spec.fork_name_at_slot::(d.slot()), - ForkName::Deneb - ) { - fork_context.to_context_bytes(ForkName::Deneb) - } else { - fork_context.to_context_bytes(ForkName::Electra) - }; + RpcSuccessResponse::DataColumnsByRoot(_) + | RpcSuccessResponse::DataColumnsByRange(_) => { + return fork_context.to_context_bytes(ForkName::Fulu); } RpcSuccessResponse::LightClientBootstrap(lc_bootstrap) => { return lc_bootstrap @@ -552,6 +548,7 @@ fn handle_length( fn handle_rpc_request( versioned_protocol: SupportedProtocol, decoded_buffer: &[u8], + current_fork: ForkName, spec: &ChainSpec, ) -> Result>, RPCError> { match versioned_protocol { @@ -571,7 +568,7 @@ fn handle_rpc_request( BlocksByRootRequest::V2(BlocksByRootRequestV2 { block_roots: RuntimeVariableList::from_ssz_bytes( decoded_buffer, - spec.max_request_blocks as usize, + spec.max_request_blocks(current_fork), )?, }), ))), @@ -579,7 +576,7 @@ fn handle_rpc_request( BlocksByRootRequest::V1(BlocksByRootRequestV1 { block_roots: RuntimeVariableList::from_ssz_bytes( decoded_buffer, - spec.max_request_blocks as usize, + spec.max_request_blocks(current_fork), )?, }), ))), @@ -590,7 +587,7 @@ fn handle_rpc_request( Ok(Some(RequestType::BlobsByRoot(BlobsByRootRequest { blob_ids: RuntimeVariableList::from_ssz_bytes( decoded_buffer, - spec.max_request_blob_sidecars as usize, + spec.max_request_blob_sidecars(current_fork), )?, }))) } @@ -682,18 +679,18 @@ fn handle_rpc_response( SignedBeaconBlock::Base(SignedBeaconBlockBase::from_ssz_bytes(decoded_buffer)?), )))), SupportedProtocol::BlobsByRangeV1 => match fork_name { - Some(ForkName::Deneb) | Some(ForkName::Electra) => { - Ok(Some(RpcSuccessResponse::BlobsByRange(Arc::new( - BlobSidecar::from_ssz_bytes(decoded_buffer)?, - )))) + Some(fork_name) => { + if fork_name.deneb_enabled() { + Ok(Some(RpcSuccessResponse::BlobsByRange(Arc::new( + BlobSidecar::from_ssz_bytes(decoded_buffer)?, + )))) + } else { + Err(RPCError::ErrorResponse( + RpcErrorResponse::InvalidRequest, + "Invalid fork name for blobs by range".to_string(), + )) + } } - Some(ForkName::Base) - | Some(ForkName::Altair) - | Some(ForkName::Bellatrix) - | Some(ForkName::Capella) => Err(RPCError::ErrorResponse( - RpcErrorResponse::InvalidRequest, - "Invalid fork name for blobs by range".to_string(), - )), None => Err(RPCError::ErrorResponse( RpcErrorResponse::InvalidRequest, format!( @@ -703,18 +700,18 @@ fn handle_rpc_response( )), }, SupportedProtocol::BlobsByRootV1 => match fork_name { - Some(ForkName::Deneb) | Some(ForkName::Electra) => { - Ok(Some(RpcSuccessResponse::BlobsByRoot(Arc::new( - BlobSidecar::from_ssz_bytes(decoded_buffer)?, - )))) + Some(fork_name) => { + if fork_name.deneb_enabled() { + Ok(Some(RpcSuccessResponse::BlobsByRoot(Arc::new( + BlobSidecar::from_ssz_bytes(decoded_buffer)?, + )))) + } else { + Err(RPCError::ErrorResponse( + RpcErrorResponse::InvalidRequest, + "Invalid fork name for blobs by root".to_string(), + )) + } } - Some(ForkName::Base) - | Some(ForkName::Altair) - | Some(ForkName::Bellatrix) - | Some(ForkName::Capella) => Err(RPCError::ErrorResponse( - RpcErrorResponse::InvalidRequest, - "Invalid fork name for blobs by root".to_string(), - )), None => Err(RPCError::ErrorResponse( RpcErrorResponse::InvalidRequest, format!( @@ -725,10 +722,7 @@ fn handle_rpc_response( }, SupportedProtocol::DataColumnsByRootV1 => match fork_name { Some(fork_name) => { - // TODO(das): PeerDAS is currently supported for both deneb and electra. This check - // does not advertise the topic on deneb, simply allows it to decode it. Advertise - // logic is in `SupportedTopic::currently_supported`. - if fork_name.deneb_enabled() { + if fork_name.fulu_enabled() { Ok(Some(RpcSuccessResponse::DataColumnsByRoot(Arc::new( DataColumnSidecar::from_ssz_bytes(decoded_buffer)?, )))) @@ -749,7 +743,7 @@ fn handle_rpc_response( }, SupportedProtocol::DataColumnsByRangeV1 => match fork_name { Some(fork_name) => { - if fork_name.deneb_enabled() { + if fork_name.fulu_enabled() { Ok(Some(RpcSuccessResponse::DataColumnsByRange(Arc::new( DataColumnSidecar::from_ssz_bytes(decoded_buffer)?, )))) @@ -864,6 +858,9 @@ fn handle_rpc_response( decoded_buffer, )?), )))), + Some(ForkName::Fulu) => Ok(Some(RpcSuccessResponse::BlocksByRange(Arc::new( + SignedBeaconBlock::Fulu(SignedBeaconBlockFulu::from_ssz_bytes(decoded_buffer)?), + )))), None => Err(RPCError::ErrorResponse( RpcErrorResponse::InvalidRequest, format!( @@ -897,6 +894,9 @@ fn handle_rpc_response( decoded_buffer, )?), )))), + Some(ForkName::Fulu) => Ok(Some(RpcSuccessResponse::BlocksByRoot(Arc::new( + SignedBeaconBlock::Fulu(SignedBeaconBlockFulu::from_ssz_bytes(decoded_buffer)?), + )))), None => Err(RPCError::ErrorResponse( RpcErrorResponse::InvalidRequest, format!( @@ -934,9 +934,10 @@ mod tests { use crate::rpc::protocol::*; use crate::types::{EnrAttestationBitfield, EnrSyncCommitteeBitfield}; use types::{ - blob_sidecar::BlobIdentifier, BeaconBlock, BeaconBlockAltair, BeaconBlockBase, - BeaconBlockBellatrix, DataColumnIdentifier, EmptyBlock, Epoch, FixedBytesExtended, - FullPayload, Signature, Slot, + blob_sidecar::BlobIdentifier, data_column_sidecar::Cell, BeaconBlock, BeaconBlockAltair, + BeaconBlockBase, BeaconBlockBellatrix, BeaconBlockHeader, DataColumnIdentifier, EmptyBlock, + Epoch, FixedBytesExtended, FullPayload, KzgCommitment, KzgProof, Signature, + SignedBeaconBlockHeader, Slot, }; type Spec = types::MainnetEthSpec; @@ -948,12 +949,14 @@ mod tests { let capella_fork_epoch = Epoch::new(3); let deneb_fork_epoch = Epoch::new(4); let electra_fork_epoch = Epoch::new(5); + let fulu_fork_epoch = Epoch::new(6); chain_spec.altair_fork_epoch = Some(altair_fork_epoch); chain_spec.bellatrix_fork_epoch = Some(bellatrix_fork_epoch); chain_spec.capella_fork_epoch = Some(capella_fork_epoch); chain_spec.deneb_fork_epoch = Some(deneb_fork_epoch); chain_spec.electra_fork_epoch = Some(electra_fork_epoch); + chain_spec.fulu_fork_epoch = Some(fulu_fork_epoch); let current_slot = match fork_name { ForkName::Base => Slot::new(0), @@ -962,6 +965,7 @@ mod tests { ForkName::Capella => capella_fork_epoch.start_slot(Spec::slots_per_epoch()), ForkName::Deneb => deneb_fork_epoch.start_slot(Spec::slots_per_epoch()), ForkName::Electra => electra_fork_epoch.start_slot(Spec::slots_per_epoch()), + ForkName::Fulu => fulu_fork_epoch.start_slot(Spec::slots_per_epoch()), }; ForkContext::new::(current_slot, Hash256::zero(), &chain_spec) } @@ -984,7 +988,17 @@ mod tests { } fn empty_data_column_sidecar() -> Arc> { - Arc::new(DataColumnSidecar::empty()) + Arc::new(DataColumnSidecar { + index: 0, + column: VariableList::new(vec![Cell::::default()]).unwrap(), + kzg_commitments: VariableList::new(vec![KzgCommitment::empty_for_testing()]).unwrap(), + kzg_proofs: VariableList::new(vec![KzgProof::empty()]).unwrap(), + signed_block_header: SignedBeaconBlockHeader { + message: BeaconBlockHeader::empty(), + signature: Signature::empty(), + }, + kzg_commitments_inclusion_proof: Default::default(), + }) } /// Bellatrix block with length < max_rpc_size. @@ -1069,21 +1083,21 @@ mod tests { } } - fn bbroot_request_v1(spec: &ChainSpec) -> BlocksByRootRequest { - BlocksByRootRequest::new_v1(vec![Hash256::zero()], spec) + fn bbroot_request_v1(fork_name: ForkName) -> BlocksByRootRequest { + BlocksByRootRequest::new_v1(vec![Hash256::zero()], &fork_context(fork_name)) } - fn bbroot_request_v2(spec: &ChainSpec) -> BlocksByRootRequest { - BlocksByRootRequest::new(vec![Hash256::zero()], spec) + fn bbroot_request_v2(fork_name: ForkName) -> BlocksByRootRequest { + BlocksByRootRequest::new(vec![Hash256::zero()], &fork_context(fork_name)) } - fn blbroot_request(spec: &ChainSpec) -> BlobsByRootRequest { + fn blbroot_request(fork_name: ForkName) -> BlobsByRootRequest { BlobsByRootRequest::new( vec![BlobIdentifier { block_root: Hash256::zero(), index: 0, }], - spec, + &fork_context(fork_name), ) } @@ -1111,7 +1125,7 @@ mod tests { seq_number: 1, attnets: EnrAttestationBitfield::::default(), syncnets: EnrSyncCommitteeBitfield::::default(), - custody_subnet_count: 1, + custody_group_count: 1, }) } @@ -1396,6 +1410,16 @@ mod tests { Ok(Some(RpcSuccessResponse::BlobsByRange(empty_blob_sidecar()))), ); + assert_eq!( + encode_then_decode_response( + SupportedProtocol::BlobsByRangeV1, + RpcResponse::Success(RpcSuccessResponse::BlobsByRange(empty_blob_sidecar())), + ForkName::Fulu, + &chain_spec + ), + Ok(Some(RpcSuccessResponse::BlobsByRange(empty_blob_sidecar()))), + ); + assert_eq!( encode_then_decode_response( SupportedProtocol::BlobsByRootV1, @@ -1416,6 +1440,16 @@ mod tests { Ok(Some(RpcSuccessResponse::BlobsByRoot(empty_blob_sidecar()))), ); + assert_eq!( + encode_then_decode_response( + SupportedProtocol::BlobsByRootV1, + RpcResponse::Success(RpcSuccessResponse::BlobsByRoot(empty_blob_sidecar())), + ForkName::Fulu, + &chain_spec + ), + Ok(Some(RpcSuccessResponse::BlobsByRoot(empty_blob_sidecar()))), + ); + assert_eq!( encode_then_decode_response( SupportedProtocol::DataColumnsByRangeV1, @@ -1444,6 +1478,20 @@ mod tests { ))), ); + assert_eq!( + encode_then_decode_response( + SupportedProtocol::DataColumnsByRangeV1, + RpcResponse::Success(RpcSuccessResponse::DataColumnsByRange( + empty_data_column_sidecar() + )), + ForkName::Fulu, + &chain_spec + ), + Ok(Some(RpcSuccessResponse::DataColumnsByRange( + empty_data_column_sidecar() + ))), + ); + assert_eq!( encode_then_decode_response( SupportedProtocol::DataColumnsByRootV1, @@ -1471,6 +1519,20 @@ mod tests { empty_data_column_sidecar() ))), ); + + assert_eq!( + encode_then_decode_response( + SupportedProtocol::DataColumnsByRootV1, + RpcResponse::Success(RpcSuccessResponse::DataColumnsByRoot( + empty_data_column_sidecar() + )), + ForkName::Fulu, + &chain_spec + ), + Ok(Some(RpcSuccessResponse::DataColumnsByRoot( + empty_data_column_sidecar() + ))), + ); } // Test RPCResponse encoding/decoding for V1 messages @@ -1833,7 +1895,8 @@ mod tests { #[test] fn test_encode_then_decode_request() { - let chain_spec = Spec::default_spec(); + let fork_context = fork_context(ForkName::Electra); + let chain_spec = fork_context.spec.clone(); let requests: &[RequestType] = &[ RequestType::Ping(ping_message()), @@ -1841,21 +1904,33 @@ mod tests { RequestType::Goodbye(GoodbyeReason::Fault), RequestType::BlocksByRange(bbrange_request_v1()), RequestType::BlocksByRange(bbrange_request_v2()), - RequestType::BlocksByRoot(bbroot_request_v1(&chain_spec)), - RequestType::BlocksByRoot(bbroot_request_v2(&chain_spec)), RequestType::MetaData(MetadataRequest::new_v1()), RequestType::BlobsByRange(blbrange_request()), - RequestType::BlobsByRoot(blbroot_request(&chain_spec)), RequestType::DataColumnsByRange(dcbrange_request()), RequestType::DataColumnsByRoot(dcbroot_request(&chain_spec)), RequestType::MetaData(MetadataRequest::new_v2()), ]; - for req in requests.iter() { for fork_name in ForkName::list_all() { encode_then_decode_request(req.clone(), fork_name, &chain_spec); } } + + // Request types that have different length limits depending on the fork + // Handled separately to have consistent `ForkName` across request and responses + let fork_dependent_requests = |fork_name| { + [ + RequestType::BlobsByRoot(blbroot_request(fork_name)), + RequestType::BlocksByRoot(bbroot_request_v1(fork_name)), + RequestType::BlocksByRoot(bbroot_request_v2(fork_name)), + ] + }; + for fork_name in ForkName::list_all() { + let requests = fork_dependent_requests(fork_name); + for req in requests { + encode_then_decode_request(req.clone(), fork_name, &chain_spec); + } + } } /// Test a malicious snappy encoding for a V1 `Status` message where the attacker diff --git a/beacon_node/lighthouse_network/src/rpc/config.rs b/beacon_node/lighthouse_network/src/rpc/config.rs index 7b3a59eac7..75d49e9cb5 100644 --- a/beacon_node/lighthouse_network/src/rpc/config.rs +++ b/beacon_node/lighthouse_network/src/rpc/config.rs @@ -110,8 +110,8 @@ impl RateLimiterConfig { pub const DEFAULT_BLOCKS_BY_RANGE_QUOTA: Quota = Quota::n_every(128, 10); pub const DEFAULT_BLOCKS_BY_ROOT_QUOTA: Quota = Quota::n_every(128, 10); // `DEFAULT_BLOCKS_BY_RANGE_QUOTA` * (target + 1) to account for high usage - pub const DEFAULT_BLOBS_BY_RANGE_QUOTA: Quota = Quota::n_every(512, 10); - pub const DEFAULT_BLOBS_BY_ROOT_QUOTA: Quota = Quota::n_every(512, 10); + pub const DEFAULT_BLOBS_BY_RANGE_QUOTA: Quota = Quota::n_every(896, 10); + pub const DEFAULT_BLOBS_BY_ROOT_QUOTA: Quota = Quota::n_every(896, 10); // 320 blocks worth of columns for regular node, or 40 blocks for supernode. // Range sync load balances when requesting blocks, and each batch is 32 blocks. pub const DEFAULT_DATA_COLUMNS_BY_RANGE_QUOTA: Quota = Quota::n_every(5120, 10); diff --git a/beacon_node/lighthouse_network/src/rpc/handler.rs b/beacon_node/lighthouse_network/src/rpc/handler.rs index 0a0a6ca754..03203fcade 100644 --- a/beacon_node/lighthouse_network/src/rpc/handler.rs +++ b/beacon_node/lighthouse_network/src/rpc/handler.rs @@ -353,6 +353,7 @@ where !matches!(self.state, HandlerState::Deactivated) } + #[allow(deprecated)] fn poll( &mut self, cx: &mut Context<'_>, @@ -814,6 +815,7 @@ where Poll::Pending } + #[allow(deprecated)] fn on_connection_event( &mut self, event: ConnectionEvent< @@ -855,7 +857,47 @@ where } let (req, substream) = substream; - let max_responses = req.max_responses(); + let current_fork = self.fork_context.current_fork(); + let spec = &self.fork_context.spec; + + match &req { + RequestType::BlocksByRange(request) => { + let max_allowed = spec.max_request_blocks(current_fork) as u64; + if *request.count() > max_allowed { + self.events_out.push(HandlerEvent::Err(HandlerErr::Inbound { + id: self.current_inbound_substream_id, + proto: Protocol::BlocksByRange, + error: RPCError::InvalidData(format!( + "requested exceeded limit. allowed: {}, requested: {}", + max_allowed, + request.count() + )), + })); + return self.shutdown(None); + } + } + RequestType::BlobsByRange(request) => { + let max_requested_blobs = request + .count + .saturating_mul(spec.max_blobs_per_block_by_fork(current_fork)); + let max_allowed = spec.max_request_blob_sidecars(current_fork) as u64; + if max_requested_blobs > max_allowed { + self.events_out.push(HandlerEvent::Err(HandlerErr::Inbound { + id: self.current_inbound_substream_id, + proto: Protocol::BlobsByRange, + error: RPCError::InvalidData(format!( + "requested exceeded limit. allowed: {}, requested: {}", + max_allowed, max_requested_blobs + )), + })); + return self.shutdown(None); + } + } + _ => {} + }; + + let max_responses = + req.max_responses(self.fork_context.current_fork(), &self.fork_context.spec); // store requests that expect responses if max_responses > 0 { @@ -924,7 +966,8 @@ where } // add the stream to substreams if we expect a response, otherwise drop the stream. - let max_responses = request.max_responses(); + let max_responses = + request.max_responses(self.fork_context.current_fork(), &self.fork_context.spec); if max_responses > 0 { let max_remaining_chunks = if request.expect_exactly_one_response() { // Currently enforced only for multiple responses diff --git a/beacon_node/lighthouse_network/src/rpc/methods.rs b/beacon_node/lighthouse_network/src/rpc/methods.rs index bb8bfb0e20..ad6bea455e 100644 --- a/beacon_node/lighthouse_network/src/rpc/methods.rs +++ b/beacon_node/lighthouse_network/src/rpc/methods.rs @@ -20,6 +20,7 @@ use types::{ Epoch, EthSpec, Hash256, LightClientBootstrap, LightClientFinalityUpdate, LightClientOptimisticUpdate, LightClientUpdate, RuntimeVariableList, SignedBeaconBlock, Slot, }; +use types::{ForkContext, ForkName}; /// Maximum length of error message. pub type MaxErrorLen = U256; @@ -137,7 +138,7 @@ pub struct MetaData { #[superstruct(only(V2, V3))] pub syncnets: EnrSyncCommitteeBitfield, #[superstruct(only(V3))] - pub custody_subnet_count: u64, + pub custody_group_count: u64, } impl MetaData { @@ -180,13 +181,13 @@ impl MetaData { seq_number: metadata.seq_number, attnets: metadata.attnets.clone(), syncnets: Default::default(), - custody_subnet_count: spec.custody_requirement, + custody_group_count: spec.custody_requirement, }), MetaData::V2(metadata) => MetaData::V3(MetaDataV3 { seq_number: metadata.seq_number, attnets: metadata.attnets.clone(), syncnets: metadata.syncnets.clone(), - custody_subnet_count: spec.custody_requirement, + custody_group_count: spec.custody_requirement, }), md @ MetaData::V3(_) => md.clone(), } @@ -327,8 +328,9 @@ pub struct BlobsByRangeRequest { } impl BlobsByRangeRequest { - pub fn max_blobs_requested(&self) -> u64 { - self.count.saturating_mul(E::max_blobs_per_block() as u64) + pub fn max_blobs_requested(&self, current_fork: ForkName, spec: &ChainSpec) -> u64 { + let max_blobs_per_block = spec.max_blobs_per_block_by_fork(current_fork); + self.count.saturating_mul(max_blobs_per_block) } } @@ -362,7 +364,7 @@ impl DataColumnsByRangeRequest { DataColumnsByRangeRequest { start_slot: 0, count: 0, - columns: vec![0; spec.number_of_columns], + columns: vec![0; spec.number_of_columns as usize], } .as_ssz_bytes() .len() @@ -418,15 +420,19 @@ pub struct BlocksByRootRequest { } impl BlocksByRootRequest { - pub fn new(block_roots: Vec, spec: &ChainSpec) -> Self { - let block_roots = - RuntimeVariableList::from_vec(block_roots, spec.max_request_blocks as usize); + pub fn new(block_roots: Vec, fork_context: &ForkContext) -> Self { + let max_request_blocks = fork_context + .spec + .max_request_blocks(fork_context.current_fork()); + let block_roots = RuntimeVariableList::from_vec(block_roots, max_request_blocks); Self::V2(BlocksByRootRequestV2 { block_roots }) } - pub fn new_v1(block_roots: Vec, spec: &ChainSpec) -> Self { - let block_roots = - RuntimeVariableList::from_vec(block_roots, spec.max_request_blocks as usize); + pub fn new_v1(block_roots: Vec, fork_context: &ForkContext) -> Self { + let max_request_blocks = fork_context + .spec + .max_request_blocks(fork_context.current_fork()); + let block_roots = RuntimeVariableList::from_vec(block_roots, max_request_blocks); Self::V1(BlocksByRootRequestV1 { block_roots }) } } @@ -439,9 +445,11 @@ pub struct BlobsByRootRequest { } impl BlobsByRootRequest { - pub fn new(blob_ids: Vec, spec: &ChainSpec) -> Self { - let blob_ids = - RuntimeVariableList::from_vec(blob_ids, spec.max_request_blob_sidecars as usize); + pub fn new(blob_ids: Vec, fork_context: &ForkContext) -> Self { + let max_request_blob_sidecars = fork_context + .spec + .max_request_blob_sidecars(fork_context.current_fork()); + let blob_ids = RuntimeVariableList::from_vec(blob_ids, max_request_blob_sidecars); Self { blob_ids } } } diff --git a/beacon_node/lighthouse_network/src/rpc/mod.rs b/beacon_node/lighthouse_network/src/rpc/mod.rs index 7d091da766..03f1395b8b 100644 --- a/beacon_node/lighthouse_network/src/rpc/mod.rs +++ b/beacon_node/lighthouse_network/src/rpc/mod.rs @@ -181,12 +181,13 @@ impl RPC { let inbound_limiter = inbound_rate_limiter_config.map(|config| { debug!(log, "Using inbound rate limiting params"; "config" => ?config); - RateLimiter::new_with_config(config.0) + RateLimiter::new_with_config(config.0, fork_context.clone()) .expect("Inbound limiter configuration parameters are valid") }); let self_limiter = outbound_rate_limiter_config.map(|config| { - SelfRateLimiter::new(config, log.clone()).expect("Configuration parameters are valid") + SelfRateLimiter::new(config, fork_context.clone(), log.clone()) + .expect("Configuration parameters are valid") }); RPC { diff --git a/beacon_node/lighthouse_network/src/rpc/protocol.rs b/beacon_node/lighthouse_network/src/rpc/protocol.rs index 57c2795b04..eac7d67490 100644 --- a/beacon_node/lighthouse_network/src/rpc/protocol.rs +++ b/beacon_node/lighthouse_network/src/rpc/protocol.rs @@ -17,12 +17,11 @@ use tokio_util::{ compat::{Compat, FuturesAsyncReadCompatExt}, }; use types::{ - BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockCapella, BeaconBlockElectra, - BlobSidecar, ChainSpec, DataColumnSidecar, EmptyBlock, EthSpec, EthSpecId, ForkContext, - ForkName, LightClientBootstrap, LightClientBootstrapAltair, LightClientFinalityUpdate, - LightClientFinalityUpdateAltair, LightClientOptimisticUpdate, - LightClientOptimisticUpdateAltair, LightClientUpdate, MainnetEthSpec, MinimalEthSpec, - Signature, SignedBeaconBlock, + BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BlobSidecar, ChainSpec, DataColumnSidecar, + EmptyBlock, EthSpec, EthSpecId, ForkContext, ForkName, LightClientBootstrap, + LightClientBootstrapAltair, LightClientFinalityUpdate, LightClientFinalityUpdateAltair, + LightClientOptimisticUpdate, LightClientOptimisticUpdateAltair, LightClientUpdate, + MainnetEthSpec, MinimalEthSpec, Signature, SignedBeaconBlock, }; // Note: Hardcoding the `EthSpec` type for `SignedBeaconBlock` as min/max values is @@ -55,24 +54,6 @@ pub static SIGNED_BEACON_BLOCK_ALTAIR_MAX: LazyLock = LazyLock::new(|| { .len() }); -pub static SIGNED_BEACON_BLOCK_CAPELLA_MAX_WITHOUT_PAYLOAD: LazyLock = LazyLock::new(|| { - SignedBeaconBlock::::from_block( - BeaconBlock::Capella(BeaconBlockCapella::full(&MainnetEthSpec::default_spec())), - Signature::empty(), - ) - .as_ssz_bytes() - .len() -}); - -pub static SIGNED_BEACON_BLOCK_ELECTRA_MAX_WITHOUT_PAYLOAD: LazyLock = LazyLock::new(|| { - SignedBeaconBlock::::from_block( - BeaconBlock::Electra(BeaconBlockElectra::full(&MainnetEthSpec::default_spec())), - Signature::empty(), - ) - .as_ssz_bytes() - .len() -}); - /// The `BeaconBlockBellatrix` block has an `ExecutionPayload` field which has a max size ~16 GiB for future proofing. /// We calculate the value from its fields instead of constructing the block and checking the length. /// Note: This is only the theoretical upper bound. We further bound the max size we receive over the network @@ -83,42 +64,12 @@ pub static SIGNED_BEACON_BLOCK_BELLATRIX_MAX: LazyLock = + types::ExecutionPayload::::max_execution_payload_bellatrix_size() // adding max size of execution payload (~16gb) + ssz::BYTES_PER_LENGTH_OFFSET); // Adding the additional ssz offset for the `ExecutionPayload` field -pub static SIGNED_BEACON_BLOCK_CAPELLA_MAX: LazyLock = LazyLock::new(|| { - *SIGNED_BEACON_BLOCK_CAPELLA_MAX_WITHOUT_PAYLOAD - + types::ExecutionPayload::::max_execution_payload_capella_size() // adding max size of execution payload (~16gb) - + ssz::BYTES_PER_LENGTH_OFFSET -}); // Adding the additional ssz offset for the `ExecutionPayload` field - -pub static SIGNED_BEACON_BLOCK_DENEB_MAX: LazyLock = LazyLock::new(|| { - *SIGNED_BEACON_BLOCK_CAPELLA_MAX_WITHOUT_PAYLOAD - + types::ExecutionPayload::::max_execution_payload_deneb_size() // adding max size of execution payload (~16gb) - + ssz::BYTES_PER_LENGTH_OFFSET // Adding the additional offsets for the `ExecutionPayload` - + (::ssz_fixed_len() * ::max_blobs_per_block()) - + ssz::BYTES_PER_LENGTH_OFFSET -}); // Length offset for the blob commitments field. - // -pub static SIGNED_BEACON_BLOCK_ELECTRA_MAX: LazyLock = LazyLock::new(|| { - *SIGNED_BEACON_BLOCK_ELECTRA_MAX_WITHOUT_PAYLOAD - + types::ExecutionPayload::::max_execution_payload_electra_size() // adding max size of execution payload (~16gb) - + ssz::BYTES_PER_LENGTH_OFFSET // Adding the additional ssz offset for the `ExecutionPayload` field - + (::ssz_fixed_len() * ::max_blobs_per_block()) - + ssz::BYTES_PER_LENGTH_OFFSET -}); // Length offset for the blob commitments field. - pub static BLOB_SIDECAR_SIZE: LazyLock = LazyLock::new(BlobSidecar::::max_size); pub static BLOB_SIDECAR_SIZE_MINIMAL: LazyLock = LazyLock::new(BlobSidecar::::max_size); -pub static DATA_COLUMNS_SIDECAR_MIN: LazyLock = LazyLock::new(|| { - DataColumnSidecar::::empty() - .as_ssz_bytes() - .len() -}); -pub static DATA_COLUMNS_SIDECAR_MAX: LazyLock = - LazyLock::new(DataColumnSidecar::::max_size); - pub static ERROR_TYPE_MIN: LazyLock = LazyLock::new(|| { VariableList::::from(Vec::::new()) .as_ssz_bytes() @@ -193,22 +144,12 @@ pub fn rpc_block_limits_by_fork(current_fork: ForkName) -> RpcLimits { *SIGNED_BEACON_BLOCK_BASE_MIN, // Base block is smaller than altair blocks *SIGNED_BEACON_BLOCK_ALTAIR_MAX, // Altair block is larger than base blocks ), - ForkName::Bellatrix => RpcLimits::new( + // After the merge the max SSZ size of a block is absurdly big. The size is actually + // bound by other constants, so here we default to the bellatrix's max value + _ => RpcLimits::new( *SIGNED_BEACON_BLOCK_BASE_MIN, // Base block is smaller than altair and bellatrix blocks *SIGNED_BEACON_BLOCK_BELLATRIX_MAX, // Bellatrix block is larger than base and altair blocks ), - ForkName::Capella => RpcLimits::new( - *SIGNED_BEACON_BLOCK_BASE_MIN, // Base block is smaller than altair and bellatrix blocks - *SIGNED_BEACON_BLOCK_CAPELLA_MAX, // Capella block is larger than base, altair and merge blocks - ), - ForkName::Deneb => RpcLimits::new( - *SIGNED_BEACON_BLOCK_BASE_MIN, // Base block is smaller than altair and bellatrix blocks - *SIGNED_BEACON_BLOCK_DENEB_MAX, // Deneb block is larger than all prior fork blocks - ), - ForkName::Electra => RpcLimits::new( - *SIGNED_BEACON_BLOCK_BASE_MIN, // Base block is smaller than altair and bellatrix blocks - *SIGNED_BEACON_BLOCK_ELECTRA_MAX, // Electra block is larger than Deneb block - ), } } @@ -226,7 +167,7 @@ fn rpc_light_client_updates_by_range_limits_by_fork(current_fork: ForkName) -> R ForkName::Deneb => { RpcLimits::new(altair_fixed_len, *LIGHT_CLIENT_UPDATES_BY_RANGE_DENEB_MAX) } - ForkName::Electra => { + ForkName::Electra | ForkName::Fulu => { RpcLimits::new(altair_fixed_len, *LIGHT_CLIENT_UPDATES_BY_RANGE_ELECTRA_MAX) } } @@ -246,7 +187,7 @@ fn rpc_light_client_finality_update_limits_by_fork(current_fork: ForkName) -> Rp ForkName::Deneb => { RpcLimits::new(altair_fixed_len, *LIGHT_CLIENT_FINALITY_UPDATE_DENEB_MAX) } - ForkName::Electra => { + ForkName::Electra | ForkName::Fulu => { RpcLimits::new(altair_fixed_len, *LIGHT_CLIENT_FINALITY_UPDATE_ELECTRA_MAX) } } @@ -267,7 +208,7 @@ fn rpc_light_client_optimistic_update_limits_by_fork(current_fork: ForkName) -> ForkName::Deneb => { RpcLimits::new(altair_fixed_len, *LIGHT_CLIENT_OPTIMISTIC_UPDATE_DENEB_MAX) } - ForkName::Electra => RpcLimits::new( + ForkName::Electra | ForkName::Fulu => RpcLimits::new( altair_fixed_len, *LIGHT_CLIENT_OPTIMISTIC_UPDATE_ELECTRA_MAX, ), @@ -284,7 +225,9 @@ fn rpc_light_client_bootstrap_limits_by_fork(current_fork: ForkName) -> RpcLimit } ForkName::Capella => RpcLimits::new(altair_fixed_len, *LIGHT_CLIENT_BOOTSTRAP_CAPELLA_MAX), ForkName::Deneb => RpcLimits::new(altair_fixed_len, *LIGHT_CLIENT_BOOTSTRAP_DENEB_MAX), - ForkName::Electra => RpcLimits::new(altair_fixed_len, *LIGHT_CLIENT_BOOTSTRAP_ELECTRA_MAX), + ForkName::Electra | ForkName::Fulu => { + RpcLimits::new(altair_fixed_len, *LIGHT_CLIENT_BOOTSTRAP_ELECTRA_MAX) + } } } @@ -611,8 +554,12 @@ impl ProtocolId { Protocol::BlocksByRoot => rpc_block_limits_by_fork(fork_context.current_fork()), Protocol::BlobsByRange => rpc_blob_limits::(), Protocol::BlobsByRoot => rpc_blob_limits::(), - Protocol::DataColumnsByRoot => rpc_data_column_limits(), - Protocol::DataColumnsByRange => rpc_data_column_limits(), + Protocol::DataColumnsByRoot => { + rpc_data_column_limits::(fork_context.current_fork(), &fork_context.spec) + } + Protocol::DataColumnsByRange => { + rpc_data_column_limits::(fork_context.current_fork(), &fork_context.spec) + } Protocol::Ping => RpcLimits::new( ::ssz_fixed_len(), ::ssz_fixed_len(), @@ -692,8 +639,11 @@ pub fn rpc_blob_limits() -> RpcLimits { } } -pub fn rpc_data_column_limits() -> RpcLimits { - RpcLimits::new(*DATA_COLUMNS_SIDECAR_MIN, *DATA_COLUMNS_SIDECAR_MAX) +pub fn rpc_data_column_limits(fork_name: ForkName, spec: &ChainSpec) -> RpcLimits { + RpcLimits::new( + DataColumnSidecar::::min_size(), + DataColumnSidecar::::max_size(spec.max_blobs_per_block_by_fork(fork_name) as usize), + ) } /* Inbound upgrade */ @@ -791,13 +741,13 @@ impl RequestType { /* These functions are used in the handler for stream management */ /// Maximum number of responses expected for this request. - pub fn max_responses(&self) -> u64 { + pub fn max_responses(&self, current_fork: ForkName, spec: &ChainSpec) -> u64 { match self { RequestType::Status(_) => 1, RequestType::Goodbye(_) => 0, RequestType::BlocksByRange(req) => *req.count(), RequestType::BlocksByRoot(req) => req.block_roots().len() as u64, - RequestType::BlobsByRange(req) => req.max_blobs_requested::(), + RequestType::BlobsByRange(req) => req.max_blobs_requested(current_fork, spec), RequestType::BlobsByRoot(req) => req.blob_ids.len() as u64, RequestType::DataColumnsByRoot(req) => req.data_column_ids.len() as u64, RequestType::DataColumnsByRange(req) => req.max_requested::(), diff --git a/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs b/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs index ecbacc8c11..b9e82a5f1e 100644 --- a/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs +++ b/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs @@ -6,10 +6,11 @@ use serde::{Deserialize, Serialize}; use std::future::Future; use std::hash::Hash; use std::pin::Pin; +use std::sync::Arc; use std::task::{Context, Poll}; use std::time::{Duration, Instant}; use tokio::time::Interval; -use types::EthSpec; +use types::{ChainSpec, EthSpec, ForkContext, ForkName}; /// Nanoseconds since a given time. // Maintained as u64 to reduce footprint @@ -109,6 +110,7 @@ pub struct RPCRateLimiter { lc_finality_update_rl: Limiter, /// LightClientUpdatesByRange rate limiter. lc_updates_by_range_rl: Limiter, + fork_context: Arc, } /// Error type for non conformant requests @@ -176,7 +178,7 @@ impl RPCRateLimiterBuilder { self } - pub fn build(self) -> Result { + pub fn build(self, fork_context: Arc) -> Result { // get our quotas let ping_quota = self.ping_quota.ok_or("Ping quota not specified")?; let metadata_quota = self.metadata_quota.ok_or("MetaData quota not specified")?; @@ -253,13 +255,14 @@ impl RPCRateLimiterBuilder { lc_finality_update_rl, lc_updates_by_range_rl, init_time: Instant::now(), + fork_context, }) } } pub trait RateLimiterItem { fn protocol(&self) -> Protocol; - fn max_responses(&self) -> u64; + fn max_responses(&self, current_fork: ForkName, spec: &ChainSpec) -> u64; } impl RateLimiterItem for super::RequestType { @@ -267,13 +270,16 @@ impl RateLimiterItem for super::RequestType { self.versioned_protocol().protocol() } - fn max_responses(&self) -> u64 { - self.max_responses() + fn max_responses(&self, current_fork: ForkName, spec: &ChainSpec) -> u64 { + self.max_responses(current_fork, spec) } } impl RPCRateLimiter { - pub fn new_with_config(config: RateLimiterConfig) -> Result { + pub fn new_with_config( + config: RateLimiterConfig, + fork_context: Arc, + ) -> Result { // Destructure to make sure every configuration value is used. let RateLimiterConfig { ping_quota, @@ -316,7 +322,7 @@ impl RPCRateLimiter { Protocol::LightClientUpdatesByRange, light_client_updates_by_range_quota, ) - .build() + .build(fork_context) } /// Get a builder instance. @@ -330,7 +336,9 @@ impl RPCRateLimiter { request: &Item, ) -> Result<(), RateLimitedErr> { let time_since_start = self.init_time.elapsed(); - let tokens = request.max_responses().max(1); + let tokens = request + .max_responses(self.fork_context.current_fork(), &self.fork_context.spec) + .max(1); let check = |limiter: &mut Limiter| limiter.allows(time_since_start, peer_id, tokens); diff --git a/beacon_node/lighthouse_network/src/rpc/self_limiter.rs b/beacon_node/lighthouse_network/src/rpc/self_limiter.rs index e968ad11e3..e0c8593f29 100644 --- a/beacon_node/lighthouse_network/src/rpc/self_limiter.rs +++ b/beacon_node/lighthouse_network/src/rpc/self_limiter.rs @@ -1,5 +1,6 @@ use std::{ collections::{hash_map::Entry, HashMap, VecDeque}, + sync::Arc, task::{Context, Poll}, time::Duration, }; @@ -9,7 +10,7 @@ use libp2p::{swarm::NotifyHandler, PeerId}; use slog::{crit, debug, Logger}; use smallvec::SmallVec; use tokio_util::time::DelayQueue; -use types::EthSpec; +use types::{EthSpec, ForkContext}; use super::{ config::OutboundRateLimiterConfig, @@ -50,9 +51,13 @@ pub enum Error { impl SelfRateLimiter { /// Creates a new [`SelfRateLimiter`] based on configration values. - pub fn new(config: OutboundRateLimiterConfig, log: Logger) -> Result { + pub fn new( + config: OutboundRateLimiterConfig, + fork_context: Arc, + log: Logger, + ) -> Result { debug!(log, "Using self rate limiting params"; "config" => ?config); - let limiter = RateLimiter::new_with_config(config.0)?; + let limiter = RateLimiter::new_with_config(config.0, fork_context)?; Ok(SelfRateLimiter { delayed_requests: Default::default(), @@ -215,7 +220,7 @@ mod tests { use crate::service::api_types::{AppRequestId, RequestId, SyncRequestId}; use libp2p::PeerId; use std::time::Duration; - use types::MainnetEthSpec; + use types::{EthSpec, ForkContext, Hash256, MainnetEthSpec, Slot}; /// Test that `next_peer_request_ready` correctly maintains the queue. #[tokio::test] @@ -225,8 +230,13 @@ mod tests { ping_quota: Quota::n_every(1, 2), ..Default::default() }); + let fork_context = std::sync::Arc::new(ForkContext::new::( + Slot::new(0), + Hash256::ZERO, + &MainnetEthSpec::default_spec(), + )); let mut limiter: SelfRateLimiter = - SelfRateLimiter::new(config, log).unwrap(); + SelfRateLimiter::new(config, fork_context, log).unwrap(); let peer_id = PeerId::random(); for i in 1..=5u32 { diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index afcbfce173..354def79b0 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -16,7 +16,7 @@ use crate::rpc::{ use crate::types::{ attestation_sync_committee_topics, fork_core_topics, subnet_from_topic_hash, GossipEncoding, GossipKind, GossipTopic, SnappyTransform, Subnet, SubnetDiscovery, ALTAIR_CORE_TOPICS, - BASE_CORE_TOPICS, CAPELLA_CORE_TOPICS, DENEB_CORE_TOPICS, LIGHT_CLIENT_GOSSIP_TOPICS, + BASE_CORE_TOPICS, CAPELLA_CORE_TOPICS, LIGHT_CLIENT_GOSSIP_TOPICS, }; use crate::EnrExt; use crate::Eth2Enr; @@ -198,15 +198,12 @@ impl Network { )?; // Construct the metadata - let custody_subnet_count = ctx.chain_spec.is_peer_das_scheduled().then(|| { - if config.subscribe_all_data_column_subnets { - ctx.chain_spec.data_column_sidecar_subnet_count - } else { - ctx.chain_spec.custody_requirement - } + let custody_group_count = ctx.chain_spec.is_peer_das_scheduled().then(|| { + ctx.chain_spec + .custody_group_count(config.subscribe_all_data_column_subnets) }); let meta_data = - utils::load_or_build_metadata(&config.network_dir, custody_subnet_count, &log); + utils::load_or_build_metadata(&config.network_dir, custody_group_count, &log); let seq_number = *meta_data.seq_number(); let globals = NetworkGlobals::new( enr, @@ -285,26 +282,23 @@ impl Network { let max_topics = ctx.chain_spec.attestation_subnet_count as usize + SYNC_COMMITTEE_SUBNET_COUNT as usize - + ctx.chain_spec.blob_sidecar_subnet_count as usize + + ctx.chain_spec.blob_sidecar_subnet_count_max() as usize + ctx.chain_spec.data_column_sidecar_subnet_count as usize + BASE_CORE_TOPICS.len() + ALTAIR_CORE_TOPICS.len() - + CAPELLA_CORE_TOPICS.len() - + DENEB_CORE_TOPICS.len() + + CAPELLA_CORE_TOPICS.len() // 0 core deneb and electra topics + LIGHT_CLIENT_GOSSIP_TOPICS.len(); let possible_fork_digests = ctx.fork_context.all_fork_digests(); let filter = gossipsub::MaxCountSubscriptionFilter { filter: utils::create_whitelist_filter( possible_fork_digests, - ctx.chain_spec.attestation_subnet_count, + &ctx.chain_spec, SYNC_COMMITTEE_SUBNET_COUNT, - ctx.chain_spec.blob_sidecar_subnet_count, - ctx.chain_spec.data_column_sidecar_subnet_count, ), // during a fork we subscribe to both the old and new topics max_subscribed_topics: max_topics * 4, - // 418 in theory = (64 attestation + 4 sync committee + 7 core topics + 6 blob topics + 128 column topics) * 2 + // 424 in theory = (64 attestation + 4 sync committee + 7 core topics + 9 blob topics + 128 column topics) * 2 max_subscriptions_per_request: max_topics * 2, }; @@ -1852,7 +1846,7 @@ impl Network { None } #[allow(unreachable_patterns)] - BehaviourEvent::ConnectionLimits(le) => void::unreachable(le), + BehaviourEvent::ConnectionLimits(le) => libp2p::core::util::unreachable(le), }, SwarmEvent::ConnectionEstablished { .. } => None, SwarmEvent::ConnectionClosed { .. } => None, diff --git a/beacon_node/lighthouse_network/src/service/utils.rs b/beacon_node/lighthouse_network/src/service/utils.rs index 490928c08c..72c2b29102 100644 --- a/beacon_node/lighthouse_network/src/service/utils.rs +++ b/beacon_node/lighthouse_network/src/service/utils.rs @@ -164,8 +164,8 @@ pub fn strip_peer_id(addr: &mut Multiaddr) { /// Load metadata from persisted file. Return default metadata if loading fails. pub fn load_or_build_metadata( - network_dir: &std::path::Path, - custody_subnet_count: Option, + network_dir: &Path, + custody_group_count_opt: Option, log: &slog::Logger, ) -> MetaData { // We load a V2 metadata version by default (regardless of current fork) @@ -216,12 +216,12 @@ pub fn load_or_build_metadata( }; // Wrap the MetaData - let meta_data = if let Some(custody_count) = custody_subnet_count { + let meta_data = if let Some(custody_group_count) = custody_group_count_opt { MetaData::V3(MetaDataV3 { attnets: meta_data.attnets, seq_number: meta_data.seq_number, syncnets: meta_data.syncnets, - custody_subnet_count: custody_count, + custody_group_count, }) } else { MetaData::V2(meta_data) @@ -236,10 +236,8 @@ pub fn load_or_build_metadata( /// possible fork digests. pub(crate) fn create_whitelist_filter( possible_fork_digests: Vec<[u8; 4]>, - attestation_subnet_count: u64, + spec: &ChainSpec, sync_committee_subnet_count: u64, - blob_sidecar_subnet_count: u64, - data_column_sidecar_subnet_count: u64, ) -> gossipsub::WhitelistSubscriptionFilter { let mut possible_hashes = HashSet::new(); for fork_digest in possible_fork_digests { @@ -259,16 +257,17 @@ pub(crate) fn create_whitelist_filter( add(BlsToExecutionChange); add(LightClientFinalityUpdate); add(LightClientOptimisticUpdate); - for id in 0..attestation_subnet_count { + for id in 0..spec.attestation_subnet_count { add(Attestation(SubnetId::new(id))); } for id in 0..sync_committee_subnet_count { add(SyncCommitteeMessage(SyncSubnetId::new(id))); } - for id in 0..blob_sidecar_subnet_count { + let blob_subnet_count = spec.blob_sidecar_subnet_count_max(); + for id in 0..blob_subnet_count { add(BlobSidecar(id)); } - for id in 0..data_column_sidecar_subnet_count { + for id in 0..spec.data_column_sidecar_subnet_count { add(DataColumnSidecar(DataColumnSubnetId::new(id))); } } @@ -283,8 +282,8 @@ pub(crate) fn save_metadata_to_disk( ) { let _ = std::fs::create_dir_all(dir); // We always store the metadata v2 to disk because - // custody_subnet_count parameter doesn't need to be persisted across runs. - // custody_subnet_count is what the user sets it for the current run. + // custody_group_count parameter doesn't need to be persisted across runs. + // custody_group_count is what the user sets it for the current run. // This is to prevent ugly branching logic when reading the metadata from disk. let metadata_bytes = metadata.metadata_v2().as_ssz_bytes(); match File::create(dir.join(METADATA_FILENAME)).and_then(|mut f| f.write_all(&metadata_bytes)) { diff --git a/beacon_node/lighthouse_network/src/types/globals.rs b/beacon_node/lighthouse_network/src/types/globals.rs index 92583b7b5d..c9e84e2dd1 100644 --- a/beacon_node/lighthouse_network/src/types/globals.rs +++ b/beacon_node/lighthouse_network/src/types/globals.rs @@ -3,10 +3,13 @@ use crate::peer_manager::peerdb::PeerDB; use crate::rpc::{MetaData, MetaDataV3}; use crate::types::{BackFillState, SyncState}; use crate::{Client, Enr, EnrExt, GossipTopic, Multiaddr, NetworkConfig, PeerId}; -use itertools::Itertools; use parking_lot::RwLock; +use slog::error; use std::collections::HashSet; use std::sync::Arc; +use types::data_column_custody_group::{ + compute_columns_for_custody_group, compute_subnets_from_custody_group, get_custody_groups, +}; use types::{ChainSpec, ColumnIndex, DataColumnSubnetId, EthSpec}; pub struct NetworkGlobals { @@ -27,8 +30,8 @@ pub struct NetworkGlobals { /// The current state of the backfill sync. pub backfill_state: RwLock, /// The computed sampling subnets and columns is stored to avoid re-computing. - pub sampling_subnets: Vec, - pub sampling_columns: Vec, + pub sampling_subnets: HashSet, + pub sampling_columns: HashSet, /// Network-related configuration. Immutable after initialization. pub config: Arc, /// Ethereum chain configuration. Immutable after initialization. @@ -48,30 +51,43 @@ impl NetworkGlobals { let (sampling_subnets, sampling_columns) = if spec.is_peer_das_scheduled() { let node_id = enr.node_id().raw(); - let custody_subnet_count = local_metadata - .custody_subnet_count() - .copied() - .expect("custody subnet count must be set if PeerDAS is scheduled"); + let custody_group_count = match local_metadata.custody_group_count() { + Ok(&cgc) if cgc <= spec.number_of_custody_groups => cgc, + _ => { + error!( + log, + "custody_group_count from metadata is either invalid or not set. This is a bug!"; + "info" => "falling back to default custody requirement" + ); + spec.custody_requirement + } + }; - let subnet_sampling_size = std::cmp::max(custody_subnet_count, spec.samples_per_slot); + // The below `expect` calls will panic on start up if the chain spec config values used + // are invalid + let sampling_size = spec + .sampling_size(custody_group_count) + .expect("should compute node sampling size from valid chain spec"); + let custody_groups = get_custody_groups(node_id, sampling_size, &spec) + .expect("should compute node custody groups"); - let sampling_subnets = DataColumnSubnetId::compute_custody_subnets::( - node_id, - subnet_sampling_size, - &spec, - ) - .expect("sampling subnet count must be valid") - .collect::>(); + let mut sampling_subnets = HashSet::new(); + for custody_index in &custody_groups { + let subnets = compute_subnets_from_custody_group(*custody_index, &spec) + .expect("should compute custody subnets for node"); + sampling_subnets.extend(subnets); + } - let sampling_columns = sampling_subnets - .iter() - .flat_map(|subnet| subnet.columns::(&spec)) - .sorted() - .collect(); + let mut sampling_columns = HashSet::new(); + for custody_index in &custody_groups { + let columns = compute_columns_for_custody_group(*custody_index, &spec) + .expect("should compute custody columns for node"); + sampling_columns.extend(columns); + } (sampling_subnets, sampling_columns) } else { - (vec![], vec![]) + (HashSet::new(), HashSet::new()) }; NetworkGlobals { @@ -159,8 +175,8 @@ impl NetworkGlobals { pub fn custody_peers_for_column(&self, column_index: ColumnIndex) -> Vec { self.peers .read() - .good_custody_subnet_peer(DataColumnSubnetId::from_column_index::( - column_index as usize, + .good_custody_subnet_peer(DataColumnSubnetId::from_column_index( + column_index, &self.spec, )) .cloned() @@ -178,7 +194,7 @@ impl NetworkGlobals { seq_number: 0, attnets: Default::default(), syncnets: Default::default(), - custody_subnet_count: spec.custody_requirement, + custody_group_count: spec.custody_requirement, }); Self::new_test_globals_with_metadata(trusted_peers, metadata, log, config, spec) } @@ -207,11 +223,11 @@ mod test { fn test_sampling_subnets() { let log = logging::test_logger(); let mut spec = E::default_spec(); - spec.eip7594_fork_epoch = Some(Epoch::new(0)); + spec.fulu_fork_epoch = Some(Epoch::new(0)); - let custody_subnet_count = spec.data_column_sidecar_subnet_count / 2; - let subnet_sampling_size = std::cmp::max(custody_subnet_count, spec.samples_per_slot); - let metadata = get_metadata(custody_subnet_count); + let custody_group_count = spec.number_of_custody_groups / 2; + let subnet_sampling_size = spec.sampling_size(custody_group_count).unwrap(); + let metadata = get_metadata(custody_group_count); let config = Arc::new(NetworkConfig::default()); let globals = NetworkGlobals::::new_test_globals_with_metadata( @@ -231,11 +247,11 @@ mod test { fn test_sampling_columns() { let log = logging::test_logger(); let mut spec = E::default_spec(); - spec.eip7594_fork_epoch = Some(Epoch::new(0)); + spec.fulu_fork_epoch = Some(Epoch::new(0)); - let custody_subnet_count = spec.data_column_sidecar_subnet_count / 2; - let subnet_sampling_size = std::cmp::max(custody_subnet_count, spec.samples_per_slot); - let metadata = get_metadata(custody_subnet_count); + let custody_group_count = spec.number_of_custody_groups / 2; + let subnet_sampling_size = spec.sampling_size(custody_group_count).unwrap(); + let metadata = get_metadata(custody_group_count); let config = Arc::new(NetworkConfig::default()); let globals = NetworkGlobals::::new_test_globals_with_metadata( @@ -251,12 +267,12 @@ mod test { ); } - fn get_metadata(custody_subnet_count: u64) -> MetaData { + fn get_metadata(custody_group_count: u64) -> MetaData { MetaData::V3(MetaDataV3 { seq_number: 0, attnets: Default::default(), syncnets: Default::default(), - custody_subnet_count, + custody_group_count, }) } } diff --git a/beacon_node/lighthouse_network/src/types/mod.rs b/beacon_node/lighthouse_network/src/types/mod.rs index 6f266fd2ba..a1eedaef74 100644 --- a/beacon_node/lighthouse_network/src/types/mod.rs +++ b/beacon_node/lighthouse_network/src/types/mod.rs @@ -18,5 +18,5 @@ pub use sync_state::{BackFillState, SyncState}; pub use topics::{ attestation_sync_committee_topics, core_topics_to_subscribe, fork_core_topics, subnet_from_topic_hash, GossipEncoding, GossipKind, GossipTopic, ALTAIR_CORE_TOPICS, - BASE_CORE_TOPICS, CAPELLA_CORE_TOPICS, DENEB_CORE_TOPICS, LIGHT_CLIENT_GOSSIP_TOPICS, + BASE_CORE_TOPICS, CAPELLA_CORE_TOPICS, LIGHT_CLIENT_GOSSIP_TOPICS, }; diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index e61ab1bf2d..481540dddb 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -7,15 +7,14 @@ use ssz::{Decode, Encode}; use std::io::{Error, ErrorKind}; use std::sync::Arc; use types::{ - Attestation, AttestationBase, AttestationElectra, AttesterSlashing, AttesterSlashingBase, - AttesterSlashingElectra, BlobSidecar, DataColumnSidecar, DataColumnSubnetId, EthSpec, - ForkContext, ForkName, LightClientFinalityUpdate, LightClientOptimisticUpdate, - ProposerSlashing, SignedAggregateAndProof, SignedAggregateAndProofBase, - SignedAggregateAndProofElectra, SignedBeaconBlock, SignedBeaconBlockAltair, - SignedBeaconBlockBase, SignedBeaconBlockBellatrix, SignedBeaconBlockCapella, - SignedBeaconBlockDeneb, SignedBeaconBlockElectra, SignedBlsToExecutionChange, - SignedContributionAndProof, SignedInclusionList, SignedVoluntaryExit, SubnetId, - SyncCommitteeMessage, SyncSubnetId, + Attestation, AttestationBase, AttesterSlashing, AttesterSlashingBase, AttesterSlashingElectra, + BlobSidecar, DataColumnSidecar, DataColumnSubnetId, EthSpec, ForkContext, ForkName, + LightClientFinalityUpdate, LightClientOptimisticUpdate, ProposerSlashing, + SignedAggregateAndProof, SignedAggregateAndProofBase, SignedAggregateAndProofElectra, + SignedBeaconBlock, SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockBellatrix, + SignedBeaconBlockCapella, SignedBeaconBlockDeneb, SignedBeaconBlockElectra, + SignedBeaconBlockFulu, SignedBlsToExecutionChange, SignedContributionAndProof, + SignedVoluntaryExit, SingleAttestation, SubnetId, SyncCommitteeMessage, SyncSubnetId, SignedInclusionList, }; #[derive(Debug, Clone, PartialEq)] @@ -28,8 +27,10 @@ pub enum PubsubMessage { DataColumnSidecar(Box<(DataColumnSubnetId, Arc>)>), /// Gossipsub message providing notification of a Aggregate attestation and associated proof. AggregateAndProofAttestation(Box>), - /// Gossipsub message providing notification of a raw un-aggregated attestation with its shard id. + /// Gossipsub message providing notification of a raw un-aggregated attestation with its subnet id. Attestation(Box<(SubnetId, Attestation)>), + /// Gossipsub message providing notification of a `SingleAttestation`` with its subnet id. + SingleAttestation(Box<(SubnetId, SingleAttestation)>), /// Gossipsub message providing notification of a voluntary exit. VoluntaryExit(Box), /// Gossipsub message providing notification of a new proposer slashing. @@ -131,6 +132,9 @@ impl PubsubMessage { PubsubMessage::Attestation(attestation_data) => { GossipKind::Attestation(attestation_data.0) } + PubsubMessage::SingleAttestation(attestation_data) => { + GossipKind::Attestation(attestation_data.0) + } PubsubMessage::VoluntaryExit(_) => GossipKind::VoluntaryExit, PubsubMessage::ProposerSlashing(_) => GossipKind::ProposerSlashing, PubsubMessage::AttesterSlashing(_) => GossipKind::AttesterSlashing, @@ -192,32 +196,32 @@ impl PubsubMessage { ))) } GossipKind::Attestation(subnet_id) => { - let attestation = - match fork_context.from_context_bytes(gossip_topic.fork_digest) { - Some(&fork_name) => { - if fork_name.electra_enabled() { - Attestation::Electra( - AttestationElectra::from_ssz_bytes(data) - .map_err(|e| format!("{:?}", e))?, - ) - } else { - Attestation::Base( - AttestationBase::from_ssz_bytes(data) - .map_err(|e| format!("{:?}", e))?, - ) - } + match fork_context.from_context_bytes(gossip_topic.fork_digest) { + Some(&fork_name) => { + if fork_name.electra_enabled() { + let single_attestation = + SingleAttestation::from_ssz_bytes(data) + .map_err(|e| format!("{:?}", e))?; + Ok(PubsubMessage::SingleAttestation(Box::new(( + *subnet_id, + single_attestation, + )))) + } else { + let attestation = Attestation::Base( + AttestationBase::from_ssz_bytes(data) + .map_err(|e| format!("{:?}", e))?, + ); + Ok(PubsubMessage::Attestation(Box::new(( + *subnet_id, + attestation, + )))) } - None => { - return Err(format!( - "Unknown gossipsub fork digest: {:?}", - gossip_topic.fork_digest - )) - } - }; - Ok(PubsubMessage::Attestation(Box::new(( - *subnet_id, - attestation, - )))) + } + None => Err(format!( + "Unknown gossipsub fork digest: {:?}", + gossip_topic.fork_digest + )), + } } GossipKind::BeaconBlock => { let beacon_block = @@ -246,6 +250,10 @@ impl PubsubMessage { SignedBeaconBlockElectra::from_ssz_bytes(data) .map_err(|e| format!("{:?}", e))?, ), + Some(ForkName::Fulu) => SignedBeaconBlock::::Fulu( + SignedBeaconBlockFulu::from_ssz_bytes(data) + .map_err(|e| format!("{:?}", e))?, + ), None => { return Err(format!( "Unknown gossipsub fork digest: {:?}", @@ -278,27 +286,15 @@ impl PubsubMessage { } GossipKind::DataColumnSidecar(subnet_id) => { match fork_context.from_context_bytes(gossip_topic.fork_digest) { - // TODO(das): Remove Deneb fork - Some(fork) if fork.deneb_enabled() => { + Some(fork) if fork.fulu_enabled() => { let col_sidecar = Arc::new( DataColumnSidecar::from_ssz_bytes(data) .map_err(|e| format!("{:?}", e))?, ); - let peer_das_enabled = - fork_context.spec.is_peer_das_enabled_for_epoch( - col_sidecar.slot().epoch(E::slots_per_epoch()), - ); - if peer_das_enabled { - Ok(PubsubMessage::DataColumnSidecar(Box::new(( - *subnet_id, - col_sidecar, - )))) - } else { - Err(format!( - "data_column_sidecar topic invalid for given fork digest {:?}", - gossip_topic.fork_digest - )) - } + Ok(PubsubMessage::DataColumnSidecar(Box::new(( + *subnet_id, + col_sidecar, + )))) } Some(_) | None => Err(format!( "data_column_sidecar topic invalid for given fork digest {:?}", @@ -438,6 +434,7 @@ impl PubsubMessage { PubsubMessage::ProposerSlashing(data) => data.as_ssz_bytes(), PubsubMessage::AttesterSlashing(data) => data.as_ssz_bytes(), PubsubMessage::Attestation(data) => data.1.as_ssz_bytes(), + PubsubMessage::SingleAttestation(data) => data.1.as_ssz_bytes(), PubsubMessage::SignedContributionAndProof(data) => data.as_ssz_bytes(), PubsubMessage::SyncCommitteeMessage(data) => data.1.as_ssz_bytes(), PubsubMessage::BlsToExecutionChange(data) => data.as_ssz_bytes(), @@ -483,6 +480,14 @@ impl std::fmt::Display for PubsubMessage { data.1.data().slot, data.1.committee_index(), ), + PubsubMessage::SingleAttestation(data) => write!( + f, + "SingleAttestation: subnet_id: {}, attestation_slot: {}, committee_index: {:?}, attester_index: {:?}", + *data.0, + data.1.data.slot, + data.1.committee_index, + data.1.attester_index, + ), PubsubMessage::VoluntaryExit(_data) => write!(f, "Voluntary Exit"), PubsubMessage::ProposerSlashing(_data) => write!(f, "Proposer Slashing"), PubsubMessage::AttesterSlashing(_data) => write!(f, "Attester Slashing"), diff --git a/beacon_node/lighthouse_network/src/types/topics.rs b/beacon_node/lighthouse_network/src/types/topics.rs index 6970218883..a61da2e8fb 100644 --- a/beacon_node/lighthouse_network/src/types/topics.rs +++ b/beacon_node/lighthouse_network/src/types/topics.rs @@ -42,10 +42,6 @@ pub const LIGHT_CLIENT_GOSSIP_TOPICS: [GossipKind; 2] = [ GossipKind::LightClientOptimisticUpdate, ]; -pub const DENEB_CORE_TOPICS: [GossipKind; 0] = []; - -pub const ELECTRA_CORE_TOPICS: [GossipKind; 1] = [GossipKind::InclusionList]; - /// Returns the core topics associated with each fork that are new to the previous fork pub fn fork_core_topics(fork_name: &ForkName, spec: &ChainSpec) -> Vec { match fork_name { @@ -56,14 +52,24 @@ pub fn fork_core_topics(fork_name: &ForkName, spec: &ChainSpec) -> V ForkName::Deneb => { // All of deneb blob topics are core topics let mut deneb_blob_topics = Vec::new(); - for i in 0..spec.blob_sidecar_subnet_count { + for i in 0..spec.blob_sidecar_subnet_count(ForkName::Deneb) { deneb_blob_topics.push(GossipKind::BlobSidecar(i)); } - let mut deneb_topics = DENEB_CORE_TOPICS.to_vec(); - deneb_topics.append(&mut deneb_blob_topics); - deneb_topics + deneb_blob_topics } - ForkName::Electra => ELECTRA_CORE_TOPICS.to_vec(), + ForkName::Electra => { + // All of electra blob topics are core topics + let mut electra_blob_topics = Vec::new(); + for i in 0..spec.blob_sidecar_subnet_count(ForkName::Electra) { + electra_blob_topics.push(GossipKind::BlobSidecar(i)); + } + + if spec.is_focil_scheduled() { + electra_blob_topics.push(GossipKind::InclusionList) + } + electra_blob_topics + } + ForkName::Fulu => vec![], } } @@ -90,7 +96,12 @@ pub fn core_topics_to_subscribe( topics.extend(previous_fork_topics); current_fork = previous_fork; } + // Remove duplicates topics + .into_iter() + .collect::>() + .into_iter() + .collect() } /// A gossipsub topic which encapsulates the type of messages that should be sent and received over @@ -473,16 +484,19 @@ mod tests { type E = MainnetEthSpec; let spec = E::default_spec(); let mut all_topics = Vec::new(); + let mut electra_core_topics = fork_core_topics::(&ForkName::Electra, &spec); let mut deneb_core_topics = fork_core_topics::(&ForkName::Deneb, &spec); + all_topics.append(&mut electra_core_topics); all_topics.append(&mut deneb_core_topics); all_topics.extend(CAPELLA_CORE_TOPICS); all_topics.extend(ALTAIR_CORE_TOPICS); all_topics.extend(BASE_CORE_TOPICS); let latest_fork = *ForkName::list_all().last().unwrap(); - assert_eq!( - core_topics_to_subscribe::(latest_fork, &spec), - all_topics - ); + let core_topics = core_topics_to_subscribe::(latest_fork, &spec); + // Need to check all the topics exist in an order independent manner + for topic in all_topics { + assert!(core_topics.contains(&topic)); + } } } diff --git a/beacon_node/lighthouse_network/tests/common.rs b/beacon_node/lighthouse_network/tests/common.rs index 84e19c81d0..6a3ec6dd32 100644 --- a/beacon_node/lighthouse_network/tests/common.rs +++ b/beacon_node/lighthouse_network/tests/common.rs @@ -25,12 +25,14 @@ pub fn fork_context(fork_name: ForkName) -> ForkContext { let capella_fork_epoch = Epoch::new(3); let deneb_fork_epoch = Epoch::new(4); let electra_fork_epoch = Epoch::new(5); + let fulu_fork_epoch = Epoch::new(6); chain_spec.altair_fork_epoch = Some(altair_fork_epoch); chain_spec.bellatrix_fork_epoch = Some(bellatrix_fork_epoch); chain_spec.capella_fork_epoch = Some(capella_fork_epoch); chain_spec.deneb_fork_epoch = Some(deneb_fork_epoch); chain_spec.electra_fork_epoch = Some(electra_fork_epoch); + chain_spec.fulu_fork_epoch = Some(fulu_fork_epoch); let current_slot = match fork_name { ForkName::Base => Slot::new(0), @@ -39,6 +41,7 @@ pub fn fork_context(fork_name: ForkName) -> ForkContext { ForkName::Capella => capella_fork_epoch.start_slot(E::slots_per_epoch()), ForkName::Deneb => deneb_fork_epoch.start_slot(E::slots_per_epoch()), ForkName::Electra => electra_fork_epoch.start_slot(E::slots_per_epoch()), + ForkName::Fulu => fulu_fork_epoch.start_slot(E::slots_per_epoch()), }; ForkContext::new::(current_slot, Hash256::zero(), &chain_spec) } diff --git a/beacon_node/lighthouse_network/tests/rpc_tests.rs b/beacon_node/lighthouse_network/tests/rpc_tests.rs index f721c8477c..4b54a24ddc 100644 --- a/beacon_node/lighthouse_network/tests/rpc_tests.rs +++ b/beacon_node/lighthouse_network/tests/rpc_tests.rs @@ -16,7 +16,7 @@ use tokio::time::sleep; use types::{ BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockBellatrix, BlobSidecar, ChainSpec, EmptyBlock, Epoch, EthSpec, FixedBytesExtended, ForkContext, ForkName, Hash256, MinimalEthSpec, - Signature, SignedBeaconBlock, Slot, + RuntimeVariableList, Signature, SignedBeaconBlock, Slot, }; type E = MinimalEthSpec; @@ -810,17 +810,20 @@ fn test_tcp_blocks_by_root_chunked_rpc() { .await; // BlocksByRoot Request - let rpc_request = RequestType::BlocksByRoot(BlocksByRootRequest::new( - vec![ - Hash256::zero(), - Hash256::zero(), - Hash256::zero(), - Hash256::zero(), - Hash256::zero(), - Hash256::zero(), - ], - &spec, - )); + let rpc_request = + RequestType::BlocksByRoot(BlocksByRootRequest::V2(BlocksByRootRequestV2 { + block_roots: RuntimeVariableList::from_vec( + vec![ + Hash256::zero(), + Hash256::zero(), + Hash256::zero(), + Hash256::zero(), + Hash256::zero(), + Hash256::zero(), + ], + spec.max_request_blocks_upper_bound(), + ), + })); // BlocksByRoot Response let full_block = BeaconBlock::Base(BeaconBlockBase::::full(&spec)); @@ -953,21 +956,24 @@ fn test_tcp_blocks_by_root_chunked_rpc_terminates_correctly() { .await; // BlocksByRoot Request - let rpc_request = RequestType::BlocksByRoot(BlocksByRootRequest::new( - vec![ - Hash256::zero(), - Hash256::zero(), - Hash256::zero(), - Hash256::zero(), - Hash256::zero(), - Hash256::zero(), - Hash256::zero(), - Hash256::zero(), - Hash256::zero(), - Hash256::zero(), - ], - &spec, - )); + let rpc_request = + RequestType::BlocksByRoot(BlocksByRootRequest::V2(BlocksByRootRequestV2 { + block_roots: RuntimeVariableList::from_vec( + vec![ + Hash256::zero(), + Hash256::zero(), + Hash256::zero(), + Hash256::zero(), + Hash256::zero(), + Hash256::zero(), + Hash256::zero(), + Hash256::zero(), + Hash256::zero(), + Hash256::zero(), + ], + spec.max_request_blocks_upper_bound(), + ), + })); // BlocksByRoot Response let full_block = BeaconBlock::Base(BeaconBlockBase::::full(&spec)); diff --git a/beacon_node/network/Cargo.toml b/beacon_node/network/Cargo.toml index 44f6c54bbc..09179c4a51 100644 --- a/beacon_node/network/Cargo.toml +++ b/beacon_node/network/Cargo.toml @@ -31,7 +31,7 @@ execution_layer = { workspace = true } fnv = { workspace = true } futures = { workspace = true } hex = { workspace = true } -igd-next = "0.14" +igd-next = { version = "0.16", features = ["aio_tokio"] } itertools = { workspace = true } lighthouse_network = { workspace = true } logging = { workspace = true } diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 8640df38e1..bab0b7a07e 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -1293,13 +1293,12 @@ impl NetworkBeaconProcessor { Err(e @ BlockError::StateRootMismatch { .. }) | Err(e @ BlockError::IncorrectBlockProposer { .. }) | Err(e @ BlockError::BlockSlotLimitReached) - | Err(e @ BlockError::ProposalSignatureInvalid) | Err(e @ BlockError::NonLinearSlots) | Err(e @ BlockError::UnknownValidator(_)) | Err(e @ BlockError::PerBlockProcessingError(_)) | Err(e @ BlockError::NonLinearParentRoots) | Err(e @ BlockError::BlockIsNotLaterThanParent { .. }) - | Err(e @ BlockError::InvalidSignature) + | Err(e @ BlockError::InvalidSignature(_)) | Err(e @ BlockError::WeakSubjectivityConflict) | Err(e @ BlockError::InconsistentFork(_)) | Err(e @ BlockError::ExecutionPayloadError(_)) @@ -3166,7 +3165,7 @@ impl NetworkBeaconProcessor { .chain .slot_clock .now() - .map_or(false, |current_slot| sync_message_slot == current_slot); + .is_some_and(|current_slot| sync_message_slot == current_slot); self.propagate_if_timely(is_timely, message_id, peer_id) } diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 49564657d1..12d3f41465 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -84,6 +84,58 @@ impl NetworkBeaconProcessor { .map_err(Into::into) } + /// Create a new `Work` event for some `SingleAttestation`. + pub fn send_single_attestation( + self: &Arc, + message_id: MessageId, + peer_id: PeerId, + single_attestation: SingleAttestation, + subnet_id: SubnetId, + should_import: bool, + seen_timestamp: Duration, + ) -> Result<(), Error> { + let result = self.chain.with_committee_cache( + single_attestation.data.target.root, + single_attestation + .data + .slot + .epoch(T::EthSpec::slots_per_epoch()), + |committee_cache, _| { + let Some(committee) = committee_cache.get_beacon_committee( + single_attestation.data.slot, + single_attestation.committee_index, + ) else { + warn!( + self.log, + "No beacon committee for slot and index"; + "slot" => single_attestation.data.slot, + "index" => single_attestation.committee_index + ); + return Ok(Ok(())); + }; + + let attestation = single_attestation.to_attestation(committee.committee)?; + + Ok(self.send_unaggregated_attestation( + message_id.clone(), + peer_id, + attestation, + subnet_id, + should_import, + seen_timestamp, + )) + }, + ); + + match result { + Ok(result) => result, + Err(e) => { + warn!(self.log, "Failed to send SingleAttestation"; "error" => ?e); + Ok(()) + } + } + } + /// Create a new `Work` event for some unaggregated attestation. pub fn send_unaggregated_attestation( self: &Arc, @@ -585,6 +637,11 @@ impl NetworkBeaconProcessor { blocks: Vec>, ) -> Result<(), Error> { let is_backfill = matches!(&process_id, ChainSegmentProcessId::BackSyncBatchId { .. }); + debug!(self.log, "Batch sending for process"; + "blocks" => blocks.len(), + "id" => ?process_id, + ); + let processor = self.clone(); let process_fn = async move { let notify_execution_layer = if processor @@ -1146,10 +1203,8 @@ impl NetworkBeaconProcessor { messages: columns .into_iter() .map(|d| { - let subnet = DataColumnSubnetId::from_column_index::( - d.index as usize, - &chain.spec, - ); + let subnet = + DataColumnSubnetId::from_column_index(d.index, &chain.spec); PubsubMessage::DataColumnSidecar(Box::new((subnet, d))) }) .collect(), @@ -1163,7 +1218,8 @@ impl NetworkBeaconProcessor { let blob_publication_batch_interval = chain.config.blob_publication_batch_interval; let blob_publication_batches = chain.config.blob_publication_batches; - let batch_size = chain.spec.number_of_columns / blob_publication_batches; + let number_of_columns = chain.spec.number_of_columns as usize; + let batch_size = number_of_columns / blob_publication_batches; let mut publish_count = 0usize; for batch in data_columns_to_publish.chunks(batch_size) { diff --git a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs index c4944078fe..67a1570275 100644 --- a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs @@ -659,24 +659,6 @@ impl NetworkBeaconProcessor { "start_slot" => req.start_slot(), ); - // Should not send more than max request blocks - let max_request_size = - self.chain - .epoch() - .map_or(self.chain.spec.max_request_blocks, |epoch| { - if self.chain.spec.fork_name_at_epoch(epoch).deneb_enabled() { - self.chain.spec.max_request_blocks_deneb - } else { - self.chain.spec.max_request_blocks - } - }); - if *req.count() > max_request_size { - return Err(( - RpcErrorResponse::InvalidRequest, - "Request exceeded max size", - )); - } - let forwards_block_root_iter = match self .chain .forwards_iter_block_roots(Slot::from(*req.start_slot())) @@ -890,14 +872,6 @@ impl NetworkBeaconProcessor { "start_slot" => req.start_slot, ); - // Should not send more than max request blocks - if req.max_blobs_requested::() > self.chain.spec.max_request_blob_sidecars { - return Err(( - RpcErrorResponse::InvalidRequest, - "Request exceeded `MAX_REQUEST_BLOBS_SIDECARS`", - )); - } - let request_start_slot = Slot::from(req.start_slot); let data_availability_boundary_slot = match self.chain.data_availability_boundary() { diff --git a/beacon_node/network/src/network_beacon_processor/tests.rs b/beacon_node/network/src/network_beacon_processor/tests.rs index 7e27a91bd6..8415ece638 100644 --- a/beacon_node/network/src/network_beacon_processor/tests.rs +++ b/beacon_node/network/src/network_beacon_processor/tests.rs @@ -15,7 +15,7 @@ use beacon_chain::test_utils::{ use beacon_chain::{BeaconChain, WhenSlotSkipped}; use beacon_processor::{work_reprocessing_queue::*, *}; use lighthouse_network::discovery::ConnectionId; -use lighthouse_network::rpc::methods::BlobsByRangeRequest; +use lighthouse_network::rpc::methods::{BlobsByRangeRequest, MetaDataV3}; use lighthouse_network::rpc::{RequestId, SubstreamId}; use lighthouse_network::{ discv5::enr::{self, CombinedKey}, @@ -198,11 +198,21 @@ impl TestRig { let (sync_tx, _sync_rx) = mpsc::unbounded_channel(); // Default metadata - let meta_data = MetaData::V2(MetaDataV2 { - seq_number: SEQ_NUMBER, - attnets: EnrAttestationBitfield::::default(), - syncnets: EnrSyncCommitteeBitfield::::default(), - }); + let meta_data = if spec.is_peer_das_scheduled() { + MetaData::V3(MetaDataV3 { + seq_number: SEQ_NUMBER, + attnets: EnrAttestationBitfield::::default(), + syncnets: EnrSyncCommitteeBitfield::::default(), + custody_group_count: spec.custody_requirement, + }) + } else { + MetaData::V2(MetaDataV2 { + seq_number: SEQ_NUMBER, + attnets: EnrAttestationBitfield::::default(), + syncnets: EnrSyncCommitteeBitfield::::default(), + }) + }; + let enr_key = CombinedKey::generate_secp256k1(); let enr = enr::Enr::builder().build(&enr_key).unwrap(); let network_config = Arc::new(NetworkConfig::default()); @@ -259,7 +269,7 @@ impl TestRig { assert!(beacon_processor.is_ok()); let block = next_block_tuple.0; let blob_sidecars = if let Some((kzg_proofs, blobs)) = next_block_tuple.1 { - Some(BlobSidecar::build_sidecars(blobs, &block, kzg_proofs).unwrap()) + Some(BlobSidecar::build_sidecars(blobs, &block, kzg_proofs, &chain.spec).unwrap()) } else { None }; @@ -342,15 +352,16 @@ impl TestRig { ) .unwrap(); } + pub fn enqueue_single_lookup_rpc_blobs(&self) { if let Some(blobs) = self.next_blobs.clone() { - let blobs = FixedBlobSidecarList::from(blobs.into_iter().map(Some).collect::>()); + let blobs = FixedBlobSidecarList::new(blobs.into_iter().map(Some).collect::>()); self.network_beacon_processor .send_rpc_blobs( self.next_block.canonical_root(), blobs, std::time::Duration::default(), - BlockProcessType::SingleBlock { id: 1 }, + BlockProcessType::SingleBlob { id: 1 }, ) .unwrap(); } @@ -1130,7 +1141,12 @@ async fn test_blobs_by_range() { .block_root_at_slot(Slot::new(slot), WhenSlotSkipped::None) .unwrap(); blob_count += root - .map(|root| rig.chain.get_blobs(&root).unwrap_or_default().len()) + .map(|root| { + rig.chain + .get_blobs(&root) + .map(|list| list.len()) + .unwrap_or(0) + }) .unwrap_or(0); } let mut actual_count = 0; diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index 76291f378f..f9ec96df87 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -28,7 +28,7 @@ use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tokio::sync::mpsc; use tokio_stream::wrappers::UnboundedReceiverStream; -use types::{BlobSidecar, DataColumnSidecar, EthSpec, SignedBeaconBlock}; +use types::{BlobSidecar, DataColumnSidecar, EthSpec, ForkContext, SignedBeaconBlock}; /// Handles messages from the network and routes them to the appropriate service to be handled. pub struct Router { @@ -90,6 +90,7 @@ impl Router { invalid_block_storage: InvalidBlockStorage, beacon_processor_send: BeaconProcessorSend, beacon_processor_reprocess_tx: mpsc::Sender, + fork_context: Arc, log: slog::Logger, ) -> Result>, String> { let message_handler_log = log.new(o!("service"=> "router")); @@ -122,6 +123,7 @@ impl Router { network_send.clone(), network_beacon_processor.clone(), sync_recv, + fork_context, sync_logger, ); @@ -398,6 +400,17 @@ impl Router { timestamp_now(), ), ), + PubsubMessage::SingleAttestation(subnet_attestation) => self + .handle_beacon_processor_send_result( + self.network_beacon_processor.send_single_attestation( + message_id, + peer_id, + subnet_attestation.1, + subnet_attestation.0, + should_process, + timestamp_now(), + ), + ), PubsubMessage::BeaconBlock(block) => self.handle_beacon_processor_send_result( self.network_beacon_processor.send_gossip_beacon_block( message_id, diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index 7826807e03..49f73bf9c8 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -312,6 +312,7 @@ impl NetworkService { invalid_block_storage, beacon_processor_send, beacon_processor_reprocess_tx, + fork_context.clone(), network_log.clone(), )?; @@ -549,7 +550,23 @@ impl NetworkService { // the attestation, else we just just propagate the Attestation. let should_process = self.subnet_service.should_process_attestation( Subnet::Attestation(subnet_id), - attestation, + attestation.data(), + ); + self.send_to_router(RouterMessage::PubsubMessage( + id, + source, + message, + should_process, + )); + } + PubsubMessage::SingleAttestation(ref subnet_and_attestation) => { + let subnet_id = subnet_and_attestation.0; + let single_attestation = &subnet_and_attestation.1; + // checks if we have an aggregator for the slot. If so, we should process + // the attestation, else we just just propagate the Attestation. + let should_process = self.subnet_service.should_process_attestation( + Subnet::Attestation(subnet_id), + &single_attestation.data, ); self.send_to_router(RouterMessage::PubsubMessage( id, @@ -734,11 +751,6 @@ impl NetworkService { } } - // TODO(das): This is added here for the purpose of testing, *without* having to - // activate Electra. This should happen as part of the Electra upgrade and we should - // move the subscription logic once it's ready to rebase PeerDAS on Electra, or if - // we decide to activate via the soft fork route: - // https://github.com/sigp/lighthouse/pull/5899 if self.fork_context.spec.is_peer_das_scheduled() { self.subscribe_to_peer_das_topics(&mut subscribed_topics); } @@ -789,32 +801,32 @@ impl NetworkService { } } + /// Keeping these separate from core topics because it has custom logic: + /// 1. Data column subscription logic depends on subscription configuration. + /// 2. Data column topic subscriptions will be dynamic based on validator balances due to + /// validator custody. + /// + /// TODO(das): The downside with not including it in core fork topic is - we subscribe to + /// PeerDAS topics on startup if Fulu is scheduled, rather than waiting until the fork. + /// If this is an issue we could potentially consider adding the logic to + /// `network.subscribe_new_fork_topics()`. fn subscribe_to_peer_das_topics(&mut self, subscribed_topics: &mut Vec) { - if self.subscribe_all_data_column_subnets { - for column_subnet in 0..self.fork_context.spec.data_column_sidecar_subnet_count { - for fork_digest in self.required_gossip_fork_digests() { - let gossip_kind = - Subnet::DataColumn(DataColumnSubnetId::new(column_subnet)).into(); - let topic = - GossipTopic::new(gossip_kind, GossipEncoding::default(), fork_digest); - if self.libp2p.subscribe(topic.clone()) { - subscribed_topics.push(topic); - } else { - warn!(self.log, "Could not subscribe to topic"; "topic" => %topic); - } - } - } + let column_subnets_to_subscribe = if self.subscribe_all_data_column_subnets { + &(0..self.fork_context.spec.data_column_sidecar_subnet_count) + .map(DataColumnSubnetId::new) + .collect() } else { - for column_subnet in &self.network_globals.sampling_subnets { - for fork_digest in self.required_gossip_fork_digests() { - let gossip_kind = Subnet::DataColumn(*column_subnet).into(); - let topic = - GossipTopic::new(gossip_kind, GossipEncoding::default(), fork_digest); - if self.libp2p.subscribe(topic.clone()) { - subscribed_topics.push(topic); - } else { - warn!(self.log, "Could not subscribe to topic"; "topic" => %topic); - } + &self.network_globals.sampling_subnets + }; + + for column_subnet in column_subnets_to_subscribe.iter() { + for fork_digest in self.required_gossip_fork_digests() { + let gossip_kind = Subnet::DataColumn(*column_subnet).into(); + let topic = GossipTopic::new(gossip_kind, GossipEncoding::default(), fork_digest); + if self.libp2p.subscribe(topic.clone()) { + subscribed_topics.push(topic); + } else { + warn!(self.log, "Could not subscribe to topic"; "topic" => %topic); } } } diff --git a/beacon_node/network/src/subnet_service/mod.rs b/beacon_node/network/src/subnet_service/mod.rs index da1f220f04..33ae567eb3 100644 --- a/beacon_node/network/src/subnet_service/mod.rs +++ b/beacon_node/network/src/subnet_service/mod.rs @@ -17,7 +17,7 @@ use lighthouse_network::{discv5::enr::NodeId, NetworkConfig, Subnet, SubnetDisco use slog::{debug, error, o, warn}; use slot_clock::SlotClock; use types::{ - Attestation, EthSpec, Slot, SubnetId, SyncCommitteeSubscription, SyncSubnetId, + AttestationData, EthSpec, Slot, SubnetId, SyncCommitteeSubscription, SyncSubnetId, ValidatorSubscription, }; @@ -363,7 +363,7 @@ impl SubnetService { pub fn should_process_attestation( &self, subnet: Subnet, - attestation: &Attestation, + attestation_data: &AttestationData, ) -> bool { // Proposer-only mode does not need to process attestations if self.proposer_only { @@ -374,7 +374,7 @@ impl SubnetService { .map(|tracked_vals| { tracked_vals.contains_key(&ExactSubnet { subnet, - slot: attestation.data().slot, + slot: attestation_data.slot, }) }) .unwrap_or(true) diff --git a/beacon_node/network/src/sync/backfill_sync/mod.rs b/beacon_node/network/src/sync/backfill_sync/mod.rs index 5703ed3504..a3d2c82642 100644 --- a/beacon_node/network/src/sync/backfill_sync/mod.rs +++ b/beacon_node/network/src/sync/backfill_sync/mod.rs @@ -388,67 +388,59 @@ impl BackFillSync { blocks: Vec>, ) -> Result { // check if we have this batch - let batch = match self.batches.get_mut(&batch_id) { - None => { - if !matches!(self.state(), BackFillState::Failed) { - // A batch might get removed when the chain advances, so this is non fatal. - debug!(self.log, "Received a block for unknown batch"; "epoch" => batch_id); - } - return Ok(ProcessResult::Successful); - } - Some(batch) => { - // A batch could be retried without the peer failing the request (disconnecting/ - // sending an error /timeout) if the peer is removed from the chain for other - // reasons. Check that this block belongs to the expected peer, and that the - // request_id matches - // TODO(das): removed peer_id matching as the node may request a different peer for data - // columns. - if !batch.is_expecting_block(&request_id) { - return Ok(ProcessResult::Successful); - } - batch + let Some(batch) = self.batches.get_mut(&batch_id) else { + if !matches!(self.state(), BackFillState::Failed) { + // A batch might get removed when the chain advances, so this is non fatal. + debug!(self.log, "Received a block for unknown batch"; "epoch" => batch_id); } + return Ok(ProcessResult::Successful); }; - { - // A stream termination has been sent. This batch has ended. Process a completed batch. - // Remove the request from the peer's active batches - self.active_requests - .get_mut(peer_id) - .map(|active_requests| active_requests.remove(&batch_id)); + // A batch could be retried without the peer failing the request (disconnecting/ + // sending an error /timeout) if the peer is removed from the chain for other + // reasons. Check that this block belongs to the expected peer, and that the + // request_id matches + // TODO(das): removed peer_id matching as the node may request a different peer for data + // columns. + if !batch.is_expecting_block(&request_id) { + return Ok(ProcessResult::Successful); + } - match batch.download_completed(blocks) { - Ok(received) => { - let awaiting_batches = - self.processing_target.saturating_sub(batch_id) / BACKFILL_EPOCHS_PER_BATCH; - debug!(self.log, "Completed batch received"; "epoch" => batch_id, "blocks" => received, "awaiting_batches" => awaiting_batches); + // A stream termination has been sent. This batch has ended. Process a completed batch. + // Remove the request from the peer's active batches + self.active_requests + .get_mut(peer_id) + .map(|active_requests| active_requests.remove(&batch_id)); - // pre-emptively request more blocks from peers whilst we process current blocks, - self.request_batches(network)?; - self.process_completed_batches(network) - } - Err(result) => { - let (expected_boundary, received_boundary, outcome) = match result { - Err(e) => { - return self - .fail_sync(BackFillError::BatchInvalidState(batch_id, e.0)) - .map(|_| ProcessResult::Successful); - } - Ok(v) => v, - }; - warn!(self.log, "Batch received out of range blocks"; "expected_boundary" => expected_boundary, "received_boundary" => received_boundary, + match batch.download_completed(blocks) { + Ok(received) => { + let awaiting_batches = + self.processing_target.saturating_sub(batch_id) / BACKFILL_EPOCHS_PER_BATCH; + debug!(self.log, "Completed batch received"; "epoch" => batch_id, "blocks" => received, "awaiting_batches" => awaiting_batches); + + // pre-emptively request more blocks from peers whilst we process current blocks, + self.request_batches(network)?; + self.process_completed_batches(network) + } + Err(result) => { + let (expected_boundary, received_boundary, outcome) = match result { + Err(e) => { + self.fail_sync(BackFillError::BatchInvalidState(batch_id, e.0))?; + return Ok(ProcessResult::Successful); + } + Ok(v) => v, + }; + warn!(self.log, "Batch received out of range blocks"; "expected_boundary" => expected_boundary, "received_boundary" => received_boundary, "peer_id" => %peer_id, batch); - if let BatchOperationOutcome::Failed { blacklist: _ } = outcome { - error!(self.log, "Backfill failed"; "epoch" => batch_id, "received_boundary" => received_boundary, "expected_boundary" => expected_boundary); - return self - .fail_sync(BackFillError::BatchDownloadFailed(batch_id)) - .map(|_| ProcessResult::Successful); - } - // this batch can't be used, so we need to request it again. - self.retry_batch_download(network, batch_id) - .map(|_| ProcessResult::Successful) + if let BatchOperationOutcome::Failed { blacklist: _ } = outcome { + error!(self.log, "Backfill failed"; "epoch" => batch_id, "received_boundary" => received_boundary, "expected_boundary" => expected_boundary); + self.fail_sync(BackFillError::BatchDownloadFailed(batch_id))?; + return Ok(ProcessResult::Successful); } + // this batch can't be used, so we need to request it again. + self.retry_batch_download(network, batch_id)?; + Ok(ProcessResult::Successful) } } } @@ -582,20 +574,16 @@ impl BackFillSync { } }; - let peer = match batch.current_peer() { - Some(v) => *v, - None => { - return self - .fail_sync(BackFillError::BatchInvalidState( - batch_id, - String::from("Peer does not exist"), - )) - .map(|_| ProcessResult::Successful) - } + let Some(peer) = batch.current_peer() else { + self.fail_sync(BackFillError::BatchInvalidState( + batch_id, + String::from("Peer does not exist"), + ))?; + return Ok(ProcessResult::Successful); }; debug!(self.log, "Backfill batch processed"; "result" => ?result, &batch, - "batch_epoch" => batch_id, "peer" => %peer, "client" => %network.client_type(&peer)); + "batch_epoch" => batch_id, "peer" => %peer, "client" => %network.client_type(peer)); match result { BatchProcessResult::Success { @@ -679,8 +667,8 @@ impl BackFillSync { { self.fail_sync(BackFillError::BatchInvalidState(batch_id, e.0))?; } - self.retry_batch_download(network, batch_id) - .map(|_| ProcessResult::Successful) + self.retry_batch_download(network, batch_id)?; + Ok(ProcessResult::Successful) } } } @@ -712,11 +700,10 @@ impl BackFillSync { // - AwaitingDownload -> A recoverable failed batch should have been // re-requested. // - Processing -> `self.current_processing_batch` is None - return self - .fail_sync(BackFillError::InvalidSyncState(String::from( - "Invalid expected batch state", - ))) - .map(|_| ProcessResult::Successful); + self.fail_sync(BackFillError::InvalidSyncState(String::from( + "Invalid expected batch state", + )))?; + return Ok(ProcessResult::Successful); } BatchState::AwaitingValidation(_) => { // TODO: I don't think this state is possible, log a CRIT just in case. @@ -731,12 +718,11 @@ impl BackFillSync { } } } else { - return self - .fail_sync(BackFillError::InvalidSyncState(format!( - "Batch not found for current processing target {}", - self.processing_target - ))) - .map(|_| ProcessResult::Successful); + self.fail_sync(BackFillError::InvalidSyncState(format!( + "Batch not found for current processing target {}", + self.processing_target + )))?; + return Ok(ProcessResult::Successful); } Ok(ProcessResult::Successful) } diff --git a/beacon_node/network/src/sync/block_lookups/common.rs b/beacon_node/network/src/sync/block_lookups/common.rs index 5e336d9c38..8eefb2d675 100644 --- a/beacon_node/network/src/sync/block_lookups/common.rs +++ b/beacon_node/network/src/sync/block_lookups/common.rs @@ -9,6 +9,8 @@ use crate::sync::network_context::{LookupRequestResult, SyncNetworkContext}; use beacon_chain::block_verification_types::RpcBlock; use beacon_chain::BeaconChainTypes; use lighthouse_network::service::api_types::Id; +use parking_lot::RwLock; +use std::collections::HashSet; use std::sync::Arc; use types::blob_sidecar::FixedBlobSidecarList; use types::{DataColumnSidecarList, SignedBeaconBlock}; @@ -41,7 +43,7 @@ pub trait RequestState { fn make_request( &self, id: Id, - peer_id: PeerId, + lookup_peers: Arc>>, expected_blobs: usize, cx: &mut SyncNetworkContext, ) -> Result; @@ -76,11 +78,11 @@ impl RequestState for BlockRequestState { fn make_request( &self, id: SingleLookupId, - peer_id: PeerId, + lookup_peers: Arc>>, _: usize, cx: &mut SyncNetworkContext, ) -> Result { - cx.block_lookup_request(id, peer_id, self.requested_block_root) + cx.block_lookup_request(id, lookup_peers, self.requested_block_root) .map_err(LookupRequestError::SendFailedNetwork) } @@ -124,11 +126,11 @@ impl RequestState for BlobRequestState { fn make_request( &self, id: Id, - peer_id: PeerId, + lookup_peers: Arc>>, expected_blobs: usize, cx: &mut SyncNetworkContext, ) -> Result { - cx.blob_lookup_request(id, peer_id, self.block_root, expected_blobs) + cx.blob_lookup_request(id, lookup_peers, self.block_root, expected_blobs) .map_err(LookupRequestError::SendFailedNetwork) } @@ -172,12 +174,11 @@ impl RequestState for CustodyRequestState { fn make_request( &self, id: Id, - // TODO(das): consider selecting peers that have custody but are in this set - _peer_id: PeerId, + lookup_peers: Arc>>, _: usize, cx: &mut SyncNetworkContext, ) -> Result { - cx.custody_lookup_request(id, self.block_root) + cx.custody_lookup_request(id, self.block_root, lookup_peers) .map_err(LookupRequestError::SendFailedNetwork) } diff --git a/beacon_node/network/src/sync/block_lookups/mod.rs b/beacon_node/network/src/sync/block_lookups/mod.rs index 5a11bca481..ac4df42a4e 100644 --- a/beacon_node/network/src/sync/block_lookups/mod.rs +++ b/beacon_node/network/src/sync/block_lookups/mod.rs @@ -153,14 +153,7 @@ impl BlockLookups { pub(crate) fn active_single_lookups(&self) -> Vec { self.single_block_lookups .iter() - .map(|(id, l)| { - ( - *id, - l.block_root(), - l.awaiting_parent(), - l.all_peers().copied().collect(), - ) - }) + .map(|(id, l)| (*id, l.block_root(), l.awaiting_parent(), l.all_peers())) .collect() } @@ -283,7 +276,7 @@ impl BlockLookups { .find(|(_, l)| l.block_root() == parent_chain_tip) { cx.send_sync_message(SyncMessage::AddPeersForceRangeSync { - peers: lookup.all_peers().copied().collect(), + peers: lookup.all_peers(), head_slot: tip_lookup.peek_downloaded_block_slot(), head_root: parent_chain_tip, }); @@ -682,7 +675,7 @@ impl BlockLookups { lookup.continue_requests(cx) } Action::ParentUnknown { parent_root } => { - let peers = lookup.all_peers().copied().collect::>(); + let peers = lookup.all_peers(); lookup.set_awaiting_parent(parent_root); debug!(self.log, "Marking lookup as awaiting parent"; "id" => lookup.id, "block_root" => ?block_root, "parent_root" => ?parent_root); self.search_parent_of_child(parent_root, block_root, &peers, cx); diff --git a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs index 9bbd2bf295..3789dbe91e 100644 --- a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs +++ b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs @@ -7,7 +7,7 @@ use crate::sync::network_context::{ use beacon_chain::{BeaconChainTypes, BlockProcessStatus}; use derivative::Derivative; use lighthouse_network::service::api_types::Id; -use rand::seq::IteratorRandom; +use parking_lot::RwLock; use std::collections::HashSet; use std::fmt::Debug; use std::sync::Arc; @@ -33,8 +33,6 @@ pub enum LookupRequestError { /// The failed attempts were primarily due to processing failures. cannot_process: bool, }, - /// No peers left to serve this lookup - NoPeers, /// Error sending event to network SendFailedNetwork(RpcRequestSendError), /// Error sending event to processor @@ -63,9 +61,12 @@ pub struct SingleBlockLookup { pub id: Id, pub block_request_state: BlockRequestState, pub component_requests: ComponentRequests, - /// Peers that claim to have imported this set of block components + /// Peers that claim to have imported this set of block components. This state is shared with + /// the custody request to have an updated view of the peers that claim to have imported the + /// block associated with this lookup. The peer set of a lookup can change rapidly, and faster + /// than the lifetime of a custody request. #[derivative(Debug(format_with = "fmt_peer_set_as_len"))] - peers: HashSet, + peers: Arc>>, block_root: Hash256, awaiting_parent: Option, created: Instant, @@ -92,7 +93,7 @@ impl SingleBlockLookup { id, block_request_state: BlockRequestState::new(requested_block_root), component_requests: ComponentRequests::WaitingForBlock, - peers: HashSet::from_iter(peers.iter().copied()), + peers: Arc::new(RwLock::new(HashSet::from_iter(peers.iter().copied()))), block_root: requested_block_root, awaiting_parent, created: Instant::now(), @@ -215,8 +216,7 @@ impl SingleBlockLookup { let block_epoch = block.slot().epoch(T::EthSpec::slots_per_epoch()); if expected_blobs == 0 { self.component_requests = ComponentRequests::NotNeeded("no data"); - } - if cx.chain.should_fetch_blobs(block_epoch) { + } else if cx.chain.should_fetch_blobs(block_epoch) { self.component_requests = ComponentRequests::ActiveBlobRequest( BlobRequestState::new(self.block_root), expected_blobs, @@ -283,24 +283,11 @@ impl SingleBlockLookup { return Err(LookupRequestError::TooManyAttempts { cannot_process }); } - let Some(peer_id) = self.use_rand_available_peer() else { - // Allow lookup to not have any peers and do nothing. This is an optimization to not - // lose progress of lookups created from a block with unknown parent before we receive - // attestations for said block. - // Lookup sync event safety: If a lookup requires peers to make progress, and does - // not receive any new peers for some time it will be dropped. If it receives a new - // peer it must attempt to make progress. - R::request_state_mut(self) - .map_err(|e| LookupRequestError::BadState(e.to_owned()))? - .get_state_mut() - .update_awaiting_download_status("no peers"); - return Ok(()); - }; - + let peers = self.peers.clone(); let request = R::request_state_mut(self) .map_err(|e| LookupRequestError::BadState(e.to_owned()))?; - match request.make_request(id, peer_id, expected_blobs, cx)? { + match request.make_request(id, peers, expected_blobs, cx)? { LookupRequestResult::RequestSent(req_id) => { // Lookup sync event safety: If make_request returns `RequestSent`, we are // guaranteed that `BlockLookups::on_download_response` will be called exactly @@ -348,29 +335,24 @@ impl SingleBlockLookup { } /// Get all unique peers that claim to have imported this set of block components - pub fn all_peers(&self) -> impl Iterator + '_ { - self.peers.iter() + pub fn all_peers(&self) -> Vec { + self.peers.read().iter().copied().collect() } /// Add peer to all request states. The peer must be able to serve this request. /// Returns true if the peer was newly inserted into some request state. pub fn add_peer(&mut self, peer_id: PeerId) -> bool { - self.peers.insert(peer_id) + self.peers.write().insert(peer_id) } /// Remove peer from available peers. pub fn remove_peer(&mut self, peer_id: &PeerId) { - self.peers.remove(peer_id); + self.peers.write().remove(peer_id); } /// Returns true if this lookup has zero peers pub fn has_no_peers(&self) -> bool { - self.peers.is_empty() - } - - /// Selects a random peer from available peers if any - fn use_rand_available_peer(&mut self) -> Option { - self.peers.iter().choose(&mut rand::thread_rng()).copied() + self.peers.read().is_empty() } } @@ -689,8 +671,8 @@ impl std::fmt::Debug for State { } fn fmt_peer_set_as_len( - peer_set: &HashSet, + peer_set: &Arc>>, f: &mut std::fmt::Formatter, ) -> Result<(), std::fmt::Error> { - write!(f, "{}", peer_set.len()) + write!(f, "{}", peer_set.read().len()) } diff --git a/beacon_node/network/src/sync/block_sidecar_coupling.rs b/beacon_node/network/src/sync/block_sidecar_coupling.rs index 966ce55fab..70a3fe4f5a 100644 --- a/beacon_node/network/src/sync/block_sidecar_coupling.rs +++ b/beacon_node/network/src/sync/block_sidecar_coupling.rs @@ -2,13 +2,13 @@ use beacon_chain::{ block_verification_types::RpcBlock, data_column_verification::CustodyDataColumn, get_block_root, }; use lighthouse_network::PeerId; -use ssz_types::VariableList; use std::{ collections::{HashMap, VecDeque}, sync::Arc, }; use types::{ - BlobSidecar, ChainSpec, ColumnIndex, DataColumnSidecar, EthSpec, Hash256, SignedBeaconBlock, + BlobSidecar, ChainSpec, ColumnIndex, DataColumnSidecar, EthSpec, Hash256, RuntimeVariableList, + SignedBeaconBlock, }; #[derive(Debug)] @@ -31,6 +31,7 @@ pub struct RangeBlockComponentsRequest { num_custody_column_requests: Option, /// The peers the request was made to. pub(crate) peer_ids: Vec, + max_blobs_per_block: usize, } impl RangeBlockComponentsRequest { @@ -39,6 +40,7 @@ impl RangeBlockComponentsRequest { expects_custody_columns: Option>, num_custody_column_requests: Option, peer_ids: Vec, + max_blobs_per_block: usize, ) -> Self { Self { blocks: <_>::default(), @@ -51,6 +53,7 @@ impl RangeBlockComponentsRequest { expects_custody_columns, num_custody_column_requests, peer_ids, + max_blobs_per_block, } } @@ -100,7 +103,7 @@ impl RangeBlockComponentsRequest { let mut responses = Vec::with_capacity(blocks.len()); let mut blob_iter = blobs.into_iter().peekable(); for block in blocks.into_iter() { - let mut blob_list = Vec::with_capacity(E::max_blobs_per_block()); + let mut blob_list = Vec::with_capacity(self.max_blobs_per_block); while { let pair_next_blob = blob_iter .peek() @@ -111,7 +114,7 @@ impl RangeBlockComponentsRequest { blob_list.push(blob_iter.next().ok_or("Missing next blob".to_string())?); } - let mut blobs_buffer = vec![None; E::max_blobs_per_block()]; + let mut blobs_buffer = vec![None; self.max_blobs_per_block]; for blob in blob_list { let blob_index = blob.index as usize; let Some(blob_opt) = blobs_buffer.get_mut(blob_index) else { @@ -123,7 +126,11 @@ impl RangeBlockComponentsRequest { *blob_opt = Some(blob); } } - let blobs = VariableList::from(blobs_buffer.into_iter().flatten().collect::>()); + let blobs = RuntimeVariableList::new( + blobs_buffer.into_iter().flatten().collect::>(), + self.max_blobs_per_block, + ) + .map_err(|_| "Blobs returned exceeds max length".to_string())?; responses.push(RpcBlock::new(None, block, Some(blobs)).map_err(|e| format!("{e:?}"))?) } @@ -245,12 +252,18 @@ mod tests { #[test] fn no_blobs_into_responses() { + let spec = test_spec::(); let peer_id = PeerId::random(); - let mut info = RangeBlockComponentsRequest::::new(false, None, None, vec![peer_id]); let mut rng = XorShiftRng::from_seed([42; 16]); let blocks = (0..4) - .map(|_| generate_rand_block_and_blobs::(ForkName::Base, NumBlobs::None, &mut rng).0) + .map(|_| { + generate_rand_block_and_blobs::(ForkName::Base, NumBlobs::None, &mut rng, &spec) + .0 + }) .collect::>(); + let max_len = spec.max_blobs_per_block(blocks.first().unwrap().epoch()) as usize; + let mut info = + RangeBlockComponentsRequest::::new(false, None, None, vec![peer_id], max_len); // Send blocks and complete terminate response for block in blocks { @@ -265,15 +278,24 @@ mod tests { #[test] fn empty_blobs_into_responses() { + let spec = test_spec::(); let peer_id = PeerId::random(); - let mut info = RangeBlockComponentsRequest::::new(true, None, None, vec![peer_id]); let mut rng = XorShiftRng::from_seed([42; 16]); let blocks = (0..4) .map(|_| { // Always generate some blobs. - generate_rand_block_and_blobs::(ForkName::Deneb, NumBlobs::Number(3), &mut rng).0 + generate_rand_block_and_blobs::( + ForkName::Deneb, + NumBlobs::Number(3), + &mut rng, + &spec, + ) + .0 }) .collect::>(); + let max_len = spec.max_blobs_per_block(blocks.first().unwrap().epoch()) as usize; + let mut info = + RangeBlockComponentsRequest::::new(true, None, None, vec![peer_id], max_len); // Send blocks and complete terminate response for block in blocks { @@ -294,24 +316,26 @@ mod tests { fn rpc_block_with_custody_columns() { let spec = test_spec::(); let expects_custody_columns = vec![1, 2, 3, 4]; - let mut info = RangeBlockComponentsRequest::::new( - false, - Some(expects_custody_columns.clone()), - Some(expects_custody_columns.len()), - vec![PeerId::random()], - ); + let mut rng = XorShiftRng::from_seed([42; 16]); let blocks = (0..4) .map(|_| { generate_rand_block_and_data_columns::( - ForkName::Deneb, + ForkName::Fulu, NumBlobs::Number(1), &mut rng, &spec, ) }) .collect::>(); - + let max_len = spec.max_blobs_per_block(blocks.first().unwrap().0.epoch()) as usize; + let mut info = RangeBlockComponentsRequest::::new( + false, + Some(expects_custody_columns.clone()), + Some(expects_custody_columns.len()), + vec![PeerId::random()], + max_len, + ); // Send blocks and complete terminate response for block in &blocks { info.add_block_response(Some(block.0.clone().into())); @@ -355,24 +379,26 @@ mod tests { let spec = test_spec::(); let expects_custody_columns = vec![1, 2, 3, 4]; let num_of_data_column_requests = 2; - let mut info = RangeBlockComponentsRequest::::new( - false, - Some(expects_custody_columns.clone()), - Some(num_of_data_column_requests), - vec![PeerId::random()], - ); + let mut rng = XorShiftRng::from_seed([42; 16]); let blocks = (0..4) .map(|_| { generate_rand_block_and_data_columns::( - ForkName::Deneb, + ForkName::Fulu, NumBlobs::Number(1), &mut rng, &spec, ) }) .collect::>(); - + let max_len = spec.max_blobs_per_block(blocks.first().unwrap().0.epoch()) as usize; + let mut info = RangeBlockComponentsRequest::::new( + false, + Some(expects_custody_columns.clone()), + Some(num_of_data_column_requests), + vec![PeerId::random()], + max_len, + ); // Send blocks and complete terminate response for block in &blocks { info.add_block_response(Some(block.0.clone().into())); diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index 5d02be2b4c..fd91dc78b1 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -69,7 +69,9 @@ use std::ops::Sub; use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc; -use types::{BlobSidecar, DataColumnSidecar, EthSpec, Hash256, SignedBeaconBlock, Slot}; +use types::{ + BlobSidecar, DataColumnSidecar, EthSpec, ForkContext, Hash256, SignedBeaconBlock, Slot, +}; #[cfg(test)] use types::ColumnIndex; @@ -258,10 +260,11 @@ pub fn spawn( network_send: mpsc::UnboundedSender>, beacon_processor: Arc>, sync_recv: mpsc::UnboundedReceiver>, + fork_context: Arc, log: slog::Logger, ) { assert!( - beacon_chain.spec.max_request_blocks >= T::EthSpec::slots_per_epoch() * EPOCHS_PER_BATCH, + beacon_chain.spec.max_request_blocks(fork_context.current_fork()) as u64 >= T::EthSpec::slots_per_epoch() * EPOCHS_PER_BATCH, "Max blocks that can be requested in a single batch greater than max allowed blocks in a single request" ); @@ -272,6 +275,7 @@ pub fn spawn( beacon_processor, sync_recv, SamplingConfig::Default, + fork_context, log.clone(), ); @@ -287,6 +291,7 @@ impl SyncManager { beacon_processor: Arc>, sync_recv: mpsc::UnboundedReceiver>, sampling_config: SamplingConfig, + fork_context: Arc, log: slog::Logger, ) -> Self { let network_globals = beacon_processor.network_globals.clone(); @@ -297,6 +302,7 @@ impl SyncManager { network_send, beacon_processor.clone(), beacon_chain.clone(), + fork_context.clone(), log.clone(), ), range_sync: RangeSync::new( @@ -1234,6 +1240,7 @@ impl SyncManager { .network .range_block_and_blob_response(id, block_or_blob) { + let epoch = resp.sender_id.batch_id(); match resp.responses { Ok(blocks) => { match resp.sender_id { @@ -1277,6 +1284,7 @@ impl SyncManager { resp.expects_custody_columns, None, vec![], + self.chain.spec.max_blobs_per_block(epoch) as usize, ), ); // inform range that the request needs to be treated as failed diff --git a/beacon_node/network/src/sync/network_context.rs b/beacon_node/network/src/sync/network_context.rs index b6b7b315f3..4135f901b1 100644 --- a/beacon_node/network/src/sync/network_context.rs +++ b/beacon_node/network/src/sync/network_context.rs @@ -27,7 +27,8 @@ use lighthouse_network::service::api_types::{ DataColumnsByRootRequester, Id, SingleLookupReqId, SyncRequestId, }; use lighthouse_network::{Client, NetworkGlobals, PeerAction, PeerId, ReportSource}; -use rand::seq::SliceRandom; +use parking_lot::RwLock; +use rand::prelude::IteratorRandom; use rand::thread_rng; pub use requests::LookupVerifyError; use requests::{ @@ -36,14 +37,14 @@ use requests::{ }; use slog::{debug, error, warn}; use std::collections::hash_map::Entry; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc; use types::blob_sidecar::FixedBlobSidecarList; use types::{ - BlobSidecar, ColumnIndex, DataColumnSidecar, DataColumnSidecarList, EthSpec, Hash256, - SignedBeaconBlock, Slot, + BlobSidecar, ColumnIndex, DataColumnSidecar, DataColumnSidecarList, EthSpec, ForkContext, + Hash256, SignedBeaconBlock, Slot, }; pub mod custody; @@ -67,6 +68,15 @@ pub enum RangeRequestId { }, } +impl RangeRequestId { + pub fn batch_id(&self) -> BatchId { + match self { + RangeRequestId::RangeSync { batch_id, .. } => *batch_id, + RangeRequestId::BackfillSync { batch_id, .. } => *batch_id, + } + } +} + #[derive(Debug)] pub enum RpcEvent { StreamTermination, @@ -206,6 +216,8 @@ pub struct SyncNetworkContext { pub chain: Arc>, + fork_context: Arc, + /// Logger for the `SyncNetworkContext`. pub log: slog::Logger, } @@ -234,6 +246,7 @@ impl SyncNetworkContext { network_send: mpsc::UnboundedSender>, network_beacon_processor: Arc>, chain: Arc>, + fork_context: Arc, log: slog::Logger, ) -> Self { SyncNetworkContext { @@ -247,6 +260,7 @@ impl SyncNetworkContext { range_block_components_requests: FnvHashMap::default(), network_beacon_processor, chain, + fork_context, log, } } @@ -299,8 +313,8 @@ impl SyncNetworkContext { pub fn get_random_custodial_peer(&self, column_index: ColumnIndex) -> Option { self.get_custodial_peers(column_index) + .into_iter() .choose(&mut thread_rng()) - .cloned() } pub fn network_globals(&self) -> &NetworkGlobals { @@ -359,6 +373,7 @@ impl SyncNetworkContext { "count" => request.count(), "epoch" => epoch, "peer" => %peer_id, + "id" => id, ); let rpc_request = match request { BlocksByRangeRequest::V1(ref req) => { @@ -428,6 +443,7 @@ impl SyncNetworkContext { "epoch" => epoch, "columns" => ?columns_by_range_request.columns, "peer" => %peer_id, + "id" => id, ); self.send_network_msg(NetworkMessage::SendRequest { @@ -445,11 +461,13 @@ impl SyncNetworkContext { (None, None) }; + let max_blobs_len = self.chain.spec.max_blobs_per_block(epoch); let info = RangeBlockComponentsRequest::new( expected_blobs, - expects_columns, + expects_columns.map(|c| c.into_iter().collect()), num_of_column_req, requested_peers, + max_blobs_len as usize, ); self.range_block_components_requests .insert(id, (sender_id, info)); @@ -459,7 +477,7 @@ impl SyncNetworkContext { fn make_columns_by_range_requests( &self, request: BlocksByRangeRequest, - custody_indexes: &Vec, + custody_indexes: &HashSet, ) -> Result, RpcRequestSendError> { let mut peer_id_to_request_map = HashMap::new(); @@ -550,9 +568,24 @@ impl SyncNetworkContext { pub fn block_lookup_request( &mut self, lookup_id: SingleLookupId, - peer_id: PeerId, + lookup_peers: Arc>>, block_root: Hash256, ) -> Result { + let Some(peer_id) = lookup_peers + .read() + .iter() + .choose(&mut rand::thread_rng()) + .copied() + else { + // Allow lookup to not have any peers and do nothing. This is an optimization to not + // lose progress of lookups created from a block with unknown parent before we receive + // attestations for said block. + // Lookup sync event safety: If a lookup requires peers to make progress, and does + // not receive any new peers for some time it will be dropped. If it receives a new + // peer it must attempt to make progress. + return Ok(LookupRequestResult::Pending("no peers")); + }; + match self.chain.get_block_process_status(&block_root) { // Unknown block, continue request to download BlockProcessStatus::Unknown => {} @@ -596,7 +629,7 @@ impl SyncNetworkContext { self.network_send .send(NetworkMessage::SendRequest { peer_id, - request: RequestType::BlocksByRoot(request.into_request(&self.chain.spec)), + request: RequestType::BlocksByRoot(request.into_request(&self.fork_context)), request_id: AppRequestId::Sync(SyncRequestId::SingleBlock { id }), }) .map_err(|_| RpcRequestSendError::NetworkSendError)?; @@ -622,10 +655,25 @@ impl SyncNetworkContext { pub fn blob_lookup_request( &mut self, lookup_id: SingleLookupId, - peer_id: PeerId, + lookup_peers: Arc>>, block_root: Hash256, expected_blobs: usize, ) -> Result { + let Some(peer_id) = lookup_peers + .read() + .iter() + .choose(&mut rand::thread_rng()) + .copied() + else { + // Allow lookup to not have any peers and do nothing. This is an optimization to not + // lose progress of lookups created from a block with unknown parent before we receive + // attestations for said block. + // Lookup sync event safety: If a lookup requires peers to make progress, and does + // not receive any new peers for some time it will be dropped. If it receives a new + // peer it must attempt to make progress. + return Ok(LookupRequestResult::Pending("no peers")); + }; + let imported_blob_indexes = self .chain .data_availability_checker @@ -663,7 +711,7 @@ impl SyncNetworkContext { self.network_send .send(NetworkMessage::SendRequest { peer_id, - request: RequestType::BlobsByRoot(request.clone().into_request(&self.chain.spec)), + request: RequestType::BlobsByRoot(request.clone().into_request(&self.fork_context)), request_id: AppRequestId::Sync(SyncRequestId::SingleBlob { id }), }) .map_err(|_| RpcRequestSendError::NetworkSendError)?; @@ -728,6 +776,7 @@ impl SyncNetworkContext { &mut self, lookup_id: SingleLookupId, block_root: Hash256, + lookup_peers: Arc>>, ) -> Result { let custody_indexes_imported = self .chain @@ -765,6 +814,7 @@ impl SyncNetworkContext { block_root, CustodyId { requester }, &custody_indexes_to_fetch, + lookup_peers, self.log.clone(), ); @@ -950,12 +1000,23 @@ impl SyncNetworkContext { ) -> Option>> { let response = self.blobs_by_root_requests.on_response(id, rpc_event); let response = response.map(|res| { - res.and_then( - |(blobs, seen_timestamp)| match to_fixed_blob_sidecar_list(blobs) { - Ok(blobs) => Ok((blobs, seen_timestamp)), - Err(e) => Err(e.into()), - }, - ) + res.and_then(|(blobs, seen_timestamp)| { + if let Some(max_len) = blobs + .first() + .map(|blob| self.chain.spec.max_blobs_per_block(blob.epoch()) as usize) + { + match to_fixed_blob_sidecar_list(blobs, max_len) { + Ok(blobs) => Ok((blobs, seen_timestamp)), + Err(e) => Err(e.into()), + } + } else { + Err(RpcResponseError::VerifyError( + LookupVerifyError::InternalError( + "Requested blobs for a block that has no blobs".to_string(), + ), + )) + } + }) }); if let Some(Err(RpcResponseError::VerifyError(e))) = &response { self.report_peer(peer_id, PeerAction::LowToleranceError, e.into()); @@ -1150,8 +1211,9 @@ impl SyncNetworkContext { fn to_fixed_blob_sidecar_list( blobs: Vec>>, + max_len: usize, ) -> Result, LookupVerifyError> { - let mut fixed_list = FixedBlobSidecarList::default(); + let mut fixed_list = FixedBlobSidecarList::new(vec![None; max_len]); for blob in blobs.into_iter() { let index = blob.index as usize; *fixed_list diff --git a/beacon_node/network/src/sync/network_context/custody.rs b/beacon_node/network/src/sync/network_context/custody.rs index e4bce3dafc..8a29545c21 100644 --- a/beacon_node/network/src/sync/network_context/custody.rs +++ b/beacon_node/network/src/sync/network_context/custody.rs @@ -7,8 +7,10 @@ use fnv::FnvHashMap; use lighthouse_network::service::api_types::{CustodyId, DataColumnsByRootRequester}; use lighthouse_network::PeerId; use lru_cache::LRUTimeCache; +use parking_lot::RwLock; use rand::Rng; use slog::{debug, warn}; +use std::collections::HashSet; use std::time::{Duration, Instant}; use std::{collections::HashMap, marker::PhantomData, sync::Arc}; use types::EthSpec; @@ -32,6 +34,8 @@ pub struct ActiveCustodyRequest { /// Peers that have recently failed to successfully respond to a columns by root request. /// Having a LRUTimeCache allows this request to not have to track disconnecting peers. failed_peers: LRUTimeCache, + /// Set of peers that claim to have imported this block and their custody columns + lookup_peers: Arc>>, /// Logger for the `SyncNetworkContext`. pub log: slog::Logger, _phantom: PhantomData, @@ -64,6 +68,7 @@ impl ActiveCustodyRequest { block_root: Hash256, custody_id: CustodyId, column_indices: &[ColumnIndex], + lookup_peers: Arc>>, log: slog::Logger, ) -> Self { Self { @@ -76,6 +81,7 @@ impl ActiveCustodyRequest { ), active_batch_columns_requests: <_>::default(), failed_peers: LRUTimeCache::new(Duration::from_secs(FAILED_PEERS_CACHE_EXPIRY_SECONDS)), + lookup_peers, log, _phantom: PhantomData, } @@ -215,6 +221,7 @@ impl ActiveCustodyRequest { } let mut columns_to_request_by_peer = HashMap::>::new(); + let lookup_peers = self.lookup_peers.read(); // Need to: // - track how many active requests a peer has for load balancing @@ -244,6 +251,8 @@ impl ActiveCustodyRequest { .iter() .map(|peer| { ( + // Prioritize peers that claim to know have imported this block + if lookup_peers.contains(peer) { 0 } else { 1 }, // De-prioritize peers that have failed to successfully respond to // requests recently self.failed_peers.contains(peer), @@ -257,7 +266,7 @@ impl ActiveCustodyRequest { .collect::>(); priorized_peers.sort_unstable(); - if let Some((_, _, _, peer_id)) = priorized_peers.first() { + if let Some((_, _, _, _, peer_id)) = priorized_peers.first() { columns_to_request_by_peer .entry(*peer_id) .or_default() @@ -283,10 +292,11 @@ impl ActiveCustodyRequest { block_root: self.block_root, indices: indices.clone(), }, - // true = enforce max_requests are returned data_columns_by_root. We only issue requests - // for blocks after we know the block has data, and only request peers after they claim to - // have imported the block+columns and claim to be custodians - true, + // If peer is in the lookup peer set, it claims to have imported the block and + // must have its columns in custody. In that case, set `true = enforce max_requests` + // and downscore if data_columns_by_root does not returned the expected custody + // columns. For the rest of peers, don't downscore if columns are missing. + lookup_peers.contains(&peer_id), ) .map_err(Error::SendFailed)?; diff --git a/beacon_node/network/src/sync/network_context/requests.rs b/beacon_node/network/src/sync/network_context/requests.rs index b9214bafcd..4a5a16459d 100644 --- a/beacon_node/network/src/sync/network_context/requests.rs +++ b/beacon_node/network/src/sync/network_context/requests.rs @@ -28,6 +28,7 @@ pub enum LookupVerifyError { UnrequestedIndex(u64), InvalidInclusionProof, DuplicateData, + InternalError(String), } /// Collection of active requests of a single ReqResp method, i.e. `blocks_by_root` diff --git a/beacon_node/network/src/sync/network_context/requests/blobs_by_root.rs b/beacon_node/network/src/sync/network_context/requests/blobs_by_root.rs index fefb27a5ef..a670229884 100644 --- a/beacon_node/network/src/sync/network_context/requests/blobs_by_root.rs +++ b/beacon_node/network/src/sync/network_context/requests/blobs_by_root.rs @@ -1,6 +1,6 @@ use lighthouse_network::rpc::methods::BlobsByRootRequest; use std::sync::Arc; -use types::{blob_sidecar::BlobIdentifier, BlobSidecar, ChainSpec, EthSpec, Hash256}; +use types::{blob_sidecar::BlobIdentifier, BlobSidecar, EthSpec, ForkContext, Hash256}; use super::{ActiveRequestItems, LookupVerifyError}; @@ -11,7 +11,7 @@ pub struct BlobsByRootSingleBlockRequest { } impl BlobsByRootSingleBlockRequest { - pub fn into_request(self, spec: &ChainSpec) -> BlobsByRootRequest { + pub fn into_request(self, spec: &ForkContext) -> BlobsByRootRequest { BlobsByRootRequest::new( self.indices .into_iter() diff --git a/beacon_node/network/src/sync/network_context/requests/blocks_by_root.rs b/beacon_node/network/src/sync/network_context/requests/blocks_by_root.rs index f3cdcbe714..6d7eabf909 100644 --- a/beacon_node/network/src/sync/network_context/requests/blocks_by_root.rs +++ b/beacon_node/network/src/sync/network_context/requests/blocks_by_root.rs @@ -1,7 +1,7 @@ use beacon_chain::get_block_root; use lighthouse_network::rpc::BlocksByRootRequest; use std::sync::Arc; -use types::{ChainSpec, EthSpec, Hash256, SignedBeaconBlock}; +use types::{EthSpec, ForkContext, Hash256, SignedBeaconBlock}; use super::{ActiveRequestItems, LookupVerifyError}; @@ -9,8 +9,8 @@ use super::{ActiveRequestItems, LookupVerifyError}; pub struct BlocksByRootSingleRequest(pub Hash256); impl BlocksByRootSingleRequest { - pub fn into_request(self, spec: &ChainSpec) -> BlocksByRootRequest { - BlocksByRootRequest::new(vec![self.0], spec) + pub fn into_request(self, fork_context: &ForkContext) -> BlocksByRootRequest { + BlocksByRootRequest::new(vec![self.0], fork_context) } } diff --git a/beacon_node/network/src/sync/peer_sync_info.rs b/beacon_node/network/src/sync/peer_sync_info.rs index c01366f1be..5ea1533d35 100644 --- a/beacon_node/network/src/sync/peer_sync_info.rs +++ b/beacon_node/network/src/sync/peer_sync_info.rs @@ -30,8 +30,8 @@ pub fn remote_sync_type( ) -> PeerSyncType { // auxiliary variables for clarity: Inclusive boundaries of the range in which we consider a peer's // head "near" ours. - let near_range_start = local.head_slot - SLOT_IMPORT_TOLERANCE as u64; - let near_range_end = local.head_slot + SLOT_IMPORT_TOLERANCE as u64; + let near_range_start = local.head_slot.saturating_sub(SLOT_IMPORT_TOLERANCE); + let near_range_end = local.head_slot.saturating_add(SLOT_IMPORT_TOLERANCE); match remote.finalized_epoch.cmp(&local.finalized_epoch) { Ordering::Less => { diff --git a/beacon_node/network/src/sync/range_sync/chain.rs b/beacon_node/network/src/sync/range_sync/chain.rs index 51d9d9da37..4eb73f5483 100644 --- a/beacon_node/network/src/sync/range_sync/chain.rs +++ b/beacon_node/network/src/sync/range_sync/chain.rs @@ -15,7 +15,6 @@ use rand::seq::SliceRandom; use rand::Rng; use slog::{crit, debug, o, warn}; use std::collections::{btree_map::Entry, BTreeMap, HashSet}; -use std::hash::{Hash, Hasher}; use strum::IntoStaticStr; use types::{Epoch, EthSpec, Hash256, Slot}; @@ -56,7 +55,7 @@ pub enum RemoveChain { pub struct KeepChain; /// A chain identifier -pub type ChainId = u64; +pub type ChainId = Id; pub type BatchId = Epoch; #[derive(Debug, Copy, Clone, IntoStaticStr)] @@ -127,14 +126,9 @@ pub enum ChainSyncingState { } impl SyncingChain { - pub fn id(target_root: &Hash256, target_slot: &Slot) -> u64 { - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - (target_root, target_slot).hash(&mut hasher); - hasher.finish() - } - #[allow(clippy::too_many_arguments)] pub fn new( + id: Id, start_epoch: Epoch, target_head_slot: Slot, target_head_root: Hash256, @@ -145,8 +139,6 @@ impl SyncingChain { let mut peers = FnvHashMap::default(); peers.insert(peer_id, Default::default()); - let id = SyncingChain::::id(&target_head_root, &target_head_slot); - SyncingChain { id, chain_type, @@ -165,6 +157,11 @@ impl SyncingChain { } } + /// Returns true if this chain has the same target + pub fn has_same_target(&self, target_head_slot: Slot, target_head_root: Hash256) -> bool { + self.target_head_slot == target_head_slot && self.target_head_root == target_head_root + } + /// Check if the chain has peers from which to process batches. pub fn available_peers(&self) -> usize { self.peers.len() @@ -1258,7 +1255,7 @@ impl slog::KV for SyncingChain { serializer: &mut dyn slog::Serializer, ) -> slog::Result { use slog::Value; - serializer.emit_u64("id", self.id)?; + serializer.emit_u32("id", self.id)?; Value::serialize(&self.start_epoch, record, "from", serializer)?; Value::serialize( &self.target_head_slot.epoch(T::EthSpec::slots_per_epoch()), diff --git a/beacon_node/network/src/sync/range_sync/chain_collection.rs b/beacon_node/network/src/sync/range_sync/chain_collection.rs index c030d0a19e..15bdf85e20 100644 --- a/beacon_node/network/src/sync/range_sync/chain_collection.rs +++ b/beacon_node/network/src/sync/range_sync/chain_collection.rs @@ -9,6 +9,7 @@ use crate::metrics; use crate::sync::network_context::SyncNetworkContext; use beacon_chain::{BeaconChain, BeaconChainTypes}; use fnv::FnvHashMap; +use lighthouse_network::service::api_types::Id; use lighthouse_network::PeerId; use lighthouse_network::SyncInfo; use slog::{crit, debug, error}; @@ -29,9 +30,9 @@ const MIN_FINALIZED_CHAIN_PROCESSED_EPOCHS: u64 = 10; #[derive(Clone)] pub enum RangeSyncState { /// A finalized chain is being synced. - Finalized(u64), + Finalized(Id), /// There are no finalized chains and we are syncing one more head chains. - Head(SmallVec<[u64; PARALLEL_HEAD_CHAINS]>), + Head(SmallVec<[Id; PARALLEL_HEAD_CHAINS]>), /// There are no head or finalized chains and no long range sync is in progress. Idle, } @@ -74,7 +75,7 @@ impl ChainCollection { if syncing_id == id { // the finalized chain that was syncing was removed debug_assert!(was_syncing && sync_type == RangeSyncType::Finalized); - let syncing_head_ids: SmallVec<[u64; PARALLEL_HEAD_CHAINS]> = self + let syncing_head_ids: SmallVec<[Id; PARALLEL_HEAD_CHAINS]> = self .head_chains .iter() .filter(|(_id, chain)| chain.is_syncing()) @@ -86,7 +87,7 @@ impl ChainCollection { RangeSyncState::Head(syncing_head_ids) }; } else { - // we removed a head chain, or an stoped finalized chain + // we removed a head chain, or a stopped finalized chain debug_assert!(!was_syncing || sync_type != RangeSyncType::Finalized); } } @@ -355,7 +356,7 @@ impl ChainCollection { .collect::>(); preferred_ids.sort_unstable(); - let mut syncing_chains = SmallVec::<[u64; PARALLEL_HEAD_CHAINS]>::new(); + let mut syncing_chains = SmallVec::<[Id; PARALLEL_HEAD_CHAINS]>::new(); for (_, _, id) in preferred_ids { let chain = self.head_chains.get_mut(&id).expect("known chain"); if syncing_chains.len() < PARALLEL_HEAD_CHAINS { @@ -465,15 +466,17 @@ impl ChainCollection { sync_type: RangeSyncType, network: &mut SyncNetworkContext, ) { - let id = SyncingChain::::id(&target_head_root, &target_head_slot); let collection = if let RangeSyncType::Finalized = sync_type { &mut self.finalized_chains } else { &mut self.head_chains }; - match collection.entry(id) { - Entry::Occupied(mut entry) => { - let chain = entry.get_mut(); + + match collection + .iter_mut() + .find(|(_, chain)| chain.has_same_target(target_head_slot, target_head_root)) + { + Some((&id, chain)) => { debug!(self.log, "Adding peer to known chain"; "peer_id" => %peer, "sync_type" => ?sync_type, &chain); debug_assert_eq!(chain.target_head_root, target_head_root); debug_assert_eq!(chain.target_head_slot, target_head_slot); @@ -483,13 +486,16 @@ impl ChainCollection { } else { error!(self.log, "Chain removed after adding peer"; "chain" => id, "reason" => ?remove_reason); } - let chain = entry.remove(); - self.on_chain_removed(&id, chain.is_syncing(), sync_type); + let is_syncing = chain.is_syncing(); + collection.remove(&id); + self.on_chain_removed(&id, is_syncing, sync_type); } } - Entry::Vacant(entry) => { + None => { let peer_rpr = peer.to_string(); + let id = network.next_id(); let new_chain = SyncingChain::new( + id, start_epoch, target_head_slot, target_head_root, @@ -497,9 +503,8 @@ impl ChainCollection { sync_type.into(), &self.log, ); - debug_assert_eq!(new_chain.get_id(), id); debug!(self.log, "New chain added to sync"; "peer_id" => peer_rpr, "sync_type" => ?sync_type, &new_chain); - entry.insert(new_chain); + collection.insert(id, new_chain); metrics::inc_counter_vec(&metrics::SYNCING_CHAINS_ADDED, &[sync_type.as_str()]); self.update_metrics(); } diff --git a/beacon_node/network/src/sync/tests/lookups.rs b/beacon_node/network/src/sync/tests/lookups.rs index 94aacad3e8..9ab581950c 100644 --- a/beacon_node/network/src/sync/tests/lookups.rs +++ b/beacon_node/network/src/sync/tests/lookups.rs @@ -39,11 +39,12 @@ use lighthouse_network::{ use slog::info; use slot_clock::{SlotClock, TestingSlotClock}; use tokio::sync::mpsc; +use types::ForkContext; use types::{ data_column_sidecar::ColumnIndex, test_utils::{SeedableRng, TestRandom, XorShiftRng}, - BeaconState, BeaconStateBase, BlobSidecar, DataColumnSidecar, Epoch, EthSpec, ForkName, - Hash256, MinimalEthSpec as E, SignedBeaconBlock, Slot, + BeaconState, BeaconStateBase, BlobSidecar, DataColumnSidecar, EthSpec, ForkName, Hash256, + MinimalEthSpec as E, SignedBeaconBlock, Slot, }; const D: Duration = Duration::new(0, 0); @@ -53,12 +54,8 @@ const SAMPLING_REQUIRED_SUCCESSES: usize = 2; type DCByRootIds = Vec; type DCByRootId = (SyncRequestId, Vec); -struct TestRigConfig { - peer_das_enabled: bool, -} - impl TestRig { - fn test_setup_with_config(config: Option) -> Self { + pub fn test_setup() -> Self { let logger_type = if cfg!(feature = "test_logger") { LoggerType::Test } else if cfg!(feature = "ci_logger") { @@ -69,13 +66,7 @@ impl TestRig { let log = build_log(slog::Level::Trace, logger_type); // Use `fork_from_env` logic to set correct fork epochs - let mut spec = test_spec::(); - - if let Some(config) = config { - if config.peer_das_enabled { - spec.eip7594_fork_epoch = Some(Epoch::new(0)); - } - } + let spec = test_spec::(); // Initialise a new beacon chain let harness = BeaconChainHarness::>::builder(E) @@ -92,6 +83,11 @@ impl TestRig { .build(); let chain = harness.chain.clone(); + let fork_context = Arc::new(ForkContext::new::( + Slot::new(0), + chain.genesis_validators_root, + &chain.spec, + )); let (network_tx, network_rx) = mpsc::unbounded_channel(); let (sync_tx, sync_rx) = mpsc::unbounded_channel::>(); @@ -119,6 +115,8 @@ impl TestRig { .network_globals .set_sync_state(SyncState::Synced); + let spec = chain.spec.clone(); + let rng = XorShiftRng::from_seed([42; 16]); TestRig { beacon_processor_rx, @@ -137,32 +135,28 @@ impl TestRig { SamplingConfig::Custom { required_successes: vec![SAMPLING_REQUIRED_SUCCESSES], }, + fork_context, log.clone(), ), harness, fork_name, log, + spec, } } - pub fn test_setup() -> Self { - Self::test_setup_with_config(None) - } - - fn test_setup_after_deneb() -> Option { + fn test_setup_after_deneb_before_fulu() -> Option { let r = Self::test_setup(); - if r.after_deneb() { + if r.after_deneb() && !r.fork_name.fulu_enabled() { Some(r) } else { None } } - fn test_setup_after_peerdas() -> Option { - let r = Self::test_setup_with_config(Some(TestRigConfig { - peer_das_enabled: true, - })); - if r.after_deneb() { + fn test_setup_after_fulu() -> Option { + let r = Self::test_setup(); + if r.fork_name.fulu_enabled() { Some(r) } else { None @@ -174,7 +168,11 @@ impl TestRig { } pub fn after_deneb(&self) -> bool { - matches!(self.fork_name, ForkName::Deneb | ForkName::Electra) + self.fork_name.deneb_enabled() + } + + pub fn after_fulu(&self) -> bool { + self.fork_name.fulu_enabled() } fn trigger_unknown_parent_block(&mut self, peer_id: PeerId, block: Arc>) { @@ -213,7 +211,7 @@ impl TestRig { ) -> (SignedBeaconBlock, Vec>) { let fork_name = self.fork_name; let rng = &mut self.rng; - generate_rand_block_and_blobs::(fork_name, num_blobs, rng) + generate_rand_block_and_blobs::(fork_name, num_blobs, rng, &self.spec) } fn rand_block_and_data_columns( @@ -377,7 +375,7 @@ impl TestRig { .__add_connected_peer_testing_only(false, &self.harness.spec) } - fn new_connected_supernode_peer(&mut self) -> PeerId { + pub fn new_connected_supernode_peer(&mut self) -> PeerId { self.network_globals .peers .write() @@ -1328,8 +1326,10 @@ impl TestRig { #[test] fn stable_rng() { + let spec = types::MainnetEthSpec::default_spec(); let mut rng = XorShiftRng::from_seed([42; 16]); - let (block, _) = generate_rand_block_and_blobs::(ForkName::Base, NumBlobs::None, &mut rng); + let (block, _) = + generate_rand_block_and_blobs::(ForkName::Base, NumBlobs::None, &mut rng, &spec); assert_eq!( block.canonical_root(), Hash256::from_slice( @@ -1672,7 +1672,7 @@ fn test_parent_lookup_too_many_processing_attempts_must_blacklist() { rig.assert_not_failed_chain(block_root); // send the right parent but fail processing rig.parent_lookup_block_response(id, peer_id, Some(parent.clone().into())); - rig.parent_block_processed(block_root, BlockError::InvalidSignature.into()); + rig.parent_block_processed(block_root, BlockError::BlockSlotLimitReached.into()); rig.parent_lookup_block_response(id, peer_id, None); rig.expect_penalty(peer_id, "lookup_block_processing_failure"); } @@ -1933,7 +1933,7 @@ fn test_same_chain_race_condition() { #[test] fn block_in_da_checker_skips_download() { - let Some(mut r) = TestRig::test_setup_after_deneb() else { + let Some(mut r) = TestRig::test_setup_after_deneb_before_fulu() else { return; }; let (block, blobs) = r.rand_block_and_blobs(NumBlobs::Number(1)); @@ -1951,7 +1951,7 @@ fn block_in_da_checker_skips_download() { #[test] fn block_in_processing_cache_becomes_invalid() { - let Some(mut r) = TestRig::test_setup_after_deneb() else { + let Some(mut r) = TestRig::test_setup_after_deneb_before_fulu() else { return; }; let (block, blobs) = r.rand_block_and_blobs(NumBlobs::Number(1)); @@ -1977,7 +1977,7 @@ fn block_in_processing_cache_becomes_invalid() { #[test] fn block_in_processing_cache_becomes_valid_imported() { - let Some(mut r) = TestRig::test_setup_after_deneb() else { + let Some(mut r) = TestRig::test_setup_after_deneb_before_fulu() else { return; }; let (block, blobs) = r.rand_block_and_blobs(NumBlobs::Number(1)); @@ -2002,7 +2002,7 @@ fn block_in_processing_cache_becomes_valid_imported() { #[ignore] #[test] fn blobs_in_da_checker_skip_download() { - let Some(mut r) = TestRig::test_setup_after_deneb() else { + let Some(mut r) = TestRig::test_setup_after_deneb_before_fulu() else { return; }; let (block, blobs) = r.rand_block_and_blobs(NumBlobs::Number(1)); @@ -2021,7 +2021,7 @@ fn blobs_in_da_checker_skip_download() { #[test] fn sampling_happy_path() { - let Some(mut r) = TestRig::test_setup_after_peerdas() else { + let Some(mut r) = TestRig::test_setup_after_fulu() else { return; }; r.new_connected_peers_for_peerdas(); @@ -2038,7 +2038,7 @@ fn sampling_happy_path() { #[test] fn sampling_with_retries() { - let Some(mut r) = TestRig::test_setup_after_peerdas() else { + let Some(mut r) = TestRig::test_setup_after_fulu() else { return; }; r.new_connected_peers_for_peerdas(); @@ -2060,7 +2060,7 @@ fn sampling_with_retries() { #[test] fn sampling_avoid_retrying_same_peer() { - let Some(mut r) = TestRig::test_setup_after_peerdas() else { + let Some(mut r) = TestRig::test_setup_after_fulu() else { return; }; let peer_id_1 = r.new_connected_supernode_peer(); @@ -2081,7 +2081,7 @@ fn sampling_avoid_retrying_same_peer() { #[test] fn sampling_batch_requests() { - let Some(mut r) = TestRig::test_setup_after_peerdas() else { + let Some(mut r) = TestRig::test_setup_after_fulu() else { return; }; let _supernode = r.new_connected_supernode_peer(); @@ -2107,7 +2107,7 @@ fn sampling_batch_requests() { #[test] fn sampling_batch_requests_not_enough_responses_returned() { - let Some(mut r) = TestRig::test_setup_after_peerdas() else { + let Some(mut r) = TestRig::test_setup_after_fulu() else { return; }; let _supernode = r.new_connected_supernode_peer(); @@ -2152,7 +2152,7 @@ fn sampling_batch_requests_not_enough_responses_returned() { #[test] fn custody_lookup_happy_path() { - let Some(mut r) = TestRig::test_setup_after_peerdas() else { + let Some(mut r) = TestRig::test_setup_after_fulu() else { return; }; let spec = E::default_spec(); @@ -2165,7 +2165,7 @@ fn custody_lookup_happy_path() { let id = r.expect_block_lookup_request(block.canonical_root()); r.complete_valid_block_request(id, block.into(), true); // for each slot we download `samples_per_slot` columns - let sample_column_count = spec.samples_per_slot * spec.data_columns_per_subnet() as u64; + let sample_column_count = spec.samples_per_slot * spec.data_columns_per_group(); let custody_ids = r.expect_only_data_columns_by_root_requests(block_root, sample_column_count as usize); r.complete_valid_custody_request(custody_ids, data_columns, false); @@ -2187,8 +2187,8 @@ mod deneb_only { block_verification_types::{AsBlock, RpcBlock}, data_availability_checker::AvailabilityCheckError, }; - use ssz_types::VariableList; use std::collections::VecDeque; + use types::RuntimeVariableList; struct DenebTester { rig: TestRig, @@ -2226,7 +2226,7 @@ mod deneb_only { impl DenebTester { fn new(request_trigger: RequestTrigger) -> Option { - let Some(mut rig) = TestRig::test_setup_after_deneb() else { + let Some(mut rig) = TestRig::test_setup_after_deneb_before_fulu() else { return None; }; let (block, blobs) = rig.rand_block_and_blobs(NumBlobs::Random); @@ -2546,12 +2546,15 @@ mod deneb_only { fn parent_block_unknown_parent(mut self) -> Self { self.rig.log("parent_block_unknown_parent"); let block = self.unknown_parent_block.take().unwrap(); + let max_len = self.rig.spec.max_blobs_per_block(block.epoch()) as usize; // Now this block is the one we expect requests from self.block = block.clone(); let block = RpcBlock::new( Some(block.canonical_root()), block, - self.unknown_parent_blobs.take().map(VariableList::from), + self.unknown_parent_blobs + .take() + .map(|vec| RuntimeVariableList::from_vec(vec, max_len)), ) .unwrap(); self.rig.parent_block_processed( @@ -2567,7 +2570,7 @@ mod deneb_only { fn invalid_parent_processed(mut self) -> Self { self.rig.parent_block_processed( self.block_root, - BlockProcessingResult::Err(BlockError::ProposalSignatureInvalid), + BlockProcessingResult::Err(BlockError::BlockSlotLimitReached), ); assert_eq!(self.rig.active_parent_lookups_count(), 1); self @@ -2576,7 +2579,7 @@ mod deneb_only { fn invalid_block_processed(mut self) -> Self { self.rig.single_block_component_processed( self.block_req_id.expect("block request id").lookup_id, - BlockProcessingResult::Err(BlockError::ProposalSignatureInvalid), + BlockProcessingResult::Err(BlockError::BlockSlotLimitReached), ); self.rig.assert_single_lookups_count(1); self @@ -2948,7 +2951,7 @@ mod deneb_only { #[ignore] #[test] fn no_peer_penalty_when_rpc_response_already_known_from_gossip() { - let Some(mut r) = TestRig::test_setup_after_deneb() else { + let Some(mut r) = TestRig::test_setup_after_deneb_before_fulu() else { return; }; let (block, blobs) = r.rand_block_and_blobs(NumBlobs::Number(2)); diff --git a/beacon_node/network/src/sync/tests/mod.rs b/beacon_node/network/src/sync/tests/mod.rs index 47666b413c..6ed5c7f8fa 100644 --- a/beacon_node/network/src/sync/tests/mod.rs +++ b/beacon_node/network/src/sync/tests/mod.rs @@ -12,7 +12,7 @@ use slot_clock::ManualSlotClock; use std::sync::Arc; use store::MemoryStore; use tokio::sync::mpsc; -use types::{test_utils::XorShiftRng, ForkName, MinimalEthSpec as E}; +use types::{test_utils::XorShiftRng, ChainSpec, ForkName, MinimalEthSpec as E}; mod lookups; mod range; @@ -64,4 +64,5 @@ struct TestRig { rng: XorShiftRng, fork_name: ForkName, log: Logger, + spec: Arc, } diff --git a/beacon_node/network/src/sync/tests/range.rs b/beacon_node/network/src/sync/tests/range.rs index 6faa8b7247..cfd89f7b44 100644 --- a/beacon_node/network/src/sync/tests/range.rs +++ b/beacon_node/network/src/sync/tests/range.rs @@ -3,16 +3,65 @@ use crate::status::ToStatusMessage; use crate::sync::manager::SLOT_IMPORT_TOLERANCE; use crate::sync::range_sync::RangeSyncType; use crate::sync::SyncMessage; +use beacon_chain::data_column_verification::CustodyDataColumn; use beacon_chain::test_utils::{AttestationStrategy, BlockStrategy}; -use beacon_chain::EngineState; +use beacon_chain::{block_verification_types::RpcBlock, EngineState, NotifyExecutionLayer}; +use lighthouse_network::rpc::methods::{ + BlobsByRangeRequest, DataColumnsByRangeRequest, OldBlocksByRangeRequest, + OldBlocksByRangeRequestV2, +}; use lighthouse_network::rpc::{RequestType, StatusMessage}; use lighthouse_network::service::api_types::{AppRequestId, Id, SyncRequestId}; use lighthouse_network::{PeerId, SyncInfo}; use std::time::Duration; -use types::{EthSpec, Hash256, MinimalEthSpec as E, SignedBeaconBlock, Slot}; +use types::{ + BlobSidecarList, BlockImportSource, EthSpec, Hash256, MinimalEthSpec as E, SignedBeaconBlock, + SignedBeaconBlockHash, Slot, +}; const D: Duration = Duration::new(0, 0); +pub(crate) enum DataSidecars { + Blobs(BlobSidecarList), + DataColumns(Vec>), +} + +enum ByRangeDataRequestIds { + PreDeneb, + PrePeerDAS(Id, PeerId), + PostPeerDAS(Vec<(Id, PeerId)>), +} + +/// Sync tests are usually written in the form: +/// - Do some action +/// - Expect a request to be sent +/// - Complete the above request +/// +/// To make writting tests succint, the machinery in this testing rig automatically identifies +/// _which_ request to complete. Picking the right request is critical for tests to pass, so this +/// filter allows better expressivity on the criteria to identify the right request. +#[derive(Default)] +struct RequestFilter { + peer: Option, + epoch: Option, +} + +impl RequestFilter { + fn peer(mut self, peer: PeerId) -> Self { + self.peer = Some(peer); + self + } + + fn epoch(mut self, epoch: u64) -> Self { + self.epoch = Some(epoch); + self + } +} + +fn filter() -> RequestFilter { + RequestFilter::default() +} + impl TestRig { /// Produce a head peer with an advanced head fn add_head_peer(&mut self) -> PeerId { @@ -64,7 +113,9 @@ impl TestRig { fn add_peer(&mut self, remote_info: SyncInfo) -> PeerId { // Create valid peer known to network globals - let peer_id = self.new_connected_peer(); + // TODO(fulu): Using supernode peers to ensure we have peer across all column + // subnets for syncing. Should add tests connecting to full node peers. + let peer_id = self.new_connected_supernode_peer(); // Send peer to sync self.send_sync_message(SyncMessage::AddPeer(peer_id, remote_info.clone())); peer_id @@ -83,11 +134,13 @@ impl TestRig { } #[track_caller] - fn expect_chain_segment(&mut self) { - self.pop_received_processor_event(|ev| { - (ev.work_type() == beacon_processor::WorkType::ChainSegment).then_some(()) - }) - .unwrap_or_else(|e| panic!("Expect ChainSegment work event: {e:?}")); + fn expect_chain_segments(&mut self, count: usize) { + for i in 0..count { + self.pop_received_processor_event(|ev| { + (ev.work_type() == beacon_processor::WorkType::ChainSegment).then_some(()) + }) + .unwrap_or_else(|e| panic!("Expect ChainSegment work event count {i}: {e:?}")); + } } fn update_execution_engine_state(&mut self, state: EngineState) { @@ -95,39 +148,80 @@ impl TestRig { self.sync_manager.update_execution_engine_state(state); } - fn find_blocks_by_range_request(&mut self, target_peer_id: &PeerId) -> (Id, Option) { + fn find_blocks_by_range_request( + &mut self, + request_filter: RequestFilter, + ) -> ((Id, PeerId), ByRangeDataRequestIds) { + let filter_f = |peer: PeerId, start_slot: u64| { + if let Some(expected_epoch) = request_filter.epoch { + let epoch = Slot::new(start_slot).epoch(E::slots_per_epoch()).as_u64(); + if epoch != expected_epoch { + return false; + } + } + if let Some(expected_peer) = request_filter.peer { + if peer != expected_peer { + return false; + } + } + true + }; + let block_req_id = self .pop_received_network_event(|ev| match ev { NetworkMessage::SendRequest { peer_id, - request: RequestType::BlocksByRange(_), + request: + RequestType::BlocksByRange(OldBlocksByRangeRequest::V2( + OldBlocksByRangeRequestV2 { start_slot, .. }, + )), request_id: AppRequestId::Sync(SyncRequestId::RangeBlockAndBlobs { id }), - } if peer_id == target_peer_id => Some(*id), + } if filter_f(*peer_id, *start_slot) => Some((*id, *peer_id)), _ => None, }) .expect("Should have a blocks by range request"); - let blob_req_id = if self.after_deneb() { - Some( - self.pop_received_network_event(|ev| match ev { + let by_range_data_requests = if self.after_fulu() { + let mut data_columns_requests = vec![]; + while let Ok(data_columns_request) = self.pop_received_network_event(|ev| match ev { + NetworkMessage::SendRequest { + peer_id, + request: + RequestType::DataColumnsByRange(DataColumnsByRangeRequest { + start_slot, .. + }), + request_id: AppRequestId::Sync(SyncRequestId::RangeBlockAndBlobs { id }), + } if filter_f(*peer_id, *start_slot) => Some((*id, *peer_id)), + _ => None, + }) { + data_columns_requests.push(data_columns_request); + } + if data_columns_requests.is_empty() { + panic!("Found zero DataColumnsByRange requests"); + } + ByRangeDataRequestIds::PostPeerDAS(data_columns_requests) + } else if self.after_deneb() { + let (id, peer) = self + .pop_received_network_event(|ev| match ev { NetworkMessage::SendRequest { peer_id, - request: RequestType::BlobsByRange(_), + request: RequestType::BlobsByRange(BlobsByRangeRequest { start_slot, .. }), request_id: AppRequestId::Sync(SyncRequestId::RangeBlockAndBlobs { id }), - } if peer_id == target_peer_id => Some(*id), + } if filter_f(*peer_id, *start_slot) => Some((*id, *peer_id)), _ => None, }) - .expect("Should have a blobs by range request"), - ) + .expect("Should have a blobs by range request"); + ByRangeDataRequestIds::PrePeerDAS(id, peer) } else { - None + ByRangeDataRequestIds::PreDeneb }; - (block_req_id, blob_req_id) + (block_req_id, by_range_data_requests) } - fn find_and_complete_blocks_by_range_request(&mut self, target_peer_id: PeerId) { - let (blocks_req_id, blobs_req_id) = self.find_blocks_by_range_request(&target_peer_id); + fn find_and_complete_blocks_by_range_request(&mut self, request_filter: RequestFilter) { + let ((blocks_req_id, block_peer), by_range_data_request_ids) = + self.find_blocks_by_range_request(request_filter); // Complete the request with a single stream termination self.log(&format!( @@ -135,26 +229,43 @@ impl TestRig { )); self.send_sync_message(SyncMessage::RpcBlock { request_id: SyncRequestId::RangeBlockAndBlobs { id: blocks_req_id }, - peer_id: target_peer_id, + peer_id: block_peer, beacon_block: None, seen_timestamp: D, }); - if let Some(blobs_req_id) = blobs_req_id { - // Complete the request with a single stream termination - self.log(&format!( - "Completing BlobsByRange request {blobs_req_id} with empty stream" - )); - self.send_sync_message(SyncMessage::RpcBlob { - request_id: SyncRequestId::RangeBlockAndBlobs { id: blobs_req_id }, - peer_id: target_peer_id, - blob_sidecar: None, - seen_timestamp: D, - }); + match by_range_data_request_ids { + ByRangeDataRequestIds::PreDeneb => {} + ByRangeDataRequestIds::PrePeerDAS(id, peer_id) => { + // Complete the request with a single stream termination + self.log(&format!( + "Completing BlobsByRange request {id} with empty stream" + )); + self.send_sync_message(SyncMessage::RpcBlob { + request_id: SyncRequestId::RangeBlockAndBlobs { id }, + peer_id, + blob_sidecar: None, + seen_timestamp: D, + }); + } + ByRangeDataRequestIds::PostPeerDAS(data_column_req_ids) => { + // Complete the request with a single stream termination + for (id, peer_id) in data_column_req_ids { + self.log(&format!( + "Completing DataColumnsByRange request {id} with empty stream" + )); + self.send_sync_message(SyncMessage::RpcDataColumn { + request_id: SyncRequestId::RangeBlockAndBlobs { id }, + peer_id, + data_column: None, + seen_timestamp: D, + }); + } + } } } - async fn create_canonical_block(&mut self) -> SignedBeaconBlock { + async fn create_canonical_block(&mut self) -> (SignedBeaconBlock, Option>) { self.harness.advance_slot(); let block_root = self @@ -165,19 +276,73 @@ impl TestRig { AttestationStrategy::AllValidators, ) .await; - self.harness - .chain - .store - .get_full_block(&block_root) - .unwrap() - .unwrap() + + let store = &self.harness.chain.store; + let block = store.get_full_block(&block_root).unwrap().unwrap(); + let fork = block.fork_name_unchecked(); + + let data_sidecars = if fork.fulu_enabled() { + store + .get_data_columns(&block_root) + .unwrap() + .map(|columns| { + columns + .into_iter() + .map(CustodyDataColumn::from_asserted_custody) + .collect() + }) + .map(DataSidecars::DataColumns) + } else if fork.deneb_enabled() { + store + .get_blobs(&block_root) + .unwrap() + .blobs() + .map(DataSidecars::Blobs) + } else { + None + }; + + (block, data_sidecars) } - async fn remember_block(&mut self, block: SignedBeaconBlock) { - self.harness - .process_block(block.slot(), block.canonical_root(), (block.into(), None)) + async fn remember_block( + &mut self, + (block, data_sidecars): (SignedBeaconBlock, Option>), + ) { + // This code is kind of duplicated from Harness::process_block, but takes sidecars directly. + let block_root = block.canonical_root(); + self.harness.set_current_slot(block.slot()); + let _: SignedBeaconBlockHash = self + .harness + .chain + .process_block( + block_root, + build_rpc_block(block.into(), &data_sidecars, &self.spec), + NotifyExecutionLayer::Yes, + BlockImportSource::RangeSync, + || Ok(()), + ) .await + .unwrap() + .try_into() .unwrap(); + self.harness.chain.recompute_head_at_current_slot().await; + } +} + +fn build_rpc_block( + block: Arc>, + data_sidecars: &Option>, + spec: &ChainSpec, +) -> RpcBlock { + match data_sidecars { + Some(DataSidecars::Blobs(blobs)) => { + RpcBlock::new(None, block, Some(blobs.clone())).unwrap() + } + Some(DataSidecars::DataColumns(columns)) => { + RpcBlock::new_with_custody_columns(None, block, columns.clone(), spec).unwrap() + } + None => RpcBlock::new_without_blobs(None, block), } } @@ -192,14 +357,14 @@ fn head_chain_removed_while_finalized_syncing() { rig.assert_state(RangeSyncType::Head); // Sync should have requested a batch, grab the request. - let _ = rig.find_blocks_by_range_request(&head_peer); + let _ = rig.find_blocks_by_range_request(filter().peer(head_peer)); // Now get a peer with an advanced finalized epoch. let finalized_peer = rig.add_finalized_peer(); rig.assert_state(RangeSyncType::Finalized); // Sync should have requested a batch, grab the request - let _ = rig.find_blocks_by_range_request(&finalized_peer); + let _ = rig.find_blocks_by_range_request(filter().peer(finalized_peer)); // Fail the head chain by disconnecting the peer. rig.peer_disconnected(head_peer); @@ -217,23 +382,23 @@ async fn state_update_while_purging() { // Need to create blocks that can be inserted into the fork-choice and fit the "known // conditions" below. let head_peer_block = rig_2.create_canonical_block().await; - let head_peer_root = head_peer_block.canonical_root(); + let head_peer_root = head_peer_block.0.canonical_root(); let finalized_peer_block = rig_2.create_canonical_block().await; - let finalized_peer_root = finalized_peer_block.canonical_root(); + let finalized_peer_root = finalized_peer_block.0.canonical_root(); // Get a peer with an advanced head let head_peer = rig.add_head_peer_with_root(head_peer_root); rig.assert_state(RangeSyncType::Head); // Sync should have requested a batch, grab the request. - let _ = rig.find_blocks_by_range_request(&head_peer); + let _ = rig.find_blocks_by_range_request(filter().peer(head_peer)); // Now get a peer with an advanced finalized epoch. let finalized_peer = rig.add_finalized_peer_with_root(finalized_peer_root); rig.assert_state(RangeSyncType::Finalized); // Sync should have requested a batch, grab the request - let _ = rig.find_blocks_by_range_request(&finalized_peer); + let _ = rig.find_blocks_by_range_request(filter().peer(finalized_peer)); // Now the chain knows both chains target roots. rig.remember_block(head_peer_block).await; @@ -252,15 +417,18 @@ fn pause_and_resume_on_ee_offline() { // make the ee offline rig.update_execution_engine_state(EngineState::Offline); // send the response to the request - rig.find_and_complete_blocks_by_range_request(peer1); + rig.find_and_complete_blocks_by_range_request(filter().peer(peer1).epoch(0)); // the beacon processor shouldn't have received any work rig.expect_empty_processor(); // while the ee is offline, more peers might arrive. Add a new finalized peer. - let peer2 = rig.add_finalized_peer(); + let _peer2 = rig.add_finalized_peer(); // send the response to the request - rig.find_and_complete_blocks_by_range_request(peer2); + // Don't filter requests and the columns requests may be sent to peer1 or peer2 + // We need to filter by epoch, because the previous batch eagerly sent requests for the next + // epoch for the other batch. So we can either filter by epoch of by sync type. + rig.find_and_complete_blocks_by_range_request(filter().epoch(0)); // the beacon processor shouldn't have received any work rig.expect_empty_processor(); // make the beacon processor available again. @@ -268,6 +436,6 @@ fn pause_and_resume_on_ee_offline() { // now resume range, we should have two processing requests in the beacon processor. rig.update_execution_engine_state(EngineState::Online); - rig.expect_chain_segment(); - rig.expect_chain_segment(); + // The head chain and finalized chain (2) should be in the processing queue + rig.expect_chain_segments(2); } diff --git a/beacon_node/operation_pool/Cargo.toml b/beacon_node/operation_pool/Cargo.toml index 5b48e3f0d8..570b74226c 100644 --- a/beacon_node/operation_pool/Cargo.toml +++ b/beacon_node/operation_pool/Cargo.toml @@ -5,24 +5,24 @@ authors = ["Michael Sproul "] edition = { workspace = true } [dependencies] +bitvec = { workspace = true } derivative = { workspace = true } +ethereum_ssz = { workspace = true } +ethereum_ssz_derive = { workspace = true } itertools = { workspace = true } metrics = { workspace = true } parking_lot = { workspace = true } -types = { workspace = true } -state_processing = { workspace = true } -ethereum_ssz = { workspace = true } -ethereum_ssz_derive = { workspace = true } +rand = { workspace = true } rayon = { workspace = true } serde = { workspace = true } +state_processing = { workspace = true } store = { workspace = true } -bitvec = { workspace = true } -rand = { workspace = true } +types = { workspace = true } [dev-dependencies] beacon_chain = { workspace = true } -tokio = { workspace = true } maplit = { workspace = true } +tokio = { workspace = true } [features] portable = ["beacon_chain/portable"] diff --git a/beacon_node/operation_pool/src/attestation_storage.rs b/beacon_node/operation_pool/src/attestation_storage.rs index 083c1170f0..49ef5c279c 100644 --- a/beacon_node/operation_pool/src/attestation_storage.rs +++ b/beacon_node/operation_pool/src/attestation_storage.rs @@ -214,7 +214,7 @@ impl CompactIndexedAttestationElectra { .is_zero() } - /// Returns `true` if aggregated, otherwise `false`. + /// Returns `true` if aggregated, otherwise `false`. pub fn aggregate_same_committee(&mut self, other: &Self) -> bool { if self.committee_bits != other.committee_bits { return false; diff --git a/beacon_node/operation_pool/src/lib.rs b/beacon_node/operation_pool/src/lib.rs index d01c73118c..835133a059 100644 --- a/beacon_node/operation_pool/src/lib.rs +++ b/beacon_node/operation_pool/src/lib.rs @@ -186,7 +186,7 @@ impl OperationPool { self.sync_contributions.write().retain(|_, contributions| { // All the contributions in this bucket have the same data, so we only need to // check the first one. - contributions.first().map_or(false, |contribution| { + contributions.first().is_some_and(|contribution| { current_slot <= contribution.slot.saturating_add(Slot::new(1)) }) }); @@ -401,7 +401,7 @@ impl OperationPool { && state .validators() .get(slashing.as_inner().signed_header_1.message.proposer_index as usize) - .map_or(false, |validator| !validator.slashed) + .is_some_and(|validator| !validator.slashed) }, |slashing| slashing.as_inner().clone(), E::MaxProposerSlashings::to_usize(), @@ -484,7 +484,7 @@ impl OperationPool { validator.exit_epoch > head_state.finalized_checkpoint().epoch }, ) - .map_or(false, |indices| !indices.is_empty()); + .is_ok_and(|indices| !indices.is_empty()); signature_ok && slashing_ok }); @@ -583,9 +583,7 @@ impl OperationPool { address_change.signature_is_still_valid(&state.fork()) && state .get_validator(address_change.as_inner().message.validator_index as usize) - .map_or(false, |validator| { - !validator.has_execution_withdrawal_credential(spec) - }) + .is_ok_and(|validator| !validator.has_execution_withdrawal_credential(spec)) }, |address_change| address_change.as_inner().clone(), E::MaxBlsToExecutionChanges::to_usize(), @@ -609,9 +607,7 @@ impl OperationPool { address_change.signature_is_still_valid(&state.fork()) && state .get_validator(address_change.as_inner().message.validator_index as usize) - .map_or(false, |validator| { - !validator.has_eth1_withdrawal_credential(spec) - }) + .is_ok_and(|validator| !validator.has_eth1_withdrawal_credential(spec)) }, |address_change| address_change.as_inner().clone(), usize::MAX, @@ -1243,14 +1239,11 @@ mod release_tests { let stats = op_pool.attestation_stats(); let fork_name = state.fork_name_unchecked(); - match fork_name { - ForkName::Electra => { - assert_eq!(stats.num_attestation_data, 1); - } - _ => { - assert_eq!(stats.num_attestation_data, committees.len()); - } - }; + if fork_name.electra_enabled() { + assert_eq!(stats.num_attestation_data, 1); + } else { + assert_eq!(stats.num_attestation_data, committees.len()); + } assert_eq!( stats.num_attestations, @@ -1262,25 +1255,19 @@ mod release_tests { let best_attestations = op_pool .get_attestations(&state, |_| true, |_| true, spec) .expect("should have best attestations"); - match fork_name { - ForkName::Electra => { - assert_eq!(best_attestations.len(), 8); - } - _ => { - assert_eq!(best_attestations.len(), max_attestations); - } - }; + if fork_name.electra_enabled() { + assert_eq!(best_attestations.len(), 8); + } else { + assert_eq!(best_attestations.len(), max_attestations); + } // All the best attestations should be signed by at least `big_step_size` (4) validators. for att in &best_attestations { - match fork_name { - ForkName::Electra => { - assert!(att.num_set_aggregation_bits() >= small_step_size); - } - _ => { - assert!(att.num_set_aggregation_bits() >= big_step_size); - } - }; + if fork_name.electra_enabled() { + assert!(att.num_set_aggregation_bits() >= small_step_size); + } else { + assert!(att.num_set_aggregation_bits() >= big_step_size); + } } } @@ -1361,17 +1348,14 @@ mod release_tests { let num_big = target_committee_size / big_step_size; let fork_name = state.fork_name_unchecked(); - match fork_name { - ForkName::Electra => { - assert_eq!(op_pool.attestation_stats().num_attestation_data, 1); - } - _ => { - assert_eq!( - op_pool.attestation_stats().num_attestation_data, - committees.len() - ); - } - }; + if fork_name.electra_enabled() { + assert_eq!(op_pool.attestation_stats().num_attestation_data, 1); + } else { + assert_eq!( + op_pool.attestation_stats().num_attestation_data, + committees.len() + ); + } assert_eq!( op_pool.num_attestations(), @@ -1384,14 +1368,11 @@ mod release_tests { .get_attestations(&state, |_| true, |_| true, spec) .expect("should have valid best attestations"); - match fork_name { - ForkName::Electra => { - assert_eq!(best_attestations.len(), 8); - } - _ => { - assert_eq!(best_attestations.len(), max_attestations); - } - }; + if fork_name.electra_enabled() { + assert_eq!(best_attestations.len(), 8); + } else { + assert_eq!(best_attestations.len(), max_attestations); + } let total_active_balance = state.get_total_active_balance().unwrap(); diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index cecfcee868..1339c15825 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -1591,5 +1591,14 @@ pub fn cli_app() -> Command { .action(ArgAction::Set) .display_order(0) ) + .arg( + Arg::new("beacon-node-backend") + .long("beacon-node-backend") + .value_name("DATABASE") + .value_parser(store::config::DatabaseBackend::VARIANTS.to_vec()) + .help("Set the database backend to be used by the beacon node.") + .action(ArgAction::Set) + .display_order(0) + ) .group(ArgGroup::new("enable_http").args(["http", "gui", "staking"]).multiple(true)) } diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 8d8a44a6fd..6d3c18d363 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -432,6 +432,10 @@ pub fn get_config( warn!(log, "The slots-per-restore-point flag is deprecated"); } + if let Some(backend) = clap_utils::parse_optional(cli_args, "beacon-node-backend")? { + client_config.store.backend = backend; + } + if let Some(hierarchy_config) = clap_utils::parse_optional(cli_args, "hierarchy-exponents")? { client_config.store.hierarchy_config = hierarchy_config; } diff --git a/beacon_node/src/lib.rs b/beacon_node/src/lib.rs index cca617d8c6..e3802c837c 100644 --- a/beacon_node/src/lib.rs +++ b/beacon_node/src/lib.rs @@ -2,7 +2,6 @@ mod cli; mod config; pub use beacon_chain; -use beacon_chain::store::LevelDB; use beacon_chain::{ builder::Witness, eth1_chain::CachingEth1Backend, slot_clock::SystemTimeSlotClock, }; @@ -16,11 +15,19 @@ use slasher::{DatabaseBackendOverride, Slasher}; use slog::{info, warn}; use std::ops::{Deref, DerefMut}; use std::sync::Arc; +use store::database::interface::BeaconNodeBackend; use types::{ChainSpec, Epoch, EthSpec, ForkName}; /// A type-alias to the tighten the definition of a production-intended `Client`. -pub type ProductionClient = - Client, E, LevelDB, LevelDB>>; +pub type ProductionClient = Client< + Witness< + SystemTimeSlotClock, + CachingEth1Backend, + E, + BeaconNodeBackend, + BeaconNodeBackend, + >, +>; /// The beacon node `Client` that will be used in production. /// @@ -238,6 +245,7 @@ mod test { spec.bellatrix_fork_epoch = Some(Epoch::new(256)); spec.deneb_fork_epoch = Some(Epoch::new(257)); spec.electra_fork_epoch = None; + spec.fulu_fork_epoch = None; let result = validator_fork_epochs(&spec); assert_eq!( result, diff --git a/beacon_node/store/Cargo.toml b/beacon_node/store/Cargo.toml index 21d0cf8dec..d2f3a5c562 100644 --- a/beacon_node/store/Cargo.toml +++ b/beacon_node/store/Cargo.toml @@ -4,6 +4,11 @@ version = "0.2.0" authors = ["Paul Hauner "] edition = { workspace = true } +[features] +default = ["leveldb"] +leveldb = ["dep:leveldb"] +redb = ["dep:redb"] + [dev-dependencies] beacon_chain = { workspace = true } criterion = { workspace = true } @@ -17,11 +22,12 @@ directory = { workspace = true } ethereum_ssz = { workspace = true } ethereum_ssz_derive = { workspace = true } itertools = { workspace = true } -leveldb = { version = "0.8" } +leveldb = { version = "0.8.6", optional = true } logging = { workspace = true } lru = { workspace = true } metrics = { workspace = true } parking_lot = { workspace = true } +redb = { version = "2.1.3", optional = true } safe_arith = { workspace = true } serde = { workspace = true } slog = { workspace = true } diff --git a/beacon_node/store/src/blob_sidecar_list_from_root.rs b/beacon_node/store/src/blob_sidecar_list_from_root.rs new file mode 100644 index 0000000000..de63eaa76c --- /dev/null +++ b/beacon_node/store/src/blob_sidecar_list_from_root.rs @@ -0,0 +1,42 @@ +use std::sync::Arc; +use types::{BlobSidecar, BlobSidecarList, EthSpec}; + +#[derive(Debug, Clone)] +pub enum BlobSidecarListFromRoot { + /// Valid root that exists in the DB, but has no blobs associated with it. + NoBlobs, + /// Contains > 1 blob for the requested root. + Blobs(BlobSidecarList), + /// No root exists in the db or cache for the requested root. + NoRoot, +} + +impl From> for BlobSidecarListFromRoot { + fn from(value: BlobSidecarList) -> Self { + Self::Blobs(value) + } +} + +impl BlobSidecarListFromRoot { + pub fn blobs(self) -> Option> { + match self { + Self::NoBlobs | Self::NoRoot => None, + Self::Blobs(blobs) => Some(blobs), + } + } + + #[allow(clippy::len_without_is_empty)] + pub fn len(&self) -> usize { + match self { + Self::NoBlobs | Self::NoRoot => 0, + Self::Blobs(blobs) => blobs.len(), + } + } + + pub fn iter(&self) -> impl Iterator>> { + match self { + Self::NoBlobs | Self::NoRoot => [].iter(), + Self::Blobs(list) => list.iter(), + } + } +} diff --git a/beacon_node/store/src/chunked_vector.rs b/beacon_node/store/src/chunked_vector.rs index 83b8da2a18..90e8c17310 100644 --- a/beacon_node/store/src/chunked_vector.rs +++ b/beacon_node/store/src/chunked_vector.rs @@ -680,7 +680,7 @@ where key: &[u8], ) -> Result, Error> { store - .get_bytes(column.into(), key)? + .get_bytes(column, key)? .map(|bytes| Self::decode(&bytes)) .transpose() } @@ -691,8 +691,11 @@ where key: &[u8], ops: &mut Vec, ) -> Result<(), Error> { - let db_key = get_key_for_col(column.into(), key); - ops.push(KeyValueStoreOp::PutKeyValue(db_key, self.encode()?)); + ops.push(KeyValueStoreOp::PutKeyValue( + column, + key.to_vec(), + self.encode()?, + )); Ok(()) } diff --git a/beacon_node/store/src/config.rs b/beacon_node/store/src/config.rs index 4f67530570..64765fd66a 100644 --- a/beacon_node/store/src/config.rs +++ b/beacon_node/store/src/config.rs @@ -1,16 +1,23 @@ use crate::hdiff::HierarchyConfig; +use crate::superstruct; use crate::{AnchorInfo, DBColumn, Error, Split, StoreItem}; use serde::{Deserialize, Serialize}; use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use std::io::Write; use std::num::NonZeroUsize; -use superstruct::superstruct; +use strum::{Display, EnumString, EnumVariantNames}; use types::non_zero_usize::new_non_zero_usize; use types::EthSpec; use zstd::Encoder; -// Only used in tests. Mainnet sets a higher default on the CLI. +#[cfg(all(feature = "redb", not(feature = "leveldb")))] +pub const DEFAULT_BACKEND: DatabaseBackend = DatabaseBackend::Redb; +#[cfg(feature = "leveldb")] +pub const DEFAULT_BACKEND: DatabaseBackend = DatabaseBackend::LevelDb; + +pub const PREV_DEFAULT_SLOTS_PER_RESTORE_POINT: u64 = 2048; +pub const DEFAULT_SLOTS_PER_RESTORE_POINT: u64 = 8192; pub const DEFAULT_EPOCHS_PER_STATE_DIFF: u64 = 8; pub const DEFAULT_BLOCK_CACHE_SIZE: NonZeroUsize = new_non_zero_usize(64); pub const DEFAULT_STATE_CACHE_SIZE: NonZeroUsize = new_non_zero_usize(128); @@ -40,6 +47,8 @@ pub struct StoreConfig { pub compact_on_prune: bool, /// Whether to prune payloads on initialization and finalization. pub prune_payloads: bool, + /// Database backend to use. + pub backend: DatabaseBackend, /// State diff hierarchy. pub hierarchy_config: HierarchyConfig, /// Whether to prune blobs older than the blob data availability boundary. @@ -104,6 +113,7 @@ impl Default for StoreConfig { compact_on_init: false, compact_on_prune: true, prune_payloads: true, + backend: DEFAULT_BACKEND, hierarchy_config: HierarchyConfig::default(), prune_blobs: true, epochs_per_blob_prune: DEFAULT_EPOCHS_PER_BLOB_PRUNE, @@ -340,3 +350,14 @@ mod test { assert_eq!(config_out, config); } } + +#[derive( + Debug, Clone, Copy, Eq, PartialEq, Serialize, Deserialize, Display, EnumString, EnumVariantNames, +)] +#[strum(serialize_all = "lowercase")] +pub enum DatabaseBackend { + #[cfg(feature = "leveldb")] + LevelDb, + #[cfg(feature = "redb")] + Redb, +} diff --git a/beacon_node/store/src/database.rs b/beacon_node/store/src/database.rs new file mode 100644 index 0000000000..2232f73c5c --- /dev/null +++ b/beacon_node/store/src/database.rs @@ -0,0 +1,5 @@ +pub mod interface; +#[cfg(feature = "leveldb")] +pub mod leveldb_impl; +#[cfg(feature = "redb")] +pub mod redb_impl; diff --git a/beacon_node/store/src/database/interface.rs b/beacon_node/store/src/database/interface.rs new file mode 100644 index 0000000000..b213433241 --- /dev/null +++ b/beacon_node/store/src/database/interface.rs @@ -0,0 +1,220 @@ +#[cfg(feature = "leveldb")] +use crate::database::leveldb_impl; +#[cfg(feature = "redb")] +use crate::database::redb_impl; +use crate::{config::DatabaseBackend, KeyValueStoreOp, StoreConfig}; +use crate::{metrics, ColumnIter, ColumnKeyIter, DBColumn, Error, ItemStore, Key, KeyValueStore}; +use std::collections::HashSet; +use std::path::Path; +use types::EthSpec; + +pub enum BeaconNodeBackend { + #[cfg(feature = "leveldb")] + LevelDb(leveldb_impl::LevelDB), + #[cfg(feature = "redb")] + Redb(redb_impl::Redb), +} + +impl ItemStore for BeaconNodeBackend {} + +impl KeyValueStore for BeaconNodeBackend { + fn get_bytes(&self, column: DBColumn, key: &[u8]) -> Result>, Error> { + match self { + #[cfg(feature = "leveldb")] + BeaconNodeBackend::LevelDb(txn) => leveldb_impl::LevelDB::get_bytes(txn, column, key), + #[cfg(feature = "redb")] + BeaconNodeBackend::Redb(txn) => redb_impl::Redb::get_bytes(txn, column, key), + } + } + + fn put_bytes(&self, column: DBColumn, key: &[u8], value: &[u8]) -> Result<(), Error> { + match self { + #[cfg(feature = "leveldb")] + BeaconNodeBackend::LevelDb(txn) => leveldb_impl::LevelDB::put_bytes_with_options( + txn, + column, + key, + value, + txn.write_options(), + ), + #[cfg(feature = "redb")] + BeaconNodeBackend::Redb(txn) => redb_impl::Redb::put_bytes_with_options( + txn, + column, + key, + value, + txn.write_options(), + ), + } + } + + fn put_bytes_sync(&self, column: DBColumn, key: &[u8], value: &[u8]) -> Result<(), Error> { + match self { + #[cfg(feature = "leveldb")] + BeaconNodeBackend::LevelDb(txn) => leveldb_impl::LevelDB::put_bytes_with_options( + txn, + column, + key, + value, + txn.write_options_sync(), + ), + #[cfg(feature = "redb")] + BeaconNodeBackend::Redb(txn) => redb_impl::Redb::put_bytes_with_options( + txn, + column, + key, + value, + txn.write_options_sync(), + ), + } + } + + fn sync(&self) -> Result<(), Error> { + match self { + #[cfg(feature = "leveldb")] + BeaconNodeBackend::LevelDb(txn) => leveldb_impl::LevelDB::sync(txn), + #[cfg(feature = "redb")] + BeaconNodeBackend::Redb(txn) => redb_impl::Redb::sync(txn), + } + } + + fn key_exists(&self, column: DBColumn, key: &[u8]) -> Result { + match self { + #[cfg(feature = "leveldb")] + BeaconNodeBackend::LevelDb(txn) => leveldb_impl::LevelDB::key_exists(txn, column, key), + #[cfg(feature = "redb")] + BeaconNodeBackend::Redb(txn) => redb_impl::Redb::key_exists(txn, column, key), + } + } + + fn key_delete(&self, column: DBColumn, key: &[u8]) -> Result<(), Error> { + match self { + #[cfg(feature = "leveldb")] + BeaconNodeBackend::LevelDb(txn) => leveldb_impl::LevelDB::key_delete(txn, column, key), + #[cfg(feature = "redb")] + BeaconNodeBackend::Redb(txn) => redb_impl::Redb::key_delete(txn, column, key), + } + } + + fn do_atomically(&self, batch: Vec) -> Result<(), Error> { + match self { + #[cfg(feature = "leveldb")] + BeaconNodeBackend::LevelDb(txn) => leveldb_impl::LevelDB::do_atomically(txn, batch), + #[cfg(feature = "redb")] + BeaconNodeBackend::Redb(txn) => redb_impl::Redb::do_atomically(txn, batch), + } + } + + fn begin_rw_transaction(&self) -> parking_lot::MutexGuard<()> { + match self { + #[cfg(feature = "leveldb")] + BeaconNodeBackend::LevelDb(txn) => leveldb_impl::LevelDB::begin_rw_transaction(txn), + #[cfg(feature = "redb")] + BeaconNodeBackend::Redb(txn) => redb_impl::Redb::begin_rw_transaction(txn), + } + } + + fn compact(&self) -> Result<(), Error> { + match self { + #[cfg(feature = "leveldb")] + BeaconNodeBackend::LevelDb(txn) => leveldb_impl::LevelDB::compact(txn), + #[cfg(feature = "redb")] + BeaconNodeBackend::Redb(txn) => redb_impl::Redb::compact(txn), + } + } + + fn iter_column_keys_from(&self, _column: DBColumn, from: &[u8]) -> ColumnKeyIter { + match self { + #[cfg(feature = "leveldb")] + BeaconNodeBackend::LevelDb(txn) => { + leveldb_impl::LevelDB::iter_column_keys_from(txn, _column, from) + } + #[cfg(feature = "redb")] + BeaconNodeBackend::Redb(txn) => { + redb_impl::Redb::iter_column_keys_from(txn, _column, from) + } + } + } + + fn iter_column_keys(&self, column: DBColumn) -> ColumnKeyIter { + match self { + #[cfg(feature = "leveldb")] + BeaconNodeBackend::LevelDb(txn) => leveldb_impl::LevelDB::iter_column_keys(txn, column), + #[cfg(feature = "redb")] + BeaconNodeBackend::Redb(txn) => redb_impl::Redb::iter_column_keys(txn, column), + } + } + + fn iter_column_from(&self, column: DBColumn, from: &[u8]) -> ColumnIter { + match self { + #[cfg(feature = "leveldb")] + BeaconNodeBackend::LevelDb(txn) => { + leveldb_impl::LevelDB::iter_column_from(txn, column, from) + } + #[cfg(feature = "redb")] + BeaconNodeBackend::Redb(txn) => redb_impl::Redb::iter_column_from(txn, column, from), + } + } + + fn compact_column(&self, _column: DBColumn) -> Result<(), Error> { + match self { + #[cfg(feature = "leveldb")] + BeaconNodeBackend::LevelDb(txn) => leveldb_impl::LevelDB::compact_column(txn, _column), + #[cfg(feature = "redb")] + BeaconNodeBackend::Redb(txn) => redb_impl::Redb::compact(txn), + } + } + + fn delete_batch(&self, col: DBColumn, ops: HashSet<&[u8]>) -> Result<(), Error> { + match self { + #[cfg(feature = "leveldb")] + BeaconNodeBackend::LevelDb(txn) => leveldb_impl::LevelDB::delete_batch(txn, col, ops), + #[cfg(feature = "redb")] + BeaconNodeBackend::Redb(txn) => redb_impl::Redb::delete_batch(txn, col, ops), + } + } + + fn delete_if( + &self, + column: DBColumn, + f: impl FnMut(&[u8]) -> Result, + ) -> Result<(), Error> { + match self { + #[cfg(feature = "leveldb")] + BeaconNodeBackend::LevelDb(txn) => leveldb_impl::LevelDB::delete_if(txn, column, f), + #[cfg(feature = "redb")] + BeaconNodeBackend::Redb(txn) => redb_impl::Redb::delete_if(txn, column, f), + } + } +} + +impl BeaconNodeBackend { + pub fn open(config: &StoreConfig, path: &Path) -> Result { + metrics::inc_counter_vec(&metrics::DISK_DB_TYPE, &[&config.backend.to_string()]); + match config.backend { + #[cfg(feature = "leveldb")] + DatabaseBackend::LevelDb => { + leveldb_impl::LevelDB::open(path).map(BeaconNodeBackend::LevelDb) + } + #[cfg(feature = "redb")] + DatabaseBackend::Redb => redb_impl::Redb::open(path).map(BeaconNodeBackend::Redb), + } + } +} + +pub struct WriteOptions { + /// fsync before acknowledging a write operation. + pub sync: bool, +} + +impl WriteOptions { + pub fn new() -> Self { + WriteOptions { sync: false } + } +} + +impl Default for WriteOptions { + fn default() -> Self { + Self::new() + } +} diff --git a/beacon_node/store/src/database/leveldb_impl.rs b/beacon_node/store/src/database/leveldb_impl.rs new file mode 100644 index 0000000000..3d8bbe1473 --- /dev/null +++ b/beacon_node/store/src/database/leveldb_impl.rs @@ -0,0 +1,304 @@ +use crate::hot_cold_store::{BytesKey, HotColdDBError}; +use crate::Key; +use crate::{ + get_key_for_col, metrics, ColumnIter, ColumnKeyIter, DBColumn, Error, KeyValueStoreOp, +}; +use leveldb::{ + compaction::Compaction, + database::{ + batch::{Batch, Writebatch}, + kv::KV, + Database, + }, + iterator::{Iterable, LevelDBIterator}, + options::{Options, ReadOptions}, +}; +use parking_lot::{Mutex, MutexGuard}; +use std::collections::HashSet; +use std::marker::PhantomData; +use std::path::Path; +use types::{EthSpec, FixedBytesExtended, Hash256}; + +use super::interface::WriteOptions; + +pub struct LevelDB { + db: Database, + /// A mutex to synchronise sensitive read-write transactions. + transaction_mutex: Mutex<()>, + _phantom: PhantomData, +} + +impl From for leveldb::options::WriteOptions { + fn from(options: WriteOptions) -> Self { + let mut opts = leveldb::options::WriteOptions::new(); + opts.sync = options.sync; + opts + } +} + +impl LevelDB { + pub fn open(path: &Path) -> Result { + let mut options = Options::new(); + + options.create_if_missing = true; + + let db = Database::open(path, options)?; + let transaction_mutex = Mutex::new(()); + + Ok(Self { + db, + transaction_mutex, + _phantom: PhantomData, + }) + } + + pub fn read_options(&self) -> ReadOptions { + ReadOptions::new() + } + + pub fn write_options(&self) -> WriteOptions { + WriteOptions::new() + } + + pub fn write_options_sync(&self) -> WriteOptions { + let mut opts = WriteOptions::new(); + opts.sync = true; + opts + } + + pub fn put_bytes_with_options( + &self, + col: DBColumn, + key: &[u8], + val: &[u8], + opts: WriteOptions, + ) -> Result<(), Error> { + let column_key = get_key_for_col(col, key); + + metrics::inc_counter_vec(&metrics::DISK_DB_WRITE_COUNT, &[col.into()]); + metrics::inc_counter_vec_by( + &metrics::DISK_DB_WRITE_BYTES, + &[col.into()], + val.len() as u64, + ); + let timer = metrics::start_timer(&metrics::DISK_DB_WRITE_TIMES); + + self.db + .put(opts.into(), BytesKey::from_vec(column_key), val) + .map_err(Into::into) + .map(|()| { + metrics::stop_timer(timer); + }) + } + + /// Store some `value` in `column`, indexed with `key`. + pub fn put_bytes(&self, col: DBColumn, key: &[u8], val: &[u8]) -> Result<(), Error> { + self.put_bytes_with_options(col, key, val, self.write_options()) + } + + pub fn put_bytes_sync(&self, col: DBColumn, key: &[u8], val: &[u8]) -> Result<(), Error> { + self.put_bytes_with_options(col, key, val, self.write_options_sync()) + } + + pub fn sync(&self) -> Result<(), Error> { + self.put_bytes_sync(DBColumn::Dummy, b"sync", b"sync") + } + + // Retrieve some bytes in `column` with `key`. + pub fn get_bytes(&self, col: DBColumn, key: &[u8]) -> Result>, Error> { + let column_key = get_key_for_col(col, key); + + metrics::inc_counter_vec(&metrics::DISK_DB_READ_COUNT, &[col.into()]); + let timer = metrics::start_timer(&metrics::DISK_DB_READ_TIMES); + + self.db + .get(self.read_options(), BytesKey::from_vec(column_key)) + .map_err(Into::into) + .map(|opt| { + opt.inspect(|bytes| { + metrics::inc_counter_vec_by( + &metrics::DISK_DB_READ_BYTES, + &[col.into()], + bytes.len() as u64, + ); + metrics::stop_timer(timer); + }) + }) + } + + /// Return `true` if `key` exists in `column`. + pub fn key_exists(&self, col: DBColumn, key: &[u8]) -> Result { + let column_key = get_key_for_col(col, key); + + metrics::inc_counter_vec(&metrics::DISK_DB_EXISTS_COUNT, &[col.into()]); + + self.db + .get(self.read_options(), BytesKey::from_vec(column_key)) + .map_err(Into::into) + .map(|val| val.is_some()) + } + + /// Removes `key` from `column`. + pub fn key_delete(&self, col: DBColumn, key: &[u8]) -> Result<(), Error> { + let column_key = get_key_for_col(col, key); + + metrics::inc_counter_vec(&metrics::DISK_DB_DELETE_COUNT, &[col.into()]); + + self.db + .delete(self.write_options().into(), BytesKey::from_vec(column_key)) + .map_err(Into::into) + } + + pub fn do_atomically(&self, ops_batch: Vec) -> Result<(), Error> { + let mut leveldb_batch = Writebatch::new(); + for op in ops_batch { + match op { + KeyValueStoreOp::PutKeyValue(col, key, value) => { + let _timer = metrics::start_timer(&metrics::DISK_DB_WRITE_TIMES); + metrics::inc_counter_vec_by( + &metrics::DISK_DB_WRITE_BYTES, + &[col.into()], + value.len() as u64, + ); + metrics::inc_counter_vec(&metrics::DISK_DB_WRITE_COUNT, &[col.into()]); + let column_key = get_key_for_col(col, &key); + leveldb_batch.put(BytesKey::from_vec(column_key), &value); + } + + KeyValueStoreOp::DeleteKey(col, key) => { + let _timer = metrics::start_timer(&metrics::DISK_DB_DELETE_TIMES); + metrics::inc_counter_vec(&metrics::DISK_DB_DELETE_COUNT, &[col.into()]); + let column_key = get_key_for_col(col, &key); + leveldb_batch.delete(BytesKey::from_vec(column_key)); + } + } + } + self.db.write(self.write_options().into(), &leveldb_batch)?; + Ok(()) + } + + pub fn begin_rw_transaction(&self) -> MutexGuard<()> { + self.transaction_mutex.lock() + } + + /// Compact all values in the states and states flag columns. + pub fn compact(&self) -> Result<(), Error> { + let _timer = metrics::start_timer(&metrics::DISK_DB_COMPACT_TIMES); + let endpoints = |column: DBColumn| { + ( + BytesKey::from_vec(get_key_for_col(column, Hash256::zero().as_slice())), + BytesKey::from_vec(get_key_for_col( + column, + Hash256::repeat_byte(0xff).as_slice(), + )), + ) + }; + + for (start_key, end_key) in [ + endpoints(DBColumn::BeaconStateTemporary), + endpoints(DBColumn::BeaconState), + endpoints(DBColumn::BeaconStateSummary), + ] { + self.db.compact(&start_key, &end_key); + } + + Ok(()) + } + + pub fn compact_column(&self, column: DBColumn) -> Result<(), Error> { + // Use key-size-agnostic keys [] and 0xff..ff with a minimum of 32 bytes to account for + // columns that may change size between sub-databases or schema versions. + let start_key = BytesKey::from_vec(get_key_for_col(column, &[])); + let end_key = BytesKey::from_vec(get_key_for_col( + column, + &vec![0xff; std::cmp::max(column.key_size(), 32)], + )); + self.db.compact(&start_key, &end_key); + Ok(()) + } + + pub fn iter_column_from(&self, column: DBColumn, from: &[u8]) -> ColumnIter { + let start_key = BytesKey::from_vec(get_key_for_col(column, from)); + let iter = self.db.iter(self.read_options()); + iter.seek(&start_key); + + Box::new( + iter.take_while(move |(key, _)| key.matches_column(column)) + .map(move |(bytes_key, value)| { + metrics::inc_counter_vec(&metrics::DISK_DB_READ_COUNT, &[column.into()]); + metrics::inc_counter_vec_by( + &metrics::DISK_DB_READ_BYTES, + &[column.into()], + value.len() as u64, + ); + let key = bytes_key.remove_column_variable(column).ok_or_else(|| { + HotColdDBError::IterationError { + unexpected_key: bytes_key.clone(), + } + })?; + Ok((K::from_bytes(key)?, value)) + }), + ) + } + + pub fn iter_column_keys_from(&self, column: DBColumn, from: &[u8]) -> ColumnKeyIter { + let start_key = BytesKey::from_vec(get_key_for_col(column, from)); + + let iter = self.db.keys_iter(self.read_options()); + iter.seek(&start_key); + + Box::new( + iter.take_while(move |key| key.matches_column(column)) + .map(move |bytes_key| { + metrics::inc_counter_vec(&metrics::DISK_DB_KEY_READ_COUNT, &[column.into()]); + metrics::inc_counter_vec_by( + &metrics::DISK_DB_KEY_READ_BYTES, + &[column.into()], + bytes_key.key.len() as u64, + ); + let key = &bytes_key.key[column.as_bytes().len()..]; + K::from_bytes(key) + }), + ) + } + + /// Iterate through all keys and values in a particular column. + pub fn iter_column_keys(&self, column: DBColumn) -> ColumnKeyIter { + self.iter_column_keys_from(column, &vec![0; column.key_size()]) + } + + pub fn iter_column(&self, column: DBColumn) -> ColumnIter { + self.iter_column_from(column, &vec![0; column.key_size()]) + } + + pub fn delete_batch(&self, col: DBColumn, ops: HashSet<&[u8]>) -> Result<(), Error> { + let mut leveldb_batch = Writebatch::new(); + for op in ops { + let column_key = get_key_for_col(col, op); + leveldb_batch.delete(BytesKey::from_vec(column_key)); + } + self.db.write(self.write_options().into(), &leveldb_batch)?; + Ok(()) + } + + pub fn delete_if( + &self, + column: DBColumn, + mut f: impl FnMut(&[u8]) -> Result, + ) -> Result<(), Error> { + let mut leveldb_batch = Writebatch::new(); + let iter = self.db.iter(self.read_options()); + + iter.take_while(move |(key, _)| key.matches_column(column)) + .for_each(|(key, value)| { + if f(&value).unwrap_or(false) { + let _timer = metrics::start_timer(&metrics::DISK_DB_DELETE_TIMES); + metrics::inc_counter_vec(&metrics::DISK_DB_DELETE_COUNT, &[column.into()]); + leveldb_batch.delete(key); + } + }); + + self.db.write(self.write_options().into(), &leveldb_batch)?; + Ok(()) + } +} diff --git a/beacon_node/store/src/database/redb_impl.rs b/beacon_node/store/src/database/redb_impl.rs new file mode 100644 index 0000000000..cbe575d184 --- /dev/null +++ b/beacon_node/store/src/database/redb_impl.rs @@ -0,0 +1,319 @@ +use crate::{metrics, ColumnIter, ColumnKeyIter, Key}; +use crate::{DBColumn, Error, KeyValueStoreOp}; +use parking_lot::{Mutex, MutexGuard, RwLock}; +use redb::TableDefinition; +use std::collections::HashSet; +use std::{borrow::BorrowMut, marker::PhantomData, path::Path}; +use strum::IntoEnumIterator; +use types::EthSpec; + +use super::interface::WriteOptions; + +pub const DB_FILE_NAME: &str = "database.redb"; + +pub struct Redb { + db: RwLock, + transaction_mutex: Mutex<()>, + _phantom: PhantomData, +} + +impl From for redb::Durability { + fn from(options: WriteOptions) -> Self { + if options.sync { + redb::Durability::Immediate + } else { + redb::Durability::Eventual + } + } +} + +impl Redb { + pub fn open(path: &Path) -> Result { + let db_file = path.join(DB_FILE_NAME); + let db = redb::Database::create(db_file)?; + let transaction_mutex = Mutex::new(()); + + for column in DBColumn::iter() { + Redb::::create_table(&db, column.into())?; + } + + Ok(Self { + db: db.into(), + transaction_mutex, + _phantom: PhantomData, + }) + } + + fn create_table(db: &redb::Database, table_name: &str) -> Result<(), Error> { + let table_definition: TableDefinition<'_, &[u8], &[u8]> = TableDefinition::new(table_name); + let tx = db.begin_write()?; + tx.open_table(table_definition)?; + tx.commit().map_err(Into::into) + } + + pub fn write_options(&self) -> WriteOptions { + WriteOptions::new() + } + + pub fn write_options_sync(&self) -> WriteOptions { + let mut opts = WriteOptions::new(); + opts.sync = true; + opts + } + + pub fn begin_rw_transaction(&self) -> MutexGuard<()> { + self.transaction_mutex.lock() + } + + pub fn put_bytes_with_options( + &self, + col: DBColumn, + key: &[u8], + val: &[u8], + opts: WriteOptions, + ) -> Result<(), Error> { + metrics::inc_counter_vec(&metrics::DISK_DB_WRITE_COUNT, &[col.into()]); + metrics::inc_counter_vec_by( + &metrics::DISK_DB_WRITE_BYTES, + &[col.into()], + val.len() as u64, + ); + let timer = metrics::start_timer(&metrics::DISK_DB_WRITE_TIMES); + + let table_definition: TableDefinition<'_, &[u8], &[u8]> = TableDefinition::new(col.into()); + let open_db = self.db.read(); + let mut tx = open_db.begin_write()?; + tx.set_durability(opts.into()); + let mut table = tx.open_table(table_definition)?; + + table.insert(key, val).map(|_| { + metrics::stop_timer(timer); + })?; + drop(table); + tx.commit().map_err(Into::into) + } + + /// Store some `value` in `column`, indexed with `key`. + pub fn put_bytes(&self, col: DBColumn, key: &[u8], val: &[u8]) -> Result<(), Error> { + self.put_bytes_with_options(col, key, val, self.write_options()) + } + + pub fn put_bytes_sync(&self, col: DBColumn, key: &[u8], val: &[u8]) -> Result<(), Error> { + self.put_bytes_with_options(col, key, val, self.write_options_sync()) + } + + pub fn sync(&self) -> Result<(), Error> { + self.put_bytes_sync(DBColumn::Dummy, b"sync", b"sync") + } + + // Retrieve some bytes in `column` with `key`. + pub fn get_bytes(&self, col: DBColumn, key: &[u8]) -> Result>, Error> { + metrics::inc_counter_vec(&metrics::DISK_DB_READ_COUNT, &[col.into()]); + let timer = metrics::start_timer(&metrics::DISK_DB_READ_TIMES); + + let table_definition: TableDefinition<'_, &[u8], &[u8]> = TableDefinition::new(col.into()); + let open_db = self.db.read(); + let tx = open_db.begin_read()?; + let table = tx.open_table(table_definition)?; + + let result = table.get(key)?; + + match result { + Some(access_guard) => { + let value = access_guard.value().to_vec(); + metrics::inc_counter_vec_by( + &metrics::DISK_DB_READ_BYTES, + &[col.into()], + value.len() as u64, + ); + metrics::stop_timer(timer); + Ok(Some(value)) + } + None => { + metrics::stop_timer(timer); + Ok(None) + } + } + } + + /// Return `true` if `key` exists in `column`. + pub fn key_exists(&self, col: DBColumn, key: &[u8]) -> Result { + metrics::inc_counter_vec(&metrics::DISK_DB_EXISTS_COUNT, &[col.into()]); + + let table_definition: TableDefinition<'_, &[u8], &[u8]> = TableDefinition::new(col.into()); + let open_db = self.db.read(); + let tx = open_db.begin_read()?; + let table = tx.open_table(table_definition)?; + + table + .get(key) + .map_err(Into::into) + .map(|access_guard| access_guard.is_some()) + } + + /// Removes `key` from `column`. + pub fn key_delete(&self, col: DBColumn, key: &[u8]) -> Result<(), Error> { + let table_definition: TableDefinition<'_, &[u8], &[u8]> = TableDefinition::new(col.into()); + let open_db = self.db.read(); + let tx = open_db.begin_write()?; + let mut table = tx.open_table(table_definition)?; + metrics::inc_counter_vec(&metrics::DISK_DB_DELETE_COUNT, &[col.into()]); + + table.remove(key).map(|_| ())?; + drop(table); + tx.commit().map_err(Into::into) + } + + pub fn do_atomically(&self, ops_batch: Vec) -> Result<(), Error> { + let open_db = self.db.read(); + let mut tx = open_db.begin_write()?; + tx.set_durability(self.write_options().into()); + for op in ops_batch { + match op { + KeyValueStoreOp::PutKeyValue(column, key, value) => { + let _timer = metrics::start_timer(&metrics::DISK_DB_WRITE_TIMES); + metrics::inc_counter_vec_by( + &metrics::DISK_DB_WRITE_BYTES, + &[column.into()], + value.len() as u64, + ); + metrics::inc_counter_vec(&metrics::DISK_DB_WRITE_COUNT, &[column.into()]); + let table_definition: TableDefinition<'_, &[u8], &[u8]> = + TableDefinition::new(column.into()); + + let mut table = tx.open_table(table_definition)?; + table.insert(key.as_slice(), value.as_slice())?; + drop(table); + } + + KeyValueStoreOp::DeleteKey(column, key) => { + metrics::inc_counter_vec(&metrics::DISK_DB_DELETE_COUNT, &[column.into()]); + let _timer = metrics::start_timer(&metrics::DISK_DB_DELETE_TIMES); + let table_definition: TableDefinition<'_, &[u8], &[u8]> = + TableDefinition::new(column.into()); + + let mut table = tx.open_table(table_definition)?; + table.remove(key.as_slice())?; + drop(table); + } + } + } + + tx.commit()?; + Ok(()) + } + + /// Compact all values in the states and states flag columns. + pub fn compact(&self) -> Result<(), Error> { + let _timer = metrics::start_timer(&metrics::DISK_DB_COMPACT_TIMES); + let mut open_db = self.db.write(); + let mut_db = open_db.borrow_mut(); + mut_db.compact().map_err(Into::into).map(|_| ()) + } + + pub fn iter_column_keys_from(&self, column: DBColumn, from: &[u8]) -> ColumnKeyIter { + let table_definition: TableDefinition<'_, &[u8], &[u8]> = + TableDefinition::new(column.into()); + + let result = (|| { + let open_db = self.db.read(); + let read_txn = open_db.begin_read()?; + let table = read_txn.open_table(table_definition)?; + let range = table.range(from..)?; + Ok(range.map(move |res| { + let (key, _) = res?; + metrics::inc_counter_vec(&metrics::DISK_DB_KEY_READ_COUNT, &[column.into()]); + metrics::inc_counter_vec_by( + &metrics::DISK_DB_KEY_READ_BYTES, + &[column.into()], + key.value().len() as u64, + ); + K::from_bytes(key.value()) + })) + })(); + + match result { + Ok(iter) => Box::new(iter), + Err(err) => Box::new(std::iter::once(Err(err))), + } + } + + /// Iterate through all keys and values in a particular column. + pub fn iter_column_keys(&self, column: DBColumn) -> ColumnKeyIter { + self.iter_column_keys_from(column, &vec![0; column.key_size()]) + } + + pub fn iter_column_from(&self, column: DBColumn, from: &[u8]) -> ColumnIter { + let table_definition: TableDefinition<'_, &[u8], &[u8]> = + TableDefinition::new(column.into()); + + let result = (|| { + let open_db = self.db.read(); + let read_txn = open_db.begin_read()?; + let table = read_txn.open_table(table_definition)?; + let range = table.range(from..)?; + + Ok(range + .take_while(move |res| match res.as_ref() { + Ok((_, _)) => true, + Err(_) => false, + }) + .map(move |res| { + let (key, value) = res?; + metrics::inc_counter_vec(&metrics::DISK_DB_READ_COUNT, &[column.into()]); + metrics::inc_counter_vec_by( + &metrics::DISK_DB_READ_BYTES, + &[column.into()], + value.value().len() as u64, + ); + Ok((K::from_bytes(key.value())?, value.value().to_vec())) + })) + })(); + + match result { + Ok(iter) => Box::new(iter), + Err(err) => Box::new(std::iter::once(Err(err))), + } + } + + pub fn iter_column(&self, column: DBColumn) -> ColumnIter { + self.iter_column_from(column, &vec![0; column.key_size()]) + } + + pub fn delete_batch(&self, col: DBColumn, ops: HashSet<&[u8]>) -> Result<(), Error> { + let open_db = self.db.read(); + let mut tx = open_db.begin_write()?; + + tx.set_durability(redb::Durability::None); + + let table_definition: TableDefinition<'_, &[u8], &[u8]> = TableDefinition::new(col.into()); + + let mut table = tx.open_table(table_definition)?; + table.retain(|key, _| !ops.contains(key))?; + + drop(table); + tx.commit()?; + Ok(()) + } + + pub fn delete_if( + &self, + column: DBColumn, + mut f: impl FnMut(&[u8]) -> Result, + ) -> Result<(), Error> { + let open_db = self.db.read(); + let mut tx = open_db.begin_write()?; + + tx.set_durability(redb::Durability::None); + + let table_definition: TableDefinition<'_, &[u8], &[u8]> = + TableDefinition::new(column.into()); + + let mut table = tx.open_table(table_definition)?; + table.retain(|_, value| !f(value).unwrap_or(false))?; + + drop(table); + tx.commit()?; + Ok(()) + } +} diff --git a/beacon_node/store/src/errors.rs b/beacon_node/store/src/errors.rs index 6bb4edee6b..41fd17ef43 100644 --- a/beacon_node/store/src/errors.rs +++ b/beacon_node/store/src/errors.rs @@ -2,6 +2,8 @@ use crate::chunked_vector::ChunkError; use crate::config::StoreConfigError; use crate::hot_cold_store::HotColdDBError; use crate::{hdiff, DBColumn}; +#[cfg(feature = "leveldb")] +use leveldb::error::Error as LevelDBError; use ssz::DecodeError; use state_processing::BlockReplayError; use types::{milhouse, BeaconStateError, EpochCacheError, Hash256, InconsistentFork, Slot}; @@ -48,6 +50,16 @@ pub enum Error { MissingGenesisState, MissingSnapshot(Slot), BlockReplayError(BlockReplayError), + AddPayloadLogicError, + InvalidKey, + InvalidBytes, + InconsistentFork(InconsistentFork), + #[cfg(feature = "leveldb")] + LevelDbError(LevelDBError), + #[cfg(feature = "redb")] + RedbError(redb::Error), + CacheBuildError(EpochCacheError), + RandaoMixOutOfBounds, MilhouseError(milhouse::Error), Compression(std::io::Error), FinalizedStateDecreasingSlot, @@ -56,17 +68,11 @@ pub enum Error { state_root: Hash256, slot: Slot, }, - AddPayloadLogicError, - InvalidKey, - InvalidBytes, - InconsistentFork(InconsistentFork), Hdiff(hdiff::Error), - CacheBuildError(EpochCacheError), ForwardsIterInvalidColumn(DBColumn), ForwardsIterGap(DBColumn, Slot, Slot), StateShouldNotBeRequired(Slot), MissingBlock(Hash256), - RandaoMixOutOfBounds, GenesisStateUnknown, ArithError(safe_arith::ArithError), } @@ -145,6 +151,62 @@ impl From for Error { } } +#[cfg(feature = "leveldb")] +impl From for Error { + fn from(e: LevelDBError) -> Error { + Error::LevelDbError(e) + } +} + +#[cfg(feature = "redb")] +impl From for Error { + fn from(e: redb::Error) -> Self { + Error::RedbError(e) + } +} + +#[cfg(feature = "redb")] +impl From for Error { + fn from(e: redb::TableError) -> Self { + Error::RedbError(e.into()) + } +} + +#[cfg(feature = "redb")] +impl From for Error { + fn from(e: redb::TransactionError) -> Self { + Error::RedbError(e.into()) + } +} + +#[cfg(feature = "redb")] +impl From for Error { + fn from(e: redb::DatabaseError) -> Self { + Error::RedbError(e.into()) + } +} + +#[cfg(feature = "redb")] +impl From for Error { + fn from(e: redb::StorageError) -> Self { + Error::RedbError(e.into()) + } +} + +#[cfg(feature = "redb")] +impl From for Error { + fn from(e: redb::CommitError) -> Self { + Error::RedbError(e.into()) + } +} + +#[cfg(feature = "redb")] +impl From for Error { + fn from(e: redb::CompactionError) -> Self { + Error::RedbError(e.into()) + } +} + impl From for Error { fn from(e: EpochCacheError) -> Error { Error::CacheBuildError(e) diff --git a/beacon_node/store/src/forwards_iter.rs b/beacon_node/store/src/forwards_iter.rs index 27769a310a..255b7d8eac 100644 --- a/beacon_node/store/src/forwards_iter.rs +++ b/beacon_node/store/src/forwards_iter.rs @@ -4,7 +4,6 @@ use crate::{ColumnIter, DBColumn, HotColdDB, ItemStore}; use itertools::process_results; use std::marker::PhantomData; use types::{BeaconState, EthSpec, Hash256, Slot}; - pub type HybridForwardsBlockRootsIterator<'a, E, Hot, Cold> = HybridForwardsIterator<'a, E, Hot, Cold>; pub type HybridForwardsStateRootsIterator<'a, E, Hot, Cold> = @@ -159,6 +158,7 @@ impl, Cold: ItemStore> Iterator return None; } self.inner + .as_mut() .next()? .and_then(|(slot_bytes, root_bytes)| { let slot = slot_bytes @@ -265,7 +265,7 @@ impl<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> // `end_slot`. If it tries to continue further a `NoContinuationData` error will be // returned. let continuation_data = - if end_slot.map_or(false, |end_slot| end_slot < freezer_upper_bound) { + if end_slot.is_some_and(|end_slot| end_slot < freezer_upper_bound) { None } else { Some(Box::new(get_state()?)) @@ -306,7 +306,7 @@ impl<'a, E: EthSpec, Hot: ItemStore, Cold: ItemStore> None => { // If the iterator has an end slot (inclusive) which has already been // covered by the (exclusive) frozen forwards iterator, then we're done! - if end_slot.map_or(false, |end_slot| iter.end_slot == end_slot + 1) { + if end_slot.is_some_and(|end_slot| iter.end_slot == end_slot + 1) { *self = Finished; return Ok(None); } diff --git a/beacon_node/store/src/garbage_collection.rs b/beacon_node/store/src/garbage_collection.rs index 5f8ed8f5e7..06393f2d21 100644 --- a/beacon_node/store/src/garbage_collection.rs +++ b/beacon_node/store/src/garbage_collection.rs @@ -1,10 +1,11 @@ //! Garbage collection process that runs at start-up to clean up the database. +use crate::database::interface::BeaconNodeBackend; use crate::hot_cold_store::HotColdDB; -use crate::{Error, LevelDB, StoreOp}; +use crate::{DBColumn, Error}; use slog::debug; use types::EthSpec; -impl HotColdDB, LevelDB> +impl HotColdDB, BeaconNodeBackend> where E: EthSpec, { @@ -16,21 +17,22 @@ where /// Delete the temporary states that were leftover by failed block imports. pub fn delete_temp_states(&self) -> Result<(), Error> { - let delete_ops = - self.iter_temporary_state_roots() - .try_fold(vec![], |mut ops, state_root| { - let state_root = state_root?; - ops.push(StoreOp::DeleteState(state_root, None)); - Result::<_, Error>::Ok(ops) - })?; - - if !delete_ops.is_empty() { + let mut ops = vec![]; + self.iter_temporary_state_roots().for_each(|state_root| { + if let Ok(state_root) = state_root { + ops.push(state_root); + } + }); + if !ops.is_empty() { debug!( self.log, "Garbage collecting {} temporary states", - delete_ops.len() + ops.len() ); - self.do_atomically_with_block_and_blobs_cache(delete_ops)?; + + self.delete_batch(DBColumn::BeaconState, ops.clone())?; + self.delete_batch(DBColumn::BeaconStateSummary, ops.clone())?; + self.delete_batch(DBColumn::BeaconStateTemporary, ops)?; } Ok(()) diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index da3e6d4ebc..134be9ec0d 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -1,10 +1,10 @@ use crate::config::{OnDiskStoreConfig, StoreConfig}; +use crate::database::interface::BeaconNodeBackend; use crate::forwards_iter::{HybridForwardsBlockRootsIterator, HybridForwardsStateRootsIterator}; use crate::hdiff::{HDiff, HDiffBuffer, HierarchyModuli, StorageStrategy}; use crate::historic_state_cache::HistoricStateCache; use crate::impls::beacon_state::{get_full_state, store_full_state}; use crate::iter::{BlockRootsIterator, ParentRootBlockIterator, RootsIterator}; -use crate::leveldb_store::{BytesKey, LevelDB}; use crate::memory_store::MemoryStore; use crate::metadata::{ AnchorInfo, BlobInfo, CompactionTimestamp, DataColumnInfo, PruningCheckpoint, SchemaVersion, @@ -14,12 +14,10 @@ use crate::metadata::{ }; use crate::state_cache::{PutStateOutcome, StateCache}; use crate::{ - get_data_column_key, get_key_for_col, DBColumn, DatabaseBlock, Error, ItemStore, - KeyValueStoreOp, StoreItem, StoreOp, + get_data_column_key, metrics, parse_data_column_key, BlobSidecarListFromRoot, ColumnKeyIter, + DBColumn, DatabaseBlock, Error, ItemStore, KeyValueStore, KeyValueStoreOp, StoreItem, StoreOp, }; -use crate::{metrics, parse_data_column_key}; use itertools::{process_results, Itertools}; -use leveldb::iterator::LevelDBIterator; use lru::LruCache; use parking_lot::{Mutex, RwLock}; use safe_arith::SafeArith; @@ -121,6 +119,11 @@ impl BlockCache { pub fn get_blobs<'a>(&'a mut self, block_root: &Hash256) -> Option<&'a BlobSidecarList> { self.blob_cache.get(block_root) } + pub fn get_data_columns(&mut self, block_root: &Hash256) -> Option> { + self.data_column_cache + .get(block_root) + .map(|map| map.values().cloned().collect::>()) + } pub fn get_data_column<'a>( &'a mut self, block_root: &Hash256, @@ -231,7 +234,7 @@ impl HotColdDB, MemoryStore> { } } -impl HotColdDB, LevelDB> { +impl HotColdDB, BeaconNodeBackend> { /// Open a new or existing database, with the given paths to the hot and cold DBs. /// /// The `migrate_schema` function is passed in so that the parent `BeaconChain` can provide @@ -249,7 +252,7 @@ impl HotColdDB, LevelDB> { let hierarchy = config.hierarchy_config.to_moduli()?; - let hot_db = LevelDB::open(hot_path)?; + let hot_db = BeaconNodeBackend::open(&config, hot_path)?; let anchor_info = RwLock::new(Self::load_anchor_info(&hot_db)?); let db = HotColdDB { @@ -257,8 +260,8 @@ impl HotColdDB, LevelDB> { anchor_info, blob_info: RwLock::new(BlobInfo::default()), data_column_info: RwLock::new(DataColumnInfo::default()), - cold_db: LevelDB::open(cold_path)?, - blobs_db: LevelDB::open(blobs_db_path)?, + blobs_db: BeaconNodeBackend::open(&config, blobs_db_path)?, + cold_db: BeaconNodeBackend::open(&config, cold_path)?, hot_db, block_cache: Mutex::new(BlockCache::new(config.block_cache_size)), state_cache: Mutex::new(StateCache::new(config.state_cache_size)), @@ -324,16 +327,15 @@ impl HotColdDB, LevelDB> { db.compare_and_set_blob_info_with_write(<_>::default(), new_blob_info.clone())?; let data_column_info = db.load_data_column_info()?; - let eip7594_fork_slot = db + let fulu_fork_slot = db .spec - .eip7594_fork_epoch + .fulu_fork_epoch .map(|epoch| epoch.start_slot(E::slots_per_epoch())); let new_data_column_info = match &data_column_info { Some(data_column_info) => { // Set the oldest data column slot to the fork slot if it is not yet set. - let oldest_data_column_slot = data_column_info - .oldest_data_column_slot - .or(eip7594_fork_slot); + let oldest_data_column_slot = + data_column_info.oldest_data_column_slot.or(fulu_fork_slot); DataColumnInfo { oldest_data_column_slot, } @@ -341,7 +343,7 @@ impl HotColdDB, LevelDB> { // First start. None => DataColumnInfo { // Set the oldest data column slot to the fork slot if it is not yet set. - oldest_data_column_slot: eip7594_fork_slot, + oldest_data_column_slot: fulu_fork_slot, }, }; db.compare_and_set_data_column_info_with_write( @@ -407,24 +409,9 @@ impl HotColdDB, LevelDB> { } /// Return an iterator over the state roots of all temporary states. - pub fn iter_temporary_state_roots(&self) -> impl Iterator> + '_ { - let column = DBColumn::BeaconStateTemporary; - let start_key = - BytesKey::from_vec(get_key_for_col(column.into(), Hash256::zero().as_slice())); - - let keys_iter = self.hot_db.keys_iter(); - keys_iter.seek(&start_key); - - keys_iter - .take_while(move |key| key.matches_column(column)) - .map(move |bytes_key| { - bytes_key.remove_column(column).ok_or_else(|| { - HotColdDBError::IterationError { - unexpected_key: bytes_key, - } - .into() - }) - }) + pub fn iter_temporary_state_roots(&self) -> ColumnKeyIter { + self.hot_db + .iter_column_keys::(DBColumn::BeaconStateTemporary) } } @@ -536,9 +523,9 @@ impl, Cold: ItemStore> HotColdDB blinded_block: &SignedBeaconBlock>, ops: &mut Vec, ) { - let db_key = get_key_for_col(DBColumn::BeaconBlock.into(), key.as_slice()); ops.push(KeyValueStoreOp::PutKeyValue( - db_key, + DBColumn::BeaconBlock, + key.as_slice().into(), blinded_block.as_ssz_bytes(), )); } @@ -660,7 +647,7 @@ impl, Cold: ItemStore> HotColdDB decoder: impl FnOnce(&[u8]) -> Result, ssz::DecodeError>, ) -> Result>, Error> { self.hot_db - .get_bytes(DBColumn::BeaconBlock.into(), block_root.as_slice())? + .get_bytes(DBColumn::BeaconBlock, block_root.as_slice())? .map(|block_bytes| decoder(&block_bytes)) .transpose() .map_err(|e| e.into()) @@ -673,10 +660,12 @@ impl, Cold: ItemStore> HotColdDB block_root: &Hash256, fork_name: ForkName, ) -> Result>, Error> { - let column = ExecutionPayload::::db_column().into(); let key = block_root.as_slice(); - match self.hot_db.get_bytes(column, key)? { + match self + .hot_db + .get_bytes(ExecutionPayload::::db_column(), key)? + { Some(bytes) => Ok(Some(ExecutionPayload::from_ssz_bytes(&bytes, fork_name)?)), None => Ok(None), } @@ -705,10 +694,7 @@ impl, Cold: ItemStore> HotColdDB ) -> Result, Error> { let column = DBColumn::SyncCommitteeBranch; - if let Some(bytes) = self - .hot_db - .get_bytes(column.into(), &block_root.as_ssz_bytes())? - { + if let Some(bytes) = self.hot_db.get_bytes(column, &block_root.as_ssz_bytes())? { let sync_committee_branch = Vec::::from_ssz_bytes(&bytes)?; return Ok(Some(sync_committee_branch)); } @@ -725,7 +711,7 @@ impl, Cold: ItemStore> HotColdDB if let Some(bytes) = self .hot_db - .get_bytes(column.into(), &sync_committee_period.as_ssz_bytes())? + .get_bytes(column, &sync_committee_period.as_ssz_bytes())? { let sync_committee: SyncCommittee = SyncCommittee::from_ssz_bytes(&bytes)?; return Ok(Some(sync_committee)); @@ -741,7 +727,7 @@ impl, Cold: ItemStore> HotColdDB ) -> Result<(), Error> { let column = DBColumn::SyncCommitteeBranch; self.hot_db.put_bytes( - column.into(), + column, &block_root.as_ssz_bytes(), &sync_committee_branch.as_ssz_bytes(), )?; @@ -755,7 +741,7 @@ impl, Cold: ItemStore> HotColdDB ) -> Result<(), Error> { let column = DBColumn::SyncCommittee; self.hot_db.put_bytes( - column.into(), + column, &sync_committee_period.to_le_bytes(), &sync_committee.as_ssz_bytes(), )?; @@ -767,10 +753,10 @@ impl, Cold: ItemStore> HotColdDB &self, sync_committee_period: u64, ) -> Result>, Error> { - let column = DBColumn::LightClientUpdate; - let res = self - .hot_db - .get_bytes(column.into(), &sync_committee_period.to_le_bytes())?; + let res = self.hot_db.get_bytes( + DBColumn::LightClientUpdate, + &sync_committee_period.to_le_bytes(), + )?; if let Some(light_client_update_bytes) = res { let epoch = sync_committee_period @@ -822,10 +808,8 @@ impl, Cold: ItemStore> HotColdDB sync_committee_period: u64, light_client_update: &LightClientUpdate, ) -> Result<(), Error> { - let column = DBColumn::LightClientUpdate; - self.hot_db.put_bytes( - column.into(), + DBColumn::LightClientUpdate, &sync_committee_period.to_le_bytes(), &light_client_update.as_ssz_bytes(), )?; @@ -836,29 +820,29 @@ impl, Cold: ItemStore> HotColdDB /// Check if the blobs for a block exists on disk. pub fn blobs_exist(&self, block_root: &Hash256) -> Result { self.blobs_db - .key_exists(DBColumn::BeaconBlob.into(), block_root.as_slice()) + .key_exists(DBColumn::BeaconBlob, block_root.as_slice()) } /// Determine whether a block exists in the database. pub fn block_exists(&self, block_root: &Hash256) -> Result { self.hot_db - .key_exists(DBColumn::BeaconBlock.into(), block_root.as_slice()) + .key_exists(DBColumn::BeaconBlock, block_root.as_slice()) } /// Delete a block from the store and the block cache. pub fn delete_block(&self, block_root: &Hash256) -> Result<(), Error> { self.block_cache.lock().delete(block_root); self.hot_db - .key_delete(DBColumn::BeaconBlock.into(), block_root.as_slice())?; + .key_delete(DBColumn::BeaconBlock, block_root.as_slice())?; self.hot_db - .key_delete(DBColumn::ExecPayload.into(), block_root.as_slice())?; + .key_delete(DBColumn::ExecPayload, block_root.as_slice())?; self.blobs_db - .key_delete(DBColumn::BeaconBlob.into(), block_root.as_slice()) + .key_delete(DBColumn::BeaconBlob, block_root.as_slice()) } pub fn put_blobs(&self, block_root: &Hash256, blobs: BlobSidecarList) -> Result<(), Error> { self.blobs_db.put_bytes( - DBColumn::BeaconBlob.into(), + DBColumn::BeaconBlob, block_root.as_slice(), &blobs.as_ssz_bytes(), )?; @@ -872,8 +856,29 @@ impl, Cold: ItemStore> HotColdDB blobs: BlobSidecarList, ops: &mut Vec, ) { - let db_key = get_key_for_col(DBColumn::BeaconBlob.into(), key.as_slice()); - ops.push(KeyValueStoreOp::PutKeyValue(db_key, blobs.as_ssz_bytes())); + ops.push(KeyValueStoreOp::PutKeyValue( + DBColumn::BeaconBlob, + key.as_slice().to_vec(), + blobs.as_ssz_bytes(), + )); + } + + pub fn put_data_columns( + &self, + block_root: &Hash256, + data_columns: DataColumnSidecarList, + ) -> Result<(), Error> { + for data_column in data_columns { + self.blobs_db.put_bytes( + DBColumn::BeaconDataColumn, + &get_data_column_key(block_root, &data_column.index), + &data_column.as_ssz_bytes(), + )?; + self.block_cache + .lock() + .put_data_column(*block_root, data_column); + } + Ok(()) } pub fn data_columns_as_kv_store_ops( @@ -883,12 +888,9 @@ impl, Cold: ItemStore> HotColdDB ops: &mut Vec, ) { for data_column in data_columns { - let db_key = get_key_for_col( - DBColumn::BeaconDataColumn.into(), - &get_data_column_key(block_root, &data_column.index), - ); ops.push(KeyValueStoreOp::PutKeyValue( - db_key, + DBColumn::BeaconDataColumn, + get_data_column_key(block_root, &data_column.index), data_column.as_ssz_bytes(), )); } @@ -1202,63 +1204,68 @@ impl, Cold: ItemStore> HotColdDB } StoreOp::DeleteStateTemporaryFlag(state_root) => { - let db_key = - get_key_for_col(TemporaryFlag::db_column().into(), state_root.as_slice()); - key_value_batch.push(KeyValueStoreOp::DeleteKey(db_key)); + key_value_batch.push(KeyValueStoreOp::DeleteKey( + TemporaryFlag::db_column(), + state_root.as_slice().to_vec(), + )); } StoreOp::DeleteBlock(block_root) => { - let key = get_key_for_col(DBColumn::BeaconBlock.into(), block_root.as_slice()); - key_value_batch.push(KeyValueStoreOp::DeleteKey(key)); + key_value_batch.push(KeyValueStoreOp::DeleteKey( + DBColumn::BeaconBlock, + block_root.as_slice().to_vec(), + )); } StoreOp::DeleteBlobs(block_root) => { - let key = get_key_for_col(DBColumn::BeaconBlob.into(), block_root.as_slice()); - key_value_batch.push(KeyValueStoreOp::DeleteKey(key)); + key_value_batch.push(KeyValueStoreOp::DeleteKey( + DBColumn::BeaconBlob, + block_root.as_slice().to_vec(), + )); } StoreOp::DeleteDataColumns(block_root, column_indices) => { for index in column_indices { - let key = get_key_for_col( - DBColumn::BeaconDataColumn.into(), - &get_data_column_key(&block_root, &index), - ); - key_value_batch.push(KeyValueStoreOp::DeleteKey(key)); + let key = get_data_column_key(&block_root, &index); + key_value_batch + .push(KeyValueStoreOp::DeleteKey(DBColumn::BeaconDataColumn, key)); } } StoreOp::DeleteState(state_root, slot) => { // Delete the hot state summary. - let state_summary_key = - get_key_for_col(DBColumn::BeaconStateSummary.into(), state_root.as_slice()); - key_value_batch.push(KeyValueStoreOp::DeleteKey(state_summary_key)); + key_value_batch.push(KeyValueStoreOp::DeleteKey( + DBColumn::BeaconStateSummary, + state_root.as_slice().to_vec(), + )); // Delete the state temporary flag (if any). Temporary flags are commonly // created by the state advance routine. - let state_temp_key = get_key_for_col( - DBColumn::BeaconStateTemporary.into(), - state_root.as_slice(), - ); - key_value_batch.push(KeyValueStoreOp::DeleteKey(state_temp_key)); + key_value_batch.push(KeyValueStoreOp::DeleteKey( + DBColumn::BeaconStateTemporary, + state_root.as_slice().to_vec(), + )); if slot.map_or(true, |slot| slot % E::slots_per_epoch() == 0) { - let state_key = - get_key_for_col(DBColumn::BeaconState.into(), state_root.as_slice()); - key_value_batch.push(KeyValueStoreOp::DeleteKey(state_key)); + key_value_batch.push(KeyValueStoreOp::DeleteKey( + DBColumn::BeaconState, + state_root.as_slice().to_vec(), + )); } } StoreOp::DeleteExecutionPayload(block_root) => { - let key = get_key_for_col(DBColumn::ExecPayload.into(), block_root.as_slice()); - key_value_batch.push(KeyValueStoreOp::DeleteKey(key)); + key_value_batch.push(KeyValueStoreOp::DeleteKey( + DBColumn::ExecPayload, + block_root.as_slice().to_vec(), + )); } StoreOp::DeleteSyncCommitteeBranch(block_root) => { - let key = get_key_for_col( - DBColumn::SyncCommitteeBranch.into(), - block_root.as_slice(), - ); - key_value_batch.push(KeyValueStoreOp::DeleteKey(key)); + key_value_batch.push(KeyValueStoreOp::DeleteKey( + DBColumn::SyncCommitteeBranch, + block_root.as_slice().to_vec(), + )); } StoreOp::KeyValueOp(kv_op) => { @@ -1269,6 +1276,19 @@ impl, Cold: ItemStore> HotColdDB Ok(key_value_batch) } + pub fn delete_batch(&self, col: DBColumn, ops: Vec) -> Result<(), Error> { + let new_ops: HashSet<&[u8]> = ops.iter().map(|v| v.as_slice()).collect(); + self.hot_db.delete_batch(col, new_ops) + } + + pub fn delete_if( + &self, + column: DBColumn, + f: impl Fn(&[u8]) -> Result, + ) -> Result<(), Error> { + self.hot_db.delete_if(column, f) + } + pub fn do_atomically_with_block_and_blobs_cache( &self, batch: Vec>, @@ -1280,9 +1300,10 @@ impl, Cold: ItemStore> HotColdDB StoreOp::PutBlobs(_, _) | StoreOp::PutDataColumns(_, _) => true, StoreOp::DeleteBlobs(block_root) => { match self.get_blobs(block_root) { - Ok(Some(blob_sidecar_list)) => { + Ok(BlobSidecarListFromRoot::Blobs(blob_sidecar_list)) => { blobs_to_delete.push((*block_root, blob_sidecar_list)); } + Ok(BlobSidecarListFromRoot::NoBlobs | BlobSidecarListFromRoot::NoRoot) => {} Err(e) => { error!( self.log, "Error getting blobs"; @@ -1290,7 +1311,6 @@ impl, Cold: ItemStore> HotColdDB "error" => ?e ); } - _ => (), } true } @@ -1608,10 +1628,8 @@ impl, Cold: ItemStore> HotColdDB ) -> Result<(), Error> { ops.push(ColdStateSummary { slot }.as_kv_store_op(*state_root)); ops.push(KeyValueStoreOp::PutKeyValue( - get_key_for_col( - DBColumn::BeaconStateRoots.into(), - &slot.as_u64().to_be_bytes(), - ), + DBColumn::BeaconStateRoots, + slot.as_u64().to_be_bytes().to_vec(), state_root.as_slice().to_vec(), )); Ok(()) @@ -1678,19 +1696,19 @@ impl, Cold: ItemStore> HotColdDB out }; - let key = get_key_for_col( - DBColumn::BeaconStateSnapshot.into(), - &state.slot().as_u64().to_be_bytes(), - ); - ops.push(KeyValueStoreOp::PutKeyValue(key, compressed_value)); + ops.push(KeyValueStoreOp::PutKeyValue( + DBColumn::BeaconStateSnapshot, + state.slot().as_u64().to_be_bytes().to_vec(), + compressed_value, + )); Ok(()) } fn load_cold_state_bytes_as_snapshot(&self, slot: Slot) -> Result>, Error> { - match self.cold_db.get_bytes( - DBColumn::BeaconStateSnapshot.into(), - &slot.as_u64().to_be_bytes(), - )? { + match self + .cold_db + .get_bytes(DBColumn::BeaconStateSnapshot, &slot.as_u64().to_be_bytes())? + { Some(bytes) => { let _timer = metrics::start_timer(&metrics::STORE_BEACON_STATE_FREEZER_DECOMPRESS_TIME); @@ -1731,11 +1749,11 @@ impl, Cold: ItemStore> HotColdDB }; let diff_bytes = diff.as_ssz_bytes(); - let key = get_key_for_col( - DBColumn::BeaconStateDiff.into(), - &state.slot().as_u64().to_be_bytes(), - ); - ops.push(KeyValueStoreOp::PutKeyValue(key, diff_bytes)); + ops.push(KeyValueStoreOp::PutKeyValue( + DBColumn::BeaconStateDiff, + state.slot().as_u64().to_be_bytes().to_vec(), + diff_bytes, + )); Ok(()) } @@ -1858,10 +1876,7 @@ impl, Cold: ItemStore> HotColdDB let bytes = { let _t = metrics::start_timer(&metrics::BEACON_HDIFF_READ_TIMES); self.cold_db - .get_bytes( - DBColumn::BeaconStateDiff.into(), - &slot.as_u64().to_be_bytes(), - )? + .get_bytes(DBColumn::BeaconStateDiff, &slot.as_u64().to_be_bytes())? .ok_or(HotColdDBError::MissingHDiff(slot))? }; let hdiff = { @@ -2044,34 +2059,86 @@ impl, Cold: ItemStore> HotColdDB }) } + /// Fetch columns for a given block from the store. + pub fn get_data_columns( + &self, + block_root: &Hash256, + ) -> Result>, Error> { + if let Some(columns) = self.block_cache.lock().get_data_columns(block_root) { + metrics::inc_counter(&metrics::BEACON_DATA_COLUMNS_CACHE_HIT_COUNT); + return Ok(Some(columns)); + } + + let columns = self + .blobs_db + .iter_column_from::>(DBColumn::BeaconDataColumn, block_root.as_slice()) + .take_while(|res| { + res.as_ref() + .is_ok_and(|(key, _)| key.starts_with(block_root.as_slice())) + }) + .map(|result| { + let (_key, value) = result?; + let column = DataColumnSidecar::::from_ssz_bytes(&value).map(Arc::new)?; + self.block_cache + .lock() + .put_data_column(*block_root, column.clone()); + Ok(column) + }) + .collect::, Error>>()?; + + if columns.is_empty() { + Ok(None) + } else { + Ok(Some(columns)) + } + } + /// Fetch blobs for a given block from the store. - pub fn get_blobs(&self, block_root: &Hash256) -> Result>, Error> { + pub fn get_blobs(&self, block_root: &Hash256) -> Result, Error> { // Check the cache. if let Some(blobs) = self.block_cache.lock().get_blobs(block_root) { metrics::inc_counter(&metrics::BEACON_BLOBS_CACHE_HIT_COUNT); - return Ok(Some(blobs.clone())); + return Ok(blobs.clone().into()); } match self .blobs_db - .get_bytes(DBColumn::BeaconBlob.into(), block_root.as_slice())? + .get_bytes(DBColumn::BeaconBlob, block_root.as_slice())? { Some(ref blobs_bytes) => { - let blobs = BlobSidecarList::from_ssz_bytes(blobs_bytes)?; - self.block_cache - .lock() - .put_blobs(*block_root, blobs.clone()); - Ok(Some(blobs)) + // We insert a VariableList of BlobSidecars into the db, but retrieve + // a plain vec since we don't know the length limit of the list without + // knowing the slot. + // The encoding of a VariableList is the same as a regular vec. + let blobs: Vec>> = Vec::<_>::from_ssz_bytes(blobs_bytes)?; + if let Some(max_blobs_per_block) = blobs + .first() + .map(|blob| self.spec.max_blobs_per_block(blob.epoch())) + { + let blobs = BlobSidecarList::from_vec(blobs, max_blobs_per_block as usize); + self.block_cache + .lock() + .put_blobs(*block_root, blobs.clone()); + + Ok(BlobSidecarListFromRoot::Blobs(blobs)) + } else { + // This always implies that there were no blobs for this block_root + Ok(BlobSidecarListFromRoot::NoBlobs) + } } - None => Ok(None), + None => Ok(BlobSidecarListFromRoot::NoRoot), } } /// Fetch all keys in the data_column column with prefix `block_root` pub fn get_data_column_keys(&self, block_root: Hash256) -> Result, Error> { self.blobs_db - .iter_raw_keys(DBColumn::BeaconDataColumn, block_root.as_slice()) - .map(|key| key.and_then(|key| parse_data_column_key(key).map(|key| key.1))) + .iter_column_from::>(DBColumn::BeaconDataColumn, block_root.as_slice()) + .take_while(|res| { + res.as_ref() + .is_ok_and(|(key, _)| key.starts_with(block_root.as_slice())) + }) + .map(|key| key.and_then(|(key, _)| parse_data_column_key(key).map(|key| key.1))) .collect() } @@ -2092,7 +2159,7 @@ impl, Cold: ItemStore> HotColdDB } match self.blobs_db.get_bytes( - DBColumn::BeaconDataColumn.into(), + DBColumn::BeaconDataColumn, &get_data_column_key(block_root, column_index), )? { Some(ref data_column_bytes) => { @@ -2150,10 +2217,12 @@ impl, Cold: ItemStore> HotColdDB schema_version: SchemaVersion, mut ops: Vec, ) -> Result<(), Error> { - let column = SchemaVersion::db_column().into(); let key = SCHEMA_VERSION_KEY.as_slice(); - let db_key = get_key_for_col(column, key); - let op = KeyValueStoreOp::PutKeyValue(db_key, schema_version.as_store_bytes()); + let op = KeyValueStoreOp::PutKeyValue( + SchemaVersion::db_column(), + key.to_vec(), + schema_version.as_store_bytes(), + ); ops.push(op); self.hot_db.do_atomically(ops) @@ -2264,7 +2333,7 @@ impl, Cold: ItemStore> HotColdDB /// Initialize the `DataColumnInfo` when starting from genesis or a checkpoint. pub fn init_data_column_info(&self, anchor_slot: Slot) -> Result { - let oldest_data_column_slot = self.spec.eip7594_fork_epoch.map(|fork_epoch| { + let oldest_data_column_slot = self.spec.fulu_fork_epoch.map(|fork_epoch| { std::cmp::max(anchor_slot, fork_epoch.start_slot(E::slots_per_epoch())) }); let data_column_info = DataColumnInfo { @@ -2575,7 +2644,8 @@ impl, Cold: ItemStore> HotColdDB let mut ops = vec![]; for slot in start_slot.as_u64()..end_slot.as_u64() { ops.push(KeyValueStoreOp::PutKeyValue( - get_key_for_col(DBColumn::BeaconBlockRoots.into(), &slot.to_be_bytes()), + DBColumn::BeaconBlockRoots, + slot.to_be_bytes().to_vec(), block_root.as_slice().to_vec(), )); } @@ -2629,7 +2699,7 @@ impl, Cold: ItemStore> HotColdDB "Pruning finalized payloads"; "info" => "you may notice degraded I/O performance while this runs" ); - let anchor_slot = self.get_anchor_info().anchor_slot; + let anchor_info = self.get_anchor_info(); let mut ops = vec![]; let mut last_pruned_block_root = None; @@ -2670,10 +2740,10 @@ impl, Cold: ItemStore> HotColdDB ops.push(StoreOp::DeleteExecutionPayload(block_root)); } - if slot == anchor_slot { + if slot <= anchor_info.oldest_block_slot { info!( self.log, - "Payload pruning reached anchor state"; + "Payload pruning reached anchor oldest block slot"; "slot" => slot ); break; @@ -2797,77 +2867,62 @@ impl, Cold: ItemStore> HotColdDB "data_availability_boundary" => data_availability_boundary, ); - let mut ops = vec![]; - let mut last_pruned_block_root = None; + // We collect block roots of deleted blobs in memory. Even for 10y of blob history this + // vec won't go beyond 1GB. We can probably optimise this out eventually. + let mut removed_block_roots = vec![]; - for res in self.forwards_block_roots_iterator_until(oldest_blob_slot, end_slot, || { - let (_, split_state) = self - .get_advanced_hot_state(split.block_root, split.slot, split.state_root)? - .ok_or(HotColdDBError::MissingSplitState( - split.state_root, - split.slot, - ))?; - - Ok((split_state, split.block_root)) - })? { - let (block_root, slot) = match res { - Ok(tuple) => tuple, - Err(e) => { - warn!( - self.log, - "Stopping blob pruning early"; - "error" => ?e, - ); - break; - } + let remove_blob_if = |blobs_bytes: &[u8]| { + let blobs = Vec::from_ssz_bytes(blobs_bytes)?; + let Some(blob): Option<&Arc>> = blobs.first() else { + return Ok(false); }; - if Some(block_root) != last_pruned_block_root { - if self - .spec - .is_peer_das_enabled_for_epoch(slot.epoch(E::slots_per_epoch())) - { - // data columns - let indices = self.get_data_column_keys(block_root)?; - if !indices.is_empty() { - trace!( - self.log, - "Pruning data columns of block"; - "slot" => slot, - "block_root" => ?block_root, - ); - last_pruned_block_root = Some(block_root); - ops.push(StoreOp::DeleteDataColumns(block_root, indices)); - } - } else if self.blobs_exist(&block_root)? { - trace!( - self.log, - "Pruning blobs of block"; - "slot" => slot, - "block_root" => ?block_root, - ); - last_pruned_block_root = Some(block_root); - ops.push(StoreOp::DeleteBlobs(block_root)); - } - } + if blob.slot() <= end_slot { + // Store the block root so we can delete from the blob cache + removed_block_roots.push(blob.block_root()); + // Delete from the on-disk db + return Ok(true); + }; + Ok(false) + }; - if slot >= end_slot { - break; - } + self.blobs_db + .delete_if(DBColumn::BeaconBlob, remove_blob_if)?; + + if self.spec.is_peer_das_enabled_for_epoch(start_epoch) { + let remove_data_column_if = |blobs_bytes: &[u8]| { + let data_column: DataColumnSidecar = + DataColumnSidecar::from_ssz_bytes(blobs_bytes)?; + + if data_column.slot() <= end_slot { + return Ok(true); + }; + + Ok(false) + }; + + self.blobs_db + .delete_if(DBColumn::BeaconDataColumn, remove_data_column_if)?; } - let blob_lists_pruned = ops.len(); + + // Remove deleted blobs from the cache. + let mut block_cache = self.block_cache.lock(); + for block_root in removed_block_roots { + block_cache.delete_blobs(&block_root); + } + drop(block_cache); + let new_blob_info = BlobInfo { oldest_blob_slot: Some(end_slot + 1), blobs_db: blob_info.blobs_db, }; - let update_blob_info = self.compare_and_set_blob_info(blob_info, new_blob_info)?; - ops.push(StoreOp::KeyValueOp(update_blob_info)); - self.do_atomically_with_block_and_blobs_cache(ops)?; + let op = self.compare_and_set_blob_info(blob_info, new_blob_info)?; + self.do_atomically_with_block_and_blobs_cache(vec![StoreOp::KeyValueOp(op)])?; + debug!( self.log, "Blob pruning complete"; - "blob_lists_pruned" => blob_lists_pruned, ); Ok(()) @@ -2930,10 +2985,7 @@ impl, Cold: ItemStore> HotColdDB for column in columns { for res in self.cold_db.iter_column_keys::>(column) { let key = res?; - cold_ops.push(KeyValueStoreOp::DeleteKey(get_key_for_col( - column.as_str(), - &key, - ))); + cold_ops.push(KeyValueStoreOp::DeleteKey(column, key)); } } let delete_ops = cold_ops.len(); @@ -3071,10 +3123,8 @@ pub fn migrate_database, Cold: ItemStore>( // Store the slot to block root mapping. cold_db_block_ops.push(KeyValueStoreOp::PutKeyValue( - get_key_for_col( - DBColumn::BeaconBlockRoots.into(), - &slot.as_u64().to_be_bytes(), - ), + DBColumn::BeaconBlockRoots, + slot.as_u64().to_be_bytes().to_vec(), block_root.as_slice().to_vec(), )); @@ -3325,3 +3375,57 @@ impl StoreItem for TemporaryFlag { Ok(TemporaryFlag) } } + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct BytesKey { + pub key: Vec, +} + +impl db_key::Key for BytesKey { + fn from_u8(key: &[u8]) -> Self { + Self { key: key.to_vec() } + } + + fn as_slice T>(&self, f: F) -> T { + f(self.key.as_slice()) + } +} + +impl BytesKey { + pub fn starts_with(&self, prefix: &Self) -> bool { + self.key.starts_with(&prefix.key) + } + + /// Return `true` iff this `BytesKey` was created with the given `column`. + pub fn matches_column(&self, column: DBColumn) -> bool { + self.key.starts_with(column.as_bytes()) + } + + /// Remove the column from a key, returning its `Hash256` portion. + pub fn remove_column(&self, column: DBColumn) -> Option { + if self.matches_column(column) { + let subkey = &self.key[column.as_bytes().len()..]; + if subkey.len() == 32 { + return Some(Hash256::from_slice(subkey)); + } + } + None + } + + /// Remove the column from a key. + /// + /// Will return `None` if the value doesn't match the column or has the wrong length. + pub fn remove_column_variable(&self, column: DBColumn) -> Option<&[u8]> { + if self.matches_column(column) { + let subkey = &self.key[column.as_bytes().len()..]; + if subkey.len() == column.key_size() { + return Some(subkey); + } + } + None + } + + pub fn from_vec(key: Vec) -> Self { + Self { key } + } +} diff --git a/beacon_node/store/src/impls/beacon_state.rs b/beacon_node/store/src/impls/beacon_state.rs index 48c289f2b2..fd08e547f1 100644 --- a/beacon_node/store/src/impls/beacon_state.rs +++ b/beacon_node/store/src/impls/beacon_state.rs @@ -13,8 +13,11 @@ pub fn store_full_state( }; metrics::inc_counter_by(&metrics::BEACON_STATE_WRITE_BYTES, bytes.len() as u64); metrics::inc_counter(&metrics::BEACON_STATE_WRITE_COUNT); - let key = get_key_for_col(DBColumn::BeaconState.into(), state_root.as_slice()); - ops.push(KeyValueStoreOp::PutKeyValue(key, bytes)); + ops.push(KeyValueStoreOp::PutKeyValue( + DBColumn::BeaconState, + state_root.as_slice().to_vec(), + bytes, + )); Ok(()) } @@ -25,7 +28,7 @@ pub fn get_full_state, E: EthSpec>( ) -> Result>, Error> { let total_timer = metrics::start_timer(&metrics::BEACON_STATE_READ_TIMES); - match db.get_bytes(DBColumn::BeaconState.into(), state_root.as_slice())? { + match db.get_bytes(DBColumn::BeaconState, state_root.as_slice())? { Some(bytes) => { let overhead_timer = metrics::start_timer(&metrics::BEACON_STATE_READ_OVERHEAD_TIMES); let container = StorageContainer::from_ssz_bytes(&bytes, spec)?; diff --git a/beacon_node/store/src/impls/execution_payload.rs b/beacon_node/store/src/impls/execution_payload.rs index 14fc10ad6d..097b069a66 100644 --- a/beacon_node/store/src/impls/execution_payload.rs +++ b/beacon_node/store/src/impls/execution_payload.rs @@ -1,8 +1,8 @@ use crate::{DBColumn, Error, StoreItem}; use ssz::{Decode, Encode}; use types::{ - BlobSidecarList, EthSpec, ExecutionPayload, ExecutionPayloadBellatrix, ExecutionPayloadCapella, - ExecutionPayloadDeneb, ExecutionPayloadElectra, + EthSpec, ExecutionPayload, ExecutionPayloadBellatrix, ExecutionPayloadCapella, + ExecutionPayloadDeneb, ExecutionPayloadElectra, ExecutionPayloadFulu, }; macro_rules! impl_store_item { @@ -26,7 +26,7 @@ impl_store_item!(ExecutionPayloadBellatrix); impl_store_item!(ExecutionPayloadCapella); impl_store_item!(ExecutionPayloadDeneb); impl_store_item!(ExecutionPayloadElectra); -impl_store_item!(BlobSidecarList); +impl_store_item!(ExecutionPayloadFulu); /// This fork-agnostic implementation should be only used for writing. /// @@ -42,17 +42,21 @@ impl StoreItem for ExecutionPayload { } fn from_store_bytes(bytes: &[u8]) -> Result { - ExecutionPayloadElectra::from_ssz_bytes(bytes) - .map(Self::Electra) + ExecutionPayloadFulu::from_ssz_bytes(bytes) + .map(Self::Fulu) .or_else(|_| { - ExecutionPayloadDeneb::from_ssz_bytes(bytes) - .map(Self::Deneb) + ExecutionPayloadElectra::from_ssz_bytes(bytes) + .map(Self::Electra) .or_else(|_| { - ExecutionPayloadCapella::from_ssz_bytes(bytes) - .map(Self::Capella) + ExecutionPayloadDeneb::from_ssz_bytes(bytes) + .map(Self::Deneb) .or_else(|_| { - ExecutionPayloadBellatrix::from_ssz_bytes(bytes) - .map(Self::Bellatrix) + ExecutionPayloadCapella::from_ssz_bytes(bytes) + .map(Self::Capella) + .or_else(|_| { + ExecutionPayloadBellatrix::from_ssz_bytes(bytes) + .map(Self::Bellatrix) + }) }) }) }) diff --git a/beacon_node/store/src/leveldb_store.rs b/beacon_node/store/src/leveldb_store.rs deleted file mode 100644 index 720afd0f3f..0000000000 --- a/beacon_node/store/src/leveldb_store.rs +++ /dev/null @@ -1,310 +0,0 @@ -use super::*; -use crate::hot_cold_store::HotColdDBError; -use leveldb::compaction::Compaction; -use leveldb::database::batch::{Batch, Writebatch}; -use leveldb::database::kv::KV; -use leveldb::database::Database; -use leveldb::error::Error as LevelDBError; -use leveldb::iterator::{Iterable, KeyIterator, LevelDBIterator}; -use leveldb::options::{Options, ReadOptions, WriteOptions}; -use parking_lot::Mutex; -use std::marker::PhantomData; -use std::path::Path; - -/// A wrapped leveldb database. -pub struct LevelDB { - db: Database, - /// A mutex to synchronise sensitive read-write transactions. - transaction_mutex: Mutex<()>, - _phantom: PhantomData, -} - -impl LevelDB { - /// Open a database at `path`, creating a new database if one does not already exist. - pub fn open(path: &Path) -> Result { - let mut options = Options::new(); - - options.create_if_missing = true; - - let db = Database::open(path, options)?; - let transaction_mutex = Mutex::new(()); - - Ok(Self { - db, - transaction_mutex, - _phantom: PhantomData, - }) - } - - fn read_options(&self) -> ReadOptions { - ReadOptions::new() - } - - fn write_options(&self) -> WriteOptions { - WriteOptions::new() - } - - fn write_options_sync(&self) -> WriteOptions { - let mut opts = WriteOptions::new(); - opts.sync = true; - opts - } - - fn put_bytes_with_options( - &self, - col: &str, - key: &[u8], - val: &[u8], - opts: WriteOptions, - ) -> Result<(), Error> { - let column_key = get_key_for_col(col, key); - - metrics::inc_counter_vec(&metrics::DISK_DB_WRITE_COUNT, &[col]); - metrics::inc_counter_vec_by(&metrics::DISK_DB_WRITE_BYTES, &[col], val.len() as u64); - let _timer = metrics::start_timer(&metrics::DISK_DB_WRITE_TIMES); - - self.db - .put(opts, BytesKey::from_vec(column_key), val) - .map_err(Into::into) - } - - pub fn keys_iter(&self) -> KeyIterator { - self.db.keys_iter(self.read_options()) - } -} - -impl KeyValueStore for LevelDB { - /// Store some `value` in `column`, indexed with `key`. - fn put_bytes(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), Error> { - self.put_bytes_with_options(col, key, val, self.write_options()) - } - - fn put_bytes_sync(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), Error> { - self.put_bytes_with_options(col, key, val, self.write_options_sync()) - } - - fn sync(&self) -> Result<(), Error> { - self.put_bytes_sync("sync", b"sync", b"sync") - } - - /// Retrieve some bytes in `column` with `key`. - fn get_bytes(&self, col: &str, key: &[u8]) -> Result>, Error> { - let column_key = get_key_for_col(col, key); - - metrics::inc_counter_vec(&metrics::DISK_DB_READ_COUNT, &[col]); - let timer = metrics::start_timer(&metrics::DISK_DB_READ_TIMES); - - self.db - .get(self.read_options(), BytesKey::from_vec(column_key)) - .map_err(Into::into) - .map(|opt| { - opt.inspect(|bytes| { - metrics::inc_counter_vec_by( - &metrics::DISK_DB_READ_BYTES, - &[col], - bytes.len() as u64, - ); - metrics::stop_timer(timer); - }) - }) - } - - /// Return `true` if `key` exists in `column`. - fn key_exists(&self, col: &str, key: &[u8]) -> Result { - let column_key = get_key_for_col(col, key); - - metrics::inc_counter_vec(&metrics::DISK_DB_EXISTS_COUNT, &[col]); - - self.db - .get(self.read_options(), BytesKey::from_vec(column_key)) - .map_err(Into::into) - .map(|val| val.is_some()) - } - - /// Removes `key` from `column`. - fn key_delete(&self, col: &str, key: &[u8]) -> Result<(), Error> { - let column_key = get_key_for_col(col, key); - - metrics::inc_counter_vec(&metrics::DISK_DB_DELETE_COUNT, &[col]); - - self.db - .delete(self.write_options(), BytesKey::from_vec(column_key)) - .map_err(Into::into) - } - - fn do_atomically(&self, ops_batch: Vec) -> Result<(), Error> { - let mut leveldb_batch = Writebatch::new(); - for op in ops_batch { - match op { - KeyValueStoreOp::PutKeyValue(key, value) => { - let col = get_col_from_key(&key).unwrap_or("unknown".to_owned()); - metrics::inc_counter_vec(&metrics::DISK_DB_WRITE_COUNT, &[&col]); - metrics::inc_counter_vec_by( - &metrics::DISK_DB_WRITE_BYTES, - &[&col], - value.len() as u64, - ); - - leveldb_batch.put(BytesKey::from_vec(key), &value); - } - - KeyValueStoreOp::DeleteKey(key) => { - let col = get_col_from_key(&key).unwrap_or("unknown".to_owned()); - metrics::inc_counter_vec(&metrics::DISK_DB_DELETE_COUNT, &[&col]); - - leveldb_batch.delete(BytesKey::from_vec(key)); - } - } - } - - let _timer = metrics::start_timer(&metrics::DISK_DB_WRITE_TIMES); - - self.db.write(self.write_options(), &leveldb_batch)?; - Ok(()) - } - - fn begin_rw_transaction(&self) -> MutexGuard<()> { - self.transaction_mutex.lock() - } - - fn compact_column(&self, column: DBColumn) -> Result<(), Error> { - // Use key-size-agnostic keys [] and 0xff..ff with a minimum of 32 bytes to account for - // columns that may change size between sub-databases or schema versions. - let start_key = BytesKey::from_vec(get_key_for_col(column.as_str(), &[])); - let end_key = BytesKey::from_vec(get_key_for_col( - column.as_str(), - &vec![0xff; std::cmp::max(column.key_size(), 32)], - )); - self.db.compact(&start_key, &end_key); - Ok(()) - } - - fn iter_column_from(&self, column: DBColumn, from: &[u8]) -> ColumnIter { - let start_key = BytesKey::from_vec(get_key_for_col(column.into(), from)); - let iter = self.db.iter(self.read_options()); - iter.seek(&start_key); - - Box::new( - iter.take_while(move |(key, _)| key.matches_column(column)) - .map(move |(bytes_key, value)| { - let key = bytes_key.remove_column_variable(column).ok_or_else(|| { - HotColdDBError::IterationError { - unexpected_key: bytes_key.clone(), - } - })?; - Ok((K::from_bytes(key)?, value)) - }), - ) - } - - fn iter_raw_entries(&self, column: DBColumn, prefix: &[u8]) -> RawEntryIter { - let start_key = BytesKey::from_vec(get_key_for_col(column.into(), prefix)); - - let iter = self.db.iter(self.read_options()); - iter.seek(&start_key); - - Box::new( - iter.take_while(move |(key, _)| key.key.starts_with(start_key.key.as_slice())) - .map(move |(bytes_key, value)| { - let subkey = &bytes_key.key[column.as_bytes().len()..]; - Ok((Vec::from(subkey), value)) - }), - ) - } - - fn iter_raw_keys(&self, column: DBColumn, prefix: &[u8]) -> RawKeyIter { - let start_key = BytesKey::from_vec(get_key_for_col(column.into(), prefix)); - - let iter = self.db.keys_iter(self.read_options()); - iter.seek(&start_key); - - Box::new( - iter.take_while(move |key| key.key.starts_with(start_key.key.as_slice())) - .map(move |bytes_key| { - let subkey = &bytes_key.key[column.as_bytes().len()..]; - Ok(Vec::from(subkey)) - }), - ) - } - - /// Iterate through all keys and values in a particular column. - fn iter_column_keys(&self, column: DBColumn) -> ColumnKeyIter { - let start_key = - BytesKey::from_vec(get_key_for_col(column.into(), &vec![0; column.key_size()])); - - let iter = self.db.keys_iter(self.read_options()); - iter.seek(&start_key); - - Box::new( - iter.take_while(move |key| key.matches_column(column)) - .map(move |bytes_key| { - let key = bytes_key.remove_column_variable(column).ok_or_else(|| { - HotColdDBError::IterationError { - unexpected_key: bytes_key.clone(), - } - })?; - K::from_bytes(key) - }), - ) - } -} - -impl ItemStore for LevelDB {} - -/// Used for keying leveldb. -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub struct BytesKey { - key: Vec, -} - -impl db_key::Key for BytesKey { - fn from_u8(key: &[u8]) -> Self { - Self { key: key.to_vec() } - } - - fn as_slice T>(&self, f: F) -> T { - f(self.key.as_slice()) - } -} - -impl BytesKey { - pub fn starts_with(&self, prefix: &Self) -> bool { - self.key.starts_with(&prefix.key) - } - - /// Return `true` iff this `BytesKey` was created with the given `column`. - pub fn matches_column(&self, column: DBColumn) -> bool { - self.key.starts_with(column.as_bytes()) - } - - /// Remove the column from a 32 byte key, yielding the `Hash256` key. - pub fn remove_column(&self, column: DBColumn) -> Option { - let key = self.remove_column_variable(column)?; - (column.key_size() == 32).then(|| Hash256::from_slice(key)) - } - - /// Remove the column from a key. - /// - /// Will return `None` if the value doesn't match the column or has the wrong length. - pub fn remove_column_variable(&self, column: DBColumn) -> Option<&[u8]> { - if self.matches_column(column) { - let subkey = &self.key[column.as_bytes().len()..]; - if subkey.len() == column.key_size() { - return Some(subkey); - } - } - None - } - - pub fn from_vec(key: Vec) -> Self { - Self { key } - } -} - -impl From for Error { - fn from(e: LevelDBError) -> Error { - Error::DBError { - message: format!("{:?}", e), - } - } -} diff --git a/beacon_node/store/src/lib.rs b/beacon_node/store/src/lib.rs index 09ae9a32dd..0cfc42ab15 100644 --- a/beacon_node/store/src/lib.rs +++ b/beacon_node/store/src/lib.rs @@ -7,6 +7,7 @@ //! //! Provides a simple API for storing/retrieving all types that sometimes needs type-hints. See //! tests for implementation examples. +pub mod blob_sidecar_list_from_root; pub mod chunked_iter; pub mod chunked_vector; pub mod config; @@ -18,7 +19,6 @@ pub mod hdiff; pub mod historic_state_cache; pub mod hot_cold_store; mod impls; -mod leveldb_store; mod memory_store; pub mod metadata; pub mod metrics; @@ -26,12 +26,13 @@ pub mod partial_beacon_state; pub mod reconstruct; pub mod state_cache; +pub mod database; pub mod iter; +pub use self::blob_sidecar_list_from_root::BlobSidecarListFromRoot; pub use self::config::StoreConfig; pub use self::consensus_context::OnDiskConsensusContext; pub use self::hot_cold_store::{HotColdDB, HotStateSummary, Split}; -pub use self::leveldb_store::LevelDB; pub use self::memory_store::MemoryStore; pub use crate::metadata::BlobInfo; pub use errors::Error; @@ -39,8 +40,9 @@ pub use impls::beacon_state::StorageContainer as BeaconStateStorageContainer; pub use metadata::AnchorInfo; pub use metrics::scrape_for_metrics; use parking_lot::MutexGuard; +use std::collections::HashSet; use std::sync::Arc; -use strum::{EnumString, IntoStaticStr}; +use strum::{EnumIter, EnumString, IntoStaticStr}; pub use types::*; const DATA_COLUMN_DB_KEY_SIZE: usize = 32 + 8; @@ -48,18 +50,18 @@ const DATA_COLUMN_DB_KEY_SIZE: usize = 32 + 8; pub type ColumnIter<'a, K> = Box), Error>> + 'a>; pub type ColumnKeyIter<'a, K> = Box> + 'a>; -pub type RawEntryIter<'a> = Box, Vec), Error>> + 'a>; -pub type RawKeyIter<'a> = Box, Error>> + 'a>; +pub type RawEntryIter<'a> = + Result, Vec), Error>> + 'a>, Error>; pub trait KeyValueStore: Sync + Send + Sized + 'static { /// Retrieve some bytes in `column` with `key`. - fn get_bytes(&self, column: &str, key: &[u8]) -> Result>, Error>; + fn get_bytes(&self, column: DBColumn, key: &[u8]) -> Result>, Error>; /// Store some `value` in `column`, indexed with `key`. - fn put_bytes(&self, column: &str, key: &[u8], value: &[u8]) -> Result<(), Error>; + fn put_bytes(&self, column: DBColumn, key: &[u8], value: &[u8]) -> Result<(), Error>; /// Same as put_bytes() but also force a flush to disk - fn put_bytes_sync(&self, column: &str, key: &[u8], value: &[u8]) -> Result<(), Error>; + fn put_bytes_sync(&self, column: DBColumn, key: &[u8], value: &[u8]) -> Result<(), Error>; /// Flush to disk. See /// https://chromium.googlesource.com/external/leveldb/+/HEAD/doc/index.md#synchronous-writes @@ -67,10 +69,10 @@ pub trait KeyValueStore: Sync + Send + Sized + 'static { fn sync(&self) -> Result<(), Error>; /// Return `true` if `key` exists in `column`. - fn key_exists(&self, column: &str, key: &[u8]) -> Result; + fn key_exists(&self, column: DBColumn, key: &[u8]) -> Result; /// Removes `key` from `column`. - fn key_delete(&self, column: &str, key: &[u8]) -> Result<(), Error>; + fn key_delete(&self, column: DBColumn, key: &[u8]) -> Result<(), Error>; /// Execute either all of the operations in `batch` or none at all, returning an error. fn do_atomically(&self, batch: Vec) -> Result<(), Error>; @@ -103,17 +105,21 @@ pub trait KeyValueStore: Sync + Send + Sized + 'static { self.iter_column_from(column, &vec![0; column.key_size()]) } - /// Iterate through all keys and values in a column from a given starting point. + /// Iterate through all keys and values in a column from a given starting point that fulfill the given predicate. fn iter_column_from(&self, column: DBColumn, from: &[u8]) -> ColumnIter; - fn iter_raw_entries(&self, _column: DBColumn, _prefix: &[u8]) -> RawEntryIter { - Box::new(std::iter::empty()) - } - - fn iter_raw_keys(&self, column: DBColumn, prefix: &[u8]) -> RawKeyIter; + fn iter_column_keys(&self, column: DBColumn) -> ColumnKeyIter; /// Iterate through all keys in a particular column. - fn iter_column_keys(&self, column: DBColumn) -> ColumnKeyIter; + fn iter_column_keys_from(&self, column: DBColumn, from: &[u8]) -> ColumnKeyIter; + + fn delete_batch(&self, column: DBColumn, ops: HashSet<&[u8]>) -> Result<(), Error>; + + fn delete_if( + &self, + column: DBColumn, + f: impl FnMut(&[u8]) -> Result, + ) -> Result<(), Error>; } pub trait Key: Sized + 'static { @@ -136,7 +142,7 @@ impl Key for Vec { } } -pub fn get_key_for_col(column: &str, key: &[u8]) -> Vec { +pub fn get_key_for_col(column: DBColumn, key: &[u8]) -> Vec { let mut result = column.as_bytes().to_vec(); result.extend_from_slice(key); result @@ -174,14 +180,18 @@ pub fn parse_data_column_key(data: Vec) -> Result<(Hash256, ColumnIndex), Er #[must_use] #[derive(Clone)] pub enum KeyValueStoreOp { - PutKeyValue(Vec, Vec), - DeleteKey(Vec), + // Indicate that a PUT operation should be made + // to the db store for a (Column, Key, Value) + PutKeyValue(DBColumn, Vec, Vec), + // Indicate that a DELETE operation should be made + // to the db store for a (Column, Key) + DeleteKey(DBColumn, Vec), } pub trait ItemStore: KeyValueStore + Sync + Send + Sized + 'static { /// Store an item in `Self`. fn put(&self, key: &Hash256, item: &I) -> Result<(), Error> { - let column = I::db_column().into(); + let column = I::db_column(); let key = key.as_slice(); self.put_bytes(column, key, &item.as_store_bytes()) @@ -189,7 +199,7 @@ pub trait ItemStore: KeyValueStore + Sync + Send + Sized + 'stati } fn put_sync(&self, key: &Hash256, item: &I) -> Result<(), Error> { - let column = I::db_column().into(); + let column = I::db_column(); let key = key.as_slice(); self.put_bytes_sync(column, key, &item.as_store_bytes()) @@ -198,7 +208,7 @@ pub trait ItemStore: KeyValueStore + Sync + Send + Sized + 'stati /// Retrieve an item from `Self`. fn get(&self, key: &Hash256) -> Result, Error> { - let column = I::db_column().into(); + let column = I::db_column(); let key = key.as_slice(); match self.get_bytes(column, key)? { @@ -209,7 +219,7 @@ pub trait ItemStore: KeyValueStore + Sync + Send + Sized + 'stati /// Returns `true` if the given key represents an item in `Self`. fn exists(&self, key: &Hash256) -> Result { - let column = I::db_column().into(); + let column = I::db_column(); let key = key.as_slice(); self.key_exists(column, key) @@ -217,7 +227,7 @@ pub trait ItemStore: KeyValueStore + Sync + Send + Sized + 'stati /// Remove an item from `Self`. fn delete(&self, key: &Hash256) -> Result<(), Error> { - let column = I::db_column().into(); + let column = I::db_column(); let key = key.as_slice(); self.key_delete(column, key) @@ -245,7 +255,7 @@ pub enum StoreOp<'a, E: EthSpec> { } /// A unique column identifier. -#[derive(Debug, Clone, Copy, PartialEq, IntoStaticStr, EnumString)] +#[derive(Debug, Clone, Copy, PartialEq, IntoStaticStr, EnumString, EnumIter)] pub enum DBColumn { /// For data related to the database itself. #[strum(serialize = "bma")] @@ -349,6 +359,9 @@ pub enum DBColumn { /// For helping persist eagerly computed light client bootstrap data #[strum(serialize = "scm")] SyncCommittee, + /// The dummy table is used to force the db to sync + #[strum(serialize = "dmy")] + Dummy, } /// A block from the database, which might have an execution payload or not. @@ -399,7 +412,8 @@ impl DBColumn { | Self::BeaconStateDiff | Self::SyncCommittee | Self::SyncCommitteeBranch - | Self::LightClientUpdate => 8, + | Self::LightClientUpdate + | Self::Dummy => 8, Self::BeaconDataColumn => DATA_COLUMN_DB_KEY_SIZE, } } @@ -419,13 +433,18 @@ pub trait StoreItem: Sized { fn from_store_bytes(bytes: &[u8]) -> Result; fn as_kv_store_op(&self, key: Hash256) -> KeyValueStoreOp { - let db_key = get_key_for_col(Self::db_column().into(), key.as_slice()); - KeyValueStoreOp::PutKeyValue(db_key, self.as_store_bytes()) + KeyValueStoreOp::PutKeyValue( + Self::db_column(), + key.as_slice().to_vec(), + self.as_store_bytes(), + ) } } #[cfg(test)] mod tests { + use crate::database::interface::BeaconNodeBackend; + use super::*; use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; @@ -475,7 +494,7 @@ mod tests { fn simplediskdb() { let dir = tempdir().unwrap(); let path = dir.path(); - let store = LevelDB::open(path).unwrap(); + let store = BeaconNodeBackend::open(&StoreConfig::default(), path).unwrap(); test_impl(store); } @@ -506,7 +525,7 @@ mod tests { #[test] fn test_get_col_from_key() { - let key = get_key_for_col(DBColumn::BeaconBlock.into(), &[1u8; 32]); + let key = get_key_for_col(DBColumn::BeaconBlock, &[1u8; 32]); let col = get_col_from_key(&key).unwrap(); assert_eq!(col, "blk"); } diff --git a/beacon_node/store/src/memory_store.rs b/beacon_node/store/src/memory_store.rs index 4c7bfdf10f..6070a2d3f0 100644 --- a/beacon_node/store/src/memory_store.rs +++ b/beacon_node/store/src/memory_store.rs @@ -1,9 +1,9 @@ use crate::{ - get_key_for_col, leveldb_store::BytesKey, ColumnIter, ColumnKeyIter, DBColumn, Error, - ItemStore, Key, KeyValueStore, KeyValueStoreOp, RawKeyIter, + errors::Error as DBError, get_key_for_col, hot_cold_store::BytesKey, ColumnIter, ColumnKeyIter, + DBColumn, Error, ItemStore, Key, KeyValueStore, KeyValueStoreOp, }; use parking_lot::{Mutex, MutexGuard, RwLock}; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashSet}; use std::marker::PhantomData; use types::*; @@ -29,19 +29,19 @@ impl MemoryStore { impl KeyValueStore for MemoryStore { /// Get the value of some key from the database. Returns `None` if the key does not exist. - fn get_bytes(&self, col: &str, key: &[u8]) -> Result>, Error> { + fn get_bytes(&self, col: DBColumn, key: &[u8]) -> Result>, Error> { let column_key = BytesKey::from_vec(get_key_for_col(col, key)); Ok(self.db.read().get(&column_key).cloned()) } /// Puts a key in the database. - fn put_bytes(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), Error> { + fn put_bytes(&self, col: DBColumn, key: &[u8], val: &[u8]) -> Result<(), Error> { let column_key = BytesKey::from_vec(get_key_for_col(col, key)); self.db.write().insert(column_key, val.to_vec()); Ok(()) } - fn put_bytes_sync(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), Error> { + fn put_bytes_sync(&self, col: DBColumn, key: &[u8], val: &[u8]) -> Result<(), Error> { self.put_bytes(col, key, val) } @@ -51,13 +51,13 @@ impl KeyValueStore for MemoryStore { } /// Return true if some key exists in some column. - fn key_exists(&self, col: &str, key: &[u8]) -> Result { + fn key_exists(&self, col: DBColumn, key: &[u8]) -> Result { let column_key = BytesKey::from_vec(get_key_for_col(col, key)); Ok(self.db.read().contains_key(&column_key)) } /// Delete some key from the database. - fn key_delete(&self, col: &str, key: &[u8]) -> Result<(), Error> { + fn key_delete(&self, col: DBColumn, key: &[u8]) -> Result<(), Error> { let column_key = BytesKey::from_vec(get_key_for_col(col, key)); self.db.write().remove(&column_key); Ok(()) @@ -66,12 +66,16 @@ impl KeyValueStore for MemoryStore { fn do_atomically(&self, batch: Vec) -> Result<(), Error> { for op in batch { match op { - KeyValueStoreOp::PutKeyValue(key, value) => { - self.db.write().insert(BytesKey::from_vec(key), value); + KeyValueStoreOp::PutKeyValue(col, key, value) => { + let column_key = get_key_for_col(col, &key); + self.db + .write() + .insert(BytesKey::from_vec(column_key), value); } - KeyValueStoreOp::DeleteKey(key) => { - self.db.write().remove(&BytesKey::from_vec(key)); + KeyValueStoreOp::DeleteKey(col, key) => { + let column_key = get_key_for_col(col, &key); + self.db.write().remove(&BytesKey::from_vec(column_key)); } } } @@ -82,8 +86,7 @@ impl KeyValueStore for MemoryStore { // We use this awkward pattern because we can't lock the `self.db` field *and* maintain a // reference to the lock guard across calls to `.next()`. This would be require a // struct with a field (the iterator) which references another field (the lock guard). - let start_key = BytesKey::from_vec(get_key_for_col(column.as_str(), from)); - let col = column.as_str(); + let start_key = BytesKey::from_vec(get_key_for_col(column, from)); let keys = self .db .read() @@ -92,7 +95,7 @@ impl KeyValueStore for MemoryStore { .filter_map(|(k, _)| k.remove_column_variable(column).map(|k| k.to_vec())) .collect::>(); Box::new(keys.into_iter().filter_map(move |key| { - self.get_bytes(col, &key).transpose().map(|res| { + self.get_bytes(column, &key).transpose().map(|res| { let k = K::from_bytes(&key)?; let v = res?; Ok((k, v)) @@ -100,18 +103,6 @@ impl KeyValueStore for MemoryStore { })) } - fn iter_raw_keys(&self, column: DBColumn, prefix: &[u8]) -> RawKeyIter { - let start_key = BytesKey::from_vec(get_key_for_col(column.as_str(), prefix)); - let keys = self - .db - .read() - .range(start_key.clone()..) - .take_while(|(k, _)| k.starts_with(&start_key)) - .filter_map(|(k, _)| k.remove_column_variable(column).map(|k| k.to_vec())) - .collect::>(); - Box::new(keys.into_iter().map(Ok)) - } - fn iter_column_keys(&self, column: DBColumn) -> ColumnKeyIter { Box::new(self.iter_column(column).map(|res| res.map(|(k, _)| k))) } @@ -123,6 +114,44 @@ impl KeyValueStore for MemoryStore { fn compact_column(&self, _column: DBColumn) -> Result<(), Error> { Ok(()) } + + fn iter_column_keys_from(&self, column: DBColumn, from: &[u8]) -> ColumnKeyIter { + // We use this awkward pattern because we can't lock the `self.db` field *and* maintain a + // reference to the lock guard across calls to `.next()`. This would be require a + // struct with a field (the iterator) which references another field (the lock guard). + let start_key = BytesKey::from_vec(get_key_for_col(column, from)); + let keys = self + .db + .read() + .range(start_key..) + .take_while(|(k, _)| k.remove_column_variable(column).is_some()) + .filter_map(|(k, _)| k.remove_column_variable(column).map(|k| k.to_vec())) + .collect::>(); + Box::new(keys.into_iter().map(move |key| K::from_bytes(&key))) + } + + fn delete_batch(&self, col: DBColumn, ops: HashSet<&[u8]>) -> Result<(), DBError> { + for op in ops { + let column_key = get_key_for_col(col, op); + self.db.write().remove(&BytesKey::from_vec(column_key)); + } + Ok(()) + } + + fn delete_if( + &self, + column: DBColumn, + mut f: impl FnMut(&[u8]) -> Result, + ) -> Result<(), Error> { + self.db.write().retain(|key, value| { + if key.remove_column_variable(column).is_some() { + !f(value).unwrap_or(false) + } else { + true + } + }); + Ok(()) + } } impl ItemStore for MemoryStore {} diff --git a/beacon_node/store/src/metadata.rs b/beacon_node/store/src/metadata.rs index 3f076a767a..1d70e105b9 100644 --- a/beacon_node/store/src/metadata.rs +++ b/beacon_node/store/src/metadata.rs @@ -225,10 +225,10 @@ impl StoreItem for BlobInfo { pub struct DataColumnInfo { /// The slot after which data columns are or *will be* available (>=). /// - /// If this slot is in the future, then it is the first slot of the EIP-7594 fork, from which + /// If this slot is in the future, then it is the first slot of the Fulu fork, from which /// data columns will be available. /// - /// If the `oldest_data_column_slot` is `None` then this means that the EIP-7594 fork epoch is + /// If the `oldest_data_column_slot` is `None` then this means that the Fulu fork epoch is /// not yet known. pub oldest_data_column_slot: Option, } diff --git a/beacon_node/store/src/metrics.rs b/beacon_node/store/src/metrics.rs index f0dd061790..6f9f667917 100644 --- a/beacon_node/store/src/metrics.rs +++ b/beacon_node/store/src/metrics.rs @@ -33,6 +33,13 @@ pub static DISK_DB_READ_BYTES: LazyLock> = LazyLock::new(| &["col"], ) }); +pub static DISK_DB_KEY_READ_BYTES: LazyLock> = LazyLock::new(|| { + try_create_int_counter_vec( + "store_disk_db_key_read_bytes_total", + "Number of key bytes read from the hot on-disk DB", + &["col"], + ) +}); pub static DISK_DB_READ_COUNT: LazyLock> = LazyLock::new(|| { try_create_int_counter_vec( "store_disk_db_read_count_total", @@ -40,6 +47,13 @@ pub static DISK_DB_READ_COUNT: LazyLock> = LazyLock::new(| &["col"], ) }); +pub static DISK_DB_KEY_READ_COUNT: LazyLock> = LazyLock::new(|| { + try_create_int_counter_vec( + "store_disk_db_read_count_total", + "Total number of key reads to the hot on-disk DB", + &["col"], + ) +}); pub static DISK_DB_WRITE_COUNT: LazyLock> = LazyLock::new(|| { try_create_int_counter_vec( "store_disk_db_write_count_total", @@ -66,6 +80,12 @@ pub static DISK_DB_EXISTS_COUNT: LazyLock> = LazyLock::new &["col"], ) }); +pub static DISK_DB_DELETE_TIMES: LazyLock> = LazyLock::new(|| { + try_create_histogram( + "store_disk_db_delete_seconds", + "Time taken to delete bytes from the store.", + ) +}); pub static DISK_DB_DELETE_COUNT: LazyLock> = LazyLock::new(|| { try_create_int_counter_vec( "store_disk_db_delete_count_total", @@ -73,6 +93,19 @@ pub static DISK_DB_DELETE_COUNT: LazyLock> = LazyLock::new &["col"], ) }); +pub static DISK_DB_COMPACT_TIMES: LazyLock> = LazyLock::new(|| { + try_create_histogram( + "store_disk_db_compact_seconds", + "Time taken to run compaction on the DB.", + ) +}); +pub static DISK_DB_TYPE: LazyLock> = LazyLock::new(|| { + try_create_int_counter_vec( + "store_disk_db_type", + "The on-disk database type being used", + &["db_type"], + ) +}); /* * Anchor Info */ diff --git a/beacon_node/store/src/partial_beacon_state.rs b/beacon_node/store/src/partial_beacon_state.rs index 22eecdcc60..d209512159 100644 --- a/beacon_node/store/src/partial_beacon_state.rs +++ b/beacon_node/store/src/partial_beacon_state.rs @@ -2,8 +2,8 @@ use crate::chunked_vector::{ load_variable_list_from_db, load_vector_from_db, BlockRootsChunked, HistoricalRoots, HistoricalSummaries, RandaoMixes, StateRootsChunked, }; -use crate::{Error, KeyValueStore}; -use ssz::{Decode, DecodeError}; +use crate::{DBColumn, Error, KeyValueStore, KeyValueStoreOp}; +use ssz::{Decode, DecodeError, Encode}; use ssz_derive::{Decode, Encode}; use std::sync::Arc; use types::historical_summary::HistoricalSummary; @@ -16,7 +16,7 @@ use types::*; /// /// This can be deleted once schema versions prior to V22 are no longer supported. #[superstruct( - variants(Base, Altair, Bellatrix, Capella, Deneb, Electra), + variants(Base, Altair, Bellatrix, Capella, Deneb, Electra, Fulu), variant_attributes(derive(Debug, PartialEq, Clone, Encode, Decode)) )] #[derive(Debug, PartialEq, Clone, Encode)] @@ -68,9 +68,9 @@ where pub current_epoch_attestations: List, E::MaxPendingAttestations>, // Participation (Altair and later) - #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra))] + #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra, Fulu))] pub previous_epoch_participation: List, - #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra))] + #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra, Fulu))] pub current_epoch_participation: List, // Finality @@ -80,13 +80,13 @@ where pub finalized_checkpoint: Checkpoint, // Inactivity - #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra))] + #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra, Fulu))] pub inactivity_scores: List, // Light-client sync committees - #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra))] + #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra, Fulu))] pub current_sync_committee: Arc>, - #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra))] + #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra, Fulu))] pub next_sync_committee: Arc>, // Execution @@ -110,37 +110,42 @@ where partial_getter(rename = "latest_execution_payload_header_electra") )] pub latest_execution_payload_header: ExecutionPayloadHeaderElectra, + #[superstruct( + only(Fulu), + partial_getter(rename = "latest_execution_payload_header_fulu") + )] + pub latest_execution_payload_header: ExecutionPayloadHeaderFulu, // Capella - #[superstruct(only(Capella, Deneb, Electra))] + #[superstruct(only(Capella, Deneb, Electra, Fulu))] pub next_withdrawal_index: u64, - #[superstruct(only(Capella, Deneb, Electra))] + #[superstruct(only(Capella, Deneb, Electra, Fulu))] pub next_withdrawal_validator_index: u64, #[ssz(skip_serializing, skip_deserializing)] - #[superstruct(only(Capella, Deneb, Electra))] + #[superstruct(only(Capella, Deneb, Electra, Fulu))] pub historical_summaries: Option>, // Electra - #[superstruct(only(Electra))] + #[superstruct(only(Electra, Fulu))] pub deposit_requests_start_index: u64, - #[superstruct(only(Electra))] + #[superstruct(only(Electra, Fulu))] pub deposit_balance_to_consume: u64, - #[superstruct(only(Electra))] + #[superstruct(only(Electra, Fulu))] pub exit_balance_to_consume: u64, - #[superstruct(only(Electra))] + #[superstruct(only(Electra, Fulu))] pub earliest_exit_epoch: Epoch, - #[superstruct(only(Electra))] + #[superstruct(only(Electra, Fulu))] pub consolidation_balance_to_consume: u64, - #[superstruct(only(Electra))] + #[superstruct(only(Electra, Fulu))] pub earliest_consolidation_epoch: Epoch, - #[superstruct(only(Electra))] + #[superstruct(only(Electra, Fulu))] pub pending_deposits: List, - #[superstruct(only(Electra))] + #[superstruct(only(Electra, Fulu))] pub pending_partial_withdrawals: List, - #[superstruct(only(Electra))] + #[superstruct(only(Electra, Fulu))] pub pending_consolidations: List, } @@ -167,6 +172,15 @@ impl PartialBeaconState { )) } + /// Prepare the partial state for storage in the KV database. + pub fn as_kv_store_op(&self, state_root: Hash256) -> KeyValueStoreOp { + KeyValueStoreOp::PutKeyValue( + DBColumn::BeaconState, + state_root.as_slice().to_vec(), + self.as_ssz_bytes(), + ) + } + pub fn load_block_roots>( &mut self, store: &S, @@ -409,6 +423,31 @@ impl TryInto> for PartialBeaconState { ], [historical_summaries] ), + PartialBeaconState::Fulu(inner) => impl_try_into_beacon_state!( + inner, + Fulu, + BeaconStateFulu, + [ + previous_epoch_participation, + current_epoch_participation, + current_sync_committee, + next_sync_committee, + inactivity_scores, + latest_execution_payload_header, + next_withdrawal_index, + next_withdrawal_validator_index, + deposit_requests_start_index, + deposit_balance_to_consume, + exit_balance_to_consume, + earliest_exit_epoch, + consolidation_balance_to_consume, + earliest_consolidation_epoch, + pending_deposits, + pending_partial_withdrawals, + pending_consolidations + ], + [historical_summaries] + ), }; Ok(state) } diff --git a/beacon_node/store/src/reconstruct.rs b/beacon_node/store/src/reconstruct.rs index 9bec83a35c..2a3b208aae 100644 --- a/beacon_node/store/src/reconstruct.rs +++ b/beacon_node/store/src/reconstruct.rs @@ -111,7 +111,7 @@ where self.store_cold_state(&state_root, &state, &mut io_batch)?; let batch_complete = - num_blocks.map_or(false, |n_blocks| slot == lower_limit_slot + n_blocks as u64); + num_blocks.is_some_and(|n_blocks| slot == lower_limit_slot + n_blocks as u64); let reconstruction_complete = slot + 1 == upper_limit_slot; // Commit the I/O batch if: diff --git a/beacon_node/store/src/state_cache.rs b/beacon_node/store/src/state_cache.rs index 5c1faa7f2f..96e4de4639 100644 --- a/beacon_node/store/src/state_cache.rs +++ b/beacon_node/store/src/state_cache.rs @@ -77,9 +77,7 @@ impl StateCache { if self .finalized_state .as_ref() - .map_or(false, |finalized_state| { - state.slot() < finalized_state.state.slot() - }) + .is_some_and(|finalized_state| state.slot() < finalized_state.state.slot()) { return Err(Error::FinalizedStateDecreasingSlot); } @@ -127,9 +125,7 @@ impl StateCache { if self .finalized_state .as_ref() - .map_or(false, |finalized_state| { - finalized_state.state_root == state_root - }) + .is_some_and(|finalized_state| finalized_state.state_root == state_root) { return Ok(PutStateOutcome::Finalized); } diff --git a/book/src/help_bn.md b/book/src/help_bn.md index a4ab44748c..2d12010094 100644 --- a/book/src/help_bn.md +++ b/book/src/help_bn.md @@ -11,6 +11,9 @@ Options: --auto-compact-db Enable or disable automatic compaction of the database on finalization. [default: true] + --beacon-node-backend + Set the database backend to be used by the beacon node. [possible + values: leveldb] --blob-prune-margin-epochs The margin for blob pruning in epochs. The oldest blobs are pruned up until data_availability_boundary - blob_prune_margin_epochs. [default: diff --git a/book/src/installation-source.md b/book/src/installation-source.md index 3c9f27d236..19098a5bc8 100644 --- a/book/src/installation-source.md +++ b/book/src/installation-source.md @@ -154,7 +154,7 @@ You can customise the features that Lighthouse is built with using the `FEATURES variable. E.g. ``` -FEATURES=gnosis,slasher-lmdb make +FEATURES=gnosis,slasher-lmdb,beacon-node-leveldb make ``` Commonly used features include: @@ -163,11 +163,12 @@ Commonly used features include: - `portable`: the default feature as Lighthouse now uses runtime detection of hardware CPU features. - `slasher-lmdb`: support for the LMDB slasher backend. Enabled by default. - `slasher-mdbx`: support for the MDBX slasher backend. +- `beacon-node-leveldb`: support for the leveldb backend. Enabled by default. - `jemalloc`: use [`jemalloc`][jemalloc] to allocate memory. Enabled by default on Linux and macOS. Not supported on Windows. - `spec-minimal`: support for the minimal preset (useful for testing). -Default features (e.g. `slasher-lmdb`) may be opted out of using the `--no-default-features` +Default features (e.g. `slasher-lmdb`, `beacon-node-leveldb`) may be opted out of using the `--no-default-features` argument for `cargo`, which can be plumbed in via the `CARGO_INSTALL_EXTRA_FLAGS` environment variable. E.g. diff --git a/common/account_utils/Cargo.toml b/common/account_utils/Cargo.toml index dece975d37..3ab6034688 100644 --- a/common/account_utils/Cargo.toml +++ b/common/account_utils/Cargo.toml @@ -6,7 +6,6 @@ edition = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -directory = { workspace = true } eth2_keystore = { workspace = true } eth2_wallet = { workspace = true } filesystem = { workspace = true } diff --git a/common/account_utils/src/validator_definitions.rs b/common/account_utils/src/validator_definitions.rs index a4850fc1c6..7337d6dfb4 100644 --- a/common/account_utils/src/validator_definitions.rs +++ b/common/account_utils/src/validator_definitions.rs @@ -4,13 +4,12 @@ //! attempt) to load into the `crate::intialized_validators::InitializedValidators` struct. use crate::{default_keystore_password_path, read_password_string, write_file_via_temporary}; -use directory::ensure_dir_exists; use eth2_keystore::Keystore; use regex::Regex; use serde::{Deserialize, Serialize}; use slog::{error, Logger}; use std::collections::HashSet; -use std::fs::{self, File}; +use std::fs::{self, create_dir_all, File}; use std::io; use std::path::{Path, PathBuf}; use types::{graffiti::GraffitiString, Address, PublicKey}; @@ -229,7 +228,7 @@ impl From> for ValidatorDefinitions { impl ValidatorDefinitions { /// Open an existing file or create a new, empty one if it does not exist. pub fn open_or_create>(validators_dir: P) -> Result { - ensure_dir_exists(validators_dir.as_ref()).map_err(|_| { + create_dir_all(validators_dir.as_ref()).map_err(|_| { Error::UnableToCreateValidatorDir(PathBuf::from(validators_dir.as_ref())) })?; let config_path = validators_dir.as_ref().join(CONFIG_FILENAME); @@ -435,7 +434,7 @@ pub fn recursively_find_voting_keystores>( && dir_entry .file_name() .to_str() - .map_or(false, is_voting_keystore) + .is_some_and(is_voting_keystore) { matches.push(dir_entry.path()) } diff --git a/common/directory/src/lib.rs b/common/directory/src/lib.rs index df03b4f9a4..d042f8dfad 100644 --- a/common/directory/src/lib.rs +++ b/common/directory/src/lib.rs @@ -1,6 +1,6 @@ use clap::ArgMatches; pub use eth2_network_config::DEFAULT_HARDCODED_NETWORK; -use std::fs::{self, create_dir_all}; +use std::fs; use std::path::{Path, PathBuf}; /// Names for the default directories. @@ -30,17 +30,6 @@ pub fn get_network_dir(matches: &ArgMatches) -> String { } } -/// Checks if a directory exists in the given path and creates a directory if it does not exist. -pub fn ensure_dir_exists>(path: P) -> Result<(), String> { - let path = path.as_ref(); - - if !path.exists() { - create_dir_all(path).map_err(|e| format!("Unable to create {:?}: {:?}", path, e))?; - } - - Ok(()) -} - /// If `arg` is in `matches`, parses the value as a path. /// /// Otherwise, attempts to find the default directory for the `testnet` from the `matches`. diff --git a/common/eth2/Cargo.toml b/common/eth2/Cargo.toml index 9d6dea100d..a1bc9d025b 100644 --- a/common/eth2/Cargo.toml +++ b/common/eth2/Cargo.toml @@ -7,6 +7,7 @@ edition = { workspace = true } [dependencies] derivative = { workspace = true } +either = { workspace = true } eth2_keystore = { workspace = true } ethereum_serde_utils = { workspace = true } ethereum_ssz = { workspace = true } @@ -31,10 +32,6 @@ zeroize = { workspace = true } [dev-dependencies] tokio = { workspace = true } -[target.'cfg(target_os = "linux")'.dependencies] -psutil = { version = "3.3.0", optional = true } -procfs = { version = "0.15.1", optional = true } - [features] default = ["lighthouse"] -lighthouse = ["psutil", "procfs"] +lighthouse = [] diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 9bb91b1543..b94115ced3 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -17,6 +17,7 @@ pub mod types; use self::mixin::{RequestAccept, ResponseOptional}; use self::types::{Error as ResponseError, *}; use derivative::Derivative; +use either::Either; use futures::Stream; use futures_util::StreamExt; use lighthouse_network::PeerId; @@ -1330,7 +1331,7 @@ impl BeaconNodeHttpClient { /// `POST v2/beacon/pool/attestations` pub async fn post_beacon_pool_attestations_v2( &self, - attestations: &[Attestation], + attestations: Either>, Vec>, fork_name: ForkName, ) -> Result<(), Error> { let mut path = self.eth_path(V2)?; @@ -1341,13 +1342,26 @@ impl BeaconNodeHttpClient { .push("pool") .push("attestations"); - self.post_with_timeout_and_consensus_header( - path, - &attestations, - self.timeouts.attestation, - fork_name, - ) - .await?; + match attestations { + Either::Right(attestations) => { + self.post_with_timeout_and_consensus_header( + path, + &attestations, + self.timeouts.attestation, + fork_name, + ) + .await?; + } + Either::Left(attestations) => { + self.post_with_timeout_and_consensus_header( + path, + &attestations, + self.timeouts.attestation, + fork_name, + ) + .await?; + } + }; Ok(()) } diff --git a/common/eth2/src/lighthouse.rs b/common/eth2/src/lighthouse.rs index 66dd5d779b..badc4857c4 100644 --- a/common/eth2/src/lighthouse.rs +++ b/common/eth2/src/lighthouse.rs @@ -88,12 +88,6 @@ pub struct ValidatorInclusionData { pub is_previous_epoch_head_attester: bool, } -#[cfg(target_os = "linux")] -use { - psutil::cpu::os::linux::CpuTimesExt, psutil::memory::os::linux::VirtualMemoryExt, - psutil::process::Process, -}; - /// Reports on the health of the Lighthouse instance. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct Health { @@ -164,69 +158,6 @@ pub struct SystemHealth { pub misc_os: String, } -impl SystemHealth { - #[cfg(not(target_os = "linux"))] - pub fn observe() -> Result { - Err("Health is only available on Linux".into()) - } - - #[cfg(target_os = "linux")] - pub fn observe() -> Result { - let vm = psutil::memory::virtual_memory() - .map_err(|e| format!("Unable to get virtual memory: {:?}", e))?; - let loadavg = - psutil::host::loadavg().map_err(|e| format!("Unable to get loadavg: {:?}", e))?; - - let cpu = - psutil::cpu::cpu_times().map_err(|e| format!("Unable to get cpu times: {:?}", e))?; - - let disk_usage = psutil::disk::disk_usage("/") - .map_err(|e| format!("Unable to disk usage info: {:?}", e))?; - - let disk = psutil::disk::DiskIoCountersCollector::default() - .disk_io_counters() - .map_err(|e| format!("Unable to get disk counters: {:?}", e))?; - - let net = psutil::network::NetIoCountersCollector::default() - .net_io_counters() - .map_err(|e| format!("Unable to get network io counters: {:?}", e))?; - - let boot_time = psutil::host::boot_time() - .map_err(|e| format!("Unable to get system boot time: {:?}", e))? - .duration_since(std::time::UNIX_EPOCH) - .map_err(|e| format!("Boot time is lower than unix epoch: {}", e))? - .as_secs(); - - Ok(Self { - sys_virt_mem_total: vm.total(), - sys_virt_mem_available: vm.available(), - sys_virt_mem_used: vm.used(), - sys_virt_mem_free: vm.free(), - sys_virt_mem_cached: vm.cached(), - sys_virt_mem_buffers: vm.buffers(), - sys_virt_mem_percent: vm.percent(), - sys_loadavg_1: loadavg.one, - sys_loadavg_5: loadavg.five, - sys_loadavg_15: loadavg.fifteen, - cpu_cores: psutil::cpu::cpu_count_physical(), - cpu_threads: psutil::cpu::cpu_count(), - system_seconds_total: cpu.system().as_secs(), - cpu_time_total: cpu.total().as_secs(), - user_seconds_total: cpu.user().as_secs(), - iowait_seconds_total: cpu.iowait().as_secs(), - idle_seconds_total: cpu.idle().as_secs(), - disk_node_bytes_total: disk_usage.total(), - disk_node_bytes_free: disk_usage.free(), - disk_node_reads_total: disk.read_count(), - disk_node_writes_total: disk.write_count(), - network_node_bytes_total_received: net.bytes_recv(), - network_node_bytes_total_transmit: net.bytes_sent(), - misc_node_boot_ts_seconds: boot_time, - misc_os: std::env::consts::OS.to_string(), - }) - } -} - /// Process specific health #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct ProcessHealth { @@ -244,59 +175,6 @@ pub struct ProcessHealth { pub pid_process_seconds_total: u64, } -impl ProcessHealth { - #[cfg(not(target_os = "linux"))] - pub fn observe() -> Result { - Err("Health is only available on Linux".into()) - } - - #[cfg(target_os = "linux")] - pub fn observe() -> Result { - let process = - Process::current().map_err(|e| format!("Unable to get current process: {:?}", e))?; - - let process_mem = process - .memory_info() - .map_err(|e| format!("Unable to get process memory info: {:?}", e))?; - - let me = procfs::process::Process::myself() - .map_err(|e| format!("Unable to get process: {:?}", e))?; - let stat = me - .stat() - .map_err(|e| format!("Unable to get stat: {:?}", e))?; - - let process_times = process - .cpu_times() - .map_err(|e| format!("Unable to get process cpu times : {:?}", e))?; - - Ok(Self { - pid: process.pid(), - pid_num_threads: stat.num_threads, - pid_mem_resident_set_size: process_mem.rss(), - pid_mem_virtual_memory_size: process_mem.vms(), - pid_mem_shared_memory_size: process_mem.shared(), - pid_process_seconds_total: process_times.busy().as_secs() - + process_times.children_system().as_secs() - + process_times.children_system().as_secs(), - }) - } -} - -impl Health { - #[cfg(not(target_os = "linux"))] - pub fn observe() -> Result { - Err("Health is only available on Linux".into()) - } - - #[cfg(target_os = "linux")] - pub fn observe() -> Result { - Ok(Self { - process: ProcessHealth::observe()?, - system: SystemHealth::observe()?, - }) - } -} - /// Indicates how up-to-date the Eth1 caches are. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct Eth1SyncStatusData { diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 68e6a927c7..202cd36f81 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -584,12 +584,20 @@ pub struct IdentityData { pub metadata: MetaData, } +#[superstruct( + variants(V2, V3), + variant_attributes(derive(Clone, Debug, PartialEq, Serialize, Deserialize)) +)] #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] pub struct MetaData { #[serde(with = "serde_utils::quoted_u64")] pub seq_number: u64, pub attnets: String, pub syncnets: String, + #[superstruct(only(V3))] + #[serde(with = "serde_utils::quoted_u64")] + pub custody_group_count: u64, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -1091,6 +1099,9 @@ impl ForkVersionDeserialize for SsePayloadAttributes { ForkName::Electra => serde_json::from_value(value) .map(Self::V3) .map_err(serde::de::Error::custom), + ForkName::Fulu => serde_json::from_value(value) + .map(Self::V3) + .map_err(serde::de::Error::custom), ForkName::Base | ForkName::Altair => Err(serde::de::Error::custom(format!( "SsePayloadAttributes deserialization for {fork_name} not implemented" ))), @@ -1123,6 +1134,7 @@ impl ForkVersionDeserialize for SseExtendedPayloadAttributes { #[serde(bound = "E: EthSpec", untagged)] pub enum EventKind { Attestation(Box>), + SingleAttestation(Box), Block(SseBlock), BlobSidecar(SseBlobSidecar), FinalizedCheckpoint(SseFinalizedCheckpoint), @@ -1149,6 +1161,7 @@ impl EventKind { EventKind::Block(_) => "block", EventKind::BlobSidecar(_) => "blob_sidecar", EventKind::Attestation(_) => "attestation", + EventKind::SingleAttestation(_) => "single_attestation", EventKind::VoluntaryExit(_) => "voluntary_exit", EventKind::FinalizedCheckpoint(_) => "finalized_checkpoint", EventKind::ChainReorg(_) => "chain_reorg", @@ -1171,6 +1184,11 @@ impl EventKind { "attestation" => Ok(EventKind::Attestation(serde_json::from_str(data).map_err( |e| ServerError::InvalidServerSentEvent(format!("Attestation: {:?}", e)), )?)), + "single_attestation" => Ok(EventKind::SingleAttestation( + serde_json::from_str(data).map_err(|e| { + ServerError::InvalidServerSentEvent(format!("SingleAttestation: {:?}", e)) + })?, + )), "block" => Ok(EventKind::Block(serde_json::from_str(data).map_err( |e| ServerError::InvalidServerSentEvent(format!("Block: {:?}", e)), )?)), @@ -1265,6 +1283,7 @@ pub enum EventTopic { Block, BlobSidecar, Attestation, + SingleAttestation, VoluntaryExit, FinalizedCheckpoint, ChainReorg, @@ -1290,6 +1309,7 @@ impl FromStr for EventTopic { "block" => Ok(EventTopic::Block), "blob_sidecar" => Ok(EventTopic::BlobSidecar), "attestation" => Ok(EventTopic::Attestation), + "single_attestation" => Ok(EventTopic::SingleAttestation), "voluntary_exit" => Ok(EventTopic::VoluntaryExit), "finalized_checkpoint" => Ok(EventTopic::FinalizedCheckpoint), "chain_reorg" => Ok(EventTopic::ChainReorg), @@ -1316,6 +1336,7 @@ impl fmt::Display for EventTopic { EventTopic::Block => write!(f, "block"), EventTopic::BlobSidecar => write!(f, "blob_sidecar"), EventTopic::Attestation => write!(f, "attestation"), + EventTopic::SingleAttestation => write!(f, "single_attestation"), EventTopic::VoluntaryExit => write!(f, "voluntary_exit"), EventTopic::FinalizedCheckpoint => write!(f, "finalized_checkpoint"), EventTopic::ChainReorg => write!(f, "chain_reorg"), @@ -1874,14 +1895,10 @@ impl PublishBlockRequest { impl TryFrom>> for PublishBlockRequest { type Error = &'static str; fn try_from(block: Arc>) -> Result { - match *block { - SignedBeaconBlock::Base(_) - | SignedBeaconBlock::Altair(_) - | SignedBeaconBlock::Bellatrix(_) - | SignedBeaconBlock::Capella(_) => Ok(PublishBlockRequest::Block(block)), - SignedBeaconBlock::Deneb(_) | SignedBeaconBlock::Electra(_) => Err( - "post-Deneb block contents cannot be fully constructed from just the signed block", - ), + if block.message().fork_name_unchecked().deneb_enabled() { + Err("post-Deneb block contents cannot be fully constructed from just the signed block") + } else { + Ok(PublishBlockRequest::Block(block)) } } } @@ -1985,16 +2002,18 @@ impl ForkVersionDeserialize for FullPayloadContents { value: Value, fork_name: ForkName, ) -> Result { - match fork_name { - ForkName::Bellatrix | ForkName::Capella => serde_json::from_value(value) - .map(Self::Payload) - .map_err(serde::de::Error::custom), - ForkName::Deneb | ForkName::Electra => serde_json::from_value(value) + if fork_name.deneb_enabled() { + serde_json::from_value(value) .map(Self::PayloadAndBlobs) - .map_err(serde::de::Error::custom), - ForkName::Base | ForkName::Altair => Err(serde::de::Error::custom(format!( + .map_err(serde::de::Error::custom) + } else if fork_name.bellatrix_enabled() { + serde_json::from_value(value) + .map(Self::Payload) + .map_err(serde::de::Error::custom) + } else { + Err(serde::de::Error::custom(format!( "FullPayloadContents deserialization for {fork_name} not implemented" - ))), + ))) } } } diff --git a/common/eth2_network_config/built_in_network_configs/chiado/config.yaml b/common/eth2_network_config/built_in_network_configs/chiado/config.yaml index 1eca01bbee..35ba3af28b 100644 --- a/common/eth2_network_config/built_in_network_configs/chiado/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/chiado/config.yaml @@ -15,7 +15,6 @@ TERMINAL_TOTAL_DIFFICULTY: 231707791542740786049188744689299064356246512 TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000 TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615 - # Genesis # --------------------------------------------------------------- # *CUSTOM @@ -27,7 +26,6 @@ GENESIS_FORK_VERSION: 0x0000006f # *CUSTOM GENESIS_DELAY: 300 - # Forking # --------------------------------------------------------------- # Some forks are disabled for now: @@ -49,6 +47,9 @@ DENEB_FORK_EPOCH: 516608 # Wed Jan 31 2024 18:15:40 GMT+0000 # Electra ELECTRA_FORK_VERSION: 0x0500006f ELECTRA_FORK_EPOCH: 18446744073709551615 +# Fulu +FULU_FORK_VERSION: 0x0600006f +FULU_FORK_EPOCH: 18446744073709551615 # Time parameters # --------------------------------------------------------------- @@ -63,7 +64,6 @@ SHARD_COMMITTEE_PERIOD: 256 # 2**10 (= 1024) ~1.4 hour ETH1_FOLLOW_DISTANCE: 1024 - # Validator cycle # --------------------------------------------------------------- # 2**2 (= 4) @@ -90,7 +90,6 @@ REORG_PARENT_WEIGHT_THRESHOLD: 160 # `2` epochs REORG_MAX_EPOCHS_SINCE_FINALIZATION: 2 - # Deposit contract # --------------------------------------------------------------- # xDai Mainnet @@ -136,9 +135,12 @@ MAX_REQUEST_BLOB_SIDECARS: 768 MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 16384 # `6` BLOB_SIDECAR_SUBNET_COUNT: 6 +# `uint64(6)` +MAX_BLOBS_PER_BLOCK: 6 # DAS -CUSTODY_REQUIREMENT: 4 -DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128 NUMBER_OF_COLUMNS: 128 -SAMPLES_PER_SLOT: 8 \ No newline at end of file +NUMBER_OF_CUSTODY_GROUPS: 128 +DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128 +SAMPLES_PER_SLOT: 8 +CUSTODY_REQUIREMENT: 4 diff --git a/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml b/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml index 500555a269..9ff5a16198 100644 --- a/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml @@ -23,7 +23,6 @@ GENESIS_FORK_VERSION: 0x00000064 # 6000 seconds (100 minutes) GENESIS_DELAY: 6000 - # Forking # --------------------------------------------------------------- # Some forks are disabled for now: @@ -45,7 +44,9 @@ DENEB_FORK_EPOCH: 889856 # 2024-03-11T18:30:20.000Z # Electra ELECTRA_FORK_VERSION: 0x05000064 ELECTRA_FORK_EPOCH: 18446744073709551615 - +# Fulu +FULU_FORK_VERSION: 0x06000064 +FULU_FORK_EPOCH: 18446744073709551615 # Time parameters # --------------------------------------------------------------- @@ -60,7 +61,6 @@ SHARD_COMMITTEE_PERIOD: 256 # 2**10 (= 1024) ~1.4 hour ETH1_FOLLOW_DISTANCE: 1024 - # Validator cycle # --------------------------------------------------------------- # 2**2 (= 4) @@ -76,7 +76,6 @@ MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: 2 # 2**12 (= 4096) CHURN_LIMIT_QUOTIENT: 4096 - # Fork choice # --------------------------------------------------------------- # 40% @@ -119,9 +118,12 @@ MAX_REQUEST_BLOB_SIDECARS: 768 MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 16384 # `6` BLOB_SIDECAR_SUBNET_COUNT: 6 +# `uint64(6)` +MAX_BLOBS_PER_BLOCK: 6 # DAS -CUSTODY_REQUIREMENT: 4 -DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128 NUMBER_OF_COLUMNS: 128 -SAMPLES_PER_SLOT: 8 \ No newline at end of file +NUMBER_OF_CUSTODY_GROUPS: 128 +DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128 +SAMPLES_PER_SLOT: 8 +CUSTODY_REQUIREMENT: 4 diff --git a/common/eth2_network_config/built_in_network_configs/holesky/config.yaml b/common/eth2_network_config/built_in_network_configs/holesky/config.yaml index d67d77d3be..d0b61422e0 100644 --- a/common/eth2_network_config/built_in_network_configs/holesky/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/holesky/config.yaml @@ -12,7 +12,6 @@ GENESIS_FORK_VERSION: 0x01017000 # Genesis delay 5 mins GENESIS_DELAY: 300 - # Forking # --------------------------------------------------------------- # Some forks are disabled for now: @@ -37,6 +36,9 @@ DENEB_FORK_EPOCH: 29696 # Electra ELECTRA_FORK_VERSION: 0x06017000 ELECTRA_FORK_EPOCH: 18446744073709551615 +# Fulu +FULU_FORK_VERSION: 0x07017000 +FULU_FORK_EPOCH: 18446744073709551615 # Time parameters # --------------------------------------------------------------- @@ -51,7 +53,6 @@ SHARD_COMMITTEE_PERIOD: 256 # 2**11 (= 2,048) Eth1 blocks ~8 hours ETH1_FOLLOW_DISTANCE: 2048 - # Validator cycle # --------------------------------------------------------------- # 2**2 (= 4) @@ -123,9 +124,12 @@ MAX_REQUEST_BLOB_SIDECARS: 768 MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 # `6` BLOB_SIDECAR_SUBNET_COUNT: 6 +# `uint64(6)` +MAX_BLOBS_PER_BLOCK: 6 # DAS -CUSTODY_REQUIREMENT: 4 -DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128 NUMBER_OF_COLUMNS: 128 -SAMPLES_PER_SLOT: 8 \ No newline at end of file +NUMBER_OF_CUSTODY_GROUPS: 128 +DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128 +SAMPLES_PER_SLOT: 8 +CUSTODY_REQUIREMENT: 4 diff --git a/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml b/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml index 18591fecdc..74fe727867 100644 --- a/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml @@ -18,8 +18,6 @@ TERMINAL_TOTAL_DIFFICULTY: 58750000000000000000000 TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000 TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615 - - # Genesis # --------------------------------------------------------------- # `2**14` (= 16,384) @@ -31,7 +29,6 @@ GENESIS_FORK_VERSION: 0x00000000 # 604800 seconds (7 days) GENESIS_DELAY: 604800 - # Forking # --------------------------------------------------------------- # Some forks are disabled for now: @@ -40,22 +37,22 @@ GENESIS_DELAY: 604800 # Altair ALTAIR_FORK_VERSION: 0x01000000 -ALTAIR_FORK_EPOCH: 74240 # Oct 27, 2021, 10:56:23am UTC +ALTAIR_FORK_EPOCH: 74240 # Oct 27, 2021, 10:56:23am UTC # Bellatrix BELLATRIX_FORK_VERSION: 0x02000000 -BELLATRIX_FORK_EPOCH: 144896 # Sept 6, 2022, 11:34:47am UTC +BELLATRIX_FORK_EPOCH: 144896 # Sept 6, 2022, 11:34:47am UTC # Capella CAPELLA_FORK_VERSION: 0x03000000 -CAPELLA_FORK_EPOCH: 194048 # April 12, 2023, 10:27:35pm UTC +CAPELLA_FORK_EPOCH: 194048 # April 12, 2023, 10:27:35pm UTC # Deneb DENEB_FORK_VERSION: 0x04000000 -DENEB_FORK_EPOCH: 269568 # March 13, 2024, 01:55:35pm UTC +DENEB_FORK_EPOCH: 269568 # March 13, 2024, 01:55:35pm UTC # Electra ELECTRA_FORK_VERSION: 0x05000000 ELECTRA_FORK_EPOCH: 18446744073709551615 -# PeerDAS -EIP7594_FORK_EPOCH: 18446744073709551615 - +# Fulu +FULU_FORK_VERSION: 0x06000000 +FULU_FORK_EPOCH: 18446744073709551615 # Time parameters # --------------------------------------------------------------- @@ -70,7 +67,6 @@ SHARD_COMMITTEE_PERIOD: 256 # 2**11 (= 2,048) Eth1 blocks ~8 hours ETH1_FOLLOW_DISTANCE: 2048 - # Validator cycle # --------------------------------------------------------------- # 2**2 (= 4) @@ -97,7 +93,6 @@ REORG_PARENT_WEIGHT_THRESHOLD: 160 # `2` epochs REORG_MAX_EPOCHS_SINCE_FINALIZATION: 2 - # Deposit contract # --------------------------------------------------------------- # Ethereum PoW Mainnet @@ -105,7 +100,6 @@ DEPOSIT_CHAIN_ID: 1 DEPOSIT_NETWORK_ID: 1 DEPOSIT_CONTRACT_ADDRESS: 0x00000000219ab540356cBB839Cbe05303d7705Fa - # Networking # --------------------------------------------------------------- # `10 * 2**20` (= 10485760, 10 MiB) @@ -145,9 +139,12 @@ MAX_REQUEST_BLOB_SIDECARS: 768 MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 # `6` BLOB_SIDECAR_SUBNET_COUNT: 6 +# `uint64(6)` +MAX_BLOBS_PER_BLOCK: 6 # DAS -CUSTODY_REQUIREMENT: 4 -DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128 NUMBER_OF_COLUMNS: 128 -SAMPLES_PER_SLOT: 8 \ No newline at end of file +NUMBER_OF_CUSTODY_GROUPS: 128 +DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128 +SAMPLES_PER_SLOT: 8 +CUSTODY_REQUIREMENT: 4 diff --git a/common/eth2_network_config/built_in_network_configs/sepolia/config.yaml b/common/eth2_network_config/built_in_network_configs/sepolia/config.yaml index b08a6180bf..7564d8f0f6 100644 --- a/common/eth2_network_config/built_in_network_configs/sepolia/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/sepolia/config.yaml @@ -119,9 +119,12 @@ MAX_REQUEST_BLOB_SIDECARS: 768 MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 # `6` BLOB_SIDECAR_SUBNET_COUNT: 6 +# `uint64(6)` +MAX_BLOBS_PER_BLOCK: 6 # DAS -CUSTODY_REQUIREMENT: 4 -DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128 NUMBER_OF_COLUMNS: 128 -SAMPLES_PER_SLOT: 8 \ No newline at end of file +NUMBER_OF_CUSTODY_GROUPS: 128 +DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128 +SAMPLES_PER_SLOT: 8 +CUSTODY_REQUIREMENT: 4 diff --git a/common/filesystem/Cargo.toml b/common/filesystem/Cargo.toml index fd026bd517..1b5abf03f4 100644 --- a/common/filesystem/Cargo.toml +++ b/common/filesystem/Cargo.toml @@ -3,7 +3,6 @@ name = "filesystem" version = "0.1.0" authors = ["Mark Mackey "] edition = { workspace = true } - # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] diff --git a/common/health_metrics/Cargo.toml b/common/health_metrics/Cargo.toml new file mode 100644 index 0000000000..08591471b2 --- /dev/null +++ b/common/health_metrics/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "health_metrics" +version = "0.1.0" +edition = { workspace = true } + +[dependencies] +eth2 = { workspace = true } +metrics = { workspace = true } + +[target.'cfg(target_os = "linux")'.dependencies] +psutil = "3.3.0" +procfs = "0.15.1" diff --git a/common/health_metrics/src/lib.rs b/common/health_metrics/src/lib.rs new file mode 100644 index 0000000000..bab80fb912 --- /dev/null +++ b/common/health_metrics/src/lib.rs @@ -0,0 +1,2 @@ +pub mod metrics; +pub mod observe; diff --git a/common/warp_utils/src/metrics.rs b/common/health_metrics/src/metrics.rs similarity index 99% rename from common/warp_utils/src/metrics.rs rename to common/health_metrics/src/metrics.rs index fabcf93650..c216426b7d 100644 --- a/common/warp_utils/src/metrics.rs +++ b/common/health_metrics/src/metrics.rs @@ -1,3 +1,4 @@ +use crate::observe::Observe; use eth2::lighthouse::{ProcessHealth, SystemHealth}; use metrics::*; use std::sync::LazyLock; diff --git a/common/health_metrics/src/observe.rs b/common/health_metrics/src/observe.rs new file mode 100644 index 0000000000..81bb8e6f7e --- /dev/null +++ b/common/health_metrics/src/observe.rs @@ -0,0 +1,127 @@ +use eth2::lighthouse::{Health, ProcessHealth, SystemHealth}; + +#[cfg(target_os = "linux")] +use { + psutil::cpu::os::linux::CpuTimesExt, psutil::memory::os::linux::VirtualMemoryExt, + psutil::process::Process, +}; + +pub trait Observe: Sized { + fn observe() -> Result; +} + +impl Observe for Health { + #[cfg(not(target_os = "linux"))] + fn observe() -> Result { + Err("Health is only available on Linux".into()) + } + + #[cfg(target_os = "linux")] + fn observe() -> Result { + Ok(Self { + process: ProcessHealth::observe()?, + system: SystemHealth::observe()?, + }) + } +} + +impl Observe for SystemHealth { + #[cfg(not(target_os = "linux"))] + fn observe() -> Result { + Err("Health is only available on Linux".into()) + } + + #[cfg(target_os = "linux")] + fn observe() -> Result { + let vm = psutil::memory::virtual_memory() + .map_err(|e| format!("Unable to get virtual memory: {:?}", e))?; + let loadavg = + psutil::host::loadavg().map_err(|e| format!("Unable to get loadavg: {:?}", e))?; + + let cpu = + psutil::cpu::cpu_times().map_err(|e| format!("Unable to get cpu times: {:?}", e))?; + + let disk_usage = psutil::disk::disk_usage("/") + .map_err(|e| format!("Unable to disk usage info: {:?}", e))?; + + let disk = psutil::disk::DiskIoCountersCollector::default() + .disk_io_counters() + .map_err(|e| format!("Unable to get disk counters: {:?}", e))?; + + let net = psutil::network::NetIoCountersCollector::default() + .net_io_counters() + .map_err(|e| format!("Unable to get network io counters: {:?}", e))?; + + let boot_time = psutil::host::boot_time() + .map_err(|e| format!("Unable to get system boot time: {:?}", e))? + .duration_since(std::time::UNIX_EPOCH) + .map_err(|e| format!("Boot time is lower than unix epoch: {}", e))? + .as_secs(); + + Ok(Self { + sys_virt_mem_total: vm.total(), + sys_virt_mem_available: vm.available(), + sys_virt_mem_used: vm.used(), + sys_virt_mem_free: vm.free(), + sys_virt_mem_cached: vm.cached(), + sys_virt_mem_buffers: vm.buffers(), + sys_virt_mem_percent: vm.percent(), + sys_loadavg_1: loadavg.one, + sys_loadavg_5: loadavg.five, + sys_loadavg_15: loadavg.fifteen, + cpu_cores: psutil::cpu::cpu_count_physical(), + cpu_threads: psutil::cpu::cpu_count(), + system_seconds_total: cpu.system().as_secs(), + cpu_time_total: cpu.total().as_secs(), + user_seconds_total: cpu.user().as_secs(), + iowait_seconds_total: cpu.iowait().as_secs(), + idle_seconds_total: cpu.idle().as_secs(), + disk_node_bytes_total: disk_usage.total(), + disk_node_bytes_free: disk_usage.free(), + disk_node_reads_total: disk.read_count(), + disk_node_writes_total: disk.write_count(), + network_node_bytes_total_received: net.bytes_recv(), + network_node_bytes_total_transmit: net.bytes_sent(), + misc_node_boot_ts_seconds: boot_time, + misc_os: std::env::consts::OS.to_string(), + }) + } +} + +impl Observe for ProcessHealth { + #[cfg(not(target_os = "linux"))] + fn observe() -> Result { + Err("Health is only available on Linux".into()) + } + + #[cfg(target_os = "linux")] + fn observe() -> Result { + let process = + Process::current().map_err(|e| format!("Unable to get current process: {:?}", e))?; + + let process_mem = process + .memory_info() + .map_err(|e| format!("Unable to get process memory info: {:?}", e))?; + + let me = procfs::process::Process::myself() + .map_err(|e| format!("Unable to get process: {:?}", e))?; + let stat = me + .stat() + .map_err(|e| format!("Unable to get stat: {:?}", e))?; + + let process_times = process + .cpu_times() + .map_err(|e| format!("Unable to get process cpu times : {:?}", e))?; + + Ok(Self { + pid: process.pid(), + pid_num_threads: stat.num_threads, + pid_mem_resident_set_size: process_mem.rss(), + pid_mem_virtual_memory_size: process_mem.vms(), + pid_mem_shared_memory_size: process_mem.shared(), + pid_process_seconds_total: process_times.busy().as_secs() + + process_times.children_system().as_secs() + + process_times.children_system().as_secs(), + }) + } +} diff --git a/common/lighthouse_version/src/lib.rs b/common/lighthouse_version/src/lib.rs index 0751bdadff..a35b8c42c1 100644 --- a/common/lighthouse_version/src/lib.rs +++ b/common/lighthouse_version/src/lib.rs @@ -48,6 +48,22 @@ pub fn version_with_platform() -> String { format!("{}/{}-{}", VERSION, Target::arch(), Target::os()) } +/// Returns semantic versioning information only. +/// +/// ## Example +/// +/// `1.5.1` +pub fn version() -> &'static str { + "6.0.1" +} + +/// Returns the name of the current client running. +/// +/// This will usually be "Lighthouse" +pub fn client_name() -> &'static str { + "Lighthouse" +} + #[cfg(test)] mod test { use super::*; @@ -64,4 +80,14 @@ mod test { VERSION ); } + + #[test] + fn semantic_version_formatting() { + let re = Regex::new(r"^[0-9]+\.[0-9]+\.[0-9]+").unwrap(); + assert!( + re.is_match(version()), + "semantic version doesn't match regex: {}", + version() + ); + } } diff --git a/common/logging/src/lib.rs b/common/logging/src/lib.rs index 7fe7f79506..0ddd867c2f 100644 --- a/common/logging/src/lib.rs +++ b/common/logging/src/lib.rs @@ -204,7 +204,7 @@ impl TimeLatch { pub fn elapsed(&mut self) -> bool { let now = Instant::now(); - let is_elapsed = self.0.map_or(false, |elapse_time| now > elapse_time); + let is_elapsed = self.0.is_some_and(|elapse_time| now > elapse_time); if is_elapsed || self.0.is_none() { self.0 = Some(now + LOG_DEBOUNCE_INTERVAL); diff --git a/common/malloc_utils/src/jemalloc.rs b/common/malloc_utils/src/jemalloc.rs index 0e2e00cb0e..f3a35fc41c 100644 --- a/common/malloc_utils/src/jemalloc.rs +++ b/common/malloc_utils/src/jemalloc.rs @@ -9,7 +9,7 @@ //! B) `_RJEM_MALLOC_CONF` at runtime. use metrics::{set_gauge, try_create_int_gauge, IntGauge}; use std::sync::LazyLock; -use tikv_jemalloc_ctl::{arenas, epoch, stats, Error}; +use tikv_jemalloc_ctl::{arenas, epoch, stats, Access, AsName, Error}; #[global_allocator] static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; @@ -52,3 +52,18 @@ pub fn scrape_jemalloc_metrics_fallible() -> Result<(), Error> { Ok(()) } + +pub fn page_size() -> Result { + // Full list of keys: https://jemalloc.net/jemalloc.3.html + "arenas.page\0".name().read() +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn page_size_ok() { + assert!(page_size().is_ok()); + } +} diff --git a/common/malloc_utils/src/lib.rs b/common/malloc_utils/src/lib.rs index 3bb242369f..50d2785a74 100644 --- a/common/malloc_utils/src/lib.rs +++ b/common/malloc_utils/src/lib.rs @@ -29,10 +29,10 @@ not(target_env = "musl"), not(feature = "jemalloc") ))] -mod glibc; +pub mod glibc; #[cfg(feature = "jemalloc")] -mod jemalloc; +pub mod jemalloc; pub use interface::*; diff --git a/common/monitoring_api/Cargo.toml b/common/monitoring_api/Cargo.toml index 5008c86e85..cb52cff29a 100644 --- a/common/monitoring_api/Cargo.toml +++ b/common/monitoring_api/Cargo.toml @@ -7,6 +7,7 @@ edition = { workspace = true } [dependencies] eth2 = { workspace = true } +health_metrics = { workspace = true } lighthouse_version = { workspace = true } metrics = { workspace = true } regex = { workspace = true } diff --git a/common/monitoring_api/src/gather.rs b/common/monitoring_api/src/gather.rs index 2f6c820f56..43bea35a93 100644 --- a/common/monitoring_api/src/gather.rs +++ b/common/monitoring_api/src/gather.rs @@ -1,4 +1,5 @@ use super::types::{BeaconProcessMetrics, ValidatorProcessMetrics}; +use health_metrics::observe::Observe; use metrics::{MetricFamily, MetricType}; use serde_json::json; use std::collections::HashMap; diff --git a/common/monitoring_api/src/lib.rs b/common/monitoring_api/src/lib.rs index 9592c50a40..6f919971b0 100644 --- a/common/monitoring_api/src/lib.rs +++ b/common/monitoring_api/src/lib.rs @@ -4,6 +4,7 @@ use std::{path::PathBuf, time::Duration}; use eth2::lighthouse::SystemHealth; use gather::{gather_beacon_metrics, gather_validator_metrics}; +use health_metrics::observe::Observe; use reqwest::{IntoUrl, Response}; pub use reqwest::{StatusCode, Url}; use sensitive_url::SensitiveUrl; diff --git a/common/validator_dir/Cargo.toml b/common/validator_dir/Cargo.toml index 773431c93c..4c03b7662e 100644 --- a/common/validator_dir/Cargo.toml +++ b/common/validator_dir/Cargo.toml @@ -12,7 +12,6 @@ insecure_keys = [] bls = { workspace = true } deposit_contract = { workspace = true } derivative = { workspace = true } -directory = { workspace = true } eth2_keystore = { workspace = true } filesystem = { workspace = true } hex = { workspace = true } diff --git a/common/validator_dir/src/builder.rs b/common/validator_dir/src/builder.rs index 3d5d149608..2e971a8b1a 100644 --- a/common/validator_dir/src/builder.rs +++ b/common/validator_dir/src/builder.rs @@ -1,7 +1,6 @@ use crate::{Error as DirError, ValidatorDir}; use bls::get_withdrawal_credentials; use deposit_contract::{encode_eth1_tx_data, Error as DepositError}; -use directory::ensure_dir_exists; use eth2_keystore::{Error as KeystoreError, Keystore, KeystoreBuilder, PlainText}; use filesystem::create_with_600_perms; use rand::{distributions::Alphanumeric, Rng}; @@ -42,7 +41,7 @@ pub enum Error { #[cfg(feature = "insecure_keys")] InsecureKeysError(String), MissingPasswordDir, - UnableToCreatePasswordDir(String), + UnableToCreatePasswordDir(io::Error), } impl From for Error { @@ -163,7 +162,7 @@ impl<'a> Builder<'a> { } if let Some(password_dir) = &self.password_dir { - ensure_dir_exists(password_dir).map_err(Error::UnableToCreatePasswordDir)?; + create_dir_all(password_dir).map_err(Error::UnableToCreatePasswordDir)?; } // The withdrawal keystore must be initialized in order to store it or create an eth1 diff --git a/common/warp_utils/Cargo.toml b/common/warp_utils/Cargo.toml index 4a3cde54a9..ec2d23686b 100644 --- a/common/warp_utils/Cargo.toml +++ b/common/warp_utils/Cargo.toml @@ -6,7 +6,6 @@ edition = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -beacon_chain = { workspace = true } bytes = { workspace = true } eth2 = { workspace = true } headers = "0.3.2" @@ -15,7 +14,6 @@ safe_arith = { workspace = true } serde = { workspace = true } serde_array_query = "0.1.0" serde_json = { workspace = true } -state_processing = { workspace = true } tokio = { workspace = true } types = { workspace = true } warp = { workspace = true } diff --git a/common/warp_utils/src/lib.rs b/common/warp_utils/src/lib.rs index 55ee423fa4..c10adbac0d 100644 --- a/common/warp_utils/src/lib.rs +++ b/common/warp_utils/src/lib.rs @@ -3,7 +3,6 @@ pub mod cors; pub mod json; -pub mod metrics; pub mod query; pub mod reject; pub mod task; diff --git a/common/warp_utils/src/reject.rs b/common/warp_utils/src/reject.rs index bbd5274a7e..3c7ef5e4fa 100644 --- a/common/warp_utils/src/reject.rs +++ b/common/warp_utils/src/reject.rs @@ -2,6 +2,7 @@ use eth2::types::{ErrorMessage, Failure, IndexedErrorMessage}; use std::convert::Infallible; use std::error::Error; use std::fmt; +use std::fmt::Debug; use warp::{http::StatusCode, reject::Reject, reply::Response, Reply}; #[derive(Debug)] @@ -19,15 +20,6 @@ pub fn server_sent_event_error(s: String) -> ServerSentEventError { ServerSentEventError(s) } -#[derive(Debug)] -pub struct BeaconChainError(pub beacon_chain::BeaconChainError); - -impl Reject for BeaconChainError {} - -pub fn beacon_chain_error(e: beacon_chain::BeaconChainError) -> warp::reject::Rejection { - warp::reject::custom(BeaconChainError(e)) -} - #[derive(Debug)] pub struct BeaconStateError(pub types::BeaconStateError); @@ -47,21 +39,12 @@ pub fn arith_error(e: safe_arith::ArithError) -> warp::reject::Rejection { } #[derive(Debug)] -pub struct SlotProcessingError(pub state_processing::SlotProcessingError); +pub struct UnhandledError(pub Box); -impl Reject for SlotProcessingError {} +impl Reject for UnhandledError {} -pub fn slot_processing_error(e: state_processing::SlotProcessingError) -> warp::reject::Rejection { - warp::reject::custom(SlotProcessingError(e)) -} - -#[derive(Debug)] -pub struct BlockProductionError(pub beacon_chain::BlockProductionError); - -impl Reject for BlockProductionError {} - -pub fn block_production_error(e: beacon_chain::BlockProductionError) -> warp::reject::Rejection { - warp::reject::custom(BlockProductionError(e)) +pub fn unhandled_error(e: D) -> warp::reject::Rejection { + warp::reject::custom(UnhandledError(Box::new(e))) } #[derive(Debug)] @@ -191,16 +174,7 @@ pub async fn handle_rejection(err: warp::Rejection) -> Result() { code = StatusCode::BAD_REQUEST; message = format!("BAD_REQUEST: invalid query: {}", e); - } else if let Some(e) = err.find::() { - code = StatusCode::INTERNAL_SERVER_ERROR; - message = format!("UNHANDLED_ERROR: {:?}", e.0); - } else if let Some(e) = err.find::() { - code = StatusCode::INTERNAL_SERVER_ERROR; - message = format!("UNHANDLED_ERROR: {:?}", e.0); - } else if let Some(e) = err.find::() { - code = StatusCode::INTERNAL_SERVER_ERROR; - message = format!("UNHANDLED_ERROR: {:?}", e.0); - } else if let Some(e) = err.find::() { + } else if let Some(e) = err.find::() { code = StatusCode::INTERNAL_SERVER_ERROR; message = format!("UNHANDLED_ERROR: {:?}", e.0); } else if let Some(e) = err.find::() { diff --git a/consensus/fork_choice/src/fork_choice.rs b/consensus/fork_choice/src/fork_choice.rs index dc6ea98240..fcf6e65460 100644 --- a/consensus/fork_choice/src/fork_choice.rs +++ b/consensus/fork_choice/src/fork_choice.rs @@ -762,20 +762,15 @@ where if let Some((parent_justified, parent_finalized)) = parent_checkpoints { (parent_justified, parent_finalized) } else { - let justification_and_finalization_state = match block { - BeaconBlockRef::Electra(_) - | BeaconBlockRef::Deneb(_) - | BeaconBlockRef::Capella(_) - | BeaconBlockRef::Bellatrix(_) - | BeaconBlockRef::Altair(_) => { + let justification_and_finalization_state = + if block.fork_name_unchecked().altair_enabled() { // NOTE: Processing justification & finalization requires the progressive // balances cache, but we cannot initialize it here as we only have an // immutable reference. The state *should* have come straight from block // processing, which initialises the cache, but if we add other `on_block` // calls in future it could be worth passing a mutable reference. per_epoch_processing::altair::process_justification_and_finalization(state)? - } - BeaconBlockRef::Base(_) => { + } else { let mut validator_statuses = per_epoch_processing::base::ValidatorStatuses::new(state, spec) .map_err(Error::ValidatorStatuses)?; @@ -787,8 +782,7 @@ where &validator_statuses.total_balances, spec, )? - } - }; + }; ( justification_and_finalization_state.current_justified_checkpoint(), diff --git a/consensus/fork_choice/tests/tests.rs b/consensus/fork_choice/tests/tests.rs index ef017159a0..b224cde048 100644 --- a/consensus/fork_choice/tests/tests.rs +++ b/consensus/fork_choice/tests/tests.rs @@ -10,6 +10,7 @@ use beacon_chain::{ use fork_choice::{ ForkChoiceStore, InvalidAttestation, InvalidBlock, PayloadVerificationStatus, QueuedAttestation, }; +use state_processing::state_advance::complete_state_advance; use std::fmt; use std::sync::Mutex; use std::time::Duration; @@ -53,7 +54,7 @@ impl ForkChoiceTest { /// Creates a new tester with a custom chain config. pub fn new_with_chain_config(chain_config: ChainConfig) -> Self { // Run fork choice tests against the latest fork. - let spec = ForkName::latest().make_genesis_spec(ChainSpec::default()); + let spec = ForkName::latest_stable().make_genesis_spec(ChainSpec::default()); let harness = BeaconChainHarness::builder(MainnetEthSpec) .spec(spec.into()) .chain_config(chain_config) @@ -172,6 +173,20 @@ impl ForkChoiceTest { let validators = self.harness.get_all_validators(); loop { let slot = self.harness.get_current_slot(); + + // Skip slashed proposers, as we expect validators to get slashed in these tests. + // Presently `make_block` will panic if the proposer is slashed, so we just avoid + // calling it in this case. + complete_state_advance(&mut state, None, slot, &self.harness.spec).unwrap(); + state.build_caches(&self.harness.spec).unwrap(); + let proposer_index = state + .get_beacon_proposer_index(slot, &self.harness.chain.spec) + .unwrap(); + if state.validators().get(proposer_index).unwrap().slashed { + self.harness.advance_slot(); + continue; + } + let (block_contents, state_) = self.harness.make_block(state, slot).await; state = state_; if !predicate(block_contents.0.message(), &state) { @@ -196,17 +211,20 @@ impl ForkChoiceTest { } /// Apply `count` blocks to the chain (with attestations). + /// + /// Note that in the case of slashed validators, their proposals will be skipped and the chain + /// may be advanced by *more than* `count` slots. pub async fn apply_blocks(self, count: usize) -> Self { - self.harness.advance_slot(); - self.harness - .extend_chain( - count, - BlockStrategy::OnCanonicalHead, - AttestationStrategy::AllValidators, - ) - .await; - - self + // Use `Self::apply_blocks_while` which gracefully handles slashed validators. + let mut blocks_applied = 0; + self.apply_blocks_while(|_, _| { + // Blocks are applied after the predicate is called, so continue applying the block if + // less than *or equal* to the count. + blocks_applied += 1; + blocks_applied <= count + }) + .await + .unwrap() } /// Slash a validator from the previous epoch committee. @@ -244,6 +262,7 @@ impl ForkChoiceTest { /// Apply `count` blocks to the chain (without attestations). pub async fn apply_blocks_without_new_attestations(self, count: usize) -> Self { + // This function does not gracefully handle slashed proposers, but may need to in future. self.harness.advance_slot(); self.harness .extend_chain( @@ -1226,14 +1245,6 @@ async fn progressive_balances_cache_attester_slashing() { .apply_blocks_while(|_, state| state.finalized_checkpoint().epoch == 0) .await .unwrap() - // Note: This test may fail if the shuffling used changes, right now it re-runs with - // deterministic shuffling. A shuffling change my cause the slashed proposer to propose - // again in the next epoch, which results in a block processing failure - // (`HeaderInvalid::ProposerSlashed`). The harness should be re-worked to successfully skip - // the slot in this scenario rather than panic-ing. The same applies to - // `progressive_balances_cache_proposer_slashing`. - .apply_blocks(2) - .await .add_previous_epoch_attester_slashing() .await // expect fork choice to import blocks successfully after a previous epoch attester is @@ -1244,7 +1255,7 @@ async fn progressive_balances_cache_attester_slashing() { // expect fork choice to import another epoch of blocks successfully - the slashed // attester's balance should be excluded from the current epoch total balance in // `ProgressiveBalancesCache` as well. - .apply_blocks(MainnetEthSpec::slots_per_epoch() as usize) + .apply_blocks(E::slots_per_epoch() as usize) .await; } @@ -1257,15 +1268,7 @@ async fn progressive_balances_cache_proposer_slashing() { .apply_blocks_while(|_, state| state.finalized_checkpoint().epoch == 0) .await .unwrap() - // Note: This test may fail if the shuffling used changes, right now it re-runs with - // deterministic shuffling. A shuffling change may cause the slashed proposer to propose - // again in the next epoch, which results in a block processing failure - // (`HeaderInvalid::ProposerSlashed`). The harness should be re-worked to successfully skip - // the slot in this scenario rather than panic-ing. The same applies to - // `progressive_balances_cache_attester_slashing`. - .apply_blocks(2) - .await - .add_previous_epoch_proposer_slashing(MainnetEthSpec::slots_per_epoch()) + .add_previous_epoch_proposer_slashing(E::slots_per_epoch()) .await // expect fork choice to import blocks successfully after a previous epoch proposer is // slashed, i.e. the slashed proposer's balance is correctly excluded from @@ -1275,6 +1278,6 @@ async fn progressive_balances_cache_proposer_slashing() { // expect fork choice to import another epoch of blocks successfully - the slashed // proposer's balance should be excluded from the current epoch total balance in // `ProgressiveBalancesCache` as well. - .apply_blocks(MainnetEthSpec::slots_per_epoch() as usize) + .apply_blocks(E::slots_per_epoch() as usize) .await; } diff --git a/consensus/merkle_proof/Cargo.toml b/consensus/merkle_proof/Cargo.toml index c2c6bf270a..2f721d917b 100644 --- a/consensus/merkle_proof/Cargo.toml +++ b/consensus/merkle_proof/Cargo.toml @@ -7,8 +7,8 @@ edition = { workspace = true } [dependencies] alloy-primitives = { workspace = true } ethereum_hashing = { workspace = true } -safe_arith = { workspace = true } fixed_bytes = { workspace = true } +safe_arith = { workspace = true } [dev-dependencies] quickcheck = { workspace = true } diff --git a/consensus/proto_array/src/proto_array.rs b/consensus/proto_array/src/proto_array.rs index 9cb415839f..6480a84d8c 100644 --- a/consensus/proto_array/src/proto_array.rs +++ b/consensus/proto_array/src/proto_array.rs @@ -473,7 +473,7 @@ impl ProtoArray { // 1. The `head_block_root` is a descendant of `latest_valid_ancestor_hash` // 2. The `latest_valid_ancestor_hash` is equal to or a descendant of the finalized block. let latest_valid_ancestor_is_descendant = - latest_valid_ancestor_root.map_or(false, |ancestor_root| { + latest_valid_ancestor_root.is_some_and(|ancestor_root| { self.is_descendant(ancestor_root, head_block_root) && self.is_finalized_checkpoint_or_descendant::(ancestor_root) }); @@ -510,13 +510,13 @@ impl ProtoArray { // head. if node .best_child - .map_or(false, |i| invalidated_indices.contains(&i)) + .is_some_and(|i| invalidated_indices.contains(&i)) { node.best_child = None } if node .best_descendant - .map_or(false, |i| invalidated_indices.contains(&i)) + .is_some_and(|i| invalidated_indices.contains(&i)) { node.best_descendant = None } @@ -1008,7 +1008,7 @@ impl ProtoArray { node.unrealized_finalized_checkpoint, node.unrealized_justified_checkpoint, ] { - if checkpoint.map_or(false, |cp| cp == self.finalized_checkpoint) { + if checkpoint.is_some_and(|cp| cp == self.finalized_checkpoint) { return true; } } @@ -1046,7 +1046,7 @@ impl ProtoArray { .find(|node| { node.execution_status .block_hash() - .map_or(false, |node_block_hash| node_block_hash == *block_hash) + .is_some_and(|node_block_hash| node_block_hash == *block_hash) }) .map(|node| node.root) } diff --git a/consensus/state_processing/src/common/get_attestation_participation.rs b/consensus/state_processing/src/common/get_attestation_participation.rs index fc09dad1f4..2c6fd3b215 100644 --- a/consensus/state_processing/src/common/get_attestation_participation.rs +++ b/consensus/state_processing/src/common/get_attestation_participation.rs @@ -44,22 +44,15 @@ pub fn get_attestation_participation_flag_indices( if is_matching_source && inclusion_delay <= E::slots_per_epoch().integer_sqrt() { participation_flag_indices.push(TIMELY_SOURCE_FLAG_INDEX); } - match state { - &BeaconState::Base(_) - | &BeaconState::Altair(_) - | &BeaconState::Bellatrix(_) - | &BeaconState::Capella(_) => { - if is_matching_target && inclusion_delay <= E::slots_per_epoch() { - participation_flag_indices.push(TIMELY_TARGET_FLAG_INDEX); - } - } - &BeaconState::Deneb(_) | &BeaconState::Electra(_) => { - if is_matching_target { - // [Modified in Deneb:EIP7045] - participation_flag_indices.push(TIMELY_TARGET_FLAG_INDEX); - } + if state.fork_name_unchecked().deneb_enabled() { + if is_matching_target { + // [Modified in Deneb:EIP7045] + participation_flag_indices.push(TIMELY_TARGET_FLAG_INDEX); } + } else if is_matching_target && inclusion_delay <= E::slots_per_epoch() { + participation_flag_indices.push(TIMELY_TARGET_FLAG_INDEX); } + if is_matching_head && inclusion_delay == spec.min_attestation_inclusion_delay { participation_flag_indices.push(TIMELY_HEAD_FLAG_INDEX); } diff --git a/consensus/state_processing/src/common/slash_validator.rs b/consensus/state_processing/src/common/slash_validator.rs index 80d857cc00..bd60f16014 100644 --- a/consensus/state_processing/src/common/slash_validator.rs +++ b/consensus/state_processing/src/common/slash_validator.rs @@ -55,15 +55,12 @@ pub fn slash_validator( let whistleblower_index = opt_whistleblower_index.unwrap_or(proposer_index); let whistleblower_reward = validator_effective_balance .safe_div(spec.whistleblower_reward_quotient_for_state(state))?; - let proposer_reward = match state { - BeaconState::Base(_) => whistleblower_reward.safe_div(spec.proposer_reward_quotient)?, - BeaconState::Altair(_) - | BeaconState::Bellatrix(_) - | BeaconState::Capella(_) - | BeaconState::Deneb(_) - | BeaconState::Electra(_) => whistleblower_reward + let proposer_reward = if state.fork_name_unchecked().altair_enabled() { + whistleblower_reward .safe_mul(PROPOSER_WEIGHT)? - .safe_div(WEIGHT_DENOMINATOR)?, + .safe_div(WEIGHT_DENOMINATOR)? + } else { + whistleblower_reward.safe_div(spec.proposer_reward_quotient)? }; // Ensure the whistleblower index is in the validator registry. diff --git a/consensus/state_processing/src/genesis.rs b/consensus/state_processing/src/genesis.rs index 00697def5d..10723ecc51 100644 --- a/consensus/state_processing/src/genesis.rs +++ b/consensus/state_processing/src/genesis.rs @@ -4,7 +4,7 @@ use super::per_block_processing::{ use crate::common::DepositDataTree; use crate::upgrade::electra::upgrade_state_to_electra; use crate::upgrade::{ - upgrade_to_altair, upgrade_to_bellatrix, upgrade_to_capella, upgrade_to_deneb, + upgrade_to_altair, upgrade_to_bellatrix, upgrade_to_capella, upgrade_to_deneb, upgrade_to_fulu, }; use safe_arith::{ArithError, SafeArith}; use std::sync::Arc; @@ -53,7 +53,7 @@ pub fn initialize_beacon_state_from_eth1( // https://github.com/ethereum/eth2.0-specs/pull/2323 if spec .altair_fork_epoch - .map_or(false, |fork_epoch| fork_epoch == E::genesis_epoch()) + .is_some_and(|fork_epoch| fork_epoch == E::genesis_epoch()) { upgrade_to_altair(&mut state, spec)?; @@ -63,7 +63,7 @@ pub fn initialize_beacon_state_from_eth1( // Similarly, perform an upgrade to the merge if configured from genesis. if spec .bellatrix_fork_epoch - .map_or(false, |fork_epoch| fork_epoch == E::genesis_epoch()) + .is_some_and(|fork_epoch| fork_epoch == E::genesis_epoch()) { // this will set state.latest_execution_payload_header = ExecutionPayloadHeaderBellatrix::default() upgrade_to_bellatrix(&mut state, spec)?; @@ -81,7 +81,7 @@ pub fn initialize_beacon_state_from_eth1( // Upgrade to capella if configured from genesis if spec .capella_fork_epoch - .map_or(false, |fork_epoch| fork_epoch == E::genesis_epoch()) + .is_some_and(|fork_epoch| fork_epoch == E::genesis_epoch()) { upgrade_to_capella(&mut state, spec)?; @@ -98,7 +98,7 @@ pub fn initialize_beacon_state_from_eth1( // Upgrade to deneb if configured from genesis if spec .deneb_fork_epoch - .map_or(false, |fork_epoch| fork_epoch == E::genesis_epoch()) + .is_some_and(|fork_epoch| fork_epoch == E::genesis_epoch()) { upgrade_to_deneb(&mut state, spec)?; @@ -115,7 +115,7 @@ pub fn initialize_beacon_state_from_eth1( // Upgrade to electra if configured from genesis. if spec .electra_fork_epoch - .map_or(false, |fork_epoch| fork_epoch == E::genesis_epoch()) + .is_some_and(|fork_epoch| fork_epoch == E::genesis_epoch()) { let post = upgrade_state_to_electra(&mut state, Epoch::new(0), Epoch::new(0), spec)?; state = post; @@ -135,11 +135,27 @@ pub fn initialize_beacon_state_from_eth1( // Override latest execution payload header. // See https://github.com/ethereum/consensus-specs/blob/dev/specs/capella/beacon-chain.md#testing - if let Some(ExecutionPayloadHeader::Electra(header)) = execution_payload_header { + if let Some(ExecutionPayloadHeader::Electra(ref header)) = execution_payload_header { *state.latest_execution_payload_header_electra_mut()? = header.clone(); } } + // Upgrade to fulu if configured from genesis. + if spec + .fulu_fork_epoch + .is_some_and(|fork_epoch| fork_epoch == E::genesis_epoch()) + { + upgrade_to_fulu(&mut state, spec)?; + + // Remove intermediate Electra fork from `state.fork`. + state.fork_mut().previous_version = spec.fulu_fork_version; + + // Override latest execution payload header. + if let Some(ExecutionPayloadHeader::Fulu(header)) = execution_payload_header { + *state.latest_execution_payload_header_fulu_mut()? = header.clone(); + } + } + // Now that we have our validators, initialize the caches (including the committees) state.build_caches(spec)?; @@ -153,7 +169,7 @@ pub fn initialize_beacon_state_from_eth1( pub fn is_valid_genesis_state(state: &BeaconState, spec: &ChainSpec) -> bool { state .get_active_validator_indices(E::genesis_epoch(), spec) - .map_or(false, |active_validators| { + .is_ok_and(|active_validators| { state.genesis_time() >= spec.min_genesis_time && active_validators.len() as u64 >= spec.min_genesis_active_validator_count }) diff --git a/consensus/state_processing/src/per_block_processing.rs b/consensus/state_processing/src/per_block_processing.rs index 436f4934b9..ef4799c245 100644 --- a/consensus/state_processing/src/per_block_processing.rs +++ b/consensus/state_processing/src/per_block_processing.rs @@ -1,7 +1,7 @@ use crate::consensus_context::ConsensusContext; use errors::{BlockOperationError, BlockProcessingError, HeaderInvalid}; use rayon::prelude::*; -use safe_arith::{ArithError, SafeArith}; +use safe_arith::{ArithError, SafeArith, SafeArithIter}; use signature_sets::{block_proposal_signature_set, get_pubkey_from_state, randao_signature_set}; use std::borrow::Cow; use tree_hash::TreeHash; @@ -391,10 +391,12 @@ pub fn partially_verify_execution_payload>( _ => return Err(BlockProcessingError::IncorrectStateType), } } + ExecutionPayloadHeaderRefMut::Fulu(header_mut) => { + match payload.to_execution_payload_header() { + ExecutionPayloadHeader::Fulu(header) => *header_mut = header, + _ => return Err(BlockProcessingError::IncorrectStateType), + } + } } Ok(()) @@ -453,15 +461,17 @@ pub fn process_execution_payload>( /// repeatedly write code to treat these errors as false. /// https://github.com/ethereum/consensus-specs/blob/dev/specs/bellatrix/beacon-chain.md#is_merge_transition_complete pub fn is_merge_transition_complete(state: &BeaconState) -> bool { - match state { + if state.fork_name_unchecked().capella_enabled() { + true + } else if state.fork_name_unchecked().bellatrix_enabled() { // We must check defaultness against the payload header with 0x0 roots, as that's what's meant // by `ExecutionPayloadHeader()` in the spec. - BeaconState::Bellatrix(_) => state + state .latest_execution_payload_header() .map(|header| !header.is_default_with_zero_roots()) - .unwrap_or(false), - BeaconState::Electra(_) | BeaconState::Deneb(_) | BeaconState::Capella(_) => true, - BeaconState::Base(_) | BeaconState::Altair(_) => false, + .unwrap_or(false) + } else { + false } } /// https://github.com/ethereum/consensus-specs/blob/dev/specs/bellatrix/beacon-chain.md#is_merge_transition_block @@ -499,7 +509,7 @@ pub fn compute_timestamp_at_slot( /// Compute the next batch of withdrawals which should be included in a block. /// -/// https://github.com/ethereum/consensus-specs/blob/dev/specs/capella/beacon-chain.md#new-get_expected_withdrawals +/// https://github.com/ethereum/consensus-specs/blob/dev/specs/electra/beacon-chain.md#new-get_expected_withdrawals pub fn get_expected_withdrawals( state: &BeaconState, spec: &ChainSpec, @@ -512,18 +522,18 @@ pub fn get_expected_withdrawals( // [New in Electra:EIP7251] // Consume pending partial withdrawals - let partial_withdrawals_count = - if let Ok(partial_withdrawals) = state.pending_partial_withdrawals() { - let mut partial_withdrawals_count = 0; - for withdrawal in partial_withdrawals { + let processed_partial_withdrawals_count = + if let Ok(pending_partial_withdrawals) = state.pending_partial_withdrawals() { + let mut processed_partial_withdrawals_count = 0; + for withdrawal in pending_partial_withdrawals { if withdrawal.withdrawable_epoch > epoch || withdrawals.len() == spec.max_pending_partials_per_withdrawals_sweep as usize { break; } - let withdrawal_balance = state.get_balance(withdrawal.index as usize)?; - let validator = state.get_validator(withdrawal.index as usize)?; + let withdrawal_balance = state.get_balance(withdrawal.validator_index as usize)?; + let validator = state.get_validator(withdrawal.validator_index as usize)?; let has_sufficient_effective_balance = validator.effective_balance >= spec.min_activation_balance; @@ -539,17 +549,17 @@ pub fn get_expected_withdrawals( ); withdrawals.push(Withdrawal { index: withdrawal_index, - validator_index: withdrawal.index, + validator_index: withdrawal.validator_index, address: validator .get_execution_withdrawal_address(spec) - .ok_or(BeaconStateError::NonExecutionAddresWithdrawalCredential)?, + .ok_or(BeaconStateError::NonExecutionAddressWithdrawalCredential)?, amount: withdrawable_balance, }); withdrawal_index.safe_add_assign(1)?; } - partial_withdrawals_count.safe_add_assign(1)?; + processed_partial_withdrawals_count.safe_add_assign(1)?; } - Some(partial_withdrawals_count) + Some(processed_partial_withdrawals_count) } else { None }; @@ -560,10 +570,20 @@ pub fn get_expected_withdrawals( ); for _ in 0..bound { let validator = state.get_validator(validator_index as usize)?; - let balance = *state.balances().get(validator_index as usize).ok_or( - BeaconStateError::BalancesOutOfBounds(validator_index as usize), - )?; - if validator.is_fully_withdrawable_at(balance, epoch, spec, fork_name) { + let partially_withdrawn_balance = withdrawals + .iter() + .filter_map(|withdrawal| { + (withdrawal.validator_index == validator_index).then_some(withdrawal.amount) + }) + .safe_sum()?; + let balance = state + .balances() + .get(validator_index as usize) + .ok_or(BeaconStateError::BalancesOutOfBounds( + validator_index as usize, + ))? + .safe_sub(partially_withdrawn_balance)?; + if validator.is_fully_withdrawable_validator(balance, epoch, spec, fork_name) { withdrawals.push(Withdrawal { index: withdrawal_index, validator_index, @@ -580,9 +600,7 @@ pub fn get_expected_withdrawals( address: validator .get_execution_withdrawal_address(spec) .ok_or(BlockProcessingError::WithdrawalCredentialsInvalid)?, - amount: balance.safe_sub( - validator.get_max_effective_balance(spec, state.fork_name_unchecked()), - )?, + amount: balance.safe_sub(validator.get_max_effective_balance(spec, fork_name))?, }); withdrawal_index.safe_add_assign(1)?; } @@ -594,7 +612,7 @@ pub fn get_expected_withdrawals( .safe_rem(state.validators().len() as u64)?; } - Ok((withdrawals.into(), partial_withdrawals_count)) + Ok((withdrawals.into(), processed_partial_withdrawals_count)) } /// Apply withdrawals to the state. @@ -603,66 +621,61 @@ pub fn process_withdrawals>( payload: Payload::Ref<'_>, spec: &ChainSpec, ) -> Result<(), BlockProcessingError> { - match state { - BeaconState::Capella(_) | BeaconState::Deneb(_) | BeaconState::Electra(_) => { - let (expected_withdrawals, partial_withdrawals_count) = - get_expected_withdrawals(state, spec)?; - let expected_root = expected_withdrawals.tree_hash_root(); - let withdrawals_root = payload.withdrawals_root()?; + if state.fork_name_unchecked().capella_enabled() { + let (expected_withdrawals, processed_partial_withdrawals_count) = + get_expected_withdrawals(state, spec)?; + let expected_root = expected_withdrawals.tree_hash_root(); + let withdrawals_root = payload.withdrawals_root()?; - if expected_root != withdrawals_root { - return Err(BlockProcessingError::WithdrawalsRootMismatch { - expected: expected_root, - found: withdrawals_root, - }); - } + if expected_root != withdrawals_root { + return Err(BlockProcessingError::WithdrawalsRootMismatch { + expected: expected_root, + found: withdrawals_root, + }); + } - for withdrawal in expected_withdrawals.iter() { - decrease_balance( - state, - withdrawal.validator_index as usize, - withdrawal.amount, - )?; - } + for withdrawal in expected_withdrawals.iter() { + decrease_balance( + state, + withdrawal.validator_index as usize, + withdrawal.amount, + )?; + } - // Update pending partial withdrawals [New in Electra:EIP7251] - if let Some(partial_withdrawals_count) = partial_withdrawals_count { - // TODO(electra): Use efficient pop_front after milhouse release https://github.com/sigp/milhouse/pull/38 - let new_partial_withdrawals = state - .pending_partial_withdrawals()? - .iter_from(partial_withdrawals_count)? - .cloned() - .collect::>(); - *state.pending_partial_withdrawals_mut()? = List::new(new_partial_withdrawals)?; - } + // Update pending partial withdrawals [New in Electra:EIP7251] + if let Some(processed_partial_withdrawals_count) = processed_partial_withdrawals_count { + state + .pending_partial_withdrawals_mut()? + .pop_front(processed_partial_withdrawals_count)?; + } - // Update the next withdrawal index if this block contained withdrawals - if let Some(latest_withdrawal) = expected_withdrawals.last() { - *state.next_withdrawal_index_mut()? = latest_withdrawal.index.safe_add(1)?; + // Update the next withdrawal index if this block contained withdrawals + if let Some(latest_withdrawal) = expected_withdrawals.last() { + *state.next_withdrawal_index_mut()? = latest_withdrawal.index.safe_add(1)?; - // Update the next validator index to start the next withdrawal sweep - if expected_withdrawals.len() == E::max_withdrawals_per_payload() { - // Next sweep starts after the latest withdrawal's validator index - let next_validator_index = latest_withdrawal - .validator_index - .safe_add(1)? - .safe_rem(state.validators().len() as u64)?; - *state.next_withdrawal_validator_index_mut()? = next_validator_index; - } - } - - // Advance sweep by the max length of the sweep if there was not a full set of withdrawals - if expected_withdrawals.len() != E::max_withdrawals_per_payload() { - let next_validator_index = state - .next_withdrawal_validator_index()? - .safe_add(spec.max_validators_per_withdrawals_sweep)? + // Update the next validator index to start the next withdrawal sweep + if expected_withdrawals.len() == E::max_withdrawals_per_payload() { + // Next sweep starts after the latest withdrawal's validator index + let next_validator_index = latest_withdrawal + .validator_index + .safe_add(1)? .safe_rem(state.validators().len() as u64)?; *state.next_withdrawal_validator_index_mut()? = next_validator_index; } - - Ok(()) } + + // Advance sweep by the max length of the sweep if there was not a full set of withdrawals + if expected_withdrawals.len() != E::max_withdrawals_per_payload() { + let next_validator_index = state + .next_withdrawal_validator_index()? + .safe_add(spec.max_validators_per_withdrawals_sweep)? + .safe_rem(state.validators().len() as u64)?; + *state.next_withdrawal_validator_index_mut()? = next_validator_index; + } + + Ok(()) + } else { // these shouldn't even be encountered but they're here for completeness - BeaconState::Base(_) | BeaconState::Altair(_) | BeaconState::Bellatrix(_) => Ok(()), + Ok(()) } } diff --git a/consensus/state_processing/src/per_block_processing/altair/sync_committee.rs b/consensus/state_processing/src/per_block_processing/altair/sync_committee.rs index 210db4c9c1..08cfd9cba8 100644 --- a/consensus/state_processing/src/per_block_processing/altair/sync_committee.rs +++ b/consensus/state_processing/src/per_block_processing/altair/sync_committee.rs @@ -38,7 +38,7 @@ pub fn process_sync_aggregate( )?; // If signature set is `None` then the signature is valid (infinity). - if signature_set.map_or(false, |signature| !signature.verify()) { + if signature_set.is_some_and(|signature| !signature.verify()) { return Err(SyncAggregateInvalid::SignatureInvalid.into()); } } diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 22d8592364..82dd616724 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -284,29 +284,22 @@ pub fn process_attestations>( ctxt: &mut ConsensusContext, spec: &ChainSpec, ) -> Result<(), BlockProcessingError> { - match block_body { - BeaconBlockBodyRef::Base(_) => { - base::process_attestations( - state, - block_body.attestations(), - verify_signatures, - ctxt, - spec, - )?; - } - BeaconBlockBodyRef::Altair(_) - | BeaconBlockBodyRef::Bellatrix(_) - | BeaconBlockBodyRef::Capella(_) - | BeaconBlockBodyRef::Deneb(_) - | BeaconBlockBodyRef::Electra(_) => { - altair_deneb::process_attestations( - state, - block_body.attestations(), - verify_signatures, - ctxt, - spec, - )?; - } + if state.fork_name_unchecked().altair_enabled() { + altair_deneb::process_attestations( + state, + block_body.attestations(), + verify_signatures, + ctxt, + spec, + )?; + } else { + base::process_attestations( + state, + block_body.attestations(), + verify_signatures, + ctxt, + spec, + )?; } Ok(()) } @@ -514,11 +507,11 @@ pub fn process_withdrawal_requests( } // Verify pubkey exists - let Some(index) = state.pubkey_cache().get(&request.validator_pubkey) else { + let Some(validator_index) = state.pubkey_cache().get(&request.validator_pubkey) else { continue; }; - let validator = state.get_validator(index)?; + let validator = state.get_validator(validator_index)?; // Verify withdrawal credentials let has_correct_credential = validator.has_execution_withdrawal_credential(spec); let is_correct_source_address = validator @@ -549,16 +542,16 @@ pub fn process_withdrawal_requests( continue; } - let pending_balance_to_withdraw = state.get_pending_balance_to_withdraw(index)?; + let pending_balance_to_withdraw = state.get_pending_balance_to_withdraw(validator_index)?; if is_full_exit_request { // Only exit validator if it has no pending withdrawals in the queue if pending_balance_to_withdraw == 0 { - initiate_validator_exit(state, index, spec)? + initiate_validator_exit(state, validator_index, spec)? } continue; } - let balance = state.get_balance(index)?; + let balance = state.get_balance(validator_index)?; let has_sufficient_effective_balance = validator.effective_balance >= spec.min_activation_balance; let has_excess_balance = balance @@ -583,7 +576,7 @@ pub fn process_withdrawal_requests( state .pending_partial_withdrawals_mut()? .push(PendingPartialWithdrawal { - index: index as u64, + validator_index: validator_index as u64, amount: to_withdraw, withdrawable_epoch, })?; @@ -746,8 +739,8 @@ pub fn process_consolidation_request( } let target_validator = state.get_validator(target_index)?; - // Verify the target has execution withdrawal credentials - if !target_validator.has_execution_withdrawal_credential(spec) { + // Verify the target has compounding withdrawal credentials + if !target_validator.has_compounding_withdrawal_credential(spec) { return Ok(()); } @@ -764,6 +757,18 @@ pub fn process_consolidation_request( { return Ok(()); } + // Verify the source has been active long enough + if current_epoch + < source_validator + .activation_epoch + .safe_add(spec.shard_committee_period)? + { + return Ok(()); + } + // Verify the source has no pending withdrawals in the queue + if state.get_pending_balance_to_withdraw(source_index)? > 0 { + return Ok(()); + } // Initiate source validator exit and append pending consolidation let source_exit_epoch = state @@ -779,10 +784,5 @@ pub fn process_consolidation_request( target_index: target_index as u64, })?; - let target_validator = state.get_validator(target_index)?; - // Churn any target excess active balance of target and raise its max - if target_validator.has_eth1_withdrawal_credential(spec) { - state.switch_to_compounding_validator(target_index, spec)?; - } Ok(()) } diff --git a/consensus/state_processing/src/per_block_processing/signature_sets.rs b/consensus/state_processing/src/per_block_processing/signature_sets.rs index 2e00ee0341..39f438f97f 100644 --- a/consensus/state_processing/src/per_block_processing/signature_sets.rs +++ b/consensus/state_processing/src/per_block_processing/signature_sets.rs @@ -387,22 +387,20 @@ where let exit = &signed_exit.message; let proposer_index = exit.validator_index as usize; - let domain = match state { - BeaconState::Base(_) - | BeaconState::Altair(_) - | BeaconState::Bellatrix(_) - | BeaconState::Capella(_) => spec.get_domain( + let domain = if state.fork_name_unchecked().deneb_enabled() { + // EIP-7044 + spec.compute_domain( + Domain::VoluntaryExit, + spec.capella_fork_version, + state.genesis_validators_root(), + ) + } else { + spec.get_domain( exit.epoch, Domain::VoluntaryExit, &state.fork(), state.genesis_validators_root(), - ), - // EIP-7044 - BeaconState::Deneb(_) | BeaconState::Electra(_) => spec.compute_domain( - Domain::VoluntaryExit, - spec.capella_fork_version, - state.genesis_validators_root(), - ), + ) }; let message = exit.signing_root(domain); diff --git a/consensus/state_processing/src/per_block_processing/verify_attestation.rs b/consensus/state_processing/src/per_block_processing/verify_attestation.rs index 6bfb51d475..0b399bea6c 100644 --- a/consensus/state_processing/src/per_block_processing/verify_attestation.rs +++ b/consensus/state_processing/src/per_block_processing/verify_attestation.rs @@ -32,21 +32,16 @@ pub fn verify_attestation_for_block_inclusion<'ctxt, E: EthSpec>( attestation: data.slot, } ); - match state { - BeaconState::Base(_) - | BeaconState::Altair(_) - | BeaconState::Bellatrix(_) - | BeaconState::Capella(_) => { - verify!( - state.slot() <= data.slot.safe_add(E::slots_per_epoch())?, - Invalid::IncludedTooLate { - state: state.slot(), - attestation: data.slot, - } - ); - } + if state.fork_name_unchecked().deneb_enabled() { // [Modified in Deneb:EIP7045] - BeaconState::Deneb(_) | BeaconState::Electra(_) => {} + } else { + verify!( + state.slot() <= data.slot.safe_add(E::slots_per_epoch())?, + Invalid::IncludedTooLate { + state: state.slot(), + attestation: data.slot, + } + ); } verify_attestation_for_state(state, attestation, ctxt, verify_signatures, spec) diff --git a/consensus/state_processing/src/per_epoch_processing.rs b/consensus/state_processing/src/per_epoch_processing.rs index 55e8853f3f..41c30c4931 100644 --- a/consensus/state_processing/src/per_epoch_processing.rs +++ b/consensus/state_processing/src/per_epoch_processing.rs @@ -41,13 +41,10 @@ pub fn process_epoch( .fork_name(spec) .map_err(Error::InconsistentStateFork)?; - match state { - BeaconState::Base(_) => base::process_epoch(state, spec), - BeaconState::Altair(_) - | BeaconState::Bellatrix(_) - | BeaconState::Capella(_) - | BeaconState::Deneb(_) - | BeaconState::Electra(_) => altair::process_epoch(state, spec), + if state.fork_name_unchecked().altair_enabled() { + altair::process_epoch(state, spec) + } else { + base::process_epoch(state, spec) } } diff --git a/consensus/state_processing/src/per_epoch_processing/epoch_processing_summary.rs b/consensus/state_processing/src/per_epoch_processing/epoch_processing_summary.rs index 952ab3f649..5508b80807 100644 --- a/consensus/state_processing/src/per_epoch_processing/epoch_processing_summary.rs +++ b/consensus/state_processing/src/per_epoch_processing/epoch_processing_summary.rs @@ -151,7 +151,7 @@ impl EpochProcessingSummary { match self { EpochProcessingSummary::Base { statuses, .. } => statuses .get(val_index) - .map_or(false, |s| s.is_active_in_current_epoch && !s.is_slashed), + .is_some_and(|s| s.is_active_in_current_epoch && !s.is_slashed), EpochProcessingSummary::Altair { participation, .. } => { participation.is_active_and_unslashed(val_index, participation.current_epoch) } @@ -176,7 +176,7 @@ impl EpochProcessingSummary { match self { EpochProcessingSummary::Base { statuses, .. } => Ok(statuses .get(val_index) - .map_or(false, |s| s.is_current_epoch_target_attester)), + .is_some_and(|s| s.is_current_epoch_target_attester)), EpochProcessingSummary::Altair { participation, .. } => participation .is_current_epoch_unslashed_participating_index( val_index, @@ -247,7 +247,7 @@ impl EpochProcessingSummary { match self { EpochProcessingSummary::Base { statuses, .. } => statuses .get(val_index) - .map_or(false, |s| s.is_active_in_previous_epoch && !s.is_slashed), + .is_some_and(|s| s.is_active_in_previous_epoch && !s.is_slashed), EpochProcessingSummary::Altair { participation, .. } => { participation.is_active_and_unslashed(val_index, participation.previous_epoch) } @@ -267,7 +267,7 @@ impl EpochProcessingSummary { match self { EpochProcessingSummary::Base { statuses, .. } => Ok(statuses .get(val_index) - .map_or(false, |s| s.is_previous_epoch_target_attester)), + .is_some_and(|s| s.is_previous_epoch_target_attester)), EpochProcessingSummary::Altair { participation, .. } => participation .is_previous_epoch_unslashed_participating_index( val_index, @@ -294,7 +294,7 @@ impl EpochProcessingSummary { match self { EpochProcessingSummary::Base { statuses, .. } => Ok(statuses .get(val_index) - .map_or(false, |s| s.is_previous_epoch_head_attester)), + .is_some_and(|s| s.is_previous_epoch_head_attester)), EpochProcessingSummary::Altair { participation, .. } => participation .is_previous_epoch_unslashed_participating_index(val_index, TIMELY_HEAD_FLAG_INDEX), } @@ -318,7 +318,7 @@ impl EpochProcessingSummary { match self { EpochProcessingSummary::Base { statuses, .. } => Ok(statuses .get(val_index) - .map_or(false, |s| s.is_previous_epoch_attester)), + .is_some_and(|s| s.is_previous_epoch_attester)), EpochProcessingSummary::Altair { participation, .. } => participation .is_previous_epoch_unslashed_participating_index( val_index, diff --git a/consensus/state_processing/src/per_epoch_processing/single_pass.rs b/consensus/state_processing/src/per_epoch_processing/single_pass.rs index 904e68e368..5c31669a60 100644 --- a/consensus/state_processing/src/per_epoch_processing/single_pass.rs +++ b/consensus/state_processing/src/per_epoch_processing/single_pass.rs @@ -1057,14 +1057,12 @@ fn process_pending_consolidations( } // Calculate the consolidated balance - let max_effective_balance = - source_validator.get_max_effective_balance(spec, state_ctxt.fork_name); let source_effective_balance = std::cmp::min( *state .balances() .get(source_index) .ok_or(BeaconStateError::UnknownValidator(source_index))?, - max_effective_balance, + source_validator.effective_balance, ); // Move active balance to target. Excess balance is withdrawable. @@ -1077,13 +1075,9 @@ fn process_pending_consolidations( next_pending_consolidation.safe_add_assign(1)?; } - let new_pending_consolidations = List::try_from_iter( - state - .pending_consolidations()? - .iter_from(next_pending_consolidation)? - .cloned(), - )?; - *state.pending_consolidations_mut()? = new_pending_consolidations; + state + .pending_consolidations_mut()? + .pop_front(next_pending_consolidation)?; // the spec tests require we don't perform effective balance updates when testing pending_consolidations if !perform_effective_balance_updates { diff --git a/consensus/state_processing/src/per_slot_processing.rs b/consensus/state_processing/src/per_slot_processing.rs index 6554423199..af1cce602c 100644 --- a/consensus/state_processing/src/per_slot_processing.rs +++ b/consensus/state_processing/src/per_slot_processing.rs @@ -1,6 +1,6 @@ use crate::upgrade::{ upgrade_to_altair, upgrade_to_bellatrix, upgrade_to_capella, upgrade_to_deneb, - upgrade_to_electra, + upgrade_to_electra, upgrade_to_fulu, }; use crate::{per_epoch_processing::EpochProcessingSummary, *}; use safe_arith::{ArithError, SafeArith}; @@ -71,6 +71,11 @@ pub fn per_slot_processing( upgrade_to_electra(state, spec)?; } + // Fulu. + if spec.fulu_fork_epoch == Some(state.current_epoch()) { + upgrade_to_fulu(state, spec)?; + } + // Additionally build all caches so that all valid states that are advanced always have // committee caches built, and we don't have to worry about initialising them at higher // layers. diff --git a/consensus/state_processing/src/upgrade.rs b/consensus/state_processing/src/upgrade.rs index 93cafa73d0..88bc87849f 100644 --- a/consensus/state_processing/src/upgrade.rs +++ b/consensus/state_processing/src/upgrade.rs @@ -3,9 +3,11 @@ pub mod bellatrix; pub mod capella; pub mod deneb; pub mod electra; +pub mod fulu; pub use altair::upgrade_to_altair; pub use bellatrix::upgrade_to_bellatrix; pub use capella::upgrade_to_capella; pub use deneb::upgrade_to_deneb; pub use electra::upgrade_to_electra; +pub use fulu::upgrade_to_fulu; diff --git a/consensus/state_processing/src/upgrade/electra.rs b/consensus/state_processing/src/upgrade/electra.rs index 1e64ef2897..258b28a45b 100644 --- a/consensus/state_processing/src/upgrade/electra.rs +++ b/consensus/state_processing/src/upgrade/electra.rs @@ -14,13 +14,15 @@ pub fn upgrade_to_electra( ) -> Result<(), Error> { let epoch = pre_state.current_epoch(); + let activation_exit_epoch = spec.compute_activation_exit_epoch(epoch)?; let earliest_exit_epoch = pre_state .validators() .iter() .filter(|v| v.exit_epoch != spec.far_future_epoch) .map(|v| v.exit_epoch) .max() - .unwrap_or(epoch) + .unwrap_or(activation_exit_epoch) + .max(activation_exit_epoch) .safe_add(1)?; // The total active balance cache must be built before the consolidation churn limit @@ -45,10 +47,11 @@ pub fn upgrade_to_electra( .enumerate() .filter(|(_, validator)| validator.activation_epoch == spec.far_future_epoch) .sorted_by_key(|(index, validator)| (validator.activation_eligibility_epoch, *index)) + .map(|(index, _)| index) .collect::>(); // Process validators to queue entire balance and reset them - for (index, _) in pre_activation { + for index in pre_activation { let balance = post .balances_mut() .get_mut(index) diff --git a/consensus/state_processing/src/upgrade/fulu.rs b/consensus/state_processing/src/upgrade/fulu.rs new file mode 100644 index 0000000000..6e0cd3fa9d --- /dev/null +++ b/consensus/state_processing/src/upgrade/fulu.rs @@ -0,0 +1,94 @@ +use std::mem; +use types::{BeaconState, BeaconStateError as Error, BeaconStateFulu, ChainSpec, EthSpec, Fork}; + +/// Transform a `Electra` state into an `Fulu` state. +pub fn upgrade_to_fulu( + pre_state: &mut BeaconState, + spec: &ChainSpec, +) -> Result<(), Error> { + let _epoch = pre_state.current_epoch(); + + let post = upgrade_state_to_fulu(pre_state, spec)?; + + *pre_state = post; + + Ok(()) +} + +pub fn upgrade_state_to_fulu( + pre_state: &mut BeaconState, + spec: &ChainSpec, +) -> Result, Error> { + let epoch = pre_state.current_epoch(); + let pre = pre_state.as_electra_mut()?; + // Where possible, use something like `mem::take` to move fields from behind the &mut + // reference. For other fields that don't have a good default value, use `clone`. + // + // Fixed size vectors get cloned because replacing them would require the same size + // allocation as cloning. + let post = BeaconState::Fulu(BeaconStateFulu { + // Versioning + genesis_time: pre.genesis_time, + genesis_validators_root: pre.genesis_validators_root, + slot: pre.slot, + fork: Fork { + previous_version: pre.fork.current_version, + current_version: spec.fulu_fork_version, + epoch, + }, + // History + latest_block_header: pre.latest_block_header.clone(), + block_roots: pre.block_roots.clone(), + state_roots: pre.state_roots.clone(), + historical_roots: mem::take(&mut pre.historical_roots), + // Eth1 + eth1_data: pre.eth1_data.clone(), + eth1_data_votes: mem::take(&mut pre.eth1_data_votes), + eth1_deposit_index: pre.eth1_deposit_index, + // Registry + validators: mem::take(&mut pre.validators), + balances: mem::take(&mut pre.balances), + // Randomness + randao_mixes: pre.randao_mixes.clone(), + // Slashings + slashings: pre.slashings.clone(), + // `Participation + previous_epoch_participation: mem::take(&mut pre.previous_epoch_participation), + current_epoch_participation: mem::take(&mut pre.current_epoch_participation), + // Finality + justification_bits: pre.justification_bits.clone(), + previous_justified_checkpoint: pre.previous_justified_checkpoint, + current_justified_checkpoint: pre.current_justified_checkpoint, + finalized_checkpoint: pre.finalized_checkpoint, + // Inactivity + inactivity_scores: mem::take(&mut pre.inactivity_scores), + // Sync committees + current_sync_committee: pre.current_sync_committee.clone(), + next_sync_committee: pre.next_sync_committee.clone(), + // Execution + latest_execution_payload_header: pre.latest_execution_payload_header.upgrade_to_fulu(), + // Capella + next_withdrawal_index: pre.next_withdrawal_index, + next_withdrawal_validator_index: pre.next_withdrawal_validator_index, + historical_summaries: pre.historical_summaries.clone(), + // Electra + deposit_requests_start_index: pre.deposit_requests_start_index, + deposit_balance_to_consume: pre.deposit_balance_to_consume, + exit_balance_to_consume: pre.exit_balance_to_consume, + earliest_exit_epoch: pre.earliest_exit_epoch, + consolidation_balance_to_consume: pre.consolidation_balance_to_consume, + earliest_consolidation_epoch: pre.earliest_consolidation_epoch, + pending_deposits: pre.pending_deposits.clone(), + pending_partial_withdrawals: pre.pending_partial_withdrawals.clone(), + pending_consolidations: pre.pending_consolidations.clone(), + // Caches + total_active_balance: pre.total_active_balance, + progressive_balances_cache: mem::take(&mut pre.progressive_balances_cache), + committee_caches: mem::take(&mut pre.committee_caches), + pubkey_cache: mem::take(&mut pre.pubkey_cache), + exit_cache: mem::take(&mut pre.exit_cache), + slashings_cache: mem::take(&mut pre.slashings_cache), + epoch_cache: mem::take(&mut pre.epoch_cache), + }); + Ok(post) +} diff --git a/consensus/types/Cargo.toml b/consensus/types/Cargo.toml index 21a15fc517..79beb81282 100644 --- a/consensus/types/Cargo.toml +++ b/consensus/types/Cargo.toml @@ -10,56 +10,56 @@ harness = false [dependencies] alloy-primitives = { workspace = true } -merkle_proof = { workspace = true } -bls = { workspace = true, features = ["arbitrary"] } -kzg = { workspace = true } -compare_fields = { workspace = true } -compare_fields_derive = { workspace = true } -eth2_interop_keypairs = { path = "../../common/eth2_interop_keypairs" } -ethereum_hashing = { workspace = true } -hex = { workspace = true } -int_to_bytes = { workspace = true } -log = { workspace = true } -rayon = { workspace = true } -rand = { workspace = true } -safe_arith = { workspace = true } -serde = { workspace = true, features = ["rc"] } -slog = { workspace = true } -ethereum_ssz = { workspace = true, features = ["arbitrary"] } -ethereum_ssz_derive = { workspace = true } -ssz_types = { workspace = true, features = ["arbitrary"] } -swap_or_not_shuffle = { workspace = true, features = ["arbitrary"] } -test_random_derive = { path = "../../common/test_random_derive" } -tree_hash = { workspace = true } -tree_hash_derive = { workspace = true } -rand_xorshift = "0.3.0" -serde_yaml = { workspace = true } -tempfile = { workspace = true } -derivative = { workspace = true } -rusqlite = { workspace = true } +alloy-rlp = { version = "0.3.4", features = ["derive"] } # The arbitrary dependency is enabled by default since Capella to avoid complexity introduced by # `AbstractExecPayload` arbitrary = { workspace = true, features = ["derive"] } +bls = { workspace = true, features = ["arbitrary"] } +compare_fields = { workspace = true } +compare_fields_derive = { workspace = true } +derivative = { workspace = true } +eth2_interop_keypairs = { path = "../../common/eth2_interop_keypairs" } +ethereum_hashing = { workspace = true } ethereum_serde_utils = { workspace = true } -regex = { workspace = true } -parking_lot = { workspace = true } -itertools = { workspace = true } -superstruct = { workspace = true } -metastruct = "0.1.0" -serde_json = { workspace = true } -smallvec = { workspace = true } -maplit = { workspace = true } -alloy-rlp = { version = "0.3.4", features = ["derive"] } -milhouse = { workspace = true } -rpds = { workspace = true } +ethereum_ssz = { workspace = true, features = ["arbitrary"] } +ethereum_ssz_derive = { workspace = true } fixed_bytes = { workspace = true } +hex = { workspace = true } +int_to_bytes = { workspace = true } +itertools = { workspace = true } +kzg = { workspace = true } +log = { workspace = true } +maplit = { workspace = true } +merkle_proof = { workspace = true } +metastruct = "0.1.0" +milhouse = { workspace = true } +parking_lot = { workspace = true } +rand = { workspace = true } +rand_xorshift = "0.3.0" +rayon = { workspace = true } +regex = { workspace = true } +rpds = { workspace = true } +rusqlite = { workspace = true } +safe_arith = { workspace = true } +serde = { workspace = true, features = ["rc"] } +serde_json = { workspace = true } +serde_yaml = { workspace = true } +slog = { workspace = true } +smallvec = { workspace = true } +ssz_types = { workspace = true, features = ["arbitrary"] } +superstruct = { workspace = true } +swap_or_not_shuffle = { workspace = true, features = ["arbitrary"] } +tempfile = { workspace = true } +test_random_derive = { path = "../../common/test_random_derive" } +tree_hash = { workspace = true } +tree_hash_derive = { workspace = true } [dev-dependencies] -criterion = { workspace = true } beacon_chain = { workspace = true } +criterion = { workspace = true } +paste = { workspace = true } state_processing = { workspace = true } tokio = { workspace = true } -paste = { workspace = true } [features] default = ["sqlite", "legacy-arith"] diff --git a/consensus/types/presets/gnosis/deneb.yaml b/consensus/types/presets/gnosis/deneb.yaml index d2d7d0abed..d25c4d3d38 100644 --- a/consensus/types/presets/gnosis/deneb.yaml +++ b/consensus/types/presets/gnosis/deneb.yaml @@ -1,6 +1,4 @@ # Gnosis preset - Deneb -# NOTE: The below are PLACEHOLDER values from Mainnet. -# Gnosis preset for the Deneb fork TBD: https://github.com/gnosischain/configs/tree/main/presets/gnosis # Misc # --------------------------------------------------------------- @@ -8,7 +6,5 @@ FIELD_ELEMENTS_PER_BLOB: 4096 # `uint64(2**12)` (= 4096) MAX_BLOB_COMMITMENTS_PER_BLOCK: 4096 -# `uint64(6)` -MAX_BLOBS_PER_BLOCK: 6 # `floorlog2(BLOB_KZG_COMMITMENTS_GINDEX) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 12 = 17 KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 17 diff --git a/consensus/types/presets/gnosis/electra.yaml b/consensus/types/presets/gnosis/electra.yaml index 660ed9b64c..42afbb233e 100644 --- a/consensus/types/presets/gnosis/electra.yaml +++ b/consensus/types/presets/gnosis/electra.yaml @@ -10,7 +10,7 @@ MAX_EFFECTIVE_BALANCE_ELECTRA: 2048000000000 # State list lengths # --------------------------------------------------------------- # `uint64(2**27)` (= 134,217,728) -PENDING_BALANCE_DEPOSITS_LIMIT: 134217728 +PENDING_DEPOSITS_LIMIT: 134217728 # `uint64(2**27)` (= 134,217,728) PENDING_PARTIAL_WITHDRAWALS_LIMIT: 134217728 # `uint64(2**18)` (= 262,144) @@ -29,12 +29,12 @@ WHISTLEBLOWER_REWARD_QUOTIENT_ELECTRA: 4096 MAX_ATTESTER_SLASHINGS_ELECTRA: 1 # `uint64(2**3)` (= 8) MAX_ATTESTATIONS_ELECTRA: 8 -# `uint64(2**0)` (= 1) -MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD: 1 +# `uint64(2**1)` (= 2) +MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD: 2 # Execution # --------------------------------------------------------------- -# 2**13 (= 8192) receipts +# 2**13 (= 8192) deposit requests MAX_DEPOSIT_REQUESTS_PER_PAYLOAD: 8192 # 2**4 (= 16) withdrawal requests MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD: 16 @@ -43,3 +43,8 @@ MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD: 16 # --------------------------------------------------------------- # 2**3 ( = 8) pending withdrawals MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP: 8 + +# Pending deposits processing +# --------------------------------------------------------------- +# 2**4 ( = 4) pending deposits +MAX_PENDING_DEPOSITS_PER_EPOCH: 16 diff --git a/consensus/types/presets/gnosis/eip7594.yaml b/consensus/types/presets/gnosis/fulu.yaml similarity index 92% rename from consensus/types/presets/gnosis/eip7594.yaml rename to consensus/types/presets/gnosis/fulu.yaml index 813febf26d..e5f3ce0212 100644 --- a/consensus/types/presets/gnosis/eip7594.yaml +++ b/consensus/types/presets/gnosis/fulu.yaml @@ -1,4 +1,4 @@ -# Mainnet preset - EIP7594 +# Gnosis preset - Fulu # Misc # --------------------------------------------------------------- diff --git a/consensus/types/presets/mainnet/altair.yaml b/consensus/types/presets/mainnet/altair.yaml index 9a17b78032..813ef72122 100644 --- a/consensus/types/presets/mainnet/altair.yaml +++ b/consensus/types/presets/mainnet/altair.yaml @@ -22,3 +22,5 @@ EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 256 # --------------------------------------------------------------- # 1 MIN_SYNC_COMMITTEE_PARTICIPANTS: 1 +# SLOTS_PER_EPOCH * EPOCHS_PER_SYNC_COMMITTEE_PERIOD (= 32 * 256) +UPDATE_TIMEOUT: 8192 diff --git a/consensus/types/presets/mainnet/deneb.yaml b/consensus/types/presets/mainnet/deneb.yaml index 0f56b8bdfa..f426d3ae1a 100644 --- a/consensus/types/presets/mainnet/deneb.yaml +++ b/consensus/types/presets/mainnet/deneb.yaml @@ -6,7 +6,5 @@ FIELD_ELEMENTS_PER_BLOB: 4096 # `uint64(2**12)` (= 4096) MAX_BLOB_COMMITMENTS_PER_BLOCK: 4096 -# `uint64(6)` -MAX_BLOBS_PER_BLOCK: 6 # `floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 12 = 17 KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 17 diff --git a/consensus/types/presets/mainnet/electra.yaml b/consensus/types/presets/mainnet/electra.yaml index 660ed9b64c..42afbb233e 100644 --- a/consensus/types/presets/mainnet/electra.yaml +++ b/consensus/types/presets/mainnet/electra.yaml @@ -10,7 +10,7 @@ MAX_EFFECTIVE_BALANCE_ELECTRA: 2048000000000 # State list lengths # --------------------------------------------------------------- # `uint64(2**27)` (= 134,217,728) -PENDING_BALANCE_DEPOSITS_LIMIT: 134217728 +PENDING_DEPOSITS_LIMIT: 134217728 # `uint64(2**27)` (= 134,217,728) PENDING_PARTIAL_WITHDRAWALS_LIMIT: 134217728 # `uint64(2**18)` (= 262,144) @@ -29,12 +29,12 @@ WHISTLEBLOWER_REWARD_QUOTIENT_ELECTRA: 4096 MAX_ATTESTER_SLASHINGS_ELECTRA: 1 # `uint64(2**3)` (= 8) MAX_ATTESTATIONS_ELECTRA: 8 -# `uint64(2**0)` (= 1) -MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD: 1 +# `uint64(2**1)` (= 2) +MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD: 2 # Execution # --------------------------------------------------------------- -# 2**13 (= 8192) receipts +# 2**13 (= 8192) deposit requests MAX_DEPOSIT_REQUESTS_PER_PAYLOAD: 8192 # 2**4 (= 16) withdrawal requests MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD: 16 @@ -43,3 +43,8 @@ MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD: 16 # --------------------------------------------------------------- # 2**3 ( = 8) pending withdrawals MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP: 8 + +# Pending deposits processing +# --------------------------------------------------------------- +# 2**4 ( = 4) pending deposits +MAX_PENDING_DEPOSITS_PER_EPOCH: 16 diff --git a/consensus/types/presets/mainnet/eip7594.yaml b/consensus/types/presets/mainnet/fulu.yaml similarity index 92% rename from consensus/types/presets/mainnet/eip7594.yaml rename to consensus/types/presets/mainnet/fulu.yaml index 813febf26d..394f335f90 100644 --- a/consensus/types/presets/mainnet/eip7594.yaml +++ b/consensus/types/presets/mainnet/fulu.yaml @@ -1,4 +1,4 @@ -# Mainnet preset - EIP7594 +# Mainnet preset - Fulu # Misc # --------------------------------------------------------------- diff --git a/consensus/types/presets/mainnet/phase0.yaml b/consensus/types/presets/mainnet/phase0.yaml index 02bc96c8cd..00133ba369 100644 --- a/consensus/types/presets/mainnet/phase0.yaml +++ b/consensus/types/presets/mainnet/phase0.yaml @@ -85,4 +85,4 @@ MAX_ATTESTATIONS: 128 # 2**4 (= 16) MAX_DEPOSITS: 16 # 2**4 (= 16) -MAX_VOLUNTARY_EXITS: 16 +MAX_VOLUNTARY_EXITS: 16 \ No newline at end of file diff --git a/consensus/types/presets/minimal/altair.yaml b/consensus/types/presets/minimal/altair.yaml index 88d78bea36..5e472c49cf 100644 --- a/consensus/types/presets/minimal/altair.yaml +++ b/consensus/types/presets/minimal/altair.yaml @@ -22,3 +22,5 @@ EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 8 # --------------------------------------------------------------- # 1 MIN_SYNC_COMMITTEE_PARTICIPANTS: 1 +# SLOTS_PER_EPOCH * EPOCHS_PER_SYNC_COMMITTEE_PERIOD (= 8 * 8) +UPDATE_TIMEOUT: 64 diff --git a/consensus/types/presets/minimal/deneb.yaml b/consensus/types/presets/minimal/deneb.yaml index be2b9fadfa..c101de3162 100644 --- a/consensus/types/presets/minimal/deneb.yaml +++ b/consensus/types/presets/minimal/deneb.yaml @@ -2,11 +2,9 @@ # Misc # --------------------------------------------------------------- -# [customized] +# `uint64(4096)` FIELD_ELEMENTS_PER_BLOB: 4096 # [customized] -MAX_BLOB_COMMITMENTS_PER_BLOCK: 16 -# `uint64(6)` -MAX_BLOBS_PER_BLOCK: 6 -# [customized] `floorlog2(BLOB_KZG_COMMITMENTS_GINDEX) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 4 = 9 -KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 9 +MAX_BLOB_COMMITMENTS_PER_BLOCK: 32 +# [customized] `floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 5 = 10 +KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 10 diff --git a/consensus/types/presets/minimal/electra.yaml b/consensus/types/presets/minimal/electra.yaml index ef1ce494d8..44e4769756 100644 --- a/consensus/types/presets/minimal/electra.yaml +++ b/consensus/types/presets/minimal/electra.yaml @@ -10,7 +10,7 @@ MAX_EFFECTIVE_BALANCE_ELECTRA: 2048000000000 # State list lengths # --------------------------------------------------------------- # `uint64(2**27)` (= 134,217,728) -PENDING_BALANCE_DEPOSITS_LIMIT: 134217728 +PENDING_DEPOSITS_LIMIT: 134217728 # [customized] `uint64(2**6)` (= 64) PENDING_PARTIAL_WITHDRAWALS_LIMIT: 64 # [customized] `uint64(2**6)` (= 64) @@ -29,8 +29,8 @@ WHISTLEBLOWER_REWARD_QUOTIENT_ELECTRA: 4096 MAX_ATTESTER_SLASHINGS_ELECTRA: 1 # `uint64(2**3)` (= 8) MAX_ATTESTATIONS_ELECTRA: 8 -# `uint64(2**0)` (= 1) -MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD: 1 +# `uint64(2**1)` (= 2) +MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD: 2 # Execution # --------------------------------------------------------------- @@ -41,5 +41,10 @@ MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD: 2 # Withdrawals processing # --------------------------------------------------------------- -# 2**0 ( = 1) pending withdrawals -MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP: 1 +# 2**1 ( = 2) pending withdrawals +MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP: 2 + +# Pending deposits processing +# --------------------------------------------------------------- +# 2**4 ( = 4) pending deposits +MAX_PENDING_DEPOSITS_PER_EPOCH: 16 diff --git a/consensus/types/presets/minimal/eip7594.yaml b/consensus/types/presets/minimal/fulu.yaml similarity index 92% rename from consensus/types/presets/minimal/eip7594.yaml rename to consensus/types/presets/minimal/fulu.yaml index 847719a421..c961eb7f3c 100644 --- a/consensus/types/presets/minimal/eip7594.yaml +++ b/consensus/types/presets/minimal/fulu.yaml @@ -1,4 +1,4 @@ -# Minimal preset - EIP7594 +# Minimal preset - Fulu # Misc # --------------------------------------------------------------- diff --git a/consensus/types/presets/minimal/phase0.yaml b/consensus/types/presets/minimal/phase0.yaml index 1f75603142..d9a6a2b6c0 100644 --- a/consensus/types/presets/minimal/phase0.yaml +++ b/consensus/types/presets/minimal/phase0.yaml @@ -4,11 +4,11 @@ # --------------------------------------------------------------- # [customized] Just 4 committees for slot for testing purposes MAX_COMMITTEES_PER_SLOT: 4 -# [customized] unsecure, but fast +# [customized] insecure, but fast TARGET_COMMITTEE_SIZE: 4 # 2**11 (= 2,048) MAX_VALIDATORS_PER_COMMITTEE: 2048 -# [customized] Faster, but unsecure. +# [customized] Faster, but insecure. SHUFFLE_ROUND_COUNT: 10 # 4 HYSTERESIS_QUOTIENT: 4 @@ -85,4 +85,4 @@ MAX_ATTESTATIONS: 128 # 2**4 (= 16) MAX_DEPOSITS: 16 # 2**4 (= 16) -MAX_VOLUNTARY_EXITS: 16 +MAX_VOLUNTARY_EXITS: 16 \ No newline at end of file diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index 190964736f..276b27b0f8 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -12,8 +12,8 @@ use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use super::{ - AggregateSignature, AttestationData, BitList, ChainSpec, Domain, EthSpec, Fork, SecretKey, - Signature, SignedRoot, + AggregateSignature, AttestationData, BitList, ChainSpec, CommitteeIndex, Domain, EthSpec, Fork, + SecretKey, Signature, SignedRoot, }; #[derive(Debug, PartialEq)] @@ -24,6 +24,10 @@ pub enum Error { IncorrectStateVariant, InvalidCommitteeLength, InvalidCommitteeIndex, + AttesterNotInCommittee(u64), + InvalidCommittee, + MissingCommittee, + NoCommitteeForSlotAndIndex { slot: Slot, index: CommitteeIndex }, } impl From for Error { @@ -231,6 +235,16 @@ impl Attestation { Attestation::Electra(att) => att.aggregation_bits.get(index), } } + + pub fn to_single_attestation_with_attester_index( + &self, + attester_index: u64, + ) -> Result { + match self { + Self::Base(_) => Err(Error::IncorrectStateVariant), + Self::Electra(attn) => attn.to_single_attestation_with_attester_index(attester_index), + } + } } impl AttestationRef<'_, E> { @@ -287,6 +301,14 @@ impl AttestationElectra { self.get_committee_indices().first().cloned() } + pub fn get_aggregation_bits(&self) -> Vec { + self.aggregation_bits + .iter() + .enumerate() + .filter_map(|(index, bit)| if bit { Some(index as u64) } else { None }) + .collect() + } + pub fn get_committee_indices(&self) -> Vec { self.committee_bits .iter() @@ -350,6 +372,22 @@ impl AttestationElectra { Ok(()) } } + + pub fn to_single_attestation_with_attester_index( + &self, + attester_index: u64, + ) -> Result { + let Some(committee_index) = self.committee_index() else { + return Err(Error::InvalidCommitteeIndex); + }; + + Ok(SingleAttestation { + committee_index, + attester_index, + data: self.data.clone(), + signature: self.signature.clone(), + }) + } } impl AttestationBase { @@ -527,6 +565,60 @@ impl ForkVersionDeserialize for Vec> { } } +#[derive( + Debug, + Clone, + Serialize, + Deserialize, + Decode, + Encode, + TestRandom, + Derivative, + arbitrary::Arbitrary, + TreeHash, + PartialEq, +)] +pub struct SingleAttestation { + #[serde(with = "serde_utils::quoted_u64")] + pub committee_index: u64, + #[serde(with = "serde_utils::quoted_u64")] + pub attester_index: u64, + pub data: AttestationData, + pub signature: AggregateSignature, +} + +impl SingleAttestation { + pub fn to_attestation(&self, committee: &[usize]) -> Result, Error> { + let aggregation_bit = committee + .iter() + .enumerate() + .find_map(|(i, &validator_index)| { + if self.attester_index as usize == validator_index { + return Some(i); + } + None + }) + .ok_or(Error::AttesterNotInCommittee(self.attester_index))?; + + let mut committee_bits: BitVector = BitVector::default(); + committee_bits + .set(self.committee_index as usize, true) + .map_err(|_| Error::InvalidCommitteeIndex)?; + + let mut aggregation_bits = + BitList::with_capacity(committee.len()).map_err(|_| Error::InvalidCommitteeLength)?; + + aggregation_bits.set(aggregation_bit, true)?; + + Ok(Attestation::Electra(AttestationElectra { + aggregation_bits, + committee_bits, + data: self.data.clone(), + signature: self.signature.clone(), + })) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/consensus/types/src/beacon_block.rs b/consensus/types/src/beacon_block.rs index 801b7dd1c7..6ea897cf1a 100644 --- a/consensus/types/src/beacon_block.rs +++ b/consensus/types/src/beacon_block.rs @@ -12,11 +12,11 @@ use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; -use self::indexed_attestation::{IndexedAttestationBase, IndexedAttestationElectra}; +use self::indexed_attestation::IndexedAttestationBase; /// A block of the `BeaconChain`. #[superstruct( - variants(Base, Altair, Bellatrix, Capella, Deneb, Electra), + variants(Base, Altair, Bellatrix, Capella, Deneb, Electra, Fulu), variant_attributes( derive( Debug, @@ -75,6 +75,8 @@ pub struct BeaconBlock = FullPayload pub body: BeaconBlockBodyDeneb, #[superstruct(only(Electra), partial_getter(rename = "body_electra"))] pub body: BeaconBlockBodyElectra, + #[superstruct(only(Fulu), partial_getter(rename = "body_fulu"))] + pub body: BeaconBlockBodyFulu, } pub type BlindedBeaconBlock = BeaconBlock>; @@ -127,8 +129,9 @@ impl> BeaconBlock { /// Usually it's better to prefer `from_ssz_bytes` which will decode the correct variant based /// on the fork slot. pub fn any_from_ssz_bytes(bytes: &[u8]) -> Result { - BeaconBlockElectra::from_ssz_bytes(bytes) - .map(BeaconBlock::Electra) + BeaconBlockFulu::from_ssz_bytes(bytes) + .map(BeaconBlock::Fulu) + .or_else(|_| BeaconBlockElectra::from_ssz_bytes(bytes).map(BeaconBlock::Electra)) .or_else(|_| BeaconBlockDeneb::from_ssz_bytes(bytes).map(BeaconBlock::Deneb)) .or_else(|_| BeaconBlockCapella::from_ssz_bytes(bytes).map(BeaconBlock::Capella)) .or_else(|_| BeaconBlockBellatrix::from_ssz_bytes(bytes).map(BeaconBlock::Bellatrix)) @@ -226,6 +229,7 @@ impl<'a, E: EthSpec, Payload: AbstractExecPayload> BeaconBlockRef<'a, E, Payl BeaconBlockRef::Capella { .. } => ForkName::Capella, BeaconBlockRef::Deneb { .. } => ForkName::Deneb, BeaconBlockRef::Electra { .. } => ForkName::Electra, + BeaconBlockRef::Fulu { .. } => ForkName::Fulu, } } @@ -495,52 +499,6 @@ impl> EmptyBlock for BeaconBlockBell } } -impl> BeaconBlockCapella { - /// Return a Capella block where the block has maximum size. - pub fn full(spec: &ChainSpec) -> Self { - let base_block: BeaconBlockBase<_, Payload> = BeaconBlockBase::full(spec); - let bls_to_execution_changes = vec![ - SignedBlsToExecutionChange { - message: BlsToExecutionChange { - validator_index: 0, - from_bls_pubkey: PublicKeyBytes::empty(), - to_execution_address: Address::ZERO, - }, - signature: Signature::empty() - }; - E::max_bls_to_execution_changes() - ] - .into(); - let sync_aggregate = SyncAggregate { - sync_committee_signature: AggregateSignature::empty(), - sync_committee_bits: BitVector::default(), - }; - BeaconBlockCapella { - slot: spec.genesis_slot, - proposer_index: 0, - parent_root: Hash256::zero(), - state_root: Hash256::zero(), - body: BeaconBlockBodyCapella { - proposer_slashings: base_block.body.proposer_slashings, - attester_slashings: base_block.body.attester_slashings, - attestations: base_block.body.attestations, - deposits: base_block.body.deposits, - voluntary_exits: base_block.body.voluntary_exits, - bls_to_execution_changes, - sync_aggregate, - randao_reveal: Signature::empty(), - eth1_data: Eth1Data { - deposit_root: Hash256::zero(), - block_hash: Hash256::zero(), - deposit_count: 0, - }, - graffiti: Graffiti::default(), - execution_payload: Payload::Capella::default(), - }, - } - } -} - impl> EmptyBlock for BeaconBlockCapella { /// Returns an empty Capella block to be used during genesis. fn empty(spec: &ChainSpec) -> Self { @@ -600,79 +558,6 @@ impl> EmptyBlock for BeaconBlockDene } } -impl> BeaconBlockElectra { - /// Return a Electra block where the block has maximum size. - pub fn full(spec: &ChainSpec) -> Self { - let base_block: BeaconBlockBase<_, Payload> = BeaconBlockBase::full(spec); - let indexed_attestation: IndexedAttestationElectra = IndexedAttestationElectra { - attesting_indices: VariableList::new(vec![0_u64; E::MaxValidatorsPerSlot::to_usize()]) - .unwrap(), - data: AttestationData::default(), - signature: AggregateSignature::empty(), - }; - let attester_slashings = vec![ - AttesterSlashingElectra { - attestation_1: indexed_attestation.clone(), - attestation_2: indexed_attestation, - }; - E::max_attester_slashings_electra() - ] - .into(); - let attestation = AttestationElectra { - aggregation_bits: BitList::with_capacity(E::MaxValidatorsPerSlot::to_usize()).unwrap(), - data: AttestationData::default(), - signature: AggregateSignature::empty(), - committee_bits: BitVector::new(), - }; - let mut attestations_electra = vec![]; - for _ in 0..E::MaxAttestationsElectra::to_usize() { - attestations_electra.push(attestation.clone()); - } - - let bls_to_execution_changes = vec![ - SignedBlsToExecutionChange { - message: BlsToExecutionChange { - validator_index: 0, - from_bls_pubkey: PublicKeyBytes::empty(), - to_execution_address: Address::ZERO, - }, - signature: Signature::empty() - }; - E::max_bls_to_execution_changes() - ] - .into(); - let sync_aggregate = SyncAggregate { - sync_committee_signature: AggregateSignature::empty(), - sync_committee_bits: BitVector::default(), - }; - BeaconBlockElectra { - slot: spec.genesis_slot, - proposer_index: 0, - parent_root: Hash256::zero(), - state_root: Hash256::zero(), - body: BeaconBlockBodyElectra { - proposer_slashings: base_block.body.proposer_slashings, - attester_slashings, - attestations: attestations_electra.into(), - deposits: base_block.body.deposits, - voluntary_exits: base_block.body.voluntary_exits, - bls_to_execution_changes, - sync_aggregate, - randao_reveal: Signature::empty(), - eth1_data: Eth1Data { - deposit_root: Hash256::zero(), - block_hash: Hash256::zero(), - deposit_count: 0, - }, - graffiti: Graffiti::default(), - execution_payload: Payload::Electra::default(), - blob_kzg_commitments: VariableList::empty(), - execution_requests: ExecutionRequests::default(), - }, - } - } -} - impl> EmptyBlock for BeaconBlockElectra { /// Returns an empty Electra block to be used during genesis. fn empty(spec: &ChainSpec) -> Self { @@ -704,6 +589,37 @@ impl> EmptyBlock for BeaconBlockElec } } +impl> EmptyBlock for BeaconBlockFulu { + /// Returns an empty Fulu block to be used during genesis. + fn empty(spec: &ChainSpec) -> Self { + BeaconBlockFulu { + slot: spec.genesis_slot, + proposer_index: 0, + parent_root: Hash256::zero(), + state_root: Hash256::zero(), + body: BeaconBlockBodyFulu { + randao_reveal: Signature::empty(), + eth1_data: Eth1Data { + deposit_root: Hash256::zero(), + block_hash: Hash256::zero(), + deposit_count: 0, + }, + graffiti: Graffiti::default(), + proposer_slashings: VariableList::empty(), + attester_slashings: VariableList::empty(), + attestations: VariableList::empty(), + deposits: VariableList::empty(), + voluntary_exits: VariableList::empty(), + sync_aggregate: SyncAggregate::empty(), + execution_payload: Payload::Fulu::default(), + bls_to_execution_changes: VariableList::empty(), + blob_kzg_commitments: VariableList::empty(), + execution_requests: ExecutionRequests::default(), + }, + } + } +} + // We can convert pre-Bellatrix blocks without payloads into blocks "with" payloads. impl From>> for BeaconBlockBase> @@ -785,6 +701,7 @@ impl_from!(BeaconBlockBellatrix, >, >, |b impl_from!(BeaconBlockCapella, >, >, |body: BeaconBlockBodyCapella<_, _>| body.into()); impl_from!(BeaconBlockDeneb, >, >, |body: BeaconBlockBodyDeneb<_, _>| body.into()); impl_from!(BeaconBlockElectra, >, >, |body: BeaconBlockBodyElectra<_, _>| body.into()); +impl_from!(BeaconBlockFulu, >, >, |body: BeaconBlockBodyFulu<_, _>| body.into()); // We can clone blocks with payloads to blocks without payloads, without cloning the payload. macro_rules! impl_clone_as_blinded { @@ -818,6 +735,7 @@ impl_clone_as_blinded!(BeaconBlockBellatrix, >, >, >); impl_clone_as_blinded!(BeaconBlockDeneb, >, >); impl_clone_as_blinded!(BeaconBlockElectra, >, >); +impl_clone_as_blinded!(BeaconBlockFulu, >, >); // A reference to a full beacon block can be cloned into a blinded beacon block, without cloning the // execution payload. @@ -988,6 +906,26 @@ mod tests { }); } + #[test] + fn roundtrip_fulu_block() { + let rng = &mut XorShiftRng::from_seed([42; 16]); + let spec = &ForkName::Fulu.make_genesis_spec(MainnetEthSpec::default_spec()); + + let inner_block = BeaconBlockFulu { + slot: Slot::random_for_test(rng), + proposer_index: u64::random_for_test(rng), + parent_root: Hash256::random_for_test(rng), + state_root: Hash256::random_for_test(rng), + body: BeaconBlockBodyFulu::random_for_test(rng), + }; + + let block = BeaconBlock::Fulu(inner_block.clone()); + + test_ssz_tree_hash_pair_with(&block, &inner_block, |bytes| { + BeaconBlock::from_ssz_bytes(bytes, spec) + }); + } + #[test] fn decode_base_and_altair() { type E = MainnetEthSpec; @@ -1007,11 +945,14 @@ mod tests { let deneb_slot = deneb_epoch.start_slot(E::slots_per_epoch()); let electra_epoch = deneb_epoch + 1; let electra_slot = electra_epoch.start_slot(E::slots_per_epoch()); + let fulu_epoch = electra_epoch + 1; + let fulu_slot = fulu_epoch.start_slot(E::slots_per_epoch()); spec.altair_fork_epoch = Some(altair_epoch); spec.capella_fork_epoch = Some(capella_epoch); spec.deneb_fork_epoch = Some(deneb_epoch); spec.electra_fork_epoch = Some(electra_epoch); + spec.fulu_fork_epoch = Some(fulu_epoch); // BeaconBlockBase { @@ -1122,5 +1063,29 @@ mod tests { BeaconBlock::from_ssz_bytes(&bad_block.as_ssz_bytes(), &spec) .expect_err("bad electra block cannot be decoded"); } + + // BeaconBlockFulu + { + let good_block = BeaconBlock::Fulu(BeaconBlockFulu { + slot: fulu_slot, + ..<_>::random_for_test(rng) + }); + // It's invalid to have a Fulu block with a epoch lower than the fork epoch. + let _bad_block = { + let mut bad = good_block.clone(); + *bad.slot_mut() = electra_slot; + bad + }; + + assert_eq!( + BeaconBlock::from_ssz_bytes(&good_block.as_ssz_bytes(), &spec) + .expect("good fulu block can be decoded"), + good_block + ); + // TODO(fulu): Uncomment once Fulu has features since without features + // and with an Electra slot it decodes successfully to Electra. + //BeaconBlock::from_ssz_bytes(&bad_block.as_ssz_bytes(), &spec) + // .expect_err("bad fulu block cannot be decoded"); + } } } diff --git a/consensus/types/src/beacon_block_body.rs b/consensus/types/src/beacon_block_body.rs index b896dc4693..3f75790a35 100644 --- a/consensus/types/src/beacon_block_body.rs +++ b/consensus/types/src/beacon_block_body.rs @@ -13,8 +13,6 @@ use tree_hash_derive::TreeHash; pub type KzgCommitments = VariableList::MaxBlobCommitmentsPerBlock>; -pub type KzgCommitmentOpts = - FixedVector, ::MaxBlobsPerBlock>; /// The number of leaves (including padding) on the `BeaconBlockBody` Merkle tree. /// @@ -30,7 +28,7 @@ pub const BLOB_KZG_COMMITMENTS_INDEX: usize = 11; /// /// This *superstruct* abstracts over the hard-fork. #[superstruct( - variants(Base, Altair, Bellatrix, Capella, Deneb, Electra), + variants(Base, Altair, Bellatrix, Capella, Deneb, Electra, Fulu), variant_attributes( derive( Debug, @@ -58,6 +56,7 @@ pub const BLOB_KZG_COMMITMENTS_INDEX: usize = 11; Capella(metastruct(mappings(beacon_block_body_capella_fields(groups(fields))))), Deneb(metastruct(mappings(beacon_block_body_deneb_fields(groups(fields))))), Electra(metastruct(mappings(beacon_block_body_electra_fields(groups(fields))))), + Fulu(metastruct(mappings(beacon_block_body_fulu_fields(groups(fields))))) ), cast_error(ty = "Error", expr = "Error::IncorrectStateVariant"), partial_getter_error(ty = "Error", expr = "Error::IncorrectStateVariant") @@ -77,7 +76,10 @@ pub struct BeaconBlockBody = FullPay partial_getter(rename = "attester_slashings_base") )] pub attester_slashings: VariableList, E::MaxAttesterSlashings>, - #[superstruct(only(Electra), partial_getter(rename = "attester_slashings_electra"))] + #[superstruct( + only(Electra, Fulu), + partial_getter(rename = "attester_slashings_electra") + )] pub attester_slashings: VariableList, E::MaxAttesterSlashingsElectra>, #[superstruct( @@ -85,11 +87,11 @@ pub struct BeaconBlockBody = FullPay partial_getter(rename = "attestations_base") )] pub attestations: VariableList, E::MaxAttestations>, - #[superstruct(only(Electra), partial_getter(rename = "attestations_electra"))] + #[superstruct(only(Electra, Fulu), partial_getter(rename = "attestations_electra"))] pub attestations: VariableList, E::MaxAttestationsElectra>, pub deposits: VariableList, pub voluntary_exits: VariableList, - #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra))] + #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra, Fulu))] pub sync_aggregate: SyncAggregate, // We flatten the execution payload so that serde can use the name of the inner type, // either `execution_payload` for full payloads, or `execution_payload_header` for blinded @@ -109,12 +111,15 @@ pub struct BeaconBlockBody = FullPay #[superstruct(only(Electra), partial_getter(rename = "execution_payload_electra"))] #[serde(flatten)] pub execution_payload: Payload::Electra, - #[superstruct(only(Capella, Deneb, Electra))] + #[superstruct(only(Fulu), partial_getter(rename = "execution_payload_fulu"))] + #[serde(flatten)] + pub execution_payload: Payload::Fulu, + #[superstruct(only(Capella, Deneb, Electra, Fulu))] pub bls_to_execution_changes: VariableList, - #[superstruct(only(Deneb, Electra))] + #[superstruct(only(Deneb, Electra, Fulu))] pub blob_kzg_commitments: KzgCommitments, - #[superstruct(only(Electra))] + #[superstruct(only(Electra, Fulu))] pub execution_requests: ExecutionRequests, #[superstruct(only(Base, Altair))] #[metastruct(exclude_from(fields))] @@ -144,6 +149,7 @@ impl<'a, E: EthSpec, Payload: AbstractExecPayload> BeaconBlockBodyRef<'a, E, Self::Capella(body) => Ok(Payload::Ref::from(&body.execution_payload)), Self::Deneb(body) => Ok(Payload::Ref::from(&body.execution_payload)), Self::Electra(body) => Ok(Payload::Ref::from(&body.execution_payload)), + Self::Fulu(body) => Ok(Payload::Ref::from(&body.execution_payload)), } } @@ -174,6 +180,10 @@ impl<'a, E: EthSpec, Payload: AbstractExecPayload> BeaconBlockBodyRef<'a, E, beacon_block_body_electra_fields!(body, |_, field| leaves .push(field.tree_hash_root())); } + Self::Fulu(body) => { + beacon_block_body_fulu_fields!(body, |_, field| leaves + .push(field.tree_hash_root())); + } } leaves } @@ -202,7 +212,7 @@ impl<'a, E: EthSpec, Payload: AbstractExecPayload> BeaconBlockBodyRef<'a, E, Self::Base(_) | Self::Altair(_) | Self::Bellatrix(_) | Self::Capella(_) => { Err(Error::IncorrectStateVariant) } - Self::Deneb(_) | Self::Electra(_) => { + Self::Deneb(_) | Self::Electra(_) | Self::Fulu(_) => { // We compute the branches by generating 2 merkle trees: // 1. Merkle tree for the `blob_kzg_commitments` List object // 2. Merkle tree for the `BeaconBlockBody` container @@ -283,7 +293,7 @@ impl<'a, E: EthSpec, Payload: AbstractExecPayload> BeaconBlockBodyRef<'a, E, /// Return `true` if this block body has a non-zero number of blobs. pub fn has_blobs(self) -> bool { self.blob_kzg_commitments() - .map_or(false, |blobs| !blobs.is_empty()) + .is_ok_and(|blobs| !blobs.is_empty()) } pub fn attestations_len(&self) -> usize { @@ -294,6 +304,7 @@ impl<'a, E: EthSpec, Payload: AbstractExecPayload> BeaconBlockBodyRef<'a, E, Self::Capella(body) => body.attestations.len(), Self::Deneb(body) => body.attestations.len(), Self::Electra(body) => body.attestations.len(), + Self::Fulu(body) => body.attestations.len(), } } @@ -305,6 +316,7 @@ impl<'a, E: EthSpec, Payload: AbstractExecPayload> BeaconBlockBodyRef<'a, E, Self::Capella(body) => body.attester_slashings.len(), Self::Deneb(body) => body.attester_slashings.len(), Self::Electra(body) => body.attester_slashings.len(), + Self::Fulu(body) => body.attester_slashings.len(), } } @@ -316,6 +328,7 @@ impl<'a, E: EthSpec, Payload: AbstractExecPayload> BeaconBlockBodyRef<'a, E, Self::Capella(body) => Box::new(body.attestations.iter().map(AttestationRef::Base)), Self::Deneb(body) => Box::new(body.attestations.iter().map(AttestationRef::Base)), Self::Electra(body) => Box::new(body.attestations.iter().map(AttestationRef::Electra)), + Self::Fulu(body) => Box::new(body.attestations.iter().map(AttestationRef::Electra)), } } @@ -351,6 +364,11 @@ impl<'a, E: EthSpec, Payload: AbstractExecPayload> BeaconBlockBodyRef<'a, E, .iter() .map(AttesterSlashingRef::Electra), ), + Self::Fulu(body) => Box::new( + body.attester_slashings + .iter() + .map(AttesterSlashingRef::Electra), + ), } } } @@ -376,6 +394,9 @@ impl<'a, E: EthSpec, Payload: AbstractExecPayload> BeaconBlockBodyRefMut<'a, Self::Electra(body) => { Box::new(body.attestations.iter_mut().map(AttestationRefMut::Electra)) } + Self::Fulu(body) => { + Box::new(body.attestations.iter_mut().map(AttestationRefMut::Electra)) + } } } } @@ -390,6 +411,7 @@ impl> BeaconBlockBodyRef<'_, E, Payl BeaconBlockBodyRef::Capella { .. } => ForkName::Capella, BeaconBlockBodyRef::Deneb { .. } => ForkName::Deneb, BeaconBlockBodyRef::Electra { .. } => ForkName::Electra, + BeaconBlockBodyRef::Fulu { .. } => ForkName::Fulu, } } } @@ -704,6 +726,52 @@ impl From>> } } +impl From>> + for ( + BeaconBlockBodyFulu>, + Option>, + ) +{ + fn from(body: BeaconBlockBodyFulu>) -> Self { + let BeaconBlockBodyFulu { + randao_reveal, + eth1_data, + graffiti, + proposer_slashings, + attester_slashings, + attestations, + deposits, + voluntary_exits, + sync_aggregate, + execution_payload: FullPayloadFulu { execution_payload }, + bls_to_execution_changes, + blob_kzg_commitments, + execution_requests, + } = body; + + ( + BeaconBlockBodyFulu { + randao_reveal, + eth1_data, + graffiti, + proposer_slashings, + attester_slashings, + attestations, + deposits, + voluntary_exits, + sync_aggregate, + execution_payload: BlindedPayloadFulu { + execution_payload_header: From::from(&execution_payload), + }, + bls_to_execution_changes, + blob_kzg_commitments: blob_kzg_commitments.clone(), + execution_requests, + }, + Some(execution_payload), + ) + } +} + // We can clone a full block into a blinded block, without cloning the payload. impl BeaconBlockBodyBase> { pub fn clone_as_blinded(&self) -> BeaconBlockBodyBase> { @@ -859,6 +927,44 @@ impl BeaconBlockBodyElectra> { } } +impl BeaconBlockBodyFulu> { + pub fn clone_as_blinded(&self) -> BeaconBlockBodyFulu> { + let BeaconBlockBodyFulu { + randao_reveal, + eth1_data, + graffiti, + proposer_slashings, + attester_slashings, + attestations, + deposits, + voluntary_exits, + sync_aggregate, + execution_payload: FullPayloadFulu { execution_payload }, + bls_to_execution_changes, + blob_kzg_commitments, + execution_requests, + } = self; + + BeaconBlockBodyFulu { + randao_reveal: randao_reveal.clone(), + eth1_data: eth1_data.clone(), + graffiti: *graffiti, + proposer_slashings: proposer_slashings.clone(), + attester_slashings: attester_slashings.clone(), + attestations: attestations.clone(), + deposits: deposits.clone(), + voluntary_exits: voluntary_exits.clone(), + sync_aggregate: sync_aggregate.clone(), + execution_payload: BlindedPayloadFulu { + execution_payload_header: execution_payload.into(), + }, + bls_to_execution_changes: bls_to_execution_changes.clone(), + blob_kzg_commitments: blob_kzg_commitments.clone(), + execution_requests: execution_requests.clone(), + } + } +} + impl From>> for ( BeaconBlockBody>, diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index e78febaf6c..f1adafd9ee 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -48,6 +48,7 @@ mod tests; pub const CACHED_EPOCHS: usize = 3; const MAX_RANDOM_BYTE: u64 = (1 << 8) - 1; +const MAX_RANDOM_VALUE: u64 = (1 << 16) - 1; pub type Validators = List::ValidatorRegistryLimit>; pub type Balances = List::ValidatorRegistryLimit>; @@ -162,7 +163,7 @@ pub enum Error { InvalidFlagIndex(usize), MerkleTreeError(merkle_proof::MerkleTreeError), PartialWithdrawalCountInvalid(usize), - NonExecutionAddresWithdrawalCredential, + NonExecutionAddressWithdrawalCredential, NoCommitteeFound(CommitteeIndex), InvalidCommitteeIndex(CommitteeIndex), InvalidSelectionProof { @@ -225,7 +226,7 @@ impl From for Hash256 { /// /// https://github.com/sigp/milhouse/issues/43 #[superstruct( - variants(Base, Altair, Bellatrix, Capella, Deneb, Electra), + variants(Base, Altair, Bellatrix, Capella, Deneb, Electra, Fulu), variant_attributes( derive( Derivative, @@ -328,6 +329,20 @@ impl From for Hash256 { groups(tree_lists) )), num_fields(all()), + )), + Fulu(metastruct( + mappings( + map_beacon_state_fulu_fields(), + map_beacon_state_fulu_tree_list_fields(mutable, fallible, groups(tree_lists)), + map_beacon_state_fulu_tree_list_fields_immutable(groups(tree_lists)), + ), + bimappings(bimap_beacon_state_fulu_tree_list_fields( + other_type = "BeaconStateFulu", + self_mutable, + fallible, + groups(tree_lists) + )), + num_fields(all()), )) ), cast_error(ty = "Error", expr = "Error::IncorrectStateVariant"), @@ -410,11 +425,11 @@ where // Participation (Altair and later) #[compare_fields(as_iter)] - #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra))] + #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra, Fulu))] #[test_random(default)] #[compare_fields(as_iter)] pub previous_epoch_participation: List, - #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra))] + #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra, Fulu))] #[test_random(default)] pub current_epoch_participation: List, @@ -434,15 +449,15 @@ where // Inactivity #[serde(with = "ssz_types::serde_utils::quoted_u64_var_list")] - #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra))] + #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra, Fulu))] #[test_random(default)] pub inactivity_scores: List, // Light-client sync committees - #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra))] + #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra, Fulu))] #[metastruct(exclude_from(tree_lists))] pub current_sync_committee: Arc>, - #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra))] + #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra, Fulu))] #[metastruct(exclude_from(tree_lists))] pub next_sync_committee: Arc>, @@ -471,56 +486,62 @@ where )] #[metastruct(exclude_from(tree_lists))] pub latest_execution_payload_header: ExecutionPayloadHeaderElectra, + #[superstruct( + only(Fulu), + partial_getter(rename = "latest_execution_payload_header_fulu") + )] + #[metastruct(exclude_from(tree_lists))] + pub latest_execution_payload_header: ExecutionPayloadHeaderFulu, // Capella - #[superstruct(only(Capella, Deneb, Electra), partial_getter(copy))] + #[superstruct(only(Capella, Deneb, Electra, Fulu), partial_getter(copy))] #[serde(with = "serde_utils::quoted_u64")] #[metastruct(exclude_from(tree_lists))] pub next_withdrawal_index: u64, - #[superstruct(only(Capella, Deneb, Electra), partial_getter(copy))] + #[superstruct(only(Capella, Deneb, Electra, Fulu), partial_getter(copy))] #[serde(with = "serde_utils::quoted_u64")] #[metastruct(exclude_from(tree_lists))] pub next_withdrawal_validator_index: u64, // Deep history valid from Capella onwards. - #[superstruct(only(Capella, Deneb, Electra))] + #[superstruct(only(Capella, Deneb, Electra, Fulu))] #[test_random(default)] pub historical_summaries: List, // Electra - #[superstruct(only(Electra), partial_getter(copy))] + #[superstruct(only(Electra, Fulu), partial_getter(copy))] #[metastruct(exclude_from(tree_lists))] #[serde(with = "serde_utils::quoted_u64")] pub deposit_requests_start_index: u64, - #[superstruct(only(Electra), partial_getter(copy))] + #[superstruct(only(Electra, Fulu), partial_getter(copy))] #[metastruct(exclude_from(tree_lists))] #[serde(with = "serde_utils::quoted_u64")] pub deposit_balance_to_consume: u64, - #[superstruct(only(Electra), partial_getter(copy))] + #[superstruct(only(Electra, Fulu), partial_getter(copy))] #[metastruct(exclude_from(tree_lists))] #[serde(with = "serde_utils::quoted_u64")] pub exit_balance_to_consume: u64, - #[superstruct(only(Electra), partial_getter(copy))] + #[superstruct(only(Electra, Fulu), partial_getter(copy))] #[metastruct(exclude_from(tree_lists))] pub earliest_exit_epoch: Epoch, - #[superstruct(only(Electra), partial_getter(copy))] + #[superstruct(only(Electra, Fulu), partial_getter(copy))] #[metastruct(exclude_from(tree_lists))] #[serde(with = "serde_utils::quoted_u64")] pub consolidation_balance_to_consume: u64, - #[superstruct(only(Electra), partial_getter(copy))] + #[superstruct(only(Electra, Fulu), partial_getter(copy))] #[metastruct(exclude_from(tree_lists))] pub earliest_consolidation_epoch: Epoch, #[compare_fields(as_iter)] #[test_random(default)] - #[superstruct(only(Electra))] + #[superstruct(only(Electra, Fulu))] pub pending_deposits: List, #[compare_fields(as_iter)] #[test_random(default)] - #[superstruct(only(Electra))] + #[superstruct(only(Electra, Fulu))] pub pending_partial_withdrawals: List, #[compare_fields(as_iter)] #[test_random(default)] - #[superstruct(only(Electra))] + #[superstruct(only(Electra, Fulu))] pub pending_consolidations: List, // Caching (not in the spec) @@ -661,6 +682,7 @@ impl BeaconState { BeaconState::Capella { .. } => ForkName::Capella, BeaconState::Deneb { .. } => ForkName::Deneb, BeaconState::Electra { .. } => ForkName::Electra, + BeaconState::Fulu { .. } => ForkName::Fulu, } } @@ -952,6 +974,11 @@ impl BeaconState { } let max_effective_balance = spec.max_effective_balance_for_fork(self.fork_name_unchecked()); + let max_random_value = if self.fork_name_unchecked().electra_enabled() { + MAX_RANDOM_VALUE + } else { + MAX_RANDOM_BYTE + }; let mut i = 0; loop { @@ -965,10 +992,10 @@ impl BeaconState { let candidate_index = *indices .get(shuffled_index) .ok_or(Error::ShuffleIndexOutOfBounds(shuffled_index))?; - let random_byte = Self::shuffling_random_byte(i, seed)?; + let random_value = self.shuffling_random_value(i, seed)?; let effective_balance = self.get_effective_balance(candidate_index)?; - if effective_balance.safe_mul(MAX_RANDOM_BYTE)? - >= max_effective_balance.safe_mul(u64::from(random_byte))? + if effective_balance.safe_mul(max_random_value)? + >= max_effective_balance.safe_mul(random_value)? { return Ok(candidate_index); } @@ -976,6 +1003,19 @@ impl BeaconState { } } + /// Fork-aware abstraction for the shuffling. + /// + /// In Electra and later, the random value is a 16-bit integer stored in a `u64`. + /// + /// Prior to Electra, the random value is an 8-bit integer stored in a `u64`. + fn shuffling_random_value(&self, i: usize, seed: &[u8]) -> Result { + if self.fork_name_unchecked().electra_enabled() { + Self::shuffling_random_u16_electra(i, seed).map(u64::from) + } else { + Self::shuffling_random_byte(i, seed).map(u64::from) + } + } + /// Get a random byte from the given `seed`. /// /// Used by the proposer & sync committee selection functions. @@ -989,6 +1029,21 @@ impl BeaconState { .ok_or(Error::ShuffleIndexOutOfBounds(index)) } + /// Get two random bytes from the given `seed`. + /// + /// This is used in place of `shuffling_random_byte` from Electra onwards. + fn shuffling_random_u16_electra(i: usize, seed: &[u8]) -> Result { + let mut preimage = seed.to_vec(); + preimage.append(&mut int_to_bytes8(i.safe_div(16)? as u64)); + let offset = i.safe_rem(16)?.safe_mul(2)?; + hash(&preimage) + .get(offset..offset.safe_add(2)?) + .ok_or(Error::ShuffleIndexOutOfBounds(offset))? + .try_into() + .map(u16::from_le_bytes) + .map_err(|_| Error::ShuffleIndexOutOfBounds(offset)) + } + /// Convenience accessor for the `execution_payload_header` as an `ExecutionPayloadHeaderRef`. pub fn latest_execution_payload_header(&self) -> Result, Error> { match self { @@ -1005,6 +1060,9 @@ impl BeaconState { BeaconState::Electra(state) => Ok(ExecutionPayloadHeaderRef::Electra( &state.latest_execution_payload_header, )), + BeaconState::Fulu(state) => Ok(ExecutionPayloadHeaderRef::Fulu( + &state.latest_execution_payload_header, + )), } } @@ -1025,6 +1083,9 @@ impl BeaconState { BeaconState::Electra(state) => Ok(ExecutionPayloadHeaderRefMut::Electra( &mut state.latest_execution_payload_header, )), + BeaconState::Fulu(state) => Ok(ExecutionPayloadHeaderRefMut::Fulu( + &mut state.latest_execution_payload_header, + )), } } @@ -1150,6 +1211,11 @@ impl BeaconState { let seed = self.get_seed(epoch, Domain::SyncCommittee, spec)?; let max_effective_balance = spec.max_effective_balance_for_fork(self.fork_name_unchecked()); + let max_random_value = if self.fork_name_unchecked().electra_enabled() { + MAX_RANDOM_VALUE + } else { + MAX_RANDOM_BYTE + }; let mut i = 0; let mut sync_committee_indices = Vec::with_capacity(E::SyncCommitteeSize::to_usize()); @@ -1164,10 +1230,10 @@ impl BeaconState { let candidate_index = *active_validator_indices .get(shuffled_index) .ok_or(Error::ShuffleIndexOutOfBounds(shuffled_index))?; - let random_byte = Self::shuffling_random_byte(i, seed.as_slice())?; + let random_value = self.shuffling_random_value(i, seed.as_slice())?; let effective_balance = self.get_validator(candidate_index)?.effective_balance; - if effective_balance.safe_mul(MAX_RANDOM_BYTE)? - >= max_effective_balance.safe_mul(u64::from(random_byte))? + if effective_balance.safe_mul(max_random_value)? + >= max_effective_balance.safe_mul(random_value)? { sync_committee_indices.push(candidate_index); } @@ -1538,6 +1604,16 @@ impl BeaconState { &mut state.exit_cache, &mut state.epoch_cache, )), + BeaconState::Fulu(state) => Ok(( + &mut state.validators, + &mut state.balances, + &state.previous_epoch_participation, + &state.current_epoch_participation, + &mut state.inactivity_scores, + &mut state.progressive_balances_cache, + &mut state.exit_cache, + &mut state.epoch_cache, + )), } } @@ -1719,10 +1795,12 @@ impl BeaconState { | BeaconState::Altair(_) | BeaconState::Bellatrix(_) | BeaconState::Capella(_) => self.get_validator_churn_limit(spec)?, - BeaconState::Deneb(_) | BeaconState::Electra(_) => std::cmp::min( - spec.max_per_epoch_activation_churn_limit, - self.get_validator_churn_limit(spec)?, - ), + BeaconState::Deneb(_) | BeaconState::Electra(_) | BeaconState::Fulu(_) => { + std::cmp::min( + spec.max_per_epoch_activation_churn_limit, + self.get_validator_churn_limit(spec)?, + ) + } }) } @@ -1861,6 +1939,7 @@ impl BeaconState { BeaconState::Capella(state) => Ok(&mut state.current_epoch_participation), BeaconState::Deneb(state) => Ok(&mut state.current_epoch_participation), BeaconState::Electra(state) => Ok(&mut state.current_epoch_participation), + BeaconState::Fulu(state) => Ok(&mut state.current_epoch_participation), } } else if epoch == previous_epoch { match self { @@ -1870,6 +1949,7 @@ impl BeaconState { BeaconState::Capella(state) => Ok(&mut state.previous_epoch_participation), BeaconState::Deneb(state) => Ok(&mut state.previous_epoch_participation), BeaconState::Electra(state) => Ok(&mut state.previous_epoch_participation), + BeaconState::Fulu(state) => Ok(&mut state.previous_epoch_participation), } } else { Err(BeaconStateError::EpochOutOfBounds) @@ -1934,7 +2014,7 @@ impl BeaconState { pub fn committee_cache_is_initialized(&self, relative_epoch: RelativeEpoch) -> bool { let i = Self::committee_cache_index(relative_epoch); - self.committee_cache_at_index(i).map_or(false, |cache| { + self.committee_cache_at_index(i).is_ok_and(|cache| { cache.is_initialized_at(relative_epoch.into_epoch(self.current_epoch())) }) } @@ -2123,6 +2203,11 @@ impl BeaconState { } ); } + Self::Fulu(self_inner) => { + map_beacon_state_fulu_tree_list_fields_immutable!(self_inner, |_, self_field| { + any_pending_mutations |= self_field.has_pending_updates(); + }); + } }; any_pending_mutations } @@ -2207,7 +2292,7 @@ impl BeaconState { // ******* Electra accessors ******* - /// Return the churn limit for the current epoch. + /// Return the churn limit for the current epoch. pub fn get_balance_churn_limit(&self, spec: &ChainSpec) -> Result { let total_active_balance = self.get_total_active_balance()?; let churn = std::cmp::max( @@ -2237,7 +2322,7 @@ impl BeaconState { for withdrawal in self .pending_partial_withdrawals()? .iter() - .filter(|withdrawal| withdrawal.index as usize == validator_index) + .filter(|withdrawal| withdrawal.validator_index as usize == validator_index) { pending_balance.safe_add_assign(withdrawal.amount)?; } @@ -2316,12 +2401,20 @@ impl BeaconState { exit_balance_to_consume .safe_add_assign(additional_epochs.safe_mul(per_epoch_churn)?)?; } - let state = self.as_electra_mut()?; - // Consume the balance and update state variables - state.exit_balance_to_consume = exit_balance_to_consume.safe_sub(exit_balance)?; - state.earliest_exit_epoch = earliest_exit_epoch; - - Ok(state.earliest_exit_epoch) + match self { + BeaconState::Base(_) + | BeaconState::Altair(_) + | BeaconState::Bellatrix(_) + | BeaconState::Capella(_) + | BeaconState::Deneb(_) => Err(Error::IncorrectStateVariant), + BeaconState::Electra(_) | BeaconState::Fulu(_) => { + // Consume the balance and update state variables + *self.exit_balance_to_consume_mut()? = + exit_balance_to_consume.safe_sub(exit_balance)?; + *self.earliest_exit_epoch_mut()? = earliest_exit_epoch; + self.earliest_exit_epoch() + } + } } pub fn compute_consolidation_epoch_and_update_churn( @@ -2355,13 +2448,20 @@ impl BeaconState { consolidation_balance_to_consume .safe_add_assign(additional_epochs.safe_mul(per_epoch_consolidation_churn)?)?; } - // Consume the balance and update state variables - let state = self.as_electra_mut()?; - state.consolidation_balance_to_consume = - consolidation_balance_to_consume.safe_sub(consolidation_balance)?; - state.earliest_consolidation_epoch = earliest_consolidation_epoch; - - Ok(state.earliest_consolidation_epoch) + match self { + BeaconState::Base(_) + | BeaconState::Altair(_) + | BeaconState::Bellatrix(_) + | BeaconState::Capella(_) + | BeaconState::Deneb(_) => Err(Error::IncorrectStateVariant), + BeaconState::Electra(_) | BeaconState::Fulu(_) => { + // Consume the balance and update state variables. + *self.consolidation_balance_to_consume_mut()? = + consolidation_balance_to_consume.safe_sub(consolidation_balance)?; + *self.earliest_consolidation_epoch_mut()? = earliest_consolidation_epoch; + self.earliest_consolidation_epoch() + } + } } #[allow(clippy::arithmetic_side_effects)] @@ -2417,6 +2517,14 @@ impl BeaconState { ); } (Self::Electra(_), _) => (), + (Self::Fulu(self_inner), Self::Fulu(base_inner)) => { + bimap_beacon_state_fulu_tree_list_fields!( + self_inner, + base_inner, + |_, self_field, base_field| { self_field.rebase_on(base_field) } + ); + } + (Self::Fulu(_), _) => (), } // Use sync committees from `base` if they are equal. @@ -2489,6 +2597,7 @@ impl BeaconState { ForkName::Capella => BeaconStateCapella::::NUM_FIELDS.next_power_of_two(), ForkName::Deneb => BeaconStateDeneb::::NUM_FIELDS.next_power_of_two(), ForkName::Electra => BeaconStateElectra::::NUM_FIELDS.next_power_of_two(), + ForkName::Fulu => BeaconStateFulu::::NUM_FIELDS.next_power_of_two(), } } @@ -2537,6 +2646,9 @@ impl BeaconState { Self::Electra(inner) => { map_beacon_state_electra_tree_list_fields!(inner, |_, x| { x.apply_updates() }) } + Self::Fulu(inner) => { + map_beacon_state_fulu_tree_list_fields!(inner, |_, x| { x.apply_updates() }) + } } Ok(()) } @@ -2632,6 +2744,11 @@ impl BeaconState { leaves.push(field.tree_hash_root()); }); } + BeaconState::Fulu(state) => { + map_beacon_state_fulu_fields!(state, |_, field| { + leaves.push(field.tree_hash_root()); + }); + } }; leaves @@ -2689,6 +2806,7 @@ impl CompareFields for BeaconState { (BeaconState::Capella(x), BeaconState::Capella(y)) => x.compare_fields(y), (BeaconState::Deneb(x), BeaconState::Deneb(y)) => x.compare_fields(y), (BeaconState::Electra(x), BeaconState::Electra(y)) => x.compare_fields(y), + (BeaconState::Fulu(x), BeaconState::Fulu(y)) => x.compare_fields(y), _ => panic!("compare_fields: mismatched state variants",), } } diff --git a/consensus/types/src/beacon_state/progressive_balances_cache.rs b/consensus/types/src/beacon_state/progressive_balances_cache.rs index fd5e51313f..8e8a1a6aa9 100644 --- a/consensus/types/src/beacon_state/progressive_balances_cache.rs +++ b/consensus/types/src/beacon_state/progressive_balances_cache.rs @@ -145,7 +145,7 @@ impl ProgressiveBalancesCache { pub fn is_initialized_at(&self, epoch: Epoch) -> bool { self.inner .as_ref() - .map_or(false, |inner| inner.current_epoch == epoch) + .is_some_and(|inner| inner.current_epoch == epoch) } /// When a new target attestation has been processed, we update the cached @@ -285,12 +285,5 @@ impl ProgressiveBalancesCache { /// `ProgressiveBalancesCache` is only enabled from `Altair` as it uses Altair-specific logic. pub fn is_progressive_balances_enabled(state: &BeaconState) -> bool { - match state { - BeaconState::Base(_) => false, - BeaconState::Altair(_) - | BeaconState::Bellatrix(_) - | BeaconState::Capella(_) - | BeaconState::Deneb(_) - | BeaconState::Electra(_) => true, - } + state.fork_name_unchecked().altair_enabled() } diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index 5a330388cc..ff4555747c 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -1,10 +1,10 @@ use crate::test_utils::TestRandom; -use crate::ForkName; use crate::{ - beacon_block_body::BLOB_KZG_COMMITMENTS_INDEX, BeaconBlockHeader, BeaconStateError, Blob, - Epoch, EthSpec, FixedVector, Hash256, SignedBeaconBlockHeader, Slot, VariableList, + beacon_block_body::BLOB_KZG_COMMITMENTS_INDEX, AbstractExecPayload, BeaconBlockHeader, + BeaconStateError, Blob, ChainSpec, Epoch, EthSpec, FixedVector, ForkName, + ForkVersionDeserialize, Hash256, KzgProofs, RuntimeFixedVector, RuntimeVariableList, + SignedBeaconBlock, SignedBeaconBlockHeader, Slot, VariableList, }; -use crate::{ForkVersionDeserialize, KzgProofs, SignedBeaconBlock}; use bls::Signature; use derivative::Derivative; use kzg::{Blob as KzgBlob, Kzg, KzgCommitment, KzgProof, BYTES_PER_BLOB, BYTES_PER_FIELD_ELEMENT}; @@ -30,19 +30,6 @@ pub struct BlobIdentifier { pub index: u64, } -impl BlobIdentifier { - pub fn get_all_blob_ids(block_root: Hash256) -> Vec { - let mut blob_ids = Vec::with_capacity(E::max_blobs_per_block()); - for i in 0..E::max_blobs_per_block() { - blob_ids.push(BlobIdentifier { - block_root, - index: i as u64, - }); - } - blob_ids - } -} - impl PartialOrd for BlobIdentifier { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) @@ -150,10 +137,10 @@ impl BlobSidecar { }) } - pub fn new_with_existing_proof( + pub fn new_with_existing_proof>( index: usize, blob: Blob, - signed_block: &SignedBeaconBlock, + signed_block: &SignedBeaconBlock, signed_block_header: SignedBeaconBlockHeader, kzg_commitments_inclusion_proof: &[Hash256], kzg_proof: KzgProof, @@ -291,19 +278,23 @@ impl BlobSidecar { blobs: BlobsList, block: &SignedBeaconBlock, kzg_proofs: KzgProofs, + spec: &ChainSpec, ) -> Result, BlobSidecarError> { let mut blob_sidecars = vec![]; for (i, (kzg_proof, blob)) in kzg_proofs.iter().zip(blobs).enumerate() { let blob_sidecar = BlobSidecar::new(i, blob, block, *kzg_proof)?; blob_sidecars.push(Arc::new(blob_sidecar)); } - Ok(VariableList::from(blob_sidecars)) + Ok(RuntimeVariableList::from_vec( + blob_sidecars, + spec.max_blobs_per_block(block.epoch()) as usize, + )) } } -pub type BlobSidecarList = VariableList>, ::MaxBlobsPerBlock>; -pub type FixedBlobSidecarList = - FixedVector>>, ::MaxBlobsPerBlock>; +pub type BlobSidecarList = RuntimeVariableList>>; +/// Alias for a non length-constrained list of `BlobSidecar`s. +pub type FixedBlobSidecarList = RuntimeFixedVector>>>; pub type BlobsList = VariableList, ::MaxBlobCommitmentsPerBlock>; impl ForkVersionDeserialize for BlobSidecarList { diff --git a/consensus/types/src/builder_bid.rs b/consensus/types/src/builder_bid.rs index 9885f78474..ac53c41216 100644 --- a/consensus/types/src/builder_bid.rs +++ b/consensus/types/src/builder_bid.rs @@ -1,8 +1,9 @@ use crate::beacon_block_body::KzgCommitments; use crate::{ ChainSpec, EthSpec, ExecutionPayloadHeaderBellatrix, ExecutionPayloadHeaderCapella, - ExecutionPayloadHeaderDeneb, ExecutionPayloadHeaderElectra, ExecutionPayloadHeaderRef, - ExecutionPayloadHeaderRefMut, ForkName, ForkVersionDeserialize, SignedRoot, Uint256, + ExecutionPayloadHeaderDeneb, ExecutionPayloadHeaderElectra, ExecutionPayloadHeaderFulu, + ExecutionPayloadHeaderRef, ExecutionPayloadHeaderRefMut, ExecutionRequests, ForkName, + ForkVersionDeserialize, SignedRoot, Uint256, }; use bls::PublicKeyBytes; use bls::Signature; @@ -11,7 +12,7 @@ use superstruct::superstruct; use tree_hash_derive::TreeHash; #[superstruct( - variants(Bellatrix, Capella, Deneb, Electra), + variants(Bellatrix, Capella, Deneb, Electra, Fulu), variant_attributes( derive(PartialEq, Debug, Serialize, Deserialize, TreeHash, Clone), serde(bound = "E: EthSpec", deny_unknown_fields) @@ -31,8 +32,12 @@ pub struct BuilderBid { pub header: ExecutionPayloadHeaderDeneb, #[superstruct(only(Electra), partial_getter(rename = "header_electra"))] pub header: ExecutionPayloadHeaderElectra, - #[superstruct(only(Deneb, Electra))] + #[superstruct(only(Fulu), partial_getter(rename = "header_fulu"))] + pub header: ExecutionPayloadHeaderFulu, + #[superstruct(only(Deneb, Electra, Fulu))] pub blob_kzg_commitments: KzgCommitments, + #[superstruct(only(Electra, Fulu))] + pub execution_requests: ExecutionRequests, #[serde(with = "serde_utils::quoted_u256")] pub value: Uint256, pub pubkey: PublicKeyBytes, @@ -85,6 +90,7 @@ impl ForkVersionDeserialize for BuilderBid { ForkName::Capella => Self::Capella(serde_json::from_value(value).map_err(convert_err)?), ForkName::Deneb => Self::Deneb(serde_json::from_value(value).map_err(convert_err)?), ForkName::Electra => Self::Electra(serde_json::from_value(value).map_err(convert_err)?), + ForkName::Fulu => Self::Fulu(serde_json::from_value(value).map_err(convert_err)?), ForkName::Base | ForkName::Altair => { return Err(serde::de::Error::custom(format!( "BuilderBid failed to deserialize: unsupported fork '{}'", diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index 1b0ab7bbfd..f21b6e47f5 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -198,21 +198,20 @@ pub struct ChainSpec { */ pub domain_inclusion_list_committee: u32, pub inclusion_list_committee_size: u64, - - /* - * DAS params - */ - pub eip7594_fork_epoch: Option, - pub custody_requirement: u64, - pub data_column_sidecar_subnet_count: u64, - pub number_of_columns: usize, - pub samples_per_slot: u64, - - /* - * FOCIL params - */ pub eip7805_fork_epoch: Option, + /* + * Fulu hard fork params + */ + pub fulu_fork_version: [u8; 4], + /// The Fulu fork epoch is optional, with `None` representing "Fulu never happens". + pub fulu_fork_epoch: Option, + pub number_of_columns: u64, + pub number_of_custody_groups: u64, + pub data_column_sidecar_subnet_count: u64, + pub samples_per_slot: u64, + pub custody_requirement: u64, + /* * Networking */ @@ -220,7 +219,7 @@ pub struct ChainSpec { pub network_id: u8, pub target_aggregators_per_committee: u64, pub gossip_max_size: u64, - pub max_request_blocks: u64, + max_request_blocks: u64, pub min_epochs_for_block_requests: u64, pub max_chunk_size: u64, pub ttfb_timeout: u64, @@ -236,11 +235,19 @@ pub struct ChainSpec { /* * Networking Deneb */ - pub max_request_blocks_deneb: u64, - pub max_request_blob_sidecars: u64, + max_request_blocks_deneb: u64, + max_request_blob_sidecars: u64, pub max_request_data_column_sidecars: u64, pub min_epochs_for_blob_sidecars_requests: u64, - pub blob_sidecar_subnet_count: u64, + blob_sidecar_subnet_count: u64, + max_blobs_per_block: u64, + + /* + * Networking Electra + */ + max_blobs_per_block_electra: u64, + blob_sidecar_subnet_count_electra: u64, + max_request_blob_sidecars_electra: u64, /* * Networking Derived @@ -325,17 +332,20 @@ impl ChainSpec { /// Returns the name of the fork which is active at `epoch`. pub fn fork_name_at_epoch(&self, epoch: Epoch) -> ForkName { - match self.electra_fork_epoch { - Some(fork_epoch) if epoch >= fork_epoch => ForkName::Electra, - _ => match self.deneb_fork_epoch { - Some(fork_epoch) if epoch >= fork_epoch => ForkName::Deneb, - _ => match self.capella_fork_epoch { - Some(fork_epoch) if epoch >= fork_epoch => ForkName::Capella, - _ => match self.bellatrix_fork_epoch { - Some(fork_epoch) if epoch >= fork_epoch => ForkName::Bellatrix, - _ => match self.altair_fork_epoch { - Some(fork_epoch) if epoch >= fork_epoch => ForkName::Altair, - _ => ForkName::Base, + match self.fulu_fork_epoch { + Some(fork_epoch) if epoch >= fork_epoch => ForkName::Fulu, + _ => match self.electra_fork_epoch { + Some(fork_epoch) if epoch >= fork_epoch => ForkName::Electra, + _ => match self.deneb_fork_epoch { + Some(fork_epoch) if epoch >= fork_epoch => ForkName::Deneb, + _ => match self.capella_fork_epoch { + Some(fork_epoch) if epoch >= fork_epoch => ForkName::Capella, + _ => match self.bellatrix_fork_epoch { + Some(fork_epoch) if epoch >= fork_epoch => ForkName::Bellatrix, + _ => match self.altair_fork_epoch { + Some(fork_epoch) if epoch >= fork_epoch => ForkName::Altair, + _ => ForkName::Base, + }, }, }, }, @@ -352,6 +362,7 @@ impl ChainSpec { ForkName::Capella => self.capella_fork_version, ForkName::Deneb => self.deneb_fork_version, ForkName::Electra => self.electra_fork_version, + ForkName::Fulu => self.fulu_fork_version, } } @@ -364,6 +375,7 @@ impl ChainSpec { ForkName::Capella => self.capella_fork_epoch, ForkName::Deneb => self.deneb_fork_epoch, ForkName::Electra => self.electra_fork_epoch, + ForkName::Fulu => self.fulu_fork_epoch, } } @@ -430,18 +442,16 @@ impl ChainSpec { } } - /// Returns true if the given epoch is greater than or equal to the `EIP7594_FORK_EPOCH`. + /// Returns true if the given epoch is greater than or equal to the `FULU_FORK_EPOCH`. pub fn is_peer_das_enabled_for_epoch(&self, block_epoch: Epoch) -> bool { - self.eip7594_fork_epoch.map_or(false, |eip7594_fork_epoch| { - block_epoch >= eip7594_fork_epoch - }) + self.fulu_fork_epoch + .is_some_and(|fulu_fork_epoch| block_epoch >= fulu_fork_epoch) } - /// Returns true if `EIP7594_FORK_EPOCH` is set and is not set to `FAR_FUTURE_EPOCH`. + /// Returns true if `FULU_FORK_EPOCH` is set and is not set to `FAR_FUTURE_EPOCH`. pub fn is_peer_das_scheduled(&self) -> bool { - self.eip7594_fork_epoch.map_or(false, |eip7594_fork_epoch| { - eip7594_fork_epoch != self.far_future_epoch - }) + self.fulu_fork_epoch + .is_some_and(|fulu_fork_epoch| fulu_fork_epoch != self.far_future_epoch) } /// Returns true if the given epoch is greater than or equal to the `EIP7805_FORK_EPOCH`. @@ -632,10 +642,97 @@ impl ChainSpec { } } - pub fn data_columns_per_subnet(&self) -> usize { + /// Returns the highest possible value for max_request_blocks based on enabled forks. + /// + /// This is useful for upper bounds in testing. + pub fn max_request_blocks_upper_bound(&self) -> usize { + if self.deneb_fork_epoch.is_some() { + self.max_request_blocks_deneb as usize + } else { + self.max_request_blocks as usize + } + } + + pub fn max_request_blob_sidecars(&self, fork_name: ForkName) -> usize { + if fork_name.electra_enabled() { + self.max_request_blob_sidecars_electra as usize + } else { + self.max_request_blob_sidecars as usize + } + } + + /// Returns the highest possible value for max_request_blobs based on enabled forks. + /// + /// This is useful for upper bounds in testing. + pub fn max_request_blobs_upper_bound(&self) -> usize { + if self.electra_fork_epoch.is_some() { + self.max_request_blob_sidecars_electra as usize + } else { + self.max_request_blob_sidecars as usize + } + } + + /// Return the value of `MAX_BLOBS_PER_BLOCK` appropriate for the fork at `epoch`. + pub fn max_blobs_per_block(&self, epoch: Epoch) -> u64 { + self.max_blobs_per_block_by_fork(self.fork_name_at_epoch(epoch)) + } + + /// Return the value of `MAX_BLOBS_PER_BLOCK` appropriate for `fork`. + pub fn max_blobs_per_block_by_fork(&self, fork_name: ForkName) -> u64 { + if fork_name.electra_enabled() { + self.max_blobs_per_block_electra + } else { + self.max_blobs_per_block + } + } + + /// Returns the `BLOB_SIDECAR_SUBNET_COUNT` at the given fork_name. + pub fn blob_sidecar_subnet_count(&self, fork_name: ForkName) -> u64 { + if fork_name.electra_enabled() { + self.blob_sidecar_subnet_count_electra + } else { + self.blob_sidecar_subnet_count + } + } + + /// Returns the highest possible value of blob sidecar subnet count based on enabled forks. + /// + /// This is useful for upper bounds for the subnet count during a given run of lighthouse. + pub fn blob_sidecar_subnet_count_max(&self) -> u64 { + if self.electra_fork_epoch.is_some() { + self.blob_sidecar_subnet_count_electra + } else { + self.blob_sidecar_subnet_count + } + } + + /// Returns the number of data columns per custody group. + pub fn data_columns_per_group(&self) -> u64 { self.number_of_columns - .safe_div(self.data_column_sidecar_subnet_count as usize) - .expect("Subnet count must be greater than 0") + .safe_div(self.number_of_custody_groups) + .expect("Custody group count must be greater than 0") + } + + /// Returns the number of column sidecars to sample per slot. + pub fn sampling_size(&self, custody_group_count: u64) -> Result { + let columns_per_custody_group = self + .number_of_columns + .safe_div(self.number_of_custody_groups) + .map_err(|_| "number_of_custody_groups must be greater than 0")?; + + let custody_column_count = columns_per_custody_group + .safe_mul(custody_group_count) + .map_err(|_| "Computing sampling size should not overflow")?; + + Ok(std::cmp::max(custody_column_count, self.samples_per_slot)) + } + + pub fn custody_group_count(&self, is_supernode: bool) -> u64 { + if is_supernode { + self.number_of_custody_groups + } else { + self.custody_requirement + } } /// Returns a `ChainSpec` compatible with the Ethereum Foundation specification. @@ -836,21 +933,19 @@ impl ChainSpec { */ domain_inclusion_list_committee: 13, inclusion_list_committee_size: 16, + eip7805_fork_epoch: None, /* - * DAS params + * Fulu hard fork params */ - eip7594_fork_epoch: None, + fulu_fork_version: [0x06, 0x00, 0x00, 0x00], + fulu_fork_epoch: None, custody_requirement: 4, + number_of_custody_groups: 128, data_column_sidecar_subnet_count: 128, number_of_columns: 128, samples_per_slot: 8, - /* - * FOCIL params - */ - eip7805_fork_epoch: None, - /* * Network specific */ @@ -879,6 +974,7 @@ impl ChainSpec { max_request_data_column_sidecars: default_max_request_data_column_sidecars(), min_epochs_for_blob_sidecars_requests: default_min_epochs_for_blob_sidecars_requests(), blob_sidecar_subnet_count: default_blob_sidecar_subnet_count(), + max_blobs_per_block: default_max_blobs_per_block(), /* * Derived Deneb Specific @@ -888,6 +984,13 @@ impl ChainSpec { max_blobs_by_root_request: default_max_blobs_by_root_request(), max_data_columns_by_root_request: default_data_columns_by_root_request(), + /* + * Networking Electra specific + */ + max_blobs_per_block_electra: default_max_blobs_per_block_electra(), + blob_sidecar_subnet_count_electra: default_blob_sidecar_subnet_count_electra(), + max_request_blob_sidecars_electra: default_max_request_blob_sidecars_electra(), + /* * Application specific */ @@ -947,7 +1050,7 @@ impl ChainSpec { // Electra electra_fork_version: [0x05, 0x00, 0x00, 0x01], electra_fork_epoch: None, - max_pending_partials_per_withdrawals_sweep: u64::checked_pow(2, 0) + max_pending_partials_per_withdrawals_sweep: u64::checked_pow(2, 1) .expect("pow does not overflow"), min_per_epoch_churn_limit_electra: option_wrapper(|| { u64::checked_pow(2, 6)?.checked_mul(u64::checked_pow(10, 9)?) @@ -957,10 +1060,11 @@ impl ChainSpec { u64::checked_pow(2, 7)?.checked_mul(u64::checked_pow(10, 9)?) }) .expect("calculation does not overflow"), - // PeerDAS - eip7594_fork_epoch: None, // FOCIL eip7805_fork_epoch: None, + // Fulu + fulu_fork_version: [0x06, 0x00, 0x00, 0x01], + fulu_fork_epoch: None, // Other network_id: 2, // lighthouse testnet network id deposit_chain_id: 5, @@ -1168,21 +1272,19 @@ impl ChainSpec { */ domain_inclusion_list_committee: 13, inclusion_list_committee_size: 16, + eip7805_fork_epoch: None, /* - * DAS params + * Fulu hard fork params */ - eip7594_fork_epoch: None, + fulu_fork_version: [0x06, 0x00, 0x00, 0x64], + fulu_fork_epoch: None, custody_requirement: 4, + number_of_custody_groups: 128, data_column_sidecar_subnet_count: 128, number_of_columns: 128, samples_per_slot: 8, - /* - * FOCIL params - */ - eip7805_fork_epoch: None, - /* * Network specific */ @@ -1211,6 +1313,7 @@ impl ChainSpec { max_request_data_column_sidecars: default_max_request_data_column_sidecars(), min_epochs_for_blob_sidecars_requests: 16384, blob_sidecar_subnet_count: default_blob_sidecar_subnet_count(), + max_blobs_per_block: default_max_blobs_per_block(), /* * Derived Deneb Specific @@ -1220,6 +1323,13 @@ impl ChainSpec { max_blobs_by_root_request: default_max_blobs_by_root_request(), max_data_columns_by_root_request: default_data_columns_by_root_request(), + /* + * Networking Electra specific + */ + max_blobs_per_block_electra: default_max_blobs_per_block_electra(), + blob_sidecar_subnet_count_electra: default_blob_sidecar_subnet_count_electra(), + max_request_blob_sidecars_electra: default_max_request_blob_sidecars_electra(), + /* * Application specific */ @@ -1309,10 +1419,13 @@ pub struct Config { #[serde(deserialize_with = "deserialize_fork_epoch")] pub electra_fork_epoch: Option>, + #[serde(default = "default_fulu_fork_version")] + #[serde(with = "serde_utils::bytes_4_hex")] + fulu_fork_version: [u8; 4], #[serde(default)] #[serde(serialize_with = "serialize_fork_epoch")] #[serde(deserialize_with = "deserialize_fork_epoch")] - pub eip7594_fork_epoch: Option>, + pub fulu_fork_epoch: Option>, #[serde(default)] #[serde(serialize_with = "serialize_fork_epoch")] @@ -1409,6 +1522,9 @@ pub struct Config { #[serde(default = "default_blob_sidecar_subnet_count")] #[serde(with = "serde_utils::quoted_u64")] blob_sidecar_subnet_count: u64, + #[serde(default = "default_max_blobs_per_block")] + #[serde(with = "serde_utils::quoted_u64")] + max_blobs_per_block: u64, #[serde(default = "default_min_per_epoch_churn_limit_electra")] #[serde(with = "serde_utils::quoted_u64")] @@ -1416,19 +1532,31 @@ pub struct Config { #[serde(default = "default_max_per_epoch_activation_exit_churn_limit")] #[serde(with = "serde_utils::quoted_u64")] max_per_epoch_activation_exit_churn_limit: u64, + #[serde(default = "default_max_blobs_per_block_electra")] + #[serde(with = "serde_utils::quoted_u64")] + max_blobs_per_block_electra: u64, + #[serde(default = "default_blob_sidecar_subnet_count_electra")] + #[serde(with = "serde_utils::quoted_u64")] + pub blob_sidecar_subnet_count_electra: u64, + #[serde(default = "default_max_request_blob_sidecars_electra")] + #[serde(with = "serde_utils::quoted_u64")] + max_request_blob_sidecars_electra: u64, - #[serde(default = "default_custody_requirement")] - #[serde(with = "serde_utils::quoted_u64")] - custody_requirement: u64, - #[serde(default = "default_data_column_sidecar_subnet_count")] - #[serde(with = "serde_utils::quoted_u64")] - data_column_sidecar_subnet_count: u64, #[serde(default = "default_number_of_columns")] #[serde(with = "serde_utils::quoted_u64")] number_of_columns: u64, + #[serde(default = "default_number_of_custody_groups")] + #[serde(with = "serde_utils::quoted_u64")] + number_of_custody_groups: u64, + #[serde(default = "default_data_column_sidecar_subnet_count")] + #[serde(with = "serde_utils::quoted_u64")] + data_column_sidecar_subnet_count: u64, #[serde(default = "default_samples_per_slot")] #[serde(with = "serde_utils::quoted_u64")] samples_per_slot: u64, + #[serde(default = "default_custody_requirement")] + #[serde(with = "serde_utils::quoted_u64")] + custody_requirement: u64, } fn default_bellatrix_fork_version() -> [u8; 4] { @@ -1451,6 +1579,11 @@ fn default_electra_fork_version() -> [u8; 4] { [0xff, 0xff, 0xff, 0xff] } +fn default_fulu_fork_version() -> [u8; 4] { + // This value shouldn't be used. + [0xff, 0xff, 0xff, 0xff] +} + /// Placeholder value: 2^256-2^10 (115792089237316195423570985008687907853269984665640564039457584007913129638912). /// /// Taken from https://github.com/ethereum/consensus-specs/blob/d5e4828aecafaf1c57ef67a5f23c4ae7b08c5137/configs/mainnet.yaml#L15-L16 @@ -1539,6 +1672,20 @@ const fn default_blob_sidecar_subnet_count() -> u64 { 6 } +/// Its important to keep this consistent with the deneb preset value for +/// `MAX_BLOBS_PER_BLOCK` else we might run into consensus issues. +const fn default_max_blobs_per_block() -> u64 { + 6 +} + +const fn default_blob_sidecar_subnet_count_electra() -> u64 { + 9 +} + +const fn default_max_request_blob_sidecars_electra() -> u64 { + 1152 +} + const fn default_min_per_epoch_churn_limit_electra() -> u64 { 128_000_000_000 } @@ -1547,6 +1694,10 @@ const fn default_max_per_epoch_activation_exit_churn_limit() -> u64 { 256_000_000_000 } +const fn default_max_blobs_per_block_electra() -> u64 { + 9 +} + const fn default_attestation_propagation_slot_range() -> u64 { 32 } @@ -1567,6 +1718,10 @@ const fn default_number_of_columns() -> u64 { 128 } +const fn default_number_of_custody_groups() -> u64 { + 128 +} + const fn default_samples_per_slot() -> u64 { 8 } @@ -1714,8 +1869,9 @@ impl Config { .electra_fork_epoch .map(|epoch| MaybeQuoted { value: epoch }), - eip7594_fork_epoch: spec - .eip7594_fork_epoch + fulu_fork_version: spec.fulu_fork_version, + fulu_fork_epoch: spec + .fulu_fork_epoch .map(|epoch| MaybeQuoted { value: epoch }), eip7805_fork_epoch: spec @@ -1760,15 +1916,20 @@ impl Config { max_request_data_column_sidecars: spec.max_request_data_column_sidecars, min_epochs_for_blob_sidecars_requests: spec.min_epochs_for_blob_sidecars_requests, blob_sidecar_subnet_count: spec.blob_sidecar_subnet_count, + max_blobs_per_block: spec.max_blobs_per_block, min_per_epoch_churn_limit_electra: spec.min_per_epoch_churn_limit_electra, max_per_epoch_activation_exit_churn_limit: spec .max_per_epoch_activation_exit_churn_limit, + max_blobs_per_block_electra: spec.max_blobs_per_block_electra, + blob_sidecar_subnet_count_electra: spec.blob_sidecar_subnet_count_electra, + max_request_blob_sidecars_electra: spec.max_request_blob_sidecars_electra, - custody_requirement: spec.custody_requirement, + number_of_columns: spec.number_of_columns, + number_of_custody_groups: spec.number_of_custody_groups, data_column_sidecar_subnet_count: spec.data_column_sidecar_subnet_count, - number_of_columns: spec.number_of_columns as u64, samples_per_slot: spec.samples_per_slot, + custody_requirement: spec.custody_requirement, } } @@ -1801,8 +1962,9 @@ impl Config { deneb_fork_version, electra_fork_epoch, electra_fork_version, - eip7594_fork_epoch, eip7805_fork_epoch, + fulu_fork_epoch, + fulu_fork_version, seconds_per_slot, seconds_per_eth1_block, min_validator_withdrawability_delay, @@ -1836,13 +1998,18 @@ impl Config { max_request_data_column_sidecars, min_epochs_for_blob_sidecars_requests, blob_sidecar_subnet_count, + max_blobs_per_block, min_per_epoch_churn_limit_electra, max_per_epoch_activation_exit_churn_limit, - custody_requirement, - data_column_sidecar_subnet_count, + max_blobs_per_block_electra, + blob_sidecar_subnet_count_electra, + max_request_blob_sidecars_electra, number_of_columns, + number_of_custody_groups, + data_column_sidecar_subnet_count, samples_per_slot, + custody_requirement, } = self; if preset_base != E::spec_name().to_string().as_str() { @@ -1865,8 +2032,9 @@ impl Config { deneb_fork_version, electra_fork_epoch: electra_fork_epoch.map(|q| q.value), electra_fork_version, - eip7594_fork_epoch: eip7594_fork_epoch.map(|q| q.value), eip7805_fork_epoch: eip7805_fork_epoch.map(|q| q.value), + fulu_fork_epoch: fulu_fork_epoch.map(|q| q.value), + fulu_fork_version, seconds_per_slot, seconds_per_eth1_block, min_validator_withdrawability_delay, @@ -1903,9 +2071,13 @@ impl Config { max_request_data_column_sidecars, min_epochs_for_blob_sidecars_requests, blob_sidecar_subnet_count, + max_blobs_per_block, min_per_epoch_churn_limit_electra, max_per_epoch_activation_exit_churn_limit, + max_blobs_per_block_electra, + max_request_blob_sidecars_electra, + blob_sidecar_subnet_count_electra, // We need to re-derive any values that might have changed in the config. max_blocks_by_root_request: max_blocks_by_root_request_common(max_request_blocks), @@ -1917,10 +2089,11 @@ impl Config { max_request_data_column_sidecars, ), - custody_requirement, + number_of_columns, + number_of_custody_groups, data_column_sidecar_subnet_count, - number_of_columns: number_of_columns as usize, samples_per_slot, + custody_requirement, ..chain_spec.clone() }) diff --git a/consensus/types/src/config_and_preset.rs b/consensus/types/src/config_and_preset.rs index c80d678b2a..235bf20238 100644 --- a/consensus/types/src/config_and_preset.rs +++ b/consensus/types/src/config_and_preset.rs @@ -1,6 +1,6 @@ use crate::{ consts::altair, consts::deneb, AltairPreset, BasePreset, BellatrixPreset, CapellaPreset, - ChainSpec, Config, DenebPreset, ElectraPreset, EthSpec, ForkName, + ChainSpec, Config, DenebPreset, ElectraPreset, EthSpec, ForkName, FuluPreset, }; use maplit::hashmap; use serde::{Deserialize, Serialize}; @@ -12,7 +12,7 @@ use superstruct::superstruct; /// /// Mostly useful for the API. #[superstruct( - variants(Capella, Deneb, Electra), + variants(Deneb, Electra, Fulu), variant_attributes(derive(Serialize, Deserialize, Debug, PartialEq, Clone)) )] #[derive(Serialize, Deserialize, Debug, PartialEq, Clone)] @@ -29,12 +29,14 @@ pub struct ConfigAndPreset { pub bellatrix_preset: BellatrixPreset, #[serde(flatten)] pub capella_preset: CapellaPreset, - #[superstruct(only(Deneb, Electra))] #[serde(flatten)] pub deneb_preset: DenebPreset, - #[superstruct(only(Electra))] + #[superstruct(only(Electra, Fulu))] #[serde(flatten)] pub electra_preset: ElectraPreset, + #[superstruct(only(Fulu))] + #[serde(flatten)] + pub fulu_preset: FuluPreset, /// The `extra_fields` map allows us to gracefully decode fields intended for future hard forks. #[serde(flatten)] pub extra_fields: HashMap, @@ -48,13 +50,31 @@ impl ConfigAndPreset { let altair_preset = AltairPreset::from_chain_spec::(spec); let bellatrix_preset = BellatrixPreset::from_chain_spec::(spec); let capella_preset = CapellaPreset::from_chain_spec::(spec); + let deneb_preset = DenebPreset::from_chain_spec::(spec); let extra_fields = get_extra_fields(spec); - if spec.electra_fork_epoch.is_some() + if spec.fulu_fork_epoch.is_some() + || fork_name.is_none() + || fork_name == Some(ForkName::Fulu) + { + let electra_preset = ElectraPreset::from_chain_spec::(spec); + let fulu_preset = FuluPreset::from_chain_spec::(spec); + + ConfigAndPreset::Fulu(ConfigAndPresetFulu { + config, + base_preset, + altair_preset, + bellatrix_preset, + capella_preset, + deneb_preset, + electra_preset, + fulu_preset, + extra_fields, + }) + } else if spec.electra_fork_epoch.is_some() || fork_name.is_none() || fork_name == Some(ForkName::Electra) { - let deneb_preset = DenebPreset::from_chain_spec::(spec); let electra_preset = ElectraPreset::from_chain_spec::(spec); ConfigAndPreset::Electra(ConfigAndPresetElectra { @@ -67,11 +87,7 @@ impl ConfigAndPreset { electra_preset, extra_fields, }) - } else if spec.deneb_fork_epoch.is_some() - || fork_name.is_none() - || fork_name == Some(ForkName::Deneb) - { - let deneb_preset = DenebPreset::from_chain_spec::(spec); + } else { ConfigAndPreset::Deneb(ConfigAndPresetDeneb { config, base_preset, @@ -81,15 +97,6 @@ impl ConfigAndPreset { deneb_preset, extra_fields, }) - } else { - ConfigAndPreset::Capella(ConfigAndPresetCapella { - config, - base_preset, - altair_preset, - bellatrix_preset, - capella_preset, - extra_fields, - }) } } } @@ -164,8 +171,8 @@ mod test { .write(false) .open(tmp_file.as_ref()) .expect("error while opening the file"); - let from: ConfigAndPresetElectra = + let from: ConfigAndPresetFulu = serde_yaml::from_reader(reader).expect("error while deserializing"); - assert_eq!(ConfigAndPreset::Electra(from), yamlconfig); + assert_eq!(ConfigAndPreset::Fulu(from), yamlconfig); } } diff --git a/consensus/types/src/data_column_custody_group.rs b/consensus/types/src/data_column_custody_group.rs new file mode 100644 index 0000000000..bb204c34a2 --- /dev/null +++ b/consensus/types/src/data_column_custody_group.rs @@ -0,0 +1,142 @@ +use crate::{ChainSpec, ColumnIndex, DataColumnSubnetId}; +use alloy_primitives::U256; +use itertools::Itertools; +use maplit::hashset; +use safe_arith::{ArithError, SafeArith}; +use std::collections::HashSet; + +pub type CustodyIndex = u64; + +#[derive(Debug)] +pub enum DataColumnCustodyGroupError { + InvalidCustodyGroup(CustodyIndex), + InvalidCustodyGroupCount(u64), + ArithError(ArithError), +} + +/// The `get_custody_groups` function is used to determine the custody groups that a node is +/// assigned to. +/// +/// spec: https://github.com/ethereum/consensus-specs/blob/8e0d0d48e81d6c7c5a8253ab61340f5ea5bac66a/specs/fulu/das-core.md#get_custody_groups +pub fn get_custody_groups( + raw_node_id: [u8; 32], + custody_group_count: u64, + spec: &ChainSpec, +) -> Result, DataColumnCustodyGroupError> { + if custody_group_count > spec.number_of_custody_groups { + return Err(DataColumnCustodyGroupError::InvalidCustodyGroupCount( + custody_group_count, + )); + } + + let mut custody_groups: HashSet = hashset![]; + let mut current_id = U256::from_be_slice(&raw_node_id); + while custody_groups.len() < custody_group_count as usize { + let mut node_id_bytes = [0u8; 32]; + node_id_bytes.copy_from_slice(current_id.as_le_slice()); + let hash = ethereum_hashing::hash_fixed(&node_id_bytes); + let hash_prefix: [u8; 8] = hash[0..8] + .try_into() + .expect("hash_fixed produces a 32 byte array"); + let hash_prefix_u64 = u64::from_le_bytes(hash_prefix); + let custody_group = hash_prefix_u64 + .safe_rem(spec.number_of_custody_groups) + .expect("spec.number_of_custody_groups must not be zero"); + custody_groups.insert(custody_group); + + current_id = current_id.wrapping_add(U256::from(1u64)); + } + + Ok(custody_groups) +} + +/// Returns the columns that are associated with a given custody group. +/// +/// spec: https://github.com/ethereum/consensus-specs/blob/8e0d0d48e81d6c7c5a8253ab61340f5ea5bac66a/specs/fulu/das-core.md#compute_columns_for_custody_group +pub fn compute_columns_for_custody_group( + custody_group: CustodyIndex, + spec: &ChainSpec, +) -> Result, DataColumnCustodyGroupError> { + let number_of_custody_groups = spec.number_of_custody_groups; + if custody_group >= number_of_custody_groups { + return Err(DataColumnCustodyGroupError::InvalidCustodyGroup( + custody_group, + )); + } + + let mut columns = Vec::new(); + for i in 0..spec.data_columns_per_group() { + let column = number_of_custody_groups + .safe_mul(i) + .and_then(|v| v.safe_add(custody_group)) + .map_err(DataColumnCustodyGroupError::ArithError)?; + columns.push(column); + } + + Ok(columns.into_iter()) +} + +pub fn compute_subnets_for_node( + raw_node_id: [u8; 32], + custody_group_count: u64, + spec: &ChainSpec, +) -> Result, DataColumnCustodyGroupError> { + let custody_groups = get_custody_groups(raw_node_id, custody_group_count, spec)?; + let mut subnets = HashSet::new(); + + for custody_group in custody_groups { + let custody_group_subnets = compute_subnets_from_custody_group(custody_group, spec)?; + subnets.extend(custody_group_subnets); + } + + Ok(subnets) +} + +/// Returns the subnets that are associated with a given custody group. +pub fn compute_subnets_from_custody_group( + custody_group: CustodyIndex, + spec: &ChainSpec, +) -> Result + '_, DataColumnCustodyGroupError> { + let result = compute_columns_for_custody_group(custody_group, spec)? + .map(|column_index| DataColumnSubnetId::from_column_index(column_index, spec)) + .unique(); + Ok(result) +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_compute_columns_for_custody_group() { + let mut spec = ChainSpec::mainnet(); + spec.number_of_custody_groups = 64; + spec.number_of_columns = 128; + let columns_per_custody_group = spec.number_of_columns / spec.number_of_custody_groups; + + for custody_group in 0..spec.number_of_custody_groups { + let columns = compute_columns_for_custody_group(custody_group, &spec) + .unwrap() + .collect::>(); + assert_eq!(columns.len(), columns_per_custody_group as usize); + } + } + + #[test] + fn test_compute_subnets_from_custody_group() { + let mut spec = ChainSpec::mainnet(); + spec.number_of_custody_groups = 64; + spec.number_of_columns = 256; + spec.data_column_sidecar_subnet_count = 128; + + let subnets_per_custody_group = + spec.data_column_sidecar_subnet_count / spec.number_of_custody_groups; + + for custody_group in 0..spec.number_of_custody_groups { + let subnets = compute_subnets_from_custody_group(custody_group, &spec) + .unwrap() + .collect::>(); + assert_eq!(subnets.len(), subnets_per_custody_group as usize); + } + } +} diff --git a/consensus/types/src/data_column_sidecar.rs b/consensus/types/src/data_column_sidecar.rs index 57251e319a..90a914dfae 100644 --- a/consensus/types/src/data_column_sidecar.rs +++ b/consensus/types/src/data_column_sidecar.rs @@ -1,7 +1,7 @@ use crate::beacon_block_body::{KzgCommitments, BLOB_KZG_COMMITMENTS_INDEX}; use crate::test_utils::TestRandom; use crate::BeaconStateError; -use crate::{BeaconBlockHeader, EthSpec, Hash256, KzgProofs, SignedBeaconBlockHeader, Slot}; +use crate::{BeaconBlockHeader, Epoch, EthSpec, Hash256, KzgProofs, SignedBeaconBlockHeader, Slot}; use bls::Signature; use derivative::Derivative; use kzg::Error as KzgError; @@ -11,7 +11,6 @@ use safe_arith::ArithError; use serde::{Deserialize, Serialize}; use ssz::Encode; use ssz_derive::{Decode, Encode}; -use ssz_types::typenum::Unsigned; use ssz_types::Error as SszError; use ssz_types::{FixedVector, VariableList}; use std::hash::Hash; @@ -68,6 +67,10 @@ impl DataColumnSidecar { self.signed_block_header.message.slot } + pub fn epoch(&self) -> Epoch { + self.slot().epoch(E::slots_per_epoch()) + } + pub fn block_root(&self) -> Hash256 { self.signed_block_header.message.tree_hash_root() } @@ -110,18 +113,16 @@ impl DataColumnSidecar { .len() } - pub fn max_size() -> usize { + pub fn max_size(max_blobs_per_block: usize) -> usize { Self { index: 0, - column: VariableList::new(vec![Cell::::default(); E::MaxBlobsPerBlock::to_usize()]) - .unwrap(), + column: VariableList::new(vec![Cell::::default(); max_blobs_per_block]).unwrap(), kzg_commitments: VariableList::new(vec![ KzgCommitment::empty_for_testing(); - E::MaxBlobsPerBlock::to_usize() + max_blobs_per_block ]) .unwrap(), - kzg_proofs: VariableList::new(vec![KzgProof::empty(); E::MaxBlobsPerBlock::to_usize()]) - .unwrap(), + kzg_proofs: VariableList::new(vec![KzgProof::empty(); max_blobs_per_block]).unwrap(), signed_block_header: SignedBeaconBlockHeader { message: BeaconBlockHeader::empty(), signature: Signature::empty(), @@ -132,20 +133,6 @@ impl DataColumnSidecar { .len() } - pub fn empty() -> Self { - Self { - index: 0, - column: DataColumn::::default(), - kzg_commitments: VariableList::default(), - kzg_proofs: VariableList::default(), - signed_block_header: SignedBeaconBlockHeader { - message: BeaconBlockHeader::empty(), - signature: Signature::empty(), - }, - kzg_commitments_inclusion_proof: Default::default(), - } - } - pub fn id(&self) -> DataColumnIdentifier { DataColumnIdentifier { block_root: self.block_root(), diff --git a/consensus/types/src/data_column_subnet_id.rs b/consensus/types/src/data_column_subnet_id.rs index df61d711c1..5b3eef24cc 100644 --- a/consensus/types/src/data_column_subnet_id.rs +++ b/consensus/types/src/data_column_subnet_id.rs @@ -1,11 +1,8 @@ //! Identifies each data column subnet by an integer identifier. use crate::data_column_sidecar::ColumnIndex; -use crate::{ChainSpec, EthSpec}; -use alloy_primitives::U256; -use itertools::Itertools; +use crate::ChainSpec; use safe_arith::{ArithError, SafeArith}; use serde::{Deserialize, Serialize}; -use std::collections::HashSet; use std::fmt::{self, Display}; use std::ops::{Deref, DerefMut}; @@ -18,76 +15,14 @@ impl DataColumnSubnetId { id.into() } - pub fn from_column_index(column_index: usize, spec: &ChainSpec) -> Self { - (column_index - .safe_rem(spec.data_column_sidecar_subnet_count as usize) + pub fn from_column_index(column_index: ColumnIndex, spec: &ChainSpec) -> Self { + column_index + .safe_rem(spec.data_column_sidecar_subnet_count) .expect( "data_column_sidecar_subnet_count should never be zero if this function is called", - ) as u64) + ) .into() } - - #[allow(clippy::arithmetic_side_effects)] - pub fn columns(&self, spec: &ChainSpec) -> impl Iterator { - let subnet = self.0; - let data_column_sidecar_subnet = spec.data_column_sidecar_subnet_count; - let columns_per_subnet = spec.data_columns_per_subnet() as u64; - (0..columns_per_subnet).map(move |i| data_column_sidecar_subnet * i + subnet) - } - - /// Compute required subnets to subscribe to given the node id. - #[allow(clippy::arithmetic_side_effects)] - pub fn compute_custody_subnets( - raw_node_id: [u8; 32], - custody_subnet_count: u64, - spec: &ChainSpec, - ) -> Result, Error> { - if custody_subnet_count > spec.data_column_sidecar_subnet_count { - return Err(Error::InvalidCustodySubnetCount(custody_subnet_count)); - } - - let mut subnets: HashSet = HashSet::new(); - let mut current_id = U256::from_be_slice(&raw_node_id); - while (subnets.len() as u64) < custody_subnet_count { - let mut node_id_bytes = [0u8; 32]; - node_id_bytes.copy_from_slice(current_id.as_le_slice()); - let hash = ethereum_hashing::hash_fixed(&node_id_bytes); - let hash_prefix: [u8; 8] = hash[0..8] - .try_into() - .expect("hash_fixed produces a 32 byte array"); - let hash_prefix_u64 = u64::from_le_bytes(hash_prefix); - let subnet = hash_prefix_u64 % spec.data_column_sidecar_subnet_count; - - if !subnets.contains(&subnet) { - subnets.insert(subnet); - } - - if current_id == U256::MAX { - current_id = U256::ZERO - } - current_id += U256::from(1u64) - } - Ok(subnets.into_iter().map(DataColumnSubnetId::new)) - } - - /// Compute the custody subnets for a given node id with the default `custody_requirement`. - /// This operation should be infallable, and empty iterator is returned if it fails unexpectedly. - pub fn compute_custody_requirement_subnets( - node_id: [u8; 32], - spec: &ChainSpec, - ) -> impl Iterator { - Self::compute_custody_subnets::(node_id, spec.custody_requirement, spec) - .expect("should compute default custody subnets") - } - - pub fn compute_custody_columns( - raw_node_id: [u8; 32], - custody_subnet_count: u64, - spec: &ChainSpec, - ) -> Result, Error> { - Self::compute_custody_subnets::(raw_node_id, custody_subnet_count, spec) - .map(|subnet| subnet.flat_map(|subnet| subnet.columns::(spec)).sorted()) - } } impl Display for DataColumnSubnetId { @@ -139,88 +74,3 @@ impl From for Error { Error::ArithError(e) } } - -#[cfg(test)] -mod test { - use crate::data_column_subnet_id::DataColumnSubnetId; - use crate::MainnetEthSpec; - use crate::Uint256; - use crate::{EthSpec, GnosisEthSpec, MinimalEthSpec}; - - type E = MainnetEthSpec; - - #[test] - fn test_compute_subnets_for_data_column() { - let spec = E::default_spec(); - let node_ids = [ - "0", - "88752428858350697756262172400162263450541348766581994718383409852729519486397", - "18732750322395381632951253735273868184515463718109267674920115648614659369468", - "27726842142488109545414954493849224833670205008410190955613662332153332462900", - "39755236029158558527862903296867805548949739810920318269566095185775868999998", - "31899136003441886988955119620035330314647133604576220223892254902004850516297", - "58579998103852084482416614330746509727562027284701078483890722833654510444626", - "28248042035542126088870192155378394518950310811868093527036637864276176517397", - "60930578857433095740782970114409273483106482059893286066493409689627770333527", - "103822458477361691467064888613019442068586830412598673713899771287914656699997", - ] - .into_iter() - .map(|v| Uint256::from_str_radix(v, 10).unwrap().to_be_bytes::<32>()) - .collect::>(); - - let custody_requirement = 4; - for node_id in node_ids { - let computed_subnets = DataColumnSubnetId::compute_custody_subnets::( - node_id, - custody_requirement, - &spec, - ) - .unwrap(); - let computed_subnets: Vec<_> = computed_subnets.collect(); - - // the number of subnets is equal to the custody requirement - assert_eq!(computed_subnets.len() as u64, custody_requirement); - - let subnet_count = spec.data_column_sidecar_subnet_count; - for subnet in computed_subnets { - let columns: Vec<_> = subnet.columns::(&spec).collect(); - // the number of columns is equal to the specified number of columns per subnet - assert_eq!(columns.len(), spec.data_columns_per_subnet()); - - for pair in columns.windows(2) { - // each successive column index is offset by the number of subnets - assert_eq!(pair[1] - pair[0], subnet_count); - } - } - } - } - - #[test] - fn test_compute_custody_requirement_subnets_never_panics() { - let node_id = [1u8; 32]; - test_compute_custody_requirement_subnets_with_spec::(node_id); - test_compute_custody_requirement_subnets_with_spec::(node_id); - test_compute_custody_requirement_subnets_with_spec::(node_id); - } - - fn test_compute_custody_requirement_subnets_with_spec(node_id: [u8; 32]) { - let _ = DataColumnSubnetId::compute_custody_requirement_subnets::( - node_id, - &E::default_spec(), - ); - } - - #[test] - fn test_columns_subnet_conversion() { - let spec = E::default_spec(); - for subnet in 0..spec.data_column_sidecar_subnet_count { - let subnet_id = DataColumnSubnetId::new(subnet); - for column_index in subnet_id.columns::(&spec) { - assert_eq!( - subnet_id, - DataColumnSubnetId::from_column_index::(column_index as usize, &spec) - ); - } - } - } -} diff --git a/consensus/types/src/deposit_tree_snapshot.rs b/consensus/types/src/deposit_tree_snapshot.rs index df1064daba..2f9df8758b 100644 --- a/consensus/types/src/deposit_tree_snapshot.rs +++ b/consensus/types/src/deposit_tree_snapshot.rs @@ -72,8 +72,7 @@ impl DepositTreeSnapshot { Some(Hash256::from_slice(&deposit_root)) } pub fn is_valid(&self) -> bool { - self.calculate_root() - .map_or(false, |calculated| self.deposit_root == calculated) + self.calculate_root() == Some(self.deposit_root) } } diff --git a/consensus/types/src/eth_spec.rs b/consensus/types/src/eth_spec.rs index 9c7245f4c3..1c28acccaf 100644 --- a/consensus/types/src/eth_spec.rs +++ b/consensus/types/src/eth_spec.rs @@ -3,11 +3,10 @@ use crate::*; use safe_arith::SafeArith; use serde::{Deserialize, Serialize}; use ssz_types::typenum::{ - bit::B0, UInt, U0, U1, U1024, U1048576, U1073741824, U1099511627776, U128, U131072, U134217728, - U16, U16777216, U2, U2048, U256, U262144, U32, U4, U4096, U512, U6, U625, U64, U65536, U8, - U8192, + bit::B0, UInt, U0, U1, U10, U1024, U1048576, U1073741824, U1099511627776, U128, U131072, + U134217728, U16, U16777216, U17, U2, U2048, U256, U262144, U32, U4, U4096, U512, U625, U64, + U65536, U8, U8192, }; -use ssz_types::typenum::{U17, U9}; use std::fmt::{self, Debug}; use std::str::FromStr; @@ -109,7 +108,6 @@ pub trait EthSpec: /* * New in Deneb */ - type MaxBlobsPerBlock: Unsigned + Clone + Sync + Send + Debug + PartialEq + Unpin; type MaxBlobCommitmentsPerBlock: Unsigned + Clone + Sync + Send + Debug + PartialEq + Unpin; type FieldElementsPerBlob: Unsigned + Clone + Sync + Send + Debug + PartialEq; type BytesPerFieldElement: Unsigned + Clone + Sync + Send + Debug + PartialEq; @@ -287,11 +285,6 @@ pub trait EthSpec: Self::MaxWithdrawalsPerPayload::to_usize() } - /// Returns the `MAX_BLOBS_PER_BLOCK` constant for this specification. - fn max_blobs_per_block() -> usize { - Self::MaxBlobsPerBlock::to_usize() - } - /// Returns the `MAX_BLOB_COMMITMENTS_PER_BLOCK` constant for this specification. fn max_blob_commitments_per_block() -> usize { Self::MaxBlobCommitmentsPerBlock::to_usize() @@ -437,7 +430,6 @@ impl EthSpec for MainnetEthSpec { type GasLimitDenominator = U1024; type MinGasLimit = U5000; type MaxExtraDataBytes = U32; - type MaxBlobsPerBlock = U6; type MaxBlobCommitmentsPerBlock = U4096; type BytesPerFieldElement = U32; type FieldElementsPerBlob = U4096; @@ -455,7 +447,7 @@ impl EthSpec for MainnetEthSpec { type PendingDepositsLimit = U134217728; type PendingPartialWithdrawalsLimit = U134217728; type PendingConsolidationsLimit = U262144; - type MaxConsolidationRequestsPerPayload = U1; + type MaxConsolidationRequestsPerPayload = U2; type MaxDepositRequestsPerPayload = U8192; type MaxAttesterSlashingsElectra = U1; type MaxAttestationsElectra = U8; @@ -492,8 +484,8 @@ impl EthSpec for MinimalEthSpec { type MaxWithdrawalsPerPayload = U4; type FieldElementsPerBlob = U4096; type BytesPerBlob = U131072; - type MaxBlobCommitmentsPerBlock = U16; - type KzgCommitmentInclusionProofDepth = U9; + type MaxBlobCommitmentsPerBlock = U32; + type KzgCommitmentInclusionProofDepth = U10; type PendingPartialWithdrawalsLimit = U64; type PendingConsolidationsLimit = U64; type MaxDepositRequestsPerPayload = U4; @@ -523,7 +515,6 @@ impl EthSpec for MinimalEthSpec { MinGasLimit, MaxExtraDataBytes, MaxBlsToExecutionChanges, - MaxBlobsPerBlock, BytesPerFieldElement, PendingDepositsLimit, MaxPendingDepositsPerEpoch, @@ -579,7 +570,6 @@ impl EthSpec for GnosisEthSpec { type SlotsPerEth1VotingPeriod = U1024; // 64 epochs * 16 slots per epoch type MaxBlsToExecutionChanges = U16; type MaxWithdrawalsPerPayload = U8; - type MaxBlobsPerBlock = U6; type MaxBlobCommitmentsPerBlock = U4096; type FieldElementsPerBlob = U4096; type BytesPerFieldElement = U32; @@ -588,7 +578,7 @@ impl EthSpec for GnosisEthSpec { type PendingDepositsLimit = U134217728; type PendingPartialWithdrawalsLimit = U134217728; type PendingConsolidationsLimit = U262144; - type MaxConsolidationRequestsPerPayload = U1; + type MaxConsolidationRequestsPerPayload = U2; type MaxDepositRequestsPerPayload = U8192; type MaxAttesterSlashingsElectra = U1; type MaxAttestationsElectra = U8; diff --git a/consensus/types/src/execution_payload.rs b/consensus/types/src/execution_payload.rs index 9f16b676a6..2df66343af 100644 --- a/consensus/types/src/execution_payload.rs +++ b/consensus/types/src/execution_payload.rs @@ -15,7 +15,7 @@ pub type Transactions = VariableList< pub type Withdrawals = VariableList::MaxWithdrawalsPerPayload>; #[superstruct( - variants(Bellatrix, Capella, Deneb, Electra), + variants(Bellatrix, Capella, Deneb, Electra, Fulu), variant_attributes( derive( Default, @@ -82,12 +82,12 @@ pub struct ExecutionPayload { pub block_hash: ExecutionBlockHash, #[serde(with = "ssz_types::serde_utils::list_of_hex_var_list")] pub transactions: Transactions, - #[superstruct(only(Capella, Deneb, Electra))] + #[superstruct(only(Capella, Deneb, Electra, Fulu))] pub withdrawals: Withdrawals, - #[superstruct(only(Deneb, Electra), partial_getter(copy))] + #[superstruct(only(Deneb, Electra, Fulu), partial_getter(copy))] #[serde(with = "serde_utils::quoted_u64")] pub blob_gas_used: u64, - #[superstruct(only(Deneb, Electra), partial_getter(copy))] + #[superstruct(only(Deneb, Electra, Fulu), partial_getter(copy))] #[serde(with = "serde_utils::quoted_u64")] pub excess_blob_gas: u64, } @@ -114,6 +114,7 @@ impl ExecutionPayload { ForkName::Capella => ExecutionPayloadCapella::from_ssz_bytes(bytes).map(Self::Capella), ForkName::Deneb => ExecutionPayloadDeneb::from_ssz_bytes(bytes).map(Self::Deneb), ForkName::Electra => ExecutionPayloadElectra::from_ssz_bytes(bytes).map(Self::Electra), + ForkName::Fulu => ExecutionPayloadFulu::from_ssz_bytes(bytes).map(Self::Fulu), } } @@ -127,45 +128,6 @@ impl ExecutionPayload { // Max size of variable length `transactions` field + (E::max_transactions_per_payload() * (ssz::BYTES_PER_LENGTH_OFFSET + E::max_bytes_per_transaction())) } - - #[allow(clippy::arithmetic_side_effects)] - /// Returns the maximum size of an execution payload. - pub fn max_execution_payload_capella_size() -> usize { - // Fixed part - ExecutionPayloadCapella::::default().as_ssz_bytes().len() - // Max size of variable length `extra_data` field - + (E::max_extra_data_bytes() * ::ssz_fixed_len()) - // Max size of variable length `transactions` field - + (E::max_transactions_per_payload() * (ssz::BYTES_PER_LENGTH_OFFSET + E::max_bytes_per_transaction())) - // Max size of variable length `withdrawals` field - + (E::max_withdrawals_per_payload() * ::ssz_fixed_len()) - } - - #[allow(clippy::arithmetic_side_effects)] - /// Returns the maximum size of an execution payload. - pub fn max_execution_payload_deneb_size() -> usize { - // Fixed part - ExecutionPayloadDeneb::::default().as_ssz_bytes().len() - // Max size of variable length `extra_data` field - + (E::max_extra_data_bytes() * ::ssz_fixed_len()) - // Max size of variable length `transactions` field - + (E::max_transactions_per_payload() * (ssz::BYTES_PER_LENGTH_OFFSET + E::max_bytes_per_transaction())) - // Max size of variable length `withdrawals` field - + (E::max_withdrawals_per_payload() * ::ssz_fixed_len()) - } - - #[allow(clippy::arithmetic_side_effects)] - /// Returns the maximum size of an execution payload. - pub fn max_execution_payload_electra_size() -> usize { - // Fixed part - ExecutionPayloadElectra::::default().as_ssz_bytes().len() - // Max size of variable length `extra_data` field - + (E::max_extra_data_bytes() * ::ssz_fixed_len()) - // Max size of variable length `transactions` field - + (E::max_transactions_per_payload() * (ssz::BYTES_PER_LENGTH_OFFSET + E::max_bytes_per_transaction())) - // Max size of variable length `withdrawals` field - + (E::max_withdrawals_per_payload() * ::ssz_fixed_len()) - } } impl ForkVersionDeserialize for ExecutionPayload { @@ -184,6 +146,7 @@ impl ForkVersionDeserialize for ExecutionPayload { ForkName::Capella => Self::Capella(serde_json::from_value(value).map_err(convert_err)?), ForkName::Deneb => Self::Deneb(serde_json::from_value(value).map_err(convert_err)?), ForkName::Electra => Self::Electra(serde_json::from_value(value).map_err(convert_err)?), + ForkName::Fulu => Self::Fulu(serde_json::from_value(value).map_err(convert_err)?), ForkName::Base | ForkName::Altair => { return Err(serde::de::Error::custom(format!( "ExecutionPayload failed to deserialize: unsupported fork '{}'", @@ -201,6 +164,7 @@ impl ExecutionPayload { ExecutionPayload::Capella(_) => ForkName::Capella, ExecutionPayload::Deneb(_) => ForkName::Deneb, ExecutionPayload::Electra(_) => ForkName::Electra, + ExecutionPayload::Fulu(_) => ForkName::Fulu, } } } diff --git a/consensus/types/src/execution_payload_header.rs b/consensus/types/src/execution_payload_header.rs index 4bfbfee9bf..3012041b8b 100644 --- a/consensus/types/src/execution_payload_header.rs +++ b/consensus/types/src/execution_payload_header.rs @@ -8,7 +8,7 @@ use tree_hash::TreeHash; use tree_hash_derive::TreeHash; #[superstruct( - variants(Bellatrix, Capella, Deneb, Electra), + variants(Bellatrix, Capella, Deneb, Electra, Fulu), variant_attributes( derive( Default, @@ -78,12 +78,12 @@ pub struct ExecutionPayloadHeader { pub block_hash: ExecutionBlockHash, #[superstruct(getter(copy))] pub transactions_root: Hash256, - #[superstruct(only(Capella, Deneb, Electra), partial_getter(copy))] + #[superstruct(only(Capella, Deneb, Electra, Fulu), partial_getter(copy))] pub withdrawals_root: Hash256, - #[superstruct(only(Deneb, Electra), partial_getter(copy))] + #[superstruct(only(Deneb, Electra, Fulu), partial_getter(copy))] #[serde(with = "serde_utils::quoted_u64")] pub blob_gas_used: u64, - #[superstruct(only(Deneb, Electra), partial_getter(copy))] + #[superstruct(only(Deneb, Electra, Fulu), partial_getter(copy))] #[serde(with = "serde_utils::quoted_u64")] pub excess_blob_gas: u64, } @@ -108,18 +108,18 @@ impl ExecutionPayloadHeader { ForkName::Electra => { ExecutionPayloadHeaderElectra::from_ssz_bytes(bytes).map(Self::Electra) } + ForkName::Fulu => ExecutionPayloadHeaderFulu::from_ssz_bytes(bytes).map(Self::Fulu), } } #[allow(clippy::arithmetic_side_effects)] pub fn ssz_max_var_len_for_fork(fork_name: ForkName) -> usize { - // Matching here in case variable fields are added in future forks. - match fork_name { - ForkName::Base | ForkName::Altair => 0, - ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb | ForkName::Electra => { - // Max size of variable length `extra_data` field - E::max_extra_data_bytes() * ::ssz_fixed_len() - } + // TODO(newfork): Add a new case here if there are new variable fields + if fork_name.bellatrix_enabled() { + // Max size of variable length `extra_data` field + E::max_extra_data_bytes() * ::ssz_fixed_len() + } else { + 0 } } @@ -129,6 +129,7 @@ impl ExecutionPayloadHeader { ExecutionPayloadHeader::Capella(_) => ForkName::Capella, ExecutionPayloadHeader::Deneb(_) => ForkName::Deneb, ExecutionPayloadHeader::Electra(_) => ForkName::Electra, + ExecutionPayloadHeader::Fulu(_) => ForkName::Fulu, } } } @@ -212,6 +213,30 @@ impl ExecutionPayloadHeaderDeneb { } } +impl ExecutionPayloadHeaderElectra { + pub fn upgrade_to_fulu(&self) -> ExecutionPayloadHeaderFulu { + ExecutionPayloadHeaderFulu { + parent_hash: self.parent_hash, + fee_recipient: self.fee_recipient, + state_root: self.state_root, + receipts_root: self.receipts_root, + logs_bloom: self.logs_bloom.clone(), + prev_randao: self.prev_randao, + block_number: self.block_number, + gas_limit: self.gas_limit, + gas_used: self.gas_used, + timestamp: self.timestamp, + extra_data: self.extra_data.clone(), + base_fee_per_gas: self.base_fee_per_gas, + block_hash: self.block_hash, + transactions_root: self.transactions_root, + withdrawals_root: self.withdrawals_root, + blob_gas_used: self.blob_gas_used, + excess_blob_gas: self.excess_blob_gas, + } + } +} + impl<'a, E: EthSpec> From<&'a ExecutionPayloadBellatrix> for ExecutionPayloadHeaderBellatrix { fn from(payload: &'a ExecutionPayloadBellatrix) -> Self { Self { @@ -303,6 +328,30 @@ impl<'a, E: EthSpec> From<&'a ExecutionPayloadElectra> for ExecutionPayloadHe } } +impl<'a, E: EthSpec> From<&'a ExecutionPayloadFulu> for ExecutionPayloadHeaderFulu { + fn from(payload: &'a ExecutionPayloadFulu) -> Self { + Self { + parent_hash: payload.parent_hash, + fee_recipient: payload.fee_recipient, + state_root: payload.state_root, + receipts_root: payload.receipts_root, + logs_bloom: payload.logs_bloom.clone(), + prev_randao: payload.prev_randao, + block_number: payload.block_number, + gas_limit: payload.gas_limit, + gas_used: payload.gas_used, + timestamp: payload.timestamp, + extra_data: payload.extra_data.clone(), + base_fee_per_gas: payload.base_fee_per_gas, + block_hash: payload.block_hash, + transactions_root: payload.transactions.tree_hash_root(), + withdrawals_root: payload.withdrawals.tree_hash_root(), + blob_gas_used: payload.blob_gas_used, + excess_blob_gas: payload.excess_blob_gas, + } + } +} + // These impls are required to work around an inelegance in `to_execution_payload_header`. // They only clone headers so they should be relatively cheap. impl<'a, E: EthSpec> From<&'a Self> for ExecutionPayloadHeaderBellatrix { @@ -329,6 +378,12 @@ impl<'a, E: EthSpec> From<&'a Self> for ExecutionPayloadHeaderElectra { } } +impl<'a, E: EthSpec> From<&'a Self> for ExecutionPayloadHeaderFulu { + fn from(payload: &'a Self) -> Self { + payload.clone() + } +} + impl<'a, E: EthSpec> From> for ExecutionPayloadHeader { fn from(payload: ExecutionPayloadRef<'a, E>) -> Self { map_execution_payload_ref_into_execution_payload_header!( @@ -387,6 +442,9 @@ impl ExecutionPayloadHeaderRefMut<'_, E> { ExecutionPayloadHeaderRefMut::Electra(mut_ref) => { *mut_ref = header.try_into()?; } + ExecutionPayloadHeaderRefMut::Fulu(mut_ref) => { + *mut_ref = header.try_into()?; + } } Ok(()) } @@ -404,6 +462,16 @@ impl TryFrom> for ExecutionPayloadHeaderEl } } +impl TryFrom> for ExecutionPayloadHeaderFulu { + type Error = BeaconStateError; + fn try_from(header: ExecutionPayloadHeader) -> Result { + match header { + ExecutionPayloadHeader::Fulu(execution_payload_header) => Ok(execution_payload_header), + _ => Err(BeaconStateError::IncorrectStateVariant), + } + } +} + impl ForkVersionDeserialize for ExecutionPayloadHeader { fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( value: serde_json::value::Value, @@ -423,6 +491,7 @@ impl ForkVersionDeserialize for ExecutionPayloadHeader { ForkName::Capella => Self::Capella(serde_json::from_value(value).map_err(convert_err)?), ForkName::Deneb => Self::Deneb(serde_json::from_value(value).map_err(convert_err)?), ForkName::Electra => Self::Electra(serde_json::from_value(value).map_err(convert_err)?), + ForkName::Fulu => Self::Fulu(serde_json::from_value(value).map_err(convert_err)?), ForkName::Base | ForkName::Altair => { return Err(serde::de::Error::custom(format!( "ExecutionPayloadHeader failed to deserialize: unsupported fork '{}'", diff --git a/consensus/types/src/execution_requests.rs b/consensus/types/src/execution_requests.rs index 96a3905420..223c6444cc 100644 --- a/consensus/types/src/execution_requests.rs +++ b/consensus/types/src/execution_requests.rs @@ -43,10 +43,29 @@ impl ExecutionRequests { /// Returns the encoding according to EIP-7685 to send /// to the execution layer over the engine api. pub fn get_execution_requests_list(&self) -> Vec { - let deposit_bytes = Bytes::from(self.deposits.as_ssz_bytes()); - let withdrawal_bytes = Bytes::from(self.withdrawals.as_ssz_bytes()); - let consolidation_bytes = Bytes::from(self.consolidations.as_ssz_bytes()); - vec![deposit_bytes, withdrawal_bytes, consolidation_bytes] + let mut requests_list = Vec::new(); + if !self.deposits.is_empty() { + requests_list.push(Bytes::from_iter( + [RequestType::Deposit.to_u8()] + .into_iter() + .chain(self.deposits.as_ssz_bytes()), + )); + } + if !self.withdrawals.is_empty() { + requests_list.push(Bytes::from_iter( + [RequestType::Withdrawal.to_u8()] + .into_iter() + .chain(self.withdrawals.as_ssz_bytes()), + )); + } + if !self.consolidations.is_empty() { + requests_list.push(Bytes::from_iter( + [RequestType::Consolidation.to_u8()] + .into_iter() + .chain(self.consolidations.as_ssz_bytes()), + )); + } + requests_list } /// Generate the execution layer `requests_hash` based on EIP-7685. @@ -55,9 +74,8 @@ impl ExecutionRequests { pub fn requests_hash(&self) -> Hash256 { let mut hasher = DynamicContext::new(); - for (i, request) in self.get_execution_requests_list().iter().enumerate() { + for request in self.get_execution_requests_list().iter() { let mut request_hasher = DynamicContext::new(); - request_hasher.update(&[i as u8]); request_hasher.update(request); let request_hash = request_hasher.finalize(); @@ -68,16 +86,16 @@ impl ExecutionRequests { } } -/// This is used to index into the `execution_requests` array. +/// The prefix types for `ExecutionRequest` objects. #[derive(Debug, Copy, Clone)] -pub enum RequestPrefix { +pub enum RequestType { Deposit, Withdrawal, Consolidation, } -impl RequestPrefix { - pub fn from_prefix(prefix: u8) -> Option { +impl RequestType { + pub fn from_u8(prefix: u8) -> Option { match prefix { 0 => Some(Self::Deposit), 1 => Some(Self::Withdrawal), @@ -85,6 +103,13 @@ impl RequestPrefix { _ => None, } } + pub fn to_u8(&self) -> u8 { + match self { + Self::Deposit => 0, + Self::Withdrawal => 1, + Self::Consolidation => 2, + } + } } #[cfg(test)] diff --git a/consensus/types/src/fork_context.rs b/consensus/types/src/fork_context.rs index 0f7f0eb769..33f1c51d44 100644 --- a/consensus/types/src/fork_context.rs +++ b/consensus/types/src/fork_context.rs @@ -69,6 +69,13 @@ impl ForkContext { )); } + if spec.fulu_fork_epoch.is_some() { + fork_to_digest.push(( + ForkName::Fulu, + ChainSpec::compute_fork_digest(spec.fulu_fork_version, genesis_validators_root), + )); + } + let fork_to_digest: HashMap = fork_to_digest.into_iter().collect(); let digest_to_fork = fork_to_digest diff --git a/consensus/types/src/fork_name.rs b/consensus/types/src/fork_name.rs index 51a5b3813b..40557e0cb9 100644 --- a/consensus/types/src/fork_name.rs +++ b/consensus/types/src/fork_name.rs @@ -17,6 +17,7 @@ pub enum ForkName { Capella, Deneb, Electra, + Fulu, } impl ForkName { @@ -28,6 +29,7 @@ impl ForkName { ForkName::Capella, ForkName::Deneb, ForkName::Electra, + ForkName::Fulu, ] } @@ -38,6 +40,7 @@ impl ForkName { (ForkName::Capella, spec.capella_fork_epoch), (ForkName::Deneb, spec.deneb_fork_epoch), (ForkName::Electra, spec.electra_fork_epoch), + (ForkName::Fulu, spec.fulu_fork_epoch), ] } @@ -46,6 +49,13 @@ impl ForkName { *ForkName::list_all().last().unwrap() } + /// Returns the fork primarily used for testing purposes. + /// This fork serves as the baseline for many tests, and the goal + /// is to ensure features are passing on this fork. + pub fn latest_stable() -> ForkName { + ForkName::Electra + } + /// Set the activation slots in the given `ChainSpec` so that the fork named by `self` /// is the only fork in effect from genesis. pub fn make_genesis_spec(&self, mut spec: ChainSpec) -> ChainSpec { @@ -57,6 +67,7 @@ impl ForkName { spec.capella_fork_epoch = None; spec.deneb_fork_epoch = None; spec.electra_fork_epoch = None; + spec.fulu_fork_epoch = None; spec } ForkName::Altair => { @@ -65,6 +76,7 @@ impl ForkName { spec.capella_fork_epoch = None; spec.deneb_fork_epoch = None; spec.electra_fork_epoch = None; + spec.fulu_fork_epoch = None; spec } ForkName::Bellatrix => { @@ -73,6 +85,7 @@ impl ForkName { spec.capella_fork_epoch = None; spec.deneb_fork_epoch = None; spec.electra_fork_epoch = None; + spec.fulu_fork_epoch = None; spec } ForkName::Capella => { @@ -81,6 +94,7 @@ impl ForkName { spec.capella_fork_epoch = Some(Epoch::new(0)); spec.deneb_fork_epoch = None; spec.electra_fork_epoch = None; + spec.fulu_fork_epoch = None; spec } ForkName::Deneb => { @@ -89,6 +103,7 @@ impl ForkName { spec.capella_fork_epoch = Some(Epoch::new(0)); spec.deneb_fork_epoch = Some(Epoch::new(0)); spec.electra_fork_epoch = None; + spec.fulu_fork_epoch = None; spec } ForkName::Electra => { @@ -97,6 +112,16 @@ impl ForkName { spec.capella_fork_epoch = Some(Epoch::new(0)); spec.deneb_fork_epoch = Some(Epoch::new(0)); spec.electra_fork_epoch = Some(Epoch::new(0)); + spec.fulu_fork_epoch = None; + spec + } + ForkName::Fulu => { + spec.altair_fork_epoch = Some(Epoch::new(0)); + spec.bellatrix_fork_epoch = Some(Epoch::new(0)); + spec.capella_fork_epoch = Some(Epoch::new(0)); + spec.deneb_fork_epoch = Some(Epoch::new(0)); + spec.electra_fork_epoch = Some(Epoch::new(0)); + spec.fulu_fork_epoch = Some(Epoch::new(0)); spec } } @@ -113,6 +138,7 @@ impl ForkName { ForkName::Capella => Some(ForkName::Bellatrix), ForkName::Deneb => Some(ForkName::Capella), ForkName::Electra => Some(ForkName::Deneb), + ForkName::Fulu => Some(ForkName::Electra), } } @@ -126,7 +152,8 @@ impl ForkName { ForkName::Bellatrix => Some(ForkName::Capella), ForkName::Capella => Some(ForkName::Deneb), ForkName::Deneb => Some(ForkName::Electra), - ForkName::Electra => None, + ForkName::Electra => Some(ForkName::Fulu), + ForkName::Fulu => None, } } @@ -149,6 +176,10 @@ impl ForkName { pub fn electra_enabled(self) -> bool { self >= ForkName::Electra } + + pub fn fulu_enabled(self) -> bool { + self >= ForkName::Fulu + } } /// Map a fork name into a fork-versioned superstruct type like `BeaconBlock`. @@ -200,6 +231,10 @@ macro_rules! map_fork_name_with { let (value, extra_data) = $body; ($t::Electra(value), extra_data) } + ForkName::Fulu => { + let (value, extra_data) = $body; + ($t::Fulu(value), extra_data) + } } }; } @@ -215,6 +250,7 @@ impl FromStr for ForkName { "capella" => ForkName::Capella, "deneb" => ForkName::Deneb, "electra" => ForkName::Electra, + "fulu" => ForkName::Fulu, _ => return Err(format!("unknown fork name: {}", fork_name)), }) } @@ -229,6 +265,7 @@ impl Display for ForkName { ForkName::Capella => "capella".fmt(f), ForkName::Deneb => "deneb".fmt(f), ForkName::Electra => "electra".fmt(f), + ForkName::Fulu => "fulu".fmt(f), } } } diff --git a/consensus/types/src/graffiti.rs b/consensus/types/src/graffiti.rs index 08f8573c6d..f781aacabd 100644 --- a/consensus/types/src/graffiti.rs +++ b/consensus/types/src/graffiti.rs @@ -57,7 +57,7 @@ impl FromStr for GraffitiString { type Err = String; fn from_str(s: &str) -> Result { - if s.as_bytes().len() > GRAFFITI_BYTES_LEN { + if s.len() > GRAFFITI_BYTES_LEN { return Err(format!( "Graffiti exceeds max length {}", GRAFFITI_BYTES_LEN diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 70c84bee96..8e664d99fa 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -107,10 +107,12 @@ pub mod slot_data; pub mod sqlite; pub mod blob_sidecar; +pub mod data_column_custody_group; pub mod data_column_sidecar; pub mod data_column_subnet_id; pub mod light_client_header; pub mod non_zero_usize; +pub mod runtime_fixed_vector; pub mod runtime_var_list; pub use crate::activation_queue::ActivationQueue; @@ -119,7 +121,7 @@ pub use crate::aggregate_and_proof::{ }; pub use crate::attestation::{ Attestation, AttestationBase, AttestationElectra, AttestationRef, AttestationRefMut, - Error as AttestationError, + Error as AttestationError, SingleAttestation, }; pub use crate::attestation_data::AttestationData; pub use crate::attestation_duty::AttestationDuty; @@ -129,13 +131,13 @@ pub use crate::attester_slashing::{ }; pub use crate::beacon_block::{ BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockBellatrix, BeaconBlockCapella, - BeaconBlockDeneb, BeaconBlockElectra, BeaconBlockRef, BeaconBlockRefMut, BlindedBeaconBlock, - BlockImportSource, EmptyBlock, + BeaconBlockDeneb, BeaconBlockElectra, BeaconBlockFulu, BeaconBlockRef, BeaconBlockRefMut, + BlindedBeaconBlock, BlockImportSource, EmptyBlock, }; pub use crate::beacon_block_body::{ BeaconBlockBody, BeaconBlockBodyAltair, BeaconBlockBodyBase, BeaconBlockBodyBellatrix, - BeaconBlockBodyCapella, BeaconBlockBodyDeneb, BeaconBlockBodyElectra, BeaconBlockBodyRef, - BeaconBlockBodyRefMut, + BeaconBlockBodyCapella, BeaconBlockBodyDeneb, BeaconBlockBodyElectra, BeaconBlockBodyFulu, + BeaconBlockBodyRef, BeaconBlockBodyRefMut, }; pub use crate::beacon_block_header::BeaconBlockHeader; pub use crate::beacon_committee::{BeaconCommittee, OwnedBeaconCommittee}; @@ -145,7 +147,7 @@ pub use crate::bls_to_execution_change::BlsToExecutionChange; pub use crate::chain_spec::{ChainSpec, Config, Domain}; pub use crate::checkpoint::Checkpoint; pub use crate::config_and_preset::{ - ConfigAndPreset, ConfigAndPresetCapella, ConfigAndPresetDeneb, ConfigAndPresetElectra, + ConfigAndPreset, ConfigAndPresetDeneb, ConfigAndPresetElectra, ConfigAndPresetFulu, }; pub use crate::consolidation_request::ConsolidationRequest; pub use crate::contribution_and_proof::ContributionAndProof; @@ -166,14 +168,15 @@ pub use crate::execution_block_hash::ExecutionBlockHash; pub use crate::execution_block_header::{EncodableExecutionBlockHeader, ExecutionBlockHeader}; pub use crate::execution_payload::{ ExecutionPayload, ExecutionPayloadBellatrix, ExecutionPayloadCapella, ExecutionPayloadDeneb, - ExecutionPayloadElectra, ExecutionPayloadRef, Transaction, Transactions, Withdrawals, + ExecutionPayloadElectra, ExecutionPayloadFulu, ExecutionPayloadRef, Transaction, Transactions, + Withdrawals, }; pub use crate::execution_payload_header::{ ExecutionPayloadHeader, ExecutionPayloadHeaderBellatrix, ExecutionPayloadHeaderCapella, - ExecutionPayloadHeaderDeneb, ExecutionPayloadHeaderElectra, ExecutionPayloadHeaderRef, - ExecutionPayloadHeaderRefMut, + ExecutionPayloadHeaderDeneb, ExecutionPayloadHeaderElectra, ExecutionPayloadHeaderFulu, + ExecutionPayloadHeaderRef, ExecutionPayloadHeaderRefMut, }; -pub use crate::execution_requests::{ExecutionRequests, RequestPrefix}; +pub use crate::execution_requests::{ExecutionRequests, RequestType}; pub use crate::fork::Fork; pub use crate::fork_context::ForkContext; pub use crate::fork_data::ForkData; @@ -189,31 +192,33 @@ pub use crate::indexed_attestation::{ }; pub use crate::light_client_bootstrap::{ LightClientBootstrap, LightClientBootstrapAltair, LightClientBootstrapCapella, - LightClientBootstrapDeneb, LightClientBootstrapElectra, + LightClientBootstrapDeneb, LightClientBootstrapElectra, LightClientBootstrapFulu, }; pub use crate::light_client_finality_update::{ LightClientFinalityUpdate, LightClientFinalityUpdateAltair, LightClientFinalityUpdateCapella, LightClientFinalityUpdateDeneb, LightClientFinalityUpdateElectra, + LightClientFinalityUpdateFulu, }; pub use crate::light_client_header::{ LightClientHeader, LightClientHeaderAltair, LightClientHeaderCapella, LightClientHeaderDeneb, - LightClientHeaderElectra, + LightClientHeaderElectra, LightClientHeaderFulu, }; pub use crate::light_client_optimistic_update::{ LightClientOptimisticUpdate, LightClientOptimisticUpdateAltair, LightClientOptimisticUpdateCapella, LightClientOptimisticUpdateDeneb, - LightClientOptimisticUpdateElectra, + LightClientOptimisticUpdateElectra, LightClientOptimisticUpdateFulu, }; pub use crate::light_client_update::{ Error as LightClientUpdateError, LightClientUpdate, LightClientUpdateAltair, - LightClientUpdateCapella, LightClientUpdateDeneb, LightClientUpdateElectra, MerkleProof, + LightClientUpdateCapella, LightClientUpdateDeneb, LightClientUpdateElectra, + LightClientUpdateFulu, MerkleProof, }; pub use crate::participation_flags::ParticipationFlags; pub use crate::payload::{ AbstractExecPayload, BlindedPayload, BlindedPayloadBellatrix, BlindedPayloadCapella, - BlindedPayloadDeneb, BlindedPayloadElectra, BlindedPayloadRef, BlockType, ExecPayload, - FullPayload, FullPayloadBellatrix, FullPayloadCapella, FullPayloadDeneb, FullPayloadElectra, - FullPayloadRef, OwnedExecPayload, + BlindedPayloadDeneb, BlindedPayloadElectra, BlindedPayloadFulu, BlindedPayloadRef, BlockType, + ExecPayload, FullPayload, FullPayloadBellatrix, FullPayloadCapella, FullPayloadDeneb, + FullPayloadElectra, FullPayloadFulu, FullPayloadRef, OwnedExecPayload, }; pub use crate::pending_attestation::PendingAttestation; pub use crate::pending_consolidation::PendingConsolidation; @@ -221,10 +226,12 @@ pub use crate::pending_deposit::PendingDeposit; pub use crate::pending_partial_withdrawal::PendingPartialWithdrawal; pub use crate::preset::{ AltairPreset, BasePreset, BellatrixPreset, CapellaPreset, DenebPreset, ElectraPreset, + FuluPreset, }; pub use crate::proposer_preparation_data::ProposerPreparationData; pub use crate::proposer_slashing::ProposerSlashing; pub use crate::relative_epoch::{Error as RelativeEpochError, RelativeEpoch}; +pub use crate::runtime_fixed_vector::RuntimeFixedVector; pub use crate::runtime_var_list::RuntimeVariableList; pub use crate::selection_proof::SelectionProof; pub use crate::shuffling_id::AttestationShufflingId; @@ -235,7 +242,7 @@ pub use crate::signed_beacon_block::{ ssz_tagged_signed_beacon_block, ssz_tagged_signed_beacon_block_arc, SignedBeaconBlock, SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockBellatrix, SignedBeaconBlockCapella, SignedBeaconBlockDeneb, SignedBeaconBlockElectra, - SignedBeaconBlockHash, SignedBlindedBeaconBlock, + SignedBeaconBlockFulu, SignedBeaconBlockHash, SignedBlindedBeaconBlock, }; pub use crate::signed_beacon_block_header::SignedBeaconBlockHeader; pub use crate::signed_bls_to_execution_change::SignedBlsToExecutionChange; diff --git a/consensus/types/src/light_client_bootstrap.rs b/consensus/types/src/light_client_bootstrap.rs index 21a7e5416f..aa0d8836d1 100644 --- a/consensus/types/src/light_client_bootstrap.rs +++ b/consensus/types/src/light_client_bootstrap.rs @@ -2,7 +2,7 @@ use crate::{ light_client_update::*, test_utils::TestRandom, BeaconState, ChainSpec, EthSpec, FixedVector, ForkName, ForkVersionDeserialize, Hash256, LightClientHeader, LightClientHeaderAltair, LightClientHeaderCapella, LightClientHeaderDeneb, LightClientHeaderElectra, - SignedBlindedBeaconBlock, Slot, SyncCommittee, + LightClientHeaderFulu, SignedBlindedBeaconBlock, Slot, SyncCommittee, }; use derivative::Derivative; use serde::{Deserialize, Deserializer, Serialize}; @@ -17,7 +17,7 @@ use tree_hash_derive::TreeHash; /// A LightClientBootstrap is the initializer we send over to light_client nodes /// that are trying to generate their basic storage when booting up. #[superstruct( - variants(Altair, Capella, Deneb, Electra), + variants(Altair, Capella, Deneb, Electra, Fulu), variant_attributes( derive( Debug, @@ -54,6 +54,8 @@ pub struct LightClientBootstrap { pub header: LightClientHeaderDeneb, #[superstruct(only(Electra), partial_getter(rename = "header_electra"))] pub header: LightClientHeaderElectra, + #[superstruct(only(Fulu), partial_getter(rename = "header_fulu"))] + pub header: LightClientHeaderFulu, /// The `SyncCommittee` used in the requested period. pub current_sync_committee: Arc>, /// Merkle proof for sync committee @@ -63,7 +65,7 @@ pub struct LightClientBootstrap { )] pub current_sync_committee_branch: FixedVector, #[superstruct( - only(Electra), + only(Electra, Fulu), partial_getter(rename = "current_sync_committee_branch_electra") )] pub current_sync_committee_branch: FixedVector, @@ -79,6 +81,7 @@ impl LightClientBootstrap { Self::Capella(_) => func(ForkName::Capella), Self::Deneb(_) => func(ForkName::Deneb), Self::Electra(_) => func(ForkName::Electra), + Self::Fulu(_) => func(ForkName::Fulu), } } @@ -97,6 +100,7 @@ impl LightClientBootstrap { ForkName::Capella => Self::Capella(LightClientBootstrapCapella::from_ssz_bytes(bytes)?), ForkName::Deneb => Self::Deneb(LightClientBootstrapDeneb::from_ssz_bytes(bytes)?), ForkName::Electra => Self::Electra(LightClientBootstrapElectra::from_ssz_bytes(bytes)?), + ForkName::Fulu => Self::Fulu(LightClientBootstrapFulu::from_ssz_bytes(bytes)?), ForkName::Base => { return Err(ssz::DecodeError::BytesInvalid(format!( "LightClientBootstrap decoding for {fork_name} not implemented" @@ -117,6 +121,7 @@ impl LightClientBootstrap { ForkName::Capella => as Encode>::ssz_fixed_len(), ForkName::Deneb => as Encode>::ssz_fixed_len(), ForkName::Electra => as Encode>::ssz_fixed_len(), + ForkName::Fulu => as Encode>::ssz_fixed_len(), }; fixed_len + LightClientHeader::::ssz_max_var_len_for_fork(fork_name) } @@ -152,6 +157,11 @@ impl LightClientBootstrap { current_sync_committee, current_sync_committee_branch: current_sync_committee_branch.into(), }), + ForkName::Fulu => Self::Fulu(LightClientBootstrapFulu { + header: LightClientHeaderFulu::block_to_light_client_header(block)?, + current_sync_committee, + current_sync_committee_branch: current_sync_committee_branch.into(), + }), }; Ok(light_client_bootstrap) @@ -192,6 +202,11 @@ impl LightClientBootstrap { current_sync_committee, current_sync_committee_branch: current_sync_committee_branch.into(), }), + ForkName::Fulu => Self::Fulu(LightClientBootstrapFulu { + header: LightClientHeaderFulu::block_to_light_client_header(block)?, + current_sync_committee, + current_sync_committee_branch: current_sync_committee_branch.into(), + }), }; Ok(light_client_bootstrap) @@ -241,4 +256,10 @@ mod tests { use crate::{LightClientBootstrapElectra, MainnetEthSpec}; ssz_tests!(LightClientBootstrapElectra); } + + #[cfg(test)] + mod fulu { + use crate::{LightClientBootstrapFulu, MainnetEthSpec}; + ssz_tests!(LightClientBootstrapFulu); + } } diff --git a/consensus/types/src/light_client_finality_update.rs b/consensus/types/src/light_client_finality_update.rs index ba2f2083cd..ee3b53c853 100644 --- a/consensus/types/src/light_client_finality_update.rs +++ b/consensus/types/src/light_client_finality_update.rs @@ -3,7 +3,7 @@ use crate::ChainSpec; use crate::{ light_client_update::*, test_utils::TestRandom, ForkName, ForkVersionDeserialize, LightClientHeaderAltair, LightClientHeaderCapella, LightClientHeaderDeneb, - LightClientHeaderElectra, SignedBlindedBeaconBlock, + LightClientHeaderElectra, LightClientHeaderFulu, SignedBlindedBeaconBlock, }; use derivative::Derivative; use serde::{Deserialize, Deserializer, Serialize}; @@ -16,7 +16,7 @@ use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; #[superstruct( - variants(Altair, Capella, Deneb, Electra), + variants(Altair, Capella, Deneb, Electra, Fulu), variant_attributes( derive( Debug, @@ -53,6 +53,8 @@ pub struct LightClientFinalityUpdate { pub attested_header: LightClientHeaderDeneb, #[superstruct(only(Electra), partial_getter(rename = "attested_header_electra"))] pub attested_header: LightClientHeaderElectra, + #[superstruct(only(Fulu), partial_getter(rename = "attested_header_fulu"))] + pub attested_header: LightClientHeaderFulu, /// The last `BeaconBlockHeader` from the last attested finalized block (end of epoch). #[superstruct(only(Altair), partial_getter(rename = "finalized_header_altair"))] pub finalized_header: LightClientHeaderAltair, @@ -62,13 +64,18 @@ pub struct LightClientFinalityUpdate { pub finalized_header: LightClientHeaderDeneb, #[superstruct(only(Electra), partial_getter(rename = "finalized_header_electra"))] pub finalized_header: LightClientHeaderElectra, + #[superstruct(only(Fulu), partial_getter(rename = "finalized_header_fulu"))] + pub finalized_header: LightClientHeaderFulu, /// Merkle proof attesting finalized header. #[superstruct( only(Altair, Capella, Deneb), partial_getter(rename = "finality_branch_altair") )] pub finality_branch: FixedVector, - #[superstruct(only(Electra), partial_getter(rename = "finality_branch_electra"))] + #[superstruct( + only(Electra, Fulu), + partial_getter(rename = "finality_branch_electra") + )] pub finality_branch: FixedVector, /// current sync aggregate pub sync_aggregate: SyncAggregate, @@ -135,6 +142,17 @@ impl LightClientFinalityUpdate { sync_aggregate, signature_slot, }), + ForkName::Fulu => Self::Fulu(LightClientFinalityUpdateFulu { + attested_header: LightClientHeaderFulu::block_to_light_client_header( + attested_block, + )?, + finalized_header: LightClientHeaderFulu::block_to_light_client_header( + finalized_block, + )?, + finality_branch: finality_branch.into(), + sync_aggregate, + signature_slot, + }), ForkName::Base => return Err(Error::AltairForkNotActive), }; @@ -151,6 +169,7 @@ impl LightClientFinalityUpdate { Self::Capella(_) => func(ForkName::Capella), Self::Deneb(_) => func(ForkName::Deneb), Self::Electra(_) => func(ForkName::Electra), + Self::Fulu(_) => func(ForkName::Fulu), } } @@ -173,6 +192,7 @@ impl LightClientFinalityUpdate { ForkName::Electra => { Self::Electra(LightClientFinalityUpdateElectra::from_ssz_bytes(bytes)?) } + ForkName::Fulu => Self::Fulu(LightClientFinalityUpdateFulu::from_ssz_bytes(bytes)?), ForkName::Base => { return Err(ssz::DecodeError::BytesInvalid(format!( "LightClientFinalityUpdate decoding for {fork_name} not implemented" @@ -193,6 +213,7 @@ impl LightClientFinalityUpdate { ForkName::Capella => as Encode>::ssz_fixed_len(), ForkName::Deneb => as Encode>::ssz_fixed_len(), ForkName::Electra => as Encode>::ssz_fixed_len(), + ForkName::Fulu => as Encode>::ssz_fixed_len(), }; // `2 *` because there are two headers in the update fixed_size + 2 * LightClientHeader::::ssz_max_var_len_for_fork(fork_name) @@ -255,4 +276,10 @@ mod tests { use crate::{LightClientFinalityUpdateElectra, MainnetEthSpec}; ssz_tests!(LightClientFinalityUpdateElectra); } + + #[cfg(test)] + mod fulu { + use crate::{LightClientFinalityUpdateFulu, MainnetEthSpec}; + ssz_tests!(LightClientFinalityUpdateFulu); + } } diff --git a/consensus/types/src/light_client_header.rs b/consensus/types/src/light_client_header.rs index 6655e0a093..0be26a7036 100644 --- a/consensus/types/src/light_client_header.rs +++ b/consensus/types/src/light_client_header.rs @@ -4,7 +4,8 @@ use crate::ForkVersionDeserialize; use crate::{light_client_update::*, BeaconBlockBody}; use crate::{ test_utils::TestRandom, EthSpec, ExecutionPayloadHeaderCapella, ExecutionPayloadHeaderDeneb, - ExecutionPayloadHeaderElectra, FixedVector, Hash256, SignedBlindedBeaconBlock, + ExecutionPayloadHeaderElectra, ExecutionPayloadHeaderFulu, FixedVector, Hash256, + SignedBlindedBeaconBlock, }; use crate::{BeaconBlockHeader, ExecutionPayloadHeader}; use derivative::Derivative; @@ -17,7 +18,7 @@ use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; #[superstruct( - variants(Altair, Capella, Deneb, Electra), + variants(Altair, Capella, Deneb, Electra, Fulu), variant_attributes( derive( Debug, @@ -59,8 +60,10 @@ pub struct LightClientHeader { partial_getter(rename = "execution_payload_header_electra") )] pub execution: ExecutionPayloadHeaderElectra, + #[superstruct(only(Fulu), partial_getter(rename = "execution_payload_header_fulu"))] + pub execution: ExecutionPayloadHeaderFulu, - #[superstruct(only(Capella, Deneb, Electra))] + #[superstruct(only(Capella, Deneb, Electra, Fulu))] pub execution_branch: FixedVector, #[ssz(skip_serializing, skip_deserializing)] @@ -92,6 +95,9 @@ impl LightClientHeader { ForkName::Electra => LightClientHeader::Electra( LightClientHeaderElectra::block_to_light_client_header(block)?, ), + ForkName::Fulu => { + LightClientHeader::Fulu(LightClientHeaderFulu::block_to_light_client_header(block)?) + } }; Ok(header) } @@ -110,6 +116,9 @@ impl LightClientHeader { ForkName::Electra => { LightClientHeader::Electra(LightClientHeaderElectra::from_ssz_bytes(bytes)?) } + ForkName::Fulu => { + LightClientHeader::Fulu(LightClientHeaderFulu::from_ssz_bytes(bytes)?) + } ForkName::Base => { return Err(ssz::DecodeError::BytesInvalid(format!( "LightClientHeader decoding for {fork_name} not implemented" @@ -283,6 +292,48 @@ impl Default for LightClientHeaderElectra { } } +impl LightClientHeaderFulu { + pub fn block_to_light_client_header( + block: &SignedBlindedBeaconBlock, + ) -> Result { + let payload = block + .message() + .execution_payload()? + .execution_payload_fulu()?; + + let header = ExecutionPayloadHeaderFulu::from(payload); + let beacon_block_body = BeaconBlockBody::from( + block + .message() + .body_fulu() + .map_err(|_| Error::BeaconBlockBodyError)? + .to_owned(), + ); + + let execution_branch = beacon_block_body + .to_ref() + .block_body_merkle_proof(EXECUTION_PAYLOAD_INDEX)?; + + Ok(LightClientHeaderFulu { + beacon: block.message().block_header(), + execution: header, + execution_branch: FixedVector::new(execution_branch)?, + _phantom_data: PhantomData, + }) + } +} + +impl Default for LightClientHeaderFulu { + fn default() -> Self { + Self { + beacon: BeaconBlockHeader::empty(), + execution: ExecutionPayloadHeaderFulu::default(), + execution_branch: FixedVector::default(), + _phantom_data: PhantomData, + } + } +} + impl ForkVersionDeserialize for LightClientHeader { fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( value: serde_json::value::Value, @@ -301,6 +352,9 @@ impl ForkVersionDeserialize for LightClientHeader { ForkName::Electra => serde_json::from_value(value) .map(|light_client_header| Self::Electra(light_client_header)) .map_err(serde::de::Error::custom), + ForkName::Fulu => serde_json::from_value(value) + .map(|light_client_header| Self::Fulu(light_client_header)) + .map_err(serde::de::Error::custom), ForkName::Base => Err(serde::de::Error::custom(format!( "LightClientHeader deserialization for {fork_name} not implemented" ))), diff --git a/consensus/types/src/light_client_optimistic_update.rs b/consensus/types/src/light_client_optimistic_update.rs index 209388af87..fcf357757b 100644 --- a/consensus/types/src/light_client_optimistic_update.rs +++ b/consensus/types/src/light_client_optimistic_update.rs @@ -2,7 +2,8 @@ use super::{EthSpec, ForkName, ForkVersionDeserialize, LightClientHeader, Slot, use crate::test_utils::TestRandom; use crate::{ light_client_update::*, ChainSpec, LightClientHeaderAltair, LightClientHeaderCapella, - LightClientHeaderDeneb, LightClientHeaderElectra, SignedBlindedBeaconBlock, + LightClientHeaderDeneb, LightClientHeaderElectra, LightClientHeaderFulu, + SignedBlindedBeaconBlock, }; use derivative::Derivative; use serde::{Deserialize, Deserializer, Serialize}; @@ -18,7 +19,7 @@ use tree_hash_derive::TreeHash; /// A LightClientOptimisticUpdate is the update we send on each slot, /// it is based off the current unfinalized epoch is verified only against BLS signature. #[superstruct( - variants(Altair, Capella, Deneb, Electra), + variants(Altair, Capella, Deneb, Electra, Fulu), variant_attributes( derive( Debug, @@ -55,6 +56,8 @@ pub struct LightClientOptimisticUpdate { pub attested_header: LightClientHeaderDeneb, #[superstruct(only(Electra), partial_getter(rename = "attested_header_electra"))] pub attested_header: LightClientHeaderElectra, + #[superstruct(only(Fulu), partial_getter(rename = "attested_header_fulu"))] + pub attested_header: LightClientHeaderFulu, /// current sync aggregate pub sync_aggregate: SyncAggregate, /// Slot of the sync aggregated signature @@ -102,6 +105,13 @@ impl LightClientOptimisticUpdate { sync_aggregate, signature_slot, }), + ForkName::Fulu => Self::Fulu(LightClientOptimisticUpdateFulu { + attested_header: LightClientHeaderFulu::block_to_light_client_header( + attested_block, + )?, + sync_aggregate, + signature_slot, + }), ForkName::Base => return Err(Error::AltairForkNotActive), }; @@ -117,6 +127,7 @@ impl LightClientOptimisticUpdate { Self::Capella(_) => func(ForkName::Capella), Self::Deneb(_) => func(ForkName::Deneb), Self::Electra(_) => func(ForkName::Electra), + Self::Fulu(_) => func(ForkName::Fulu), } } @@ -155,6 +166,7 @@ impl LightClientOptimisticUpdate { ForkName::Electra => { Self::Electra(LightClientOptimisticUpdateElectra::from_ssz_bytes(bytes)?) } + ForkName::Fulu => Self::Fulu(LightClientOptimisticUpdateFulu::from_ssz_bytes(bytes)?), ForkName::Base => { return Err(ssz::DecodeError::BytesInvalid(format!( "LightClientOptimisticUpdate decoding for {fork_name} not implemented" @@ -175,6 +187,7 @@ impl LightClientOptimisticUpdate { ForkName::Capella => as Encode>::ssz_fixed_len(), ForkName::Deneb => as Encode>::ssz_fixed_len(), ForkName::Electra => as Encode>::ssz_fixed_len(), + ForkName::Fulu => as Encode>::ssz_fixed_len(), }; fixed_len + LightClientHeader::::ssz_max_var_len_for_fork(fork_name) } @@ -238,4 +251,10 @@ mod tests { use crate::{LightClientOptimisticUpdateElectra, MainnetEthSpec}; ssz_tests!(LightClientOptimisticUpdateElectra); } + + #[cfg(test)] + mod fulu { + use crate::{LightClientOptimisticUpdateFulu, MainnetEthSpec}; + ssz_tests!(LightClientOptimisticUpdateFulu); + } } diff --git a/consensus/types/src/light_client_update.rs b/consensus/types/src/light_client_update.rs index c3a50e71c1..0dd91edc3c 100644 --- a/consensus/types/src/light_client_update.rs +++ b/consensus/types/src/light_client_update.rs @@ -4,7 +4,7 @@ use crate::LightClientHeader; use crate::{ beacon_state, test_utils::TestRandom, ChainSpec, Epoch, ForkName, ForkVersionDeserialize, LightClientHeaderAltair, LightClientHeaderCapella, LightClientHeaderDeneb, - SignedBlindedBeaconBlock, + LightClientHeaderFulu, SignedBlindedBeaconBlock, }; use derivative::Derivative; use safe_arith::ArithError; @@ -100,7 +100,7 @@ impl From for Error { /// or to sync up to the last committee period, we need to have one ready for each ALTAIR period /// we go over, note: there is no need to keep all of the updates from [ALTAIR_PERIOD, CURRENT_PERIOD]. #[superstruct( - variants(Altair, Capella, Deneb, Electra), + variants(Altair, Capella, Deneb, Electra, Fulu), variant_attributes( derive( Debug, @@ -137,6 +137,8 @@ pub struct LightClientUpdate { pub attested_header: LightClientHeaderDeneb, #[superstruct(only(Electra), partial_getter(rename = "attested_header_electra"))] pub attested_header: LightClientHeaderElectra, + #[superstruct(only(Fulu), partial_getter(rename = "attested_header_fulu"))] + pub attested_header: LightClientHeaderFulu, /// The `SyncCommittee` used in the next period. pub next_sync_committee: Arc>, // Merkle proof for next sync committee @@ -146,7 +148,7 @@ pub struct LightClientUpdate { )] pub next_sync_committee_branch: NextSyncCommitteeBranch, #[superstruct( - only(Electra), + only(Electra, Fulu), partial_getter(rename = "next_sync_committee_branch_electra") )] pub next_sync_committee_branch: NextSyncCommitteeBranchElectra, @@ -159,13 +161,18 @@ pub struct LightClientUpdate { pub finalized_header: LightClientHeaderDeneb, #[superstruct(only(Electra), partial_getter(rename = "finalized_header_electra"))] pub finalized_header: LightClientHeaderElectra, + #[superstruct(only(Fulu), partial_getter(rename = "finalized_header_fulu"))] + pub finalized_header: LightClientHeaderFulu, /// Merkle proof attesting finalized header. #[superstruct( only(Altair, Capella, Deneb), partial_getter(rename = "finality_branch_altair") )] pub finality_branch: FinalityBranch, - #[superstruct(only(Electra), partial_getter(rename = "finality_branch_electra"))] + #[superstruct( + only(Electra, Fulu), + partial_getter(rename = "finality_branch_electra") + )] pub finality_branch: FinalityBranchElectra, /// current sync aggreggate pub sync_aggregate: SyncAggregate, @@ -285,6 +292,26 @@ impl LightClientUpdate { sync_aggregate: sync_aggregate.clone(), signature_slot: block_slot, }) + } + ForkName::Fulu => { + let attested_header = + LightClientHeaderFulu::block_to_light_client_header(attested_block)?; + + let finalized_header = if let Some(finalized_block) = finalized_block { + LightClientHeaderFulu::block_to_light_client_header(finalized_block)? + } else { + LightClientHeaderFulu::default() + }; + + Self::Fulu(LightClientUpdateFulu { + attested_header, + next_sync_committee, + next_sync_committee_branch: next_sync_committee_branch.into(), + finalized_header, + finality_branch: finality_branch.into(), + sync_aggregate: sync_aggregate.clone(), + signature_slot: block_slot, + }) } // To add a new fork, just append the new fork variant on the latest fork. Forks that // have a distinct execution header will need a new LightClientUpdate variant only // if you need to test or support lightclient usages @@ -301,6 +328,7 @@ impl LightClientUpdate { ForkName::Capella => Self::Capella(LightClientUpdateCapella::from_ssz_bytes(bytes)?), ForkName::Deneb => Self::Deneb(LightClientUpdateDeneb::from_ssz_bytes(bytes)?), ForkName::Electra => Self::Electra(LightClientUpdateElectra::from_ssz_bytes(bytes)?), + ForkName::Fulu => Self::Fulu(LightClientUpdateFulu::from_ssz_bytes(bytes)?), ForkName::Base => { return Err(ssz::DecodeError::BytesInvalid(format!( "LightClientUpdate decoding for {fork_name} not implemented" @@ -317,6 +345,7 @@ impl LightClientUpdate { LightClientUpdate::Capella(update) => update.attested_header.beacon.slot, LightClientUpdate::Deneb(update) => update.attested_header.beacon.slot, LightClientUpdate::Electra(update) => update.attested_header.beacon.slot, + LightClientUpdate::Fulu(update) => update.attested_header.beacon.slot, } } @@ -326,6 +355,7 @@ impl LightClientUpdate { LightClientUpdate::Capella(update) => update.finalized_header.beacon.slot, LightClientUpdate::Deneb(update) => update.finalized_header.beacon.slot, LightClientUpdate::Electra(update) => update.finalized_header.beacon.slot, + LightClientUpdate::Fulu(update) => update.finalized_header.beacon.slot, } } @@ -445,6 +475,7 @@ impl LightClientUpdate { ForkName::Capella => as Encode>::ssz_fixed_len(), ForkName::Deneb => as Encode>::ssz_fixed_len(), ForkName::Electra => as Encode>::ssz_fixed_len(), + ForkName::Fulu => as Encode>::ssz_fixed_len(), }; fixed_len + 2 * LightClientHeader::::ssz_max_var_len_for_fork(fork_name) } @@ -458,6 +489,7 @@ impl LightClientUpdate { Self::Capella(_) => func(ForkName::Capella), Self::Deneb(_) => func(ForkName::Deneb), Self::Electra(_) => func(ForkName::Electra), + Self::Fulu(_) => func(ForkName::Fulu), } } } @@ -513,6 +545,13 @@ mod tests { ssz_tests!(LightClientUpdateElectra); } + #[cfg(test)] + mod fulu { + use super::*; + use crate::MainnetEthSpec; + ssz_tests!(LightClientUpdateFulu); + } + #[test] fn finalized_root_params() { assert!(2usize.pow(FINALIZED_ROOT_PROOF_LEN as u32) <= FINALIZED_ROOT_INDEX); diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index e68801840a..abc9afd34c 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -84,13 +84,15 @@ pub trait AbstractExecPayload: + TryInto + TryInto + TryInto + + TryInto { type Ref<'a>: ExecPayload + Copy + From<&'a Self::Bellatrix> + From<&'a Self::Capella> + From<&'a Self::Deneb> - + From<&'a Self::Electra>; + + From<&'a Self::Electra> + + From<&'a Self::Fulu>; type Bellatrix: OwnedExecPayload + Into @@ -108,10 +110,14 @@ pub trait AbstractExecPayload: + Into + for<'a> From>> + TryFrom>; + type Fulu: OwnedExecPayload + + Into + + for<'a> From>> + + TryFrom>; } #[superstruct( - variants(Bellatrix, Capella, Deneb, Electra), + variants(Bellatrix, Capella, Deneb, Electra, Fulu), variant_attributes( derive( Debug, @@ -157,6 +163,8 @@ pub struct FullPayload { pub execution_payload: ExecutionPayloadDeneb, #[superstruct(only(Electra), partial_getter(rename = "execution_payload_electra"))] pub execution_payload: ExecutionPayloadElectra, + #[superstruct(only(Fulu), partial_getter(rename = "execution_payload_fulu"))] + pub execution_payload: ExecutionPayloadFulu, } impl From> for ExecutionPayload { @@ -273,6 +281,9 @@ impl ExecPayload for FullPayload { FullPayload::Electra(ref inner) => { Ok(inner.execution_payload.withdrawals.tree_hash_root()) } + FullPayload::Fulu(ref inner) => { + Ok(inner.execution_payload.withdrawals.tree_hash_root()) + } } } @@ -283,6 +294,7 @@ impl ExecPayload for FullPayload { } FullPayload::Deneb(ref inner) => Ok(inner.execution_payload.blob_gas_used), FullPayload::Electra(ref inner) => Ok(inner.execution_payload.blob_gas_used), + FullPayload::Fulu(ref inner) => Ok(inner.execution_payload.blob_gas_used), } } @@ -313,6 +325,7 @@ impl FullPayload { ForkName::Capella => Ok(FullPayloadCapella::default().into()), ForkName::Deneb => Ok(FullPayloadDeneb::default().into()), ForkName::Electra => Ok(FullPayloadElectra::default().into()), + ForkName::Fulu => Ok(FullPayloadFulu::default().into()), } } } @@ -412,6 +425,7 @@ impl ExecPayload for FullPayloadRef<'_, E> { FullPayloadRef::Electra(inner) => { Ok(inner.execution_payload.withdrawals.tree_hash_root()) } + FullPayloadRef::Fulu(inner) => Ok(inner.execution_payload.withdrawals.tree_hash_root()), } } @@ -422,6 +436,7 @@ impl ExecPayload for FullPayloadRef<'_, E> { } FullPayloadRef::Deneb(inner) => Ok(inner.execution_payload.blob_gas_used), FullPayloadRef::Electra(inner) => Ok(inner.execution_payload.blob_gas_used), + FullPayloadRef::Fulu(inner) => Ok(inner.execution_payload.blob_gas_used), } } @@ -444,6 +459,7 @@ impl AbstractExecPayload for FullPayload { type Capella = FullPayloadCapella; type Deneb = FullPayloadDeneb; type Electra = FullPayloadElectra; + type Fulu = FullPayloadFulu; } impl From> for FullPayload { @@ -462,7 +478,7 @@ impl TryFrom> for FullPayload { } #[superstruct( - variants(Bellatrix, Capella, Deneb, Electra), + variants(Bellatrix, Capella, Deneb, Electra, Fulu), variant_attributes( derive( Debug, @@ -507,6 +523,8 @@ pub struct BlindedPayload { pub execution_payload_header: ExecutionPayloadHeaderDeneb, #[superstruct(only(Electra), partial_getter(rename = "execution_payload_electra"))] pub execution_payload_header: ExecutionPayloadHeaderElectra, + #[superstruct(only(Fulu), partial_getter(rename = "execution_payload_fulu"))] + pub execution_payload_header: ExecutionPayloadHeaderFulu, } impl<'a, E: EthSpec> From> for BlindedPayload { @@ -599,6 +617,7 @@ impl ExecPayload for BlindedPayload { BlindedPayload::Electra(ref inner) => { Ok(inner.execution_payload_header.withdrawals_root) } + BlindedPayload::Fulu(ref inner) => Ok(inner.execution_payload_header.withdrawals_root), } } @@ -609,6 +628,7 @@ impl ExecPayload for BlindedPayload { } BlindedPayload::Deneb(ref inner) => Ok(inner.execution_payload_header.blob_gas_used), BlindedPayload::Electra(ref inner) => Ok(inner.execution_payload_header.blob_gas_used), + BlindedPayload::Fulu(ref inner) => Ok(inner.execution_payload_header.blob_gas_used), } } @@ -707,6 +727,7 @@ impl<'b, E: EthSpec> ExecPayload for BlindedPayloadRef<'b, E> { BlindedPayloadRef::Electra(inner) => { Ok(inner.execution_payload_header.withdrawals_root) } + BlindedPayloadRef::Fulu(inner) => Ok(inner.execution_payload_header.withdrawals_root), } } @@ -717,6 +738,7 @@ impl<'b, E: EthSpec> ExecPayload for BlindedPayloadRef<'b, E> { } BlindedPayloadRef::Deneb(inner) => Ok(inner.execution_payload_header.blob_gas_used), BlindedPayloadRef::Electra(inner) => Ok(inner.execution_payload_header.blob_gas_used), + BlindedPayloadRef::Fulu(inner) => Ok(inner.execution_payload_header.blob_gas_used), } } @@ -1020,6 +1042,13 @@ impl_exec_payload_for_fork!( ExecutionPayloadElectra, Electra ); +impl_exec_payload_for_fork!( + BlindedPayloadFulu, + FullPayloadFulu, + ExecutionPayloadHeaderFulu, + ExecutionPayloadFulu, + Fulu +); impl AbstractExecPayload for BlindedPayload { type Ref<'a> = BlindedPayloadRef<'a, E>; @@ -1027,6 +1056,7 @@ impl AbstractExecPayload for BlindedPayload { type Capella = BlindedPayloadCapella; type Deneb = BlindedPayloadDeneb; type Electra = BlindedPayloadElectra; + type Fulu = BlindedPayloadFulu; } impl From> for BlindedPayload { @@ -1063,6 +1093,11 @@ impl From> for BlindedPayload { execution_payload_header, }) } + ExecutionPayloadHeader::Fulu(execution_payload_header) => { + Self::Fulu(BlindedPayloadFulu { + execution_payload_header, + }) + } } } } @@ -1082,6 +1117,9 @@ impl From> for ExecutionPayloadHeader { BlindedPayload::Electra(blinded_payload) => { ExecutionPayloadHeader::Electra(blinded_payload.execution_payload_header) } + BlindedPayload::Fulu(blinded_payload) => { + ExecutionPayloadHeader::Fulu(blinded_payload.execution_payload_header) + } } } } diff --git a/consensus/types/src/pending_partial_withdrawal.rs b/consensus/types/src/pending_partial_withdrawal.rs index e5ace7b273..846dd97360 100644 --- a/consensus/types/src/pending_partial_withdrawal.rs +++ b/consensus/types/src/pending_partial_withdrawal.rs @@ -21,7 +21,7 @@ use tree_hash_derive::TreeHash; )] pub struct PendingPartialWithdrawal { #[serde(with = "serde_utils::quoted_u64")] - pub index: u64, + pub validator_index: u64, #[serde(with = "serde_utils::quoted_u64")] pub amount: u64, pub withdrawable_epoch: Epoch, diff --git a/consensus/types/src/preset.rs b/consensus/types/src/preset.rs index b469b7b777..707d2d4697 100644 --- a/consensus/types/src/preset.rs +++ b/consensus/types/src/preset.rs @@ -205,8 +205,6 @@ impl CapellaPreset { #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] #[serde(rename_all = "UPPERCASE")] pub struct DenebPreset { - #[serde(with = "serde_utils::quoted_u64")] - pub max_blobs_per_block: u64, #[serde(with = "serde_utils::quoted_u64")] pub max_blob_commitments_per_block: u64, #[serde(with = "serde_utils::quoted_u64")] @@ -216,7 +214,6 @@ pub struct DenebPreset { impl DenebPreset { pub fn from_chain_spec(_spec: &ChainSpec) -> Self { Self { - max_blobs_per_block: E::max_blobs_per_block() as u64, max_blob_commitments_per_block: E::max_blob_commitments_per_block() as u64, field_elements_per_blob: E::field_elements_per_blob() as u64, } @@ -237,7 +234,7 @@ pub struct ElectraPreset { #[serde(with = "serde_utils::quoted_u64")] pub max_pending_partials_per_withdrawals_sweep: u64, #[serde(with = "serde_utils::quoted_u64")] - pub pending_balance_deposits_limit: u64, + pub pending_deposits_limit: u64, #[serde(with = "serde_utils::quoted_u64")] pub pending_partial_withdrawals_limit: u64, #[serde(with = "serde_utils::quoted_u64")] @@ -263,7 +260,7 @@ impl ElectraPreset { whistleblower_reward_quotient_electra: spec.whistleblower_reward_quotient_electra, max_pending_partials_per_withdrawals_sweep: spec .max_pending_partials_per_withdrawals_sweep, - pending_balance_deposits_limit: E::pending_deposits_limit() as u64, + pending_deposits_limit: E::pending_deposits_limit() as u64, pending_partial_withdrawals_limit: E::pending_partial_withdrawals_limit() as u64, pending_consolidations_limit: E::pending_consolidations_limit() as u64, max_consolidation_requests_per_payload: E::max_consolidation_requests_per_payload() @@ -278,7 +275,7 @@ impl ElectraPreset { #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] #[serde(rename_all = "UPPERCASE")] -pub struct Eip7594Preset { +pub struct FuluPreset { #[serde(with = "serde_utils::quoted_u64")] pub field_elements_per_cell: u64, #[serde(with = "serde_utils::quoted_u64")] @@ -287,7 +284,7 @@ pub struct Eip7594Preset { pub kzg_commitments_inclusion_proof_depth: u64, } -impl Eip7594Preset { +impl FuluPreset { pub fn from_chain_spec(_spec: &ChainSpec) -> Self { Self { field_elements_per_cell: E::field_elements_per_cell() as u64, @@ -343,8 +340,8 @@ mod test { let electra: ElectraPreset = preset_from_file(&preset_name, "electra.yaml"); assert_eq!(electra, ElectraPreset::from_chain_spec::(&spec)); - let eip7594: Eip7594Preset = preset_from_file(&preset_name, "eip7594.yaml"); - assert_eq!(eip7594, Eip7594Preset::from_chain_spec::(&spec)); + let fulu: FuluPreset = preset_from_file(&preset_name, "fulu.yaml"); + assert_eq!(fulu, FuluPreset::from_chain_spec::(&spec)); } #[test] diff --git a/consensus/types/src/runtime_fixed_vector.rs b/consensus/types/src/runtime_fixed_vector.rs new file mode 100644 index 0000000000..2b08b7bf70 --- /dev/null +++ b/consensus/types/src/runtime_fixed_vector.rs @@ -0,0 +1,81 @@ +//! Emulates a fixed size array but with the length set at runtime. +//! +//! The length of the list cannot be changed once it is set. + +#[derive(Clone, Debug)] +pub struct RuntimeFixedVector { + vec: Vec, + len: usize, +} + +impl RuntimeFixedVector { + pub fn new(vec: Vec) -> Self { + let len = vec.len(); + Self { vec, len } + } + + pub fn to_vec(&self) -> Vec { + self.vec.clone() + } + + pub fn as_slice(&self) -> &[T] { + self.vec.as_slice() + } + + #[allow(clippy::len_without_is_empty)] + pub fn len(&self) -> usize { + self.len + } + + pub fn into_vec(self) -> Vec { + self.vec + } + + pub fn default(max_len: usize) -> Self { + Self { + vec: vec![T::default(); max_len], + len: max_len, + } + } + + pub fn take(&mut self) -> Self { + let new = std::mem::take(&mut self.vec); + *self = Self::new(vec![T::default(); self.len]); + Self { + vec: new, + len: self.len, + } + } +} + +impl std::ops::Deref for RuntimeFixedVector { + type Target = [T]; + + fn deref(&self) -> &[T] { + &self.vec[..] + } +} + +impl std::ops::DerefMut for RuntimeFixedVector { + fn deref_mut(&mut self) -> &mut [T] { + &mut self.vec[..] + } +} + +impl IntoIterator for RuntimeFixedVector { + type Item = T; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.vec.into_iter() + } +} + +impl<'a, T> IntoIterator for &'a RuntimeFixedVector { + type Item = &'a T; + type IntoIter = std::slice::Iter<'a, T>; + + fn into_iter(self) -> Self::IntoIter { + self.vec.iter() + } +} diff --git a/consensus/types/src/runtime_var_list.rs b/consensus/types/src/runtime_var_list.rs index 8290876fa1..857073b3b8 100644 --- a/consensus/types/src/runtime_var_list.rs +++ b/consensus/types/src/runtime_var_list.rs @@ -2,7 +2,7 @@ use derivative::Derivative; use serde::{Deserialize, Serialize}; use ssz::Decode; use ssz_types::Error; -use std::ops::{Deref, DerefMut, Index, IndexMut}; +use std::ops::{Deref, Index, IndexMut}; use std::slice::SliceIndex; /// Emulates a SSZ `List`. @@ -10,6 +10,8 @@ use std::slice::SliceIndex; /// An ordered, heap-allocated, variable-length, homogeneous collection of `T`, with no more than /// `max_len` values. /// +/// To ensure there are no inconsistent states, we do not allow any mutating operation if `max_len` is not set. +/// /// ## Example /// /// ``` @@ -35,6 +37,7 @@ use std::slice::SliceIndex; /// /// // Push a value to if it _does_ exceed the maximum. /// assert!(long.push(6).is_err()); +/// /// ``` #[derive(Debug, Clone, Serialize, Deserialize, Derivative)] #[derivative(PartialEq, Eq, Hash(bound = "T: std::hash::Hash"))] @@ -65,7 +68,7 @@ impl RuntimeVariableList { Self { vec, max_len } } - /// Create an empty list. + /// Create an empty list with the given `max_len`. pub fn empty(max_len: usize) -> Self { Self { vec: vec![], @@ -77,6 +80,10 @@ impl RuntimeVariableList { self.vec.as_slice() } + pub fn as_mut_slice(&mut self) -> &mut [T] { + self.vec.as_mut_slice() + } + /// Returns the number of values presently in `self`. pub fn len(&self) -> usize { self.vec.len() @@ -88,6 +95,8 @@ impl RuntimeVariableList { } /// Returns the type-level maximum length. + /// + /// Returns `None` if self is uninitialized with a max_len. pub fn max_len(&self) -> usize { self.max_len } @@ -169,12 +178,6 @@ impl Deref for RuntimeVariableList { } } -impl DerefMut for RuntimeVariableList { - fn deref_mut(&mut self) -> &mut [T] { - &mut self.vec[..] - } -} - impl<'a, T> IntoIterator for &'a RuntimeVariableList { type Item = &'a T; type IntoIter = std::slice::Iter<'a, T>; diff --git a/consensus/types/src/signed_beacon_block.rs b/consensus/types/src/signed_beacon_block.rs index bb5e1ea34b..d9bf9bf55d 100644 --- a/consensus/types/src/signed_beacon_block.rs +++ b/consensus/types/src/signed_beacon_block.rs @@ -38,7 +38,7 @@ impl From for Hash256 { /// A `BeaconBlock` and a signature from its proposer. #[superstruct( - variants(Base, Altair, Bellatrix, Capella, Deneb, Electra), + variants(Base, Altair, Bellatrix, Capella, Deneb, Electra, Fulu), variant_attributes( derive( Debug, @@ -81,6 +81,8 @@ pub struct SignedBeaconBlock = FullP pub message: BeaconBlockDeneb, #[superstruct(only(Electra), partial_getter(rename = "message_electra"))] pub message: BeaconBlockElectra, + #[superstruct(only(Fulu), partial_getter(rename = "message_fulu"))] + pub message: BeaconBlockFulu, pub signature: Signature, } @@ -163,6 +165,9 @@ impl> SignedBeaconBlock BeaconBlock::Electra(message) => { SignedBeaconBlock::Electra(SignedBeaconBlockElectra { message, signature }) } + BeaconBlock::Fulu(message) => { + SignedBeaconBlock::Fulu(SignedBeaconBlockFulu { message, signature }) + } } } @@ -570,6 +575,64 @@ impl SignedBeaconBlockElectra> { } } +impl SignedBeaconBlockFulu> { + pub fn into_full_block( + self, + execution_payload: ExecutionPayloadFulu, + ) -> SignedBeaconBlockFulu> { + let SignedBeaconBlockFulu { + message: + BeaconBlockFulu { + slot, + proposer_index, + parent_root, + state_root, + body: + BeaconBlockBodyFulu { + randao_reveal, + eth1_data, + graffiti, + proposer_slashings, + attester_slashings, + attestations, + deposits, + voluntary_exits, + sync_aggregate, + execution_payload: BlindedPayloadFulu { .. }, + bls_to_execution_changes, + blob_kzg_commitments, + execution_requests, + }, + }, + signature, + } = self; + SignedBeaconBlockFulu { + message: BeaconBlockFulu { + slot, + proposer_index, + parent_root, + state_root, + body: BeaconBlockBodyFulu { + randao_reveal, + eth1_data, + graffiti, + proposer_slashings, + attester_slashings, + attestations, + deposits, + voluntary_exits, + sync_aggregate, + execution_payload: FullPayloadFulu { execution_payload }, + bls_to_execution_changes, + blob_kzg_commitments, + execution_requests, + }, + }, + signature, + } + } +} + impl SignedBeaconBlock> { pub fn try_into_full_block( self, @@ -590,12 +653,16 @@ impl SignedBeaconBlock> { (SignedBeaconBlock::Electra(block), Some(ExecutionPayload::Electra(payload))) => { SignedBeaconBlock::Electra(block.into_full_block(payload)) } + (SignedBeaconBlock::Fulu(block), Some(ExecutionPayload::Fulu(payload))) => { + SignedBeaconBlock::Fulu(block.into_full_block(payload)) + } // avoid wildcard matching forks so that compiler will // direct us here when a new fork has been added (SignedBeaconBlock::Bellatrix(_), _) => return None, (SignedBeaconBlock::Capella(_), _) => return None, (SignedBeaconBlock::Deneb(_), _) => return None, (SignedBeaconBlock::Electra(_), _) => return None, + (SignedBeaconBlock::Fulu(_), _) => return None, }; Some(full_block) } @@ -741,6 +808,9 @@ pub mod ssz_tagged_signed_beacon_block { ForkName::Electra => Ok(SignedBeaconBlock::Electra( SignedBeaconBlockElectra::from_ssz_bytes(body)?, )), + ForkName::Fulu => Ok(SignedBeaconBlock::Fulu( + SignedBeaconBlockFulu::from_ssz_bytes(body)?, + )), } } } @@ -841,8 +911,9 @@ mod test { ), SignedBeaconBlock::from_block( BeaconBlock::Electra(BeaconBlockElectra::empty(spec)), - sig, + sig.clone(), ), + SignedBeaconBlock::from_block(BeaconBlock::Fulu(BeaconBlockFulu::empty(spec)), sig), ]; for block in blocks { diff --git a/consensus/types/src/subnet_id.rs b/consensus/types/src/subnet_id.rs index 187b070d29..7a5357c6cc 100644 --- a/consensus/types/src/subnet_id.rs +++ b/consensus/types/src/subnet_id.rs @@ -1,4 +1,5 @@ //! Identifies each shard by an integer identifier. +use crate::SingleAttestation; use crate::{AttestationRef, ChainSpec, CommitteeIndex, EthSpec, Slot}; use alloy_primitives::{bytes::Buf, U256}; use safe_arith::{ArithError, SafeArith}; @@ -57,6 +58,21 @@ impl SubnetId { ) } + /// Compute the subnet for an attestation where each slot in the + /// attestation epoch contains `committee_count_per_slot` committees. + pub fn compute_subnet_for_single_attestation( + attestation: &SingleAttestation, + committee_count_per_slot: u64, + spec: &ChainSpec, + ) -> Result { + Self::compute_subnet::( + attestation.data.slot, + attestation.committee_index, + committee_count_per_slot, + spec, + ) + } + /// Compute the subnet for an attestation with `attestation.data.slot == slot` and /// `attestation.data.index == committee_index` where each slot in the attestation epoch /// contains `committee_count_at_slot` committees. diff --git a/consensus/types/src/validator.rs b/consensus/types/src/validator.rs index 222b9292a2..5aed90d2c1 100644 --- a/consensus/types/src/validator.rs +++ b/consensus/types/src/validator.rs @@ -56,7 +56,7 @@ impl Validator { }; let max_effective_balance = validator.get_max_effective_balance(spec, fork_name); - // safe math is unnecessary here since the spec.effecive_balance_increment is never <= 0 + // safe math is unnecessary here since the spec.effective_balance_increment is never <= 0 validator.effective_balance = std::cmp::min( amount - (amount % spec.effective_balance_increment), max_effective_balance, @@ -195,7 +195,7 @@ impl Validator { /// Returns `true` if the validator is fully withdrawable at some epoch. /// /// Calls the correct function depending on the provided `fork_name`. - pub fn is_fully_withdrawable_at( + pub fn is_fully_withdrawable_validator( &self, balance: u64, epoch: Epoch, @@ -203,14 +203,14 @@ impl Validator { current_fork: ForkName, ) -> bool { if current_fork.electra_enabled() { - self.is_fully_withdrawable_at_electra(balance, epoch, spec) + self.is_fully_withdrawable_validator_electra(balance, epoch, spec) } else { - self.is_fully_withdrawable_at_capella(balance, epoch, spec) + self.is_fully_withdrawable_validator_capella(balance, epoch, spec) } } /// Returns `true` if the validator is fully withdrawable at some epoch. - fn is_fully_withdrawable_at_capella( + fn is_fully_withdrawable_validator_capella( &self, balance: u64, epoch: Epoch, @@ -222,7 +222,7 @@ impl Validator { /// Returns `true` if the validator is fully withdrawable at some epoch. /// /// Modified in electra as part of EIP 7251. - fn is_fully_withdrawable_at_electra( + fn is_fully_withdrawable_validator_electra( &self, balance: u64, epoch: Epoch, diff --git a/crypto/kzg/Cargo.toml b/crypto/kzg/Cargo.toml index bfe0f19cd0..e26fe59413 100644 --- a/crypto/kzg/Cargo.toml +++ b/crypto/kzg/Cargo.toml @@ -8,6 +8,12 @@ edition = "2021" [dependencies] arbitrary = { workspace = true } c-kzg = { workspace = true } +# Required to maintain the pin from https://github.com/sigp/lighthouse/pull/6608 +crate_crypto_internal_eth_kzg_bls12_381 = { workspace = true } +crate_crypto_internal_eth_kzg_erasure_codes = { workspace = true } +crate_crypto_internal_eth_kzg_maybe_rayon = { workspace = true } +crate_crypto_internal_eth_kzg_polynomial = { workspace = true } +crate_crypto_kzg_multi_open_fk20 = { workspace = true } derivative = { workspace = true } ethereum_hashing = { workspace = true } ethereum_serde_utils = { workspace = true } diff --git a/crypto/kzg/src/lib.rs b/crypto/kzg/src/lib.rs index 348ed785af..2a5c6e47f5 100644 --- a/crypto/kzg/src/lib.rs +++ b/crypto/kzg/src/lib.rs @@ -21,6 +21,9 @@ pub use rust_eth_kzg::{ Cell, CellIndex as CellID, CellRef, TrustedSetup as PeerDASTrustedSetup, }; +// Note: `spec.number_of_columns` is a config and should match `CELLS_PER_EXT_BLOB` - however this +// is a constant in the KZG library - be aware that overriding `number_of_columns` will break KZG +// operations. pub type CellsAndKzgProofs = ([Cell; CELLS_PER_EXT_BLOB], [KzgProof; CELLS_PER_EXT_BLOB]); pub type KzgBlobRef<'a> = &'a [u8; BYTES_PER_BLOB]; diff --git a/database_manager/src/cli.rs b/database_manager/src/cli.rs index 4246a51f89..9db807df2c 100644 --- a/database_manager/src/cli.rs +++ b/database_manager/src/cli.rs @@ -57,6 +57,15 @@ pub struct DatabaseManager { )] pub blobs_dir: Option, + #[clap( + long, + value_name = "DATABASE", + help = "Set the database backend to be used by the beacon node.", + display_order = 0, + default_value_t = store::config::DatabaseBackend::LevelDb + )] + pub backend: store::config::DatabaseBackend, + #[clap( long, global = true, diff --git a/database_manager/src/lib.rs b/database_manager/src/lib.rs index fc15e98616..bed90df9df 100644 --- a/database_manager/src/lib.rs +++ b/database_manager/src/lib.rs @@ -16,10 +16,12 @@ use slog::{info, warn, Logger}; use std::fs; use std::io::Write; use std::path::PathBuf; +use store::KeyValueStore; use store::{ + database::interface::BeaconNodeBackend, errors::Error, metadata::{SchemaVersion, CURRENT_SCHEMA_VERSION}, - DBColumn, HotColdDB, KeyValueStore, LevelDB, + DBColumn, HotColdDB, }; use strum::{EnumString, EnumVariantNames}; use types::{BeaconState, EthSpec, Slot}; @@ -40,7 +42,7 @@ fn parse_client_config( .clone_from(&database_manager_config.blobs_dir); client_config.store.blob_prune_margin_epochs = database_manager_config.blob_prune_margin_epochs; client_config.store.hierarchy_config = database_manager_config.hierarchy_exponents.clone(); - + client_config.store.backend = database_manager_config.backend; Ok(client_config) } @@ -55,7 +57,7 @@ pub fn display_db_version( let blobs_path = client_config.get_blobs_db_path(); let mut version = CURRENT_SCHEMA_VERSION; - HotColdDB::, LevelDB>::open( + HotColdDB::, BeaconNodeBackend>::open( &hot_path, &cold_path, &blobs_path, @@ -145,11 +147,14 @@ pub fn inspect_db( let mut num_keys = 0; let sub_db = if inspect_config.freezer { - LevelDB::::open(&cold_path).map_err(|e| format!("Unable to open freezer DB: {e:?}"))? + BeaconNodeBackend::::open(&client_config.store, &cold_path) + .map_err(|e| format!("Unable to open freezer DB: {e:?}"))? } else if inspect_config.blobs_db { - LevelDB::::open(&blobs_path).map_err(|e| format!("Unable to open blobs DB: {e:?}"))? + BeaconNodeBackend::::open(&client_config.store, &blobs_path) + .map_err(|e| format!("Unable to open blobs DB: {e:?}"))? } else { - LevelDB::::open(&hot_path).map_err(|e| format!("Unable to open hot DB: {e:?}"))? + BeaconNodeBackend::::open(&client_config.store, &hot_path) + .map_err(|e| format!("Unable to open hot DB: {e:?}"))? }; let skip = inspect_config.skip.unwrap_or(0); @@ -263,11 +268,20 @@ pub fn compact_db( let column = compact_config.column; let (sub_db, db_name) = if compact_config.freezer { - (LevelDB::::open(&cold_path)?, "freezer_db") + ( + BeaconNodeBackend::::open(&client_config.store, &cold_path)?, + "freezer_db", + ) } else if compact_config.blobs_db { - (LevelDB::::open(&blobs_path)?, "blobs_db") + ( + BeaconNodeBackend::::open(&client_config.store, &blobs_path)?, + "blobs_db", + ) } else { - (LevelDB::::open(&hot_path)?, "hot_db") + ( + BeaconNodeBackend::::open(&client_config.store, &hot_path)?, + "hot_db", + ) }; info!( log, @@ -303,7 +317,7 @@ pub fn migrate_db( let mut from = CURRENT_SCHEMA_VERSION; let to = migrate_config.to; - let db = HotColdDB::, LevelDB>::open( + let db = HotColdDB::, BeaconNodeBackend>::open( &hot_path, &cold_path, &blobs_path, @@ -343,7 +357,7 @@ pub fn prune_payloads( let cold_path = client_config.get_freezer_db_path(); let blobs_path = client_config.get_blobs_db_path(); - let db = HotColdDB::, LevelDB>::open( + let db = HotColdDB::, BeaconNodeBackend>::open( &hot_path, &cold_path, &blobs_path, @@ -369,7 +383,7 @@ pub fn prune_blobs( let cold_path = client_config.get_freezer_db_path(); let blobs_path = client_config.get_blobs_db_path(); - let db = HotColdDB::, LevelDB>::open( + let db = HotColdDB::, BeaconNodeBackend>::open( &hot_path, &cold_path, &blobs_path, @@ -406,7 +420,7 @@ pub fn prune_states( let cold_path = client_config.get_freezer_db_path(); let blobs_path = client_config.get_blobs_db_path(); - let db = HotColdDB::, LevelDB>::open( + let db = HotColdDB::, BeaconNodeBackend>::open( &hot_path, &cold_path, &blobs_path, diff --git a/lcli/Dockerfile b/lcli/Dockerfile index d2cb6f6f14..67bc290112 100644 --- a/lcli/Dockerfile +++ b/lcli/Dockerfile @@ -1,7 +1,7 @@ # `lcli` requires the full project to be in scope, so this should be built either: # - from the `lighthouse` dir with the command: `docker build -f ./lcli/Dockerflie .` # - from the current directory with the command: `docker build -f ./Dockerfile ../` -FROM rust:1.80.0-bullseye AS builder +FROM rust:1.84.0-bullseye AS builder RUN apt-get update && apt-get -y upgrade && apt-get install -y cmake libclang-dev COPY . lighthouse ARG FEATURES diff --git a/lcli/src/mock_el.rs b/lcli/src/mock_el.rs index 8d3220b1df..2e2c27a2db 100644 --- a/lcli/src/mock_el.rs +++ b/lcli/src/mock_el.rs @@ -9,6 +9,7 @@ use execution_layer::{ }; use std::net::Ipv4Addr; use std::path::PathBuf; +use std::sync::Arc; use types::*; pub fn run(mut env: Environment, matches: &ArgMatches) -> Result<(), String> { @@ -19,9 +20,10 @@ pub fn run(mut env: Environment, matches: &ArgMatches) -> Result< let shanghai_time = parse_required(matches, "shanghai-time")?; let cancun_time = parse_optional(matches, "cancun-time")?; let prague_time = parse_optional(matches, "prague-time")?; + let osaka_time = parse_optional(matches, "osaka-time")?; let handle = env.core_context().executor.handle().unwrap(); - let spec = &E::default_spec(); + let spec = Arc::new(E::default_spec()); let jwt_key = JwtKey::from_slice(&DEFAULT_JWT_SECRET).unwrap(); std::fs::write(jwt_path, hex::encode(DEFAULT_JWT_SECRET)).unwrap(); @@ -37,9 +39,10 @@ pub fn run(mut env: Environment, matches: &ArgMatches) -> Result< shanghai_time: Some(shanghai_time), cancun_time, prague_time, + osaka_time, }; let kzg = None; - let server: MockServer = MockServer::new_with_config(&handle, config, kzg); + let server: MockServer = MockServer::new_with_config(&handle, config, spec, kzg); if all_payloads_valid { eprintln!( diff --git a/lcli/src/transition_blocks.rs b/lcli/src/transition_blocks.rs index 94d95a0d1c..ecfa04fc81 100644 --- a/lcli/src/transition_blocks.rs +++ b/lcli/src/transition_blocks.rs @@ -223,7 +223,7 @@ pub fn run( .update_tree_hash_cache() .map_err(|e| format!("Unable to build THC: {:?}", e))?; - if state_root_opt.map_or(false, |expected| expected != state_root) { + if state_root_opt.is_some_and(|expected| expected != state_root) { return Err(format!( "State root mismatch! Expected {}, computed {}", state_root_opt.unwrap(), @@ -331,7 +331,7 @@ fn do_transition( .map_err(|e| format!("Unable to build tree hash cache: {:?}", e))?; debug!("Initial tree hash: {:?}", t.elapsed()); - if state_root_opt.map_or(false, |expected| expected != state_root) { + if state_root_opt.is_some_and(|expected| expected != state_root) { return Err(format!( "State root mismatch! Expected {}, computed {}", state_root_opt.unwrap(), diff --git a/lighthouse/Cargo.toml b/lighthouse/Cargo.toml index eda9a2ebf2..c95735d41c 100644 --- a/lighthouse/Cargo.toml +++ b/lighthouse/Cargo.toml @@ -4,10 +4,10 @@ version = "6.0.1" authors = ["Sigma Prime "] edition = { workspace = true } autotests = false -rust-version = "1.80.0" +rust-version = "1.83.0" [features] -default = ["slasher-lmdb"] +default = ["slasher-lmdb", "beacon-node-leveldb"] # Writes debugging .ssz files to /tmp during block processing. write_ssz_files = ["beacon_node/write_ssz_files"] # Compiles the BLS crypto code so that the binary is portable across machines. @@ -24,6 +24,11 @@ slasher-mdbx = ["slasher/mdbx"] slasher-lmdb = ["slasher/lmdb"] # Support slasher redb backend. slasher-redb = ["slasher/redb"] +# Supports beacon node leveldb backend. +beacon-node-leveldb = ["store/leveldb"] +# Supports beacon node redb backend. +beacon-node-redb = ["store/redb"] + # Deprecated. This is now enabled by default on non windows targets. jemalloc = [] @@ -56,6 +61,7 @@ serde_json = { workspace = true } serde_yaml = { workspace = true } slasher = { workspace = true } slog = { workspace = true } +store = { workspace = true } task_executor = { workspace = true } types = { workspace = true } unused_port = { workspace = true } diff --git a/lighthouse/src/main.rs b/lighthouse/src/main.rs index 43c5e1107c..dd7401d49e 100644 --- a/lighthouse/src/main.rs +++ b/lighthouse/src/main.rs @@ -66,11 +66,15 @@ fn bls_hardware_acceleration() -> bool { return std::arch::is_aarch64_feature_detected!("neon"); } -fn allocator_name() -> &'static str { - if cfg!(target_os = "windows") { - "system" - } else { - "jemalloc" +fn allocator_name() -> String { + #[cfg(target_os = "windows")] + { + "system".to_string() + } + #[cfg(not(target_os = "windows"))] + match malloc_utils::jemalloc::page_size() { + Ok(page_size) => format!("jemalloc ({}K)", page_size / 1024), + Err(e) => format!("jemalloc (error: {e:?})"), } } diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 88e05dfa12..1063a80ff4 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -1,11 +1,12 @@ -use beacon_node::ClientConfig as Config; - use crate::exec::{CommandLineTestExec, CompletedTest}; use beacon_node::beacon_chain::chain_config::{ DisallowedReOrgOffsets, DEFAULT_RE_ORG_CUTOFF_DENOMINATOR, DEFAULT_RE_ORG_HEAD_THRESHOLD, DEFAULT_RE_ORG_MAX_EPOCHS_SINCE_FINALIZATION, }; -use beacon_node::beacon_chain::graffiti_calculator::GraffitiOrigin; +use beacon_node::{ + beacon_chain::graffiti_calculator::GraffitiOrigin, + beacon_chain::store::config::DatabaseBackend as BeaconNodeBackend, ClientConfig as Config, +}; use beacon_processor::BeaconProcessorConfig; use eth1::Eth1Endpoint; use lighthouse_network::PeerId; @@ -2691,3 +2692,13 @@ fn genesis_state_url_value() { assert_eq!(config.genesis_state_url_timeout, Duration::from_secs(42)); }); } + +#[test] +fn beacon_node_backend_override() { + CommandLineTest::new() + .flag("beacon-node-backend", Some("leveldb")) + .run_with_zero_port() + .with_config(|config| { + assert_eq!(config.store.backend, BeaconNodeBackend::LevelDb); + }); +} diff --git a/lighthouse/tests/validator_client.rs b/lighthouse/tests/validator_client.rs index c5b303e4d1..1945399c86 100644 --- a/lighthouse/tests/validator_client.rs +++ b/lighthouse/tests/validator_client.rs @@ -345,7 +345,7 @@ fn http_store_keystore_passwords_in_secrets_dir_present() { } #[test] -fn http_token_path_flag() { +fn http_token_path_flag_present() { let dir = TempDir::new().expect("Unable to create temporary directory"); CommandLineTest::new() .flag("http", None) @@ -359,6 +359,19 @@ fn http_token_path_flag() { }); } +#[test] +fn http_token_path_default() { + CommandLineTest::new() + .flag("http", None) + .run() + .with_config(|config| { + assert_eq!( + config.http_api.http_token_path, + config.validator_dir.join("api-token.txt") + ); + }); +} + // Tests for Metrics flags. #[test] fn metrics_flag() { diff --git a/scripts/local_testnet/network_params_das.yaml b/scripts/local_testnet/network_params_das.yaml index ab2f07a24e..030aa2b820 100644 --- a/scripts/local_testnet/network_params_das.yaml +++ b/scripts/local_testnet/network_params_das.yaml @@ -3,6 +3,7 @@ participants: cl_image: lighthouse:local cl_extra_params: - --subscribe-all-data-column-subnets + - --subscribe-all-subnets - --target-peers=3 count: 2 - cl_type: lighthouse @@ -11,11 +12,14 @@ participants: - --target-peers=3 count: 2 network_params: - eip7594_fork_epoch: 0 + electra_fork_epoch: 1 + fulu_fork_epoch: 2 seconds_per_slot: 6 snooper_enabled: false global_log_level: debug additional_services: - dora - - goomy_blob + - spamoor_blob - prometheus_grafana +dora_params: + image: ethpandaops/dora:fulu-support \ No newline at end of file diff --git a/slasher/src/database.rs b/slasher/src/database.rs index 20b4a33771..e2b49dca29 100644 --- a/slasher/src/database.rs +++ b/slasher/src/database.rs @@ -406,7 +406,7 @@ impl SlasherDB { ) -> Result<(), Error> { // Don't update maximum if new target is less than or equal to previous. In the case of // no previous we *do* want to update. - if previous_max_target.map_or(false, |prev_max| max_target <= prev_max) { + if previous_max_target.is_some_and(|prev_max| max_target <= prev_max) { return Ok(()); } diff --git a/testing/ef_tests/Makefile b/testing/ef_tests/Makefile index d5f4997bb7..7b507f8c50 100644 --- a/testing/ef_tests/Makefile +++ b/testing/ef_tests/Makefile @@ -1,4 +1,4 @@ -TESTS_TAG := v1.5.0-alpha.8 +TESTS_TAG := v1.5.0-beta.1 TESTS = general minimal mainnet TARBALLS = $(patsubst %,%-$(TESTS_TAG).tar.gz,$(TESTS)) diff --git a/testing/ef_tests/check_all_files_accessed.py b/testing/ef_tests/check_all_files_accessed.py index dacca204c1..02a01555b4 100755 --- a/testing/ef_tests/check_all_files_accessed.py +++ b/testing/ef_tests/check_all_files_accessed.py @@ -35,6 +35,8 @@ excluded_paths = [ "tests/.*/.*/ssz_static/LightClientStore", # LightClientSnapshot "tests/.*/.*/ssz_static/LightClientSnapshot", + # LightClientDataCollection + "tests/minimal/.*/light_client/data_collection", # One of the EF researchers likes to pack the tarballs on a Mac ".*\\.DS_Store.*", # More Mac weirdness. @@ -47,7 +49,10 @@ excluded_paths = [ "bls12-381-tests/hash_to_G2", "tests/.*/eip6110", "tests/.*/whisk", - "tests/.*/eip7594", + # TODO(electra): SingleAttestation tests are waiting on Eitan's PR + "tests/.*/electra/ssz_static/SingleAttestation", + "tests/.*/fulu/ssz_static/SingleAttestation", + "tests/.*/fulu/ssz_static/MatrixEntry", ] diff --git a/testing/ef_tests/src/cases.rs b/testing/ef_tests/src/cases.rs index 63274ee0c0..4a202ee3d2 100644 --- a/testing/ef_tests/src/cases.rs +++ b/testing/ef_tests/src/cases.rs @@ -13,12 +13,13 @@ mod bls_fast_aggregate_verify; mod bls_sign_msg; mod bls_verify_msg; mod common; +mod compute_columns_for_custody_groups; mod epoch_processing; mod fork; mod fork_choice; mod genesis_initialization; mod genesis_validity; -mod get_custody_columns; +mod get_custody_groups; mod kzg_blob_to_kzg_commitment; mod kzg_compute_blob_kzg_proof; mod kzg_compute_cells_and_kzg_proofs; @@ -49,11 +50,12 @@ pub use bls_fast_aggregate_verify::*; pub use bls_sign_msg::*; pub use bls_verify_msg::*; pub use common::SszStaticType; +pub use compute_columns_for_custody_groups::*; pub use epoch_processing::*; pub use fork::ForkTest; pub use genesis_initialization::*; pub use genesis_validity::*; -pub use get_custody_columns::*; +pub use get_custody_groups::*; pub use kzg_blob_to_kzg_commitment::*; pub use kzg_compute_blob_kzg_proof::*; pub use kzg_compute_cells_and_kzg_proofs::*; @@ -74,15 +76,44 @@ pub use ssz_generic::*; pub use ssz_static::*; pub use transition::TransitionTest; -#[derive(Debug, PartialEq)] +/// Used for running feature tests for future forks that have not yet been added to `ForkName`. +/// This runs tests in the directory named by the feature instead of the fork name. This has been +/// the pattern used in the `consensus-spec-tests` repository: +/// `consensus-spec-tests/tests/general/[feature_name]/[runner_name].` +/// e.g. consensus-spec-tests/tests/general/peerdas/ssz_static +/// +/// The feature tests can be run with one of the following methods: +/// 1. `handler.run_for_feature(feature_name)` for new tests that are not on existing fork, i.e. a +/// new handler. This will be temporary and the test will need to be updated to use +/// `handle.run()` once the feature is incorporated into a fork. +/// 2. `handler.run()` for tests that are already on existing forks, but with new test vectors for +/// the feature. In this case the `handler.is_enabled_for_feature` will need to be implemented +/// to return `true` for the feature in order for the feature test vector to be tested. +#[derive(Debug, PartialEq, Clone, Copy)] pub enum FeatureName { - Eip7594, + // TODO(fulu): to be removed once we start using Fulu types for test vectors. + // Existing SSZ types for PeerDAS (Fulu) are the same as Electra, so the test vectors get + // loaded as Electra types (default serde behaviour for untagged enums). + Fulu, +} + +impl FeatureName { + pub fn list_all() -> Vec { + vec![FeatureName::Fulu] + } + + /// `ForkName` to use when running the feature tests. + pub fn fork_name(&self) -> ForkName { + match self { + FeatureName::Fulu => ForkName::Electra, + } + } } impl Display for FeatureName { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { - FeatureName::Eip7594 => f.write_str("eip7594"), + FeatureName::Fulu => f.write_str("fulu"), } } } @@ -107,11 +138,13 @@ pub trait Case: Debug + Sync { true } - /// Whether or not this test exists for the given `feature_name`. + /// Whether or not this test exists for the given `feature_name`. This is intended to be used + /// for features that have not been added to a fork yet, and there is usually a separate folder + /// for the feature in the `consensus-spec-tests` repository. /// - /// Returns `true` by default. + /// Returns `false` by default. fn is_enabled_for_feature(_feature_name: FeatureName) -> bool { - true + false } /// Execute a test and return the result. diff --git a/testing/ef_tests/src/cases/common.rs b/testing/ef_tests/src/cases/common.rs index e16f5b257f..62f834820f 100644 --- a/testing/ef_tests/src/cases/common.rs +++ b/testing/ef_tests/src/cases/common.rs @@ -66,6 +66,7 @@ pub fn previous_fork(fork_name: ForkName) -> ForkName { ForkName::Capella => ForkName::Bellatrix, ForkName::Deneb => ForkName::Capella, ForkName::Electra => ForkName::Deneb, + ForkName::Fulu => ForkName::Electra, } } diff --git a/testing/ef_tests/src/cases/compute_columns_for_custody_groups.rs b/testing/ef_tests/src/cases/compute_columns_for_custody_groups.rs new file mode 100644 index 0000000000..8a6330d399 --- /dev/null +++ b/testing/ef_tests/src/cases/compute_columns_for_custody_groups.rs @@ -0,0 +1,43 @@ +use super::*; +use serde::Deserialize; +use std::marker::PhantomData; +use types::data_column_custody_group::{compute_columns_for_custody_group, CustodyIndex}; + +#[derive(Debug, Clone, Deserialize)] +#[serde(bound = "E: EthSpec", deny_unknown_fields)] +pub struct ComputeColumnsForCustodyGroups { + /// The custody group index. + pub custody_group: CustodyIndex, + /// The list of resulting custody columns. + pub result: Vec, + #[serde(skip)] + _phantom: PhantomData, +} + +impl LoadCase for ComputeColumnsForCustodyGroups { + fn load_from_dir(path: &Path, _fork_name: ForkName) -> Result { + decode::yaml_decode_file(path.join("meta.yaml").as_path()) + } +} + +impl Case for ComputeColumnsForCustodyGroups { + fn is_enabled_for_fork(fork_name: ForkName) -> bool { + fork_name.fulu_enabled() + } + + fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> { + let spec = E::default_spec(); + let computed_columns = compute_columns_for_custody_group(self.custody_group, &spec) + .expect("should compute custody columns from group") + .collect::>(); + + let expected = &self.result; + if computed_columns == *expected { + Ok(()) + } else { + Err(Error::NotEqual(format!( + "Got {computed_columns:?}\nExpected {expected:?}" + ))) + } + } +} diff --git a/testing/ef_tests/src/cases/epoch_processing.rs b/testing/ef_tests/src/cases/epoch_processing.rs index c1adf10770..e05225c171 100644 --- a/testing/ef_tests/src/cases/epoch_processing.rs +++ b/testing/ef_tests/src/cases/epoch_processing.rs @@ -100,47 +100,35 @@ type_name!(ParticipationFlagUpdates, "participation_flag_updates"); impl EpochTransition for JustificationAndFinalization { fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError> { - match state { - BeaconState::Base(_) => { - let mut validator_statuses = base::ValidatorStatuses::new(state, spec)?; - validator_statuses.process_attestations(state)?; - let justification_and_finalization_state = - base::process_justification_and_finalization( - state, - &validator_statuses.total_balances, - spec, - )?; - justification_and_finalization_state.apply_changes_to_state(state); - Ok(()) - } - BeaconState::Altair(_) - | BeaconState::Bellatrix(_) - | BeaconState::Capella(_) - | BeaconState::Deneb(_) - | BeaconState::Electra(_) => { - initialize_progressive_balances_cache(state, spec)?; - let justification_and_finalization_state = - altair::process_justification_and_finalization(state)?; - justification_and_finalization_state.apply_changes_to_state(state); - Ok(()) - } + if state.fork_name_unchecked().altair_enabled() { + initialize_progressive_balances_cache(state, spec)?; + let justification_and_finalization_state = + altair::process_justification_and_finalization(state)?; + justification_and_finalization_state.apply_changes_to_state(state); + Ok(()) + } else { + let mut validator_statuses = base::ValidatorStatuses::new(state, spec)?; + validator_statuses.process_attestations(state)?; + let justification_and_finalization_state = + base::process_justification_and_finalization( + state, + &validator_statuses.total_balances, + spec, + )?; + justification_and_finalization_state.apply_changes_to_state(state); + Ok(()) } } } impl EpochTransition for RewardsAndPenalties { fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError> { - match state { - BeaconState::Base(_) => { - let mut validator_statuses = base::ValidatorStatuses::new(state, spec)?; - validator_statuses.process_attestations(state)?; - base::process_rewards_and_penalties(state, &validator_statuses, spec) - } - BeaconState::Altair(_) - | BeaconState::Bellatrix(_) - | BeaconState::Capella(_) - | BeaconState::Deneb(_) - | BeaconState::Electra(_) => altair::process_rewards_and_penalties_slow(state, spec), + if state.fork_name_unchecked().altair_enabled() { + altair::process_rewards_and_penalties_slow(state, spec) + } else { + let mut validator_statuses = base::ValidatorStatuses::new(state, spec)?; + validator_statuses.process_attestations(state)?; + base::process_rewards_and_penalties(state, &validator_statuses, spec) } } } @@ -159,24 +147,17 @@ impl EpochTransition for RegistryUpdates { impl EpochTransition for Slashings { fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError> { - match state { - BeaconState::Base(_) => { - let mut validator_statuses = base::ValidatorStatuses::new(state, spec)?; - validator_statuses.process_attestations(state)?; - process_slashings( - state, - validator_statuses.total_balances.current_epoch(), - spec, - )?; - } - BeaconState::Altair(_) - | BeaconState::Bellatrix(_) - | BeaconState::Capella(_) - | BeaconState::Deneb(_) - | BeaconState::Electra(_) => { - process_slashings_slow(state, spec)?; - } - }; + if state.fork_name_unchecked().altair_enabled() { + process_slashings_slow(state, spec)?; + } else { + let mut validator_statuses = base::ValidatorStatuses::new(state, spec)?; + validator_statuses.process_attestations(state)?; + process_slashings( + state, + validator_statuses.total_balances.current_epoch(), + spec, + )?; + } Ok(()) } } @@ -251,11 +232,10 @@ impl EpochTransition for HistoricalRootsUpdate { impl EpochTransition for HistoricalSummariesUpdate { fn run(state: &mut BeaconState, _spec: &ChainSpec) -> Result<(), EpochProcessingError> { - match state { - BeaconState::Capella(_) | BeaconState::Deneb(_) | BeaconState::Electra(_) => { - process_historical_summaries_update(state) - } - _ => Ok(()), + if state.fork_name_unchecked().capella_enabled() { + process_historical_summaries_update(state) + } else { + Ok(()) } } } @@ -272,39 +252,30 @@ impl EpochTransition for ParticipationRecordUpdates { impl EpochTransition for SyncCommitteeUpdates { fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError> { - match state { - BeaconState::Base(_) => Ok(()), - BeaconState::Altair(_) - | BeaconState::Bellatrix(_) - | BeaconState::Capella(_) - | BeaconState::Deneb(_) - | BeaconState::Electra(_) => altair::process_sync_committee_updates(state, spec), + if state.fork_name_unchecked().altair_enabled() { + altair::process_sync_committee_updates(state, spec) + } else { + Ok(()) } } } impl EpochTransition for InactivityUpdates { fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError> { - match state { - BeaconState::Base(_) => Ok(()), - BeaconState::Altair(_) - | BeaconState::Bellatrix(_) - | BeaconState::Capella(_) - | BeaconState::Deneb(_) - | BeaconState::Electra(_) => altair::process_inactivity_updates_slow(state, spec), + if state.fork_name_unchecked().altair_enabled() { + altair::process_inactivity_updates_slow(state, spec) + } else { + Ok(()) } } } impl EpochTransition for ParticipationFlagUpdates { fn run(state: &mut BeaconState, _: &ChainSpec) -> Result<(), EpochProcessingError> { - match state { - BeaconState::Base(_) => Ok(()), - BeaconState::Altair(_) - | BeaconState::Bellatrix(_) - | BeaconState::Capella(_) - | BeaconState::Deneb(_) - | BeaconState::Electra(_) => altair::process_participation_flag_updates(state), + if state.fork_name_unchecked().altair_enabled() { + altair::process_participation_flag_updates(state) + } else { + Ok(()) } } } diff --git a/testing/ef_tests/src/cases/fork.rs b/testing/ef_tests/src/cases/fork.rs index 132cfb4c0a..85301e22f6 100644 --- a/testing/ef_tests/src/cases/fork.rs +++ b/testing/ef_tests/src/cases/fork.rs @@ -5,7 +5,7 @@ use crate::decode::{ssz_decode_state, yaml_decode_file}; use serde::Deserialize; use state_processing::upgrade::{ upgrade_to_altair, upgrade_to_bellatrix, upgrade_to_capella, upgrade_to_deneb, - upgrade_to_electra, + upgrade_to_electra, upgrade_to_fulu, }; use types::BeaconState; @@ -69,6 +69,7 @@ impl Case for ForkTest { ForkName::Capella => upgrade_to_capella(&mut result_state, spec).map(|_| result_state), ForkName::Deneb => upgrade_to_deneb(&mut result_state, spec).map(|_| result_state), ForkName::Electra => upgrade_to_electra(&mut result_state, spec).map(|_| result_state), + ForkName::Fulu => upgrade_to_fulu(&mut result_state, spec).map(|_| result_state), }; compare_beacon_state_results_without_caches(&mut result, &mut expected) diff --git a/testing/ef_tests/src/cases/fork_choice.rs b/testing/ef_tests/src/cases/fork_choice.rs index 427bcf5e9c..a1c74389a7 100644 --- a/testing/ef_tests/src/cases/fork_choice.rs +++ b/testing/ef_tests/src/cases/fork_choice.rs @@ -523,7 +523,7 @@ impl Tester { || Ok(()), ))? .map(|avail: AvailabilityProcessingStatus| avail.try_into()); - let success = blob_success && result.as_ref().map_or(false, |inner| inner.is_ok()); + let success = blob_success && result.as_ref().is_ok_and(|inner| inner.is_ok()); if success != valid { return Err(Error::DidntFail(format!( "block with root {} was valid={} whilst test expects valid={}. result: {:?}", diff --git a/testing/ef_tests/src/cases/genesis_initialization.rs b/testing/ef_tests/src/cases/genesis_initialization.rs index 11402c75e6..210e18f781 100644 --- a/testing/ef_tests/src/cases/genesis_initialization.rs +++ b/testing/ef_tests/src/cases/genesis_initialization.rs @@ -66,8 +66,7 @@ impl LoadCase for GenesisInitialization { impl Case for GenesisInitialization { fn is_enabled_for_fork(fork_name: ForkName) -> bool { - // Altair genesis and later requires real crypto. - fork_name == ForkName::Base || cfg!(not(feature = "fake_crypto")) + fork_name == ForkName::Base } fn result(&self, _case_index: usize, fork_name: ForkName) -> Result<(), Error> { diff --git a/testing/ef_tests/src/cases/genesis_validity.rs b/testing/ef_tests/src/cases/genesis_validity.rs index e977fa3d63..8fb9f2fbdc 100644 --- a/testing/ef_tests/src/cases/genesis_validity.rs +++ b/testing/ef_tests/src/cases/genesis_validity.rs @@ -39,6 +39,10 @@ impl LoadCase for GenesisValidity { } impl Case for GenesisValidity { + fn is_enabled_for_fork(fork_name: ForkName) -> bool { + fork_name == ForkName::Base + } + fn result(&self, _case_index: usize, fork_name: ForkName) -> Result<(), Error> { let spec = &testing_spec::(fork_name); diff --git a/testing/ef_tests/src/cases/get_custody_columns.rs b/testing/ef_tests/src/cases/get_custody_groups.rs similarity index 59% rename from testing/ef_tests/src/cases/get_custody_columns.rs rename to testing/ef_tests/src/cases/get_custody_groups.rs index 9665f87730..1c1294305f 100644 --- a/testing/ef_tests/src/cases/get_custody_columns.rs +++ b/testing/ef_tests/src/cases/get_custody_groups.rs @@ -2,37 +2,42 @@ use super::*; use alloy_primitives::U256; use serde::Deserialize; use std::marker::PhantomData; -use types::DataColumnSubnetId; +use types::data_column_custody_group::get_custody_groups; #[derive(Debug, Clone, Deserialize)] #[serde(bound = "E: EthSpec", deny_unknown_fields)] -pub struct GetCustodyColumns { +pub struct GetCustodyGroups { + /// The NodeID input. pub node_id: String, - pub custody_subnet_count: u64, + /// The count of custody groups. + pub custody_group_count: u64, + /// The list of resulting custody groups. pub result: Vec, #[serde(skip)] _phantom: PhantomData, } -impl LoadCase for GetCustodyColumns { +impl LoadCase for GetCustodyGroups { fn load_from_dir(path: &Path, _fork_name: ForkName) -> Result { decode::yaml_decode_file(path.join("meta.yaml").as_path()) } } -impl Case for GetCustodyColumns { +impl Case for GetCustodyGroups { + fn is_enabled_for_fork(fork_name: ForkName) -> bool { + fork_name.fulu_enabled() + } + fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> { let spec = E::default_spec(); let node_id = U256::from_str_radix(&self.node_id, 10) .map_err(|e| Error::FailedToParseTest(format!("{e:?}")))?; let raw_node_id = node_id.to_be_bytes::<32>(); - let computed = DataColumnSubnetId::compute_custody_columns::( - raw_node_id, - self.custody_subnet_count, - &spec, - ) - .expect("should compute custody columns") - .collect::>(); + let mut computed = get_custody_groups(raw_node_id, self.custody_group_count, &spec) + .map(|set| set.into_iter().collect::>()) + .expect("should compute custody groups"); + computed.sort(); + let expected = &self.result; if computed == *expected { Ok(()) diff --git a/testing/ef_tests/src/cases/kzg_blob_to_kzg_commitment.rs b/testing/ef_tests/src/cases/kzg_blob_to_kzg_commitment.rs index fa16a5fcb7..feb9a4ff5c 100644 --- a/testing/ef_tests/src/cases/kzg_blob_to_kzg_commitment.rs +++ b/testing/ef_tests/src/cases/kzg_blob_to_kzg_commitment.rs @@ -31,10 +31,6 @@ impl Case for KZGBlobToKZGCommitment { fork_name == ForkName::Deneb } - fn is_enabled_for_feature(feature_name: FeatureName) -> bool { - feature_name != FeatureName::Eip7594 - } - fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> { let kzg = get_kzg(); let commitment = parse_blob::(&self.input.blob).and_then(|blob| { diff --git a/testing/ef_tests/src/cases/kzg_compute_blob_kzg_proof.rs b/testing/ef_tests/src/cases/kzg_compute_blob_kzg_proof.rs index 694013e251..4aadc37af2 100644 --- a/testing/ef_tests/src/cases/kzg_compute_blob_kzg_proof.rs +++ b/testing/ef_tests/src/cases/kzg_compute_blob_kzg_proof.rs @@ -32,10 +32,6 @@ impl Case for KZGComputeBlobKZGProof { fork_name == ForkName::Deneb } - fn is_enabled_for_feature(feature_name: FeatureName) -> bool { - feature_name != FeatureName::Eip7594 - } - fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> { let parse_input = |input: &KZGComputeBlobKZGProofInput| -> Result<_, Error> { let blob = parse_blob::(&input.blob)?; diff --git a/testing/ef_tests/src/cases/kzg_compute_cells_and_kzg_proofs.rs b/testing/ef_tests/src/cases/kzg_compute_cells_and_kzg_proofs.rs index 2a9f8ceeef..6ab9a8db65 100644 --- a/testing/ef_tests/src/cases/kzg_compute_cells_and_kzg_proofs.rs +++ b/testing/ef_tests/src/cases/kzg_compute_cells_and_kzg_proofs.rs @@ -27,7 +27,7 @@ impl LoadCase for KZGComputeCellsAndKZGProofs { impl Case for KZGComputeCellsAndKZGProofs { fn is_enabled_for_fork(fork_name: ForkName) -> bool { - fork_name == ForkName::Deneb + fork_name.fulu_enabled() } fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> { diff --git a/testing/ef_tests/src/cases/kzg_compute_kzg_proof.rs b/testing/ef_tests/src/cases/kzg_compute_kzg_proof.rs index 6f53038f28..4a47fe35eb 100644 --- a/testing/ef_tests/src/cases/kzg_compute_kzg_proof.rs +++ b/testing/ef_tests/src/cases/kzg_compute_kzg_proof.rs @@ -39,10 +39,6 @@ impl Case for KZGComputeKZGProof { fork_name == ForkName::Deneb } - fn is_enabled_for_feature(feature_name: FeatureName) -> bool { - feature_name != FeatureName::Eip7594 - } - fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> { let parse_input = |input: &KZGComputeKZGProofInput| -> Result<_, Error> { let blob = parse_blob::(&input.blob)?; diff --git a/testing/ef_tests/src/cases/kzg_recover_cells_and_kzg_proofs.rs b/testing/ef_tests/src/cases/kzg_recover_cells_and_kzg_proofs.rs index 10cc866fbe..732cb54f31 100644 --- a/testing/ef_tests/src/cases/kzg_recover_cells_and_kzg_proofs.rs +++ b/testing/ef_tests/src/cases/kzg_recover_cells_and_kzg_proofs.rs @@ -28,7 +28,7 @@ impl LoadCase for KZGRecoverCellsAndKZGProofs { impl Case for KZGRecoverCellsAndKZGProofs { fn is_enabled_for_fork(fork_name: ForkName) -> bool { - fork_name == ForkName::Deneb + fork_name.fulu_enabled() } fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> { diff --git a/testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof.rs b/testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof.rs index 3dc955bdcc..66f50d534b 100644 --- a/testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof.rs +++ b/testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof.rs @@ -116,10 +116,6 @@ impl Case for KZGVerifyBlobKZGProof { fork_name == ForkName::Deneb } - fn is_enabled_for_feature(feature_name: FeatureName) -> bool { - feature_name != FeatureName::Eip7594 - } - fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> { let parse_input = |input: &KZGVerifyBlobKZGProofInput| -> Result<(Blob, KzgCommitment, KzgProof), Error> { let blob = parse_blob::(&input.blob)?; diff --git a/testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof_batch.rs b/testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof_batch.rs index 80cd0a2849..efd4158806 100644 --- a/testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof_batch.rs +++ b/testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof_batch.rs @@ -33,10 +33,6 @@ impl Case for KZGVerifyBlobKZGProofBatch { fork_name == ForkName::Deneb } - fn is_enabled_for_feature(feature_name: FeatureName) -> bool { - feature_name != FeatureName::Eip7594 - } - fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> { let parse_input = |input: &KZGVerifyBlobKZGProofBatchInput| -> Result<_, Error> { let blobs = input diff --git a/testing/ef_tests/src/cases/kzg_verify_cell_kzg_proof_batch.rs b/testing/ef_tests/src/cases/kzg_verify_cell_kzg_proof_batch.rs index 5887d764ca..e3edc0df0a 100644 --- a/testing/ef_tests/src/cases/kzg_verify_cell_kzg_proof_batch.rs +++ b/testing/ef_tests/src/cases/kzg_verify_cell_kzg_proof_batch.rs @@ -30,7 +30,7 @@ impl LoadCase for KZGVerifyCellKZGProofBatch { impl Case for KZGVerifyCellKZGProofBatch { fn is_enabled_for_fork(fork_name: ForkName) -> bool { - fork_name == ForkName::Deneb + fork_name.fulu_enabled() } fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> { diff --git a/testing/ef_tests/src/cases/kzg_verify_kzg_proof.rs b/testing/ef_tests/src/cases/kzg_verify_kzg_proof.rs index ed7583dbd0..07df05a6ac 100644 --- a/testing/ef_tests/src/cases/kzg_verify_kzg_proof.rs +++ b/testing/ef_tests/src/cases/kzg_verify_kzg_proof.rs @@ -33,10 +33,6 @@ impl Case for KZGVerifyKZGProof { fork_name == ForkName::Deneb } - fn is_enabled_for_feature(feature_name: FeatureName) -> bool { - feature_name != FeatureName::Eip7594 - } - fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> { let parse_input = |input: &KZGVerifyKZGProofInput| -> Result<_, Error> { let commitment = parse_commitment(&input.commitment)?; diff --git a/testing/ef_tests/src/cases/merkle_proof_validity.rs b/testing/ef_tests/src/cases/merkle_proof_validity.rs index 49c0719784..109d2cc796 100644 --- a/testing/ef_tests/src/cases/merkle_proof_validity.rs +++ b/testing/ef_tests/src/cases/merkle_proof_validity.rs @@ -4,7 +4,7 @@ use serde::Deserialize; use tree_hash::Hash256; use types::{ light_client_update, BeaconBlockBody, BeaconBlockBodyCapella, BeaconBlockBodyDeneb, - BeaconBlockBodyElectra, BeaconState, FixedVector, FullPayload, Unsigned, + BeaconBlockBodyElectra, BeaconBlockBodyFulu, BeaconState, FixedVector, FullPayload, Unsigned, }; #[derive(Debug, Clone, Deserialize)] @@ -131,6 +131,9 @@ impl LoadCase for KzgInclusionMerkleProofValidity { ssz_decode_file::>(&path.join("object.ssz_snappy"))? .into() } + ForkName::Fulu => { + ssz_decode_file::>(&path.join("object.ssz_snappy"))?.into() + } }; let merkle_proof = yaml_decode_file(&path.join("proof.yaml"))?; // Metadata does not exist in these tests but it is left like this just in case. @@ -246,6 +249,9 @@ impl LoadCase for BeaconBlockBodyMerkleProofValidity { ssz_decode_file::>(&path.join("object.ssz_snappy"))? .into() } + ForkName::Fulu => { + ssz_decode_file::>(&path.join("object.ssz_snappy"))?.into() + } }; let merkle_proof = yaml_decode_file(&path.join("proof.yaml"))?; // Metadata does not exist in these tests but it is left like this just in case. diff --git a/testing/ef_tests/src/cases/operations.rs b/testing/ef_tests/src/cases/operations.rs index 54ca52447f..adb5bee768 100644 --- a/testing/ef_tests/src/cases/operations.rs +++ b/testing/ef_tests/src/cases/operations.rs @@ -98,29 +98,24 @@ impl Operation for Attestation { ) -> Result<(), BlockProcessingError> { initialize_epoch_cache(state, spec)?; let mut ctxt = ConsensusContext::new(state.slot()); - match state { - BeaconState::Base(_) => base::process_attestations( + if state.fork_name_unchecked().altair_enabled() { + initialize_progressive_balances_cache(state, spec)?; + altair_deneb::process_attestation( + state, + self.to_ref(), + 0, + &mut ctxt, + VerifySignatures::True, + spec, + ) + } else { + base::process_attestations( state, [self.clone().to_ref()].into_iter(), VerifySignatures::True, &mut ctxt, spec, - ), - BeaconState::Altair(_) - | BeaconState::Bellatrix(_) - | BeaconState::Capella(_) - | BeaconState::Deneb(_) - | BeaconState::Electra(_) => { - initialize_progressive_balances_cache(state, spec)?; - altair_deneb::process_attestation( - state, - self.to_ref(), - 0, - &mut ctxt, - VerifySignatures::True, - spec, - ) - } + ) } } } @@ -131,14 +126,11 @@ impl Operation for AttesterSlashing { } fn decode(path: &Path, fork_name: ForkName, _spec: &ChainSpec) -> Result { - Ok(match fork_name { - ForkName::Base - | ForkName::Altair - | ForkName::Bellatrix - | ForkName::Capella - | ForkName::Deneb => Self::Base(ssz_decode_file(path)?), - ForkName::Electra => Self::Electra(ssz_decode_file(path)?), - }) + if fork_name.electra_enabled() { + Ok(Self::Electra(ssz_decode_file(path)?)) + } else { + Ok(Self::Base(ssz_decode_file(path)?)) + } } fn apply_to( @@ -308,6 +300,7 @@ impl Operation for BeaconBlockBody> { ForkName::Capella => BeaconBlockBody::Capella(<_>::from_ssz_bytes(bytes)?), ForkName::Deneb => BeaconBlockBody::Deneb(<_>::from_ssz_bytes(bytes)?), ForkName::Electra => BeaconBlockBody::Electra(<_>::from_ssz_bytes(bytes)?), + ForkName::Fulu => BeaconBlockBody::Fulu(<_>::from_ssz_bytes(bytes)?), _ => panic!(), }) }) @@ -322,7 +315,7 @@ impl Operation for BeaconBlockBody> { let valid = extra .execution_metadata .as_ref() - .map_or(false, |e| e.execution_valid); + .is_some_and(|e| e.execution_valid); if valid { process_execution_payload::>(state, self.to_ref(), spec) } else { @@ -363,6 +356,10 @@ impl Operation for BeaconBlockBody> { let inner = >>::from_ssz_bytes(bytes)?; BeaconBlockBody::Electra(inner.clone_as_blinded()) } + ForkName::Fulu => { + let inner = >>::from_ssz_bytes(bytes)?; + BeaconBlockBody::Electra(inner.clone_as_blinded()) + } _ => panic!(), }) }) @@ -377,7 +374,7 @@ impl Operation for BeaconBlockBody> { let valid = extra .execution_metadata .as_ref() - .map_or(false, |e| e.execution_valid); + .is_some_and(|e| e.execution_valid); if valid { process_execution_payload::>(state, self.to_ref(), spec) } else { diff --git a/testing/ef_tests/src/cases/transition.rs b/testing/ef_tests/src/cases/transition.rs index dc5029d53e..6d037dae87 100644 --- a/testing/ef_tests/src/cases/transition.rs +++ b/testing/ef_tests/src/cases/transition.rs @@ -60,6 +60,14 @@ impl LoadCase for TransitionTest { spec.deneb_fork_epoch = Some(Epoch::new(0)); spec.electra_fork_epoch = Some(metadata.fork_epoch); } + ForkName::Fulu => { + spec.altair_fork_epoch = Some(Epoch::new(0)); + spec.bellatrix_fork_epoch = Some(Epoch::new(0)); + spec.capella_fork_epoch = Some(Epoch::new(0)); + spec.deneb_fork_epoch = Some(Epoch::new(0)); + spec.electra_fork_epoch = Some(Epoch::new(0)); + spec.fulu_fork_epoch = Some(metadata.fork_epoch); + } } // Load blocks diff --git a/testing/ef_tests/src/decode.rs b/testing/ef_tests/src/decode.rs index 757b9bf3c4..eb88ac6af1 100644 --- a/testing/ef_tests/src/decode.rs +++ b/testing/ef_tests/src/decode.rs @@ -28,7 +28,7 @@ pub fn log_file_access>(file_accessed: P) { writeln!(&mut file, "{:?}", file_accessed.as_ref()).expect("should write to file"); - file.unlock().expect("unable to unlock file"); + fs2::FileExt::unlock(&file).expect("unable to unlock file"); } pub fn yaml_decode(string: &str) -> Result { diff --git a/testing/ef_tests/src/handler.rs b/testing/ef_tests/src/handler.rs index f4a09de32c..d1ddd6a48f 100644 --- a/testing/ef_tests/src/handler.rs +++ b/testing/ef_tests/src/handler.rs @@ -7,9 +7,6 @@ use std::marker::PhantomData; use std::path::PathBuf; use types::{BeaconState, EthSpec, ForkName}; -const EIP7594_FORK: ForkName = ForkName::Deneb; -const EIP7594_TESTS: [&str; 4] = ["ssz_static", "merkle_proof", "networking", "kzg"]; - pub trait Handler { type Case: Case + LoadCase; @@ -24,7 +21,7 @@ pub trait Handler { // Add forks here to exclude them from EF spec testing. Helpful for adding future or // unspecified forks. fn disabled_forks(&self) -> Vec { - vec![] + vec![ForkName::Fulu] } fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool { @@ -39,13 +36,16 @@ pub trait Handler { for fork_name in ForkName::list_all() { if !self.disabled_forks().contains(&fork_name) && self.is_enabled_for_fork(fork_name) { self.run_for_fork(fork_name); + } + } - if fork_name == EIP7594_FORK - && EIP7594_TESTS.contains(&Self::runner_name()) - && self.is_enabled_for_feature(FeatureName::Eip7594) - { - self.run_for_feature(EIP7594_FORK, FeatureName::Eip7594); - } + // Run feature tests for future forks that are not yet added to `ForkName`. + // This runs tests in the directory named by the feature instead of the fork name. + // e.g. consensus-spec-tests/tests/general/[feature_name]/[runner_name] + // e.g. consensus-spec-tests/tests/general/peerdas/ssz_static + for feature_name in FeatureName::list_all() { + if self.is_enabled_for_feature(feature_name) { + self.run_for_feature(feature_name); } } } @@ -96,8 +96,9 @@ pub trait Handler { crate::results::assert_tests_pass(&name, &handler_path, &results); } - fn run_for_feature(&self, fork_name: ForkName, feature_name: FeatureName) { + fn run_for_feature(&self, feature_name: FeatureName) { let feature_name_str = feature_name.to_string(); + let fork_name = feature_name.fork_name(); let handler_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("consensus-spec-tests") @@ -287,6 +288,10 @@ impl SszStaticHandler { Self::for_forks(vec![ForkName::Electra]) } + pub fn fulu_only() -> Self { + Self::for_forks(vec![ForkName::Fulu]) + } + pub fn altair_and_later() -> Self { Self::for_forks(ForkName::list_all()[1..].to_vec()) } @@ -307,6 +312,10 @@ impl SszStaticHandler { Self::for_forks(ForkName::list_all()[5..].to_vec()) } + pub fn fulu_and_later() -> Self { + Self::for_forks(ForkName::list_all()[6..].to_vec()) + } + pub fn pre_electra() -> Self { Self::for_forks(ForkName::list_all()[0..5].to_vec()) } @@ -344,6 +353,25 @@ where fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool { self.supported_forks.contains(&fork_name) } + + fn is_enabled_for_feature(&self, feature_name: FeatureName) -> bool { + // TODO(fulu): to be removed once Fulu types start differing from Electra. We currently run Fulu tests as a + // "feature" - this means we use Electra types for Fulu SSZ tests (except for PeerDAS types, e.g. `DataColumnSidecar`). + // + // This ensures we only run the tests **once** for `Fulu`, using the types matching the + // correct fork, e.g. `Fulu` uses SSZ types from `Electra` as of spec test version + // `v1.5.0-beta.0`, therefore the `Fulu` tests should get included when testing Deneb types. + // + // e.g. Fulu test vectors are executed in the 2nd line below, but excluded in the 1st + // line when testing the type `AttestationElectra`: + // + // ``` + // SszStaticHandler::, MainnetEthSpec>::pre_electra().run(); + // SszStaticHandler::, MainnetEthSpec>::electra_only().run(); + // ``` + feature_name == FeatureName::Fulu + && self.supported_forks.contains(&feature_name.fork_name()) + } } impl Handler for SszStaticTHCHandler, E> @@ -363,6 +391,10 @@ where fn handler_name(&self) -> String { BeaconState::::name().into() } + + fn is_enabled_for_feature(&self, feature_name: FeatureName) -> bool { + feature_name == FeatureName::Fulu + } } impl Handler for SszStaticWithSpecHandler @@ -384,6 +416,10 @@ where fn handler_name(&self) -> String { T::name().into() } + + fn is_enabled_for_feature(&self, feature_name: FeatureName) -> bool { + feature_name == FeatureName::Fulu + } } #[derive(Derivative)] @@ -841,10 +877,10 @@ impl Handler for KZGVerifyKZGProofHandler { #[derive(Derivative)] #[derivative(Default(bound = ""))] -pub struct GetCustodyColumnsHandler(PhantomData); +pub struct GetCustodyGroupsHandler(PhantomData); -impl Handler for GetCustodyColumnsHandler { - type Case = cases::GetCustodyColumns; +impl Handler for GetCustodyGroupsHandler { + type Case = cases::GetCustodyGroups; fn config_name() -> &'static str { E::name() @@ -855,7 +891,35 @@ impl Handler for GetCustodyColumnsHandler { } fn handler_name(&self) -> String { - "get_custody_columns".into() + "get_custody_groups".into() + } + + fn is_enabled_for_feature(&self, feature_name: FeatureName) -> bool { + feature_name == FeatureName::Fulu + } +} + +#[derive(Derivative)] +#[derivative(Default(bound = ""))] +pub struct ComputeColumnsForCustodyGroupHandler(PhantomData); + +impl Handler for ComputeColumnsForCustodyGroupHandler { + type Case = cases::ComputeColumnsForCustodyGroups; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "networking" + } + + fn handler_name(&self) -> String { + "compute_columns_for_custody_group".into() + } + + fn is_enabled_for_feature(&self, feature_name: FeatureName) -> bool { + feature_name == FeatureName::Fulu } } @@ -877,6 +941,10 @@ impl Handler for KZGComputeCellsAndKZGProofHandler { fn handler_name(&self) -> String { "compute_cells_and_kzg_proofs".into() } + + fn is_enabled_for_feature(&self, feature_name: FeatureName) -> bool { + feature_name == FeatureName::Fulu + } } #[derive(Derivative)] @@ -897,6 +965,10 @@ impl Handler for KZGVerifyCellKZGProofBatchHandler { fn handler_name(&self) -> String { "verify_cell_kzg_proof_batch".into() } + + fn is_enabled_for_feature(&self, feature_name: FeatureName) -> bool { + feature_name == FeatureName::Fulu + } } #[derive(Derivative)] @@ -917,6 +989,10 @@ impl Handler for KZGRecoverCellsAndKZGProofHandler { fn handler_name(&self) -> String { "recover_cells_and_kzg_proofs".into() } + + fn is_enabled_for_feature(&self, feature_name: FeatureName) -> bool { + feature_name == FeatureName::Fulu + } } #[derive(Derivative)] @@ -963,9 +1039,12 @@ impl Handler for KzgInclusionMerkleProofValidityHandler bool { - // Enabled in Deneb fork_name.deneb_enabled() } + + fn is_enabled_for_feature(&self, feature_name: FeatureName) -> bool { + feature_name == FeatureName::Fulu + } } #[derive(Derivative)] diff --git a/testing/ef_tests/src/type_name.rs b/testing/ef_tests/src/type_name.rs index c50032a63d..285ac951a6 100644 --- a/testing/ef_tests/src/type_name.rs +++ b/testing/ef_tests/src/type_name.rs @@ -54,6 +54,7 @@ type_name_generic!(BeaconBlockBodyBellatrix, "BeaconBlockBody"); type_name_generic!(BeaconBlockBodyCapella, "BeaconBlockBody"); type_name_generic!(BeaconBlockBodyDeneb, "BeaconBlockBody"); type_name_generic!(BeaconBlockBodyElectra, "BeaconBlockBody"); +type_name_generic!(BeaconBlockBodyFulu, "BeaconBlockBody"); type_name!(BeaconBlockHeader); type_name_generic!(BeaconState); type_name!(BlobIdentifier); @@ -74,12 +75,14 @@ type_name_generic!(ExecutionPayloadBellatrix, "ExecutionPayload"); type_name_generic!(ExecutionPayloadCapella, "ExecutionPayload"); type_name_generic!(ExecutionPayloadDeneb, "ExecutionPayload"); type_name_generic!(ExecutionPayloadElectra, "ExecutionPayload"); +type_name_generic!(ExecutionPayloadFulu, "ExecutionPayload"); type_name_generic!(FullPayload, "ExecutionPayload"); type_name_generic!(ExecutionPayloadHeader); type_name_generic!(ExecutionPayloadHeaderBellatrix, "ExecutionPayloadHeader"); type_name_generic!(ExecutionPayloadHeaderCapella, "ExecutionPayloadHeader"); type_name_generic!(ExecutionPayloadHeaderDeneb, "ExecutionPayloadHeader"); type_name_generic!(ExecutionPayloadHeaderElectra, "ExecutionPayloadHeader"); +type_name_generic!(ExecutionPayloadHeaderFulu, "ExecutionPayloadHeader"); type_name_generic!(ExecutionRequests); type_name_generic!(BlindedPayload, "ExecutionPayloadHeader"); type_name!(Fork); @@ -93,6 +96,7 @@ type_name_generic!(LightClientBootstrapAltair, "LightClientBootstrap"); type_name_generic!(LightClientBootstrapCapella, "LightClientBootstrap"); type_name_generic!(LightClientBootstrapDeneb, "LightClientBootstrap"); type_name_generic!(LightClientBootstrapElectra, "LightClientBootstrap"); +type_name_generic!(LightClientBootstrapFulu, "LightClientBootstrap"); type_name_generic!(LightClientFinalityUpdate); type_name_generic!(LightClientFinalityUpdateAltair, "LightClientFinalityUpdate"); type_name_generic!( @@ -104,11 +108,13 @@ type_name_generic!( LightClientFinalityUpdateElectra, "LightClientFinalityUpdate" ); +type_name_generic!(LightClientFinalityUpdateFulu, "LightClientFinalityUpdate"); type_name_generic!(LightClientHeader); type_name_generic!(LightClientHeaderAltair, "LightClientHeader"); type_name_generic!(LightClientHeaderCapella, "LightClientHeader"); type_name_generic!(LightClientHeaderDeneb, "LightClientHeader"); type_name_generic!(LightClientHeaderElectra, "LightClientHeader"); +type_name_generic!(LightClientHeaderFulu, "LightClientHeader"); type_name_generic!(LightClientOptimisticUpdate); type_name_generic!( LightClientOptimisticUpdateAltair, @@ -126,11 +132,16 @@ type_name_generic!( LightClientOptimisticUpdateElectra, "LightClientOptimisticUpdate" ); +type_name_generic!( + LightClientOptimisticUpdateFulu, + "LightClientOptimisticUpdate" +); type_name_generic!(LightClientUpdate); type_name_generic!(LightClientUpdateAltair, "LightClientUpdate"); type_name_generic!(LightClientUpdateCapella, "LightClientUpdate"); type_name_generic!(LightClientUpdateDeneb, "LightClientUpdate"); type_name_generic!(LightClientUpdateElectra, "LightClientUpdate"); +type_name_generic!(LightClientUpdateFulu, "LightClientUpdate"); type_name_generic!(PendingAttestation); type_name!(PendingConsolidation); type_name!(PendingPartialWithdrawal); diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index 292625a371..bba7efde49 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -276,9 +276,9 @@ mod ssz_static { fn attestation() { SszStaticHandler::, MinimalEthSpec>::pre_electra().run(); SszStaticHandler::, MainnetEthSpec>::pre_electra().run(); - SszStaticHandler::, MinimalEthSpec>::electra_only() + SszStaticHandler::, MinimalEthSpec>::electra_and_later() .run(); - SszStaticHandler::, MainnetEthSpec>::electra_only() + SszStaticHandler::, MainnetEthSpec>::electra_and_later() .run(); } @@ -288,9 +288,9 @@ mod ssz_static { .run(); SszStaticHandler::, MainnetEthSpec>::pre_electra() .run(); - SszStaticHandler::, MinimalEthSpec>::electra_only() + SszStaticHandler::, MinimalEthSpec>::electra_and_later() .run(); - SszStaticHandler::, MainnetEthSpec>::electra_only() + SszStaticHandler::, MainnetEthSpec>::electra_and_later() .run(); } @@ -300,9 +300,9 @@ mod ssz_static { .run(); SszStaticHandler::, MainnetEthSpec>::pre_electra() .run(); - SszStaticHandler::, MinimalEthSpec>::electra_only() + SszStaticHandler::, MinimalEthSpec>::electra_and_later() .run(); - SszStaticHandler::, MainnetEthSpec>::electra_only() + SszStaticHandler::, MainnetEthSpec>::electra_and_later() .run(); } @@ -314,10 +314,10 @@ mod ssz_static { SszStaticHandler::, MainnetEthSpec>::pre_electra( ) .run(); - SszStaticHandler::, MinimalEthSpec>::electra_only( + SszStaticHandler::, MinimalEthSpec>::electra_and_later( ) .run(); - SszStaticHandler::, MainnetEthSpec>::electra_only( + SszStaticHandler::, MainnetEthSpec>::electra_and_later( ) .run(); } @@ -328,10 +328,10 @@ mod ssz_static { .run(); SszStaticHandler::, MainnetEthSpec>::pre_electra() .run(); - SszStaticHandler::, MinimalEthSpec>::electra_only( + SszStaticHandler::, MinimalEthSpec>::electra_and_later( ) .run(); - SszStaticHandler::, MainnetEthSpec>::electra_only( + SszStaticHandler::, MainnetEthSpec>::electra_and_later( ) .run(); } @@ -361,6 +361,8 @@ mod ssz_static { .run(); SszStaticHandler::, MainnetEthSpec>::electra_only() .run(); + SszStaticHandler::, MinimalEthSpec>::fulu_only().run(); + SszStaticHandler::, MainnetEthSpec>::fulu_only().run(); } // Altair and later @@ -399,6 +401,10 @@ mod ssz_static { .run(); SszStaticHandler::, MainnetEthSpec>::electra_only() .run(); + SszStaticHandler::, MinimalEthSpec>::fulu_only() + .run(); + SszStaticHandler::, MainnetEthSpec>::fulu_only() + .run(); } // LightClientHeader has no internal indicator of which fork it is for, so we test it separately. @@ -430,6 +436,10 @@ mod ssz_static { SszStaticHandler::, MainnetEthSpec>::electra_only( ) .run(); + SszStaticHandler::, MinimalEthSpec>::fulu_only() + .run(); + SszStaticHandler::, MainnetEthSpec>::fulu_only() + .run(); } // LightClientOptimisticUpdate has no internal indicator of which fork it is for, so we test it separately. @@ -445,6 +455,8 @@ mod ssz_static { SszStaticHandler::, MainnetEthSpec>::deneb_only().run(); SszStaticHandler::, MinimalEthSpec>::electra_only().run(); SszStaticHandler::, MainnetEthSpec>::electra_only().run(); + SszStaticHandler::, MinimalEthSpec>::fulu_only().run(); + SszStaticHandler::, MainnetEthSpec>::fulu_only().run(); } // LightClientFinalityUpdate has no internal indicator of which fork it is for, so we test it separately. @@ -480,6 +492,12 @@ mod ssz_static { SszStaticHandler::, MainnetEthSpec>::electra_only( ) .run(); + SszStaticHandler::, MinimalEthSpec>::fulu_only( + ) + .run(); + SszStaticHandler::, MainnetEthSpec>::fulu_only( + ) + .run(); } // LightClientUpdate has no internal indicator of which fork it is for, so we test it separately. @@ -509,6 +527,10 @@ mod ssz_static { SszStaticHandler::, MainnetEthSpec>::electra_only( ) .run(); + SszStaticHandler::, MinimalEthSpec>::fulu_only() + .run(); + SszStaticHandler::, MainnetEthSpec>::fulu_only() + .run(); } #[test] @@ -566,6 +588,8 @@ mod ssz_static { .run(); SszStaticHandler::, MainnetEthSpec>::electra_only() .run(); + SszStaticHandler::, MinimalEthSpec>::fulu_only().run(); + SszStaticHandler::, MainnetEthSpec>::fulu_only().run(); } #[test] @@ -586,6 +610,10 @@ mod ssz_static { ::electra_only().run(); SszStaticHandler::, MainnetEthSpec> ::electra_only().run(); + SszStaticHandler::, MinimalEthSpec>::fulu_only() + .run(); + SszStaticHandler::, MainnetEthSpec>::fulu_only() + .run(); } #[test] @@ -626,18 +654,18 @@ mod ssz_static { #[test] fn data_column_sidecar() { - SszStaticHandler::, MinimalEthSpec>::deneb_only() - .run_for_feature(ForkName::Deneb, FeatureName::Eip7594); - SszStaticHandler::, MainnetEthSpec>::deneb_only() - .run_for_feature(ForkName::Deneb, FeatureName::Eip7594); + SszStaticHandler::, MinimalEthSpec>::default() + .run_for_feature(FeatureName::Fulu); + SszStaticHandler::, MainnetEthSpec>::default() + .run_for_feature(FeatureName::Fulu); } #[test] fn data_column_identifier() { - SszStaticHandler::::deneb_only() - .run_for_feature(ForkName::Deneb, FeatureName::Eip7594); - SszStaticHandler::::deneb_only() - .run_for_feature(ForkName::Deneb, FeatureName::Eip7594); + SszStaticHandler::::default() + .run_for_feature(FeatureName::Fulu); + SszStaticHandler::::default() + .run_for_feature(FeatureName::Fulu); } #[test] @@ -901,20 +929,17 @@ fn kzg_verify_kzg_proof() { #[test] fn kzg_compute_cells_and_proofs() { - KZGComputeCellsAndKZGProofHandler::::default() - .run_for_feature(ForkName::Deneb, FeatureName::Eip7594); + KZGComputeCellsAndKZGProofHandler::::default().run(); } #[test] fn kzg_verify_cell_proof_batch() { - KZGVerifyCellKZGProofBatchHandler::::default() - .run_for_feature(ForkName::Deneb, FeatureName::Eip7594); + KZGVerifyCellKZGProofBatchHandler::::default().run(); } #[test] fn kzg_recover_cells_and_proofs() { - KZGRecoverCellsAndKZGProofHandler::::default() - .run_for_feature(ForkName::Deneb, FeatureName::Eip7594); + KZGRecoverCellsAndKZGProofHandler::::default().run(); } #[test] @@ -948,9 +973,13 @@ fn rewards() { } #[test] -fn get_custody_columns() { - GetCustodyColumnsHandler::::default() - .run_for_feature(ForkName::Deneb, FeatureName::Eip7594); - GetCustodyColumnsHandler::::default() - .run_for_feature(ForkName::Deneb, FeatureName::Eip7594); +fn get_custody_groups() { + GetCustodyGroupsHandler::::default().run(); + GetCustodyGroupsHandler::::default().run() +} + +#[test] +fn compute_columns_for_custody_group() { + ComputeColumnsForCustodyGroupHandler::::default().run(); + ComputeColumnsForCustodyGroupHandler::::default().run(); } diff --git a/testing/node_test_rig/src/lib.rs b/testing/node_test_rig/src/lib.rs index ac01c84b9d..6e632ccf54 100644 --- a/testing/node_test_rig/src/lib.rs +++ b/testing/node_test_rig/src/lib.rs @@ -7,6 +7,7 @@ use environment::RuntimeContext; use eth2::{reqwest::ClientBuilder, BeaconNodeHttpClient, Timeouts}; use sensitive_url::SensitiveUrl; use std::path::PathBuf; +use std::sync::Arc; use std::time::Duration; use std::time::{SystemTime, UNIX_EPOCH}; use tempfile::{Builder as TempBuilder, TempDir}; @@ -248,8 +249,14 @@ impl LocalExecutionNode { if let Err(e) = std::fs::write(jwt_file_path, config.jwt_key.hex_string()) { panic!("Failed to write jwt file {}", e); } + let spec = Arc::new(E::default_spec()); Self { - server: MockServer::new_with_config(&context.executor.handle().unwrap(), config, None), + server: MockServer::new_with_config( + &context.executor.handle().unwrap(), + config, + spec, + None, + ), datadir, } } diff --git a/testing/simulator/src/basic_sim.rs b/testing/simulator/src/basic_sim.rs index 8f659a893f..82a7028582 100644 --- a/testing/simulator/src/basic_sim.rs +++ b/testing/simulator/src/basic_sim.rs @@ -22,7 +22,8 @@ const ALTAIR_FORK_EPOCH: u64 = 0; const BELLATRIX_FORK_EPOCH: u64 = 0; const CAPELLA_FORK_EPOCH: u64 = 1; const DENEB_FORK_EPOCH: u64 = 2; -//const ELECTRA_FORK_EPOCH: u64 = 3; +// const ELECTRA_FORK_EPOCH: u64 = 3; +// const FULU_FORK_EPOCH: u64 = 4; const SUGGESTED_FEE_RECIPIENT: [u8; 20] = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]; @@ -118,6 +119,7 @@ pub fn run_basic_sim(matches: &ArgMatches) -> Result<(), String> { spec.capella_fork_epoch = Some(Epoch::new(CAPELLA_FORK_EPOCH)); spec.deneb_fork_epoch = Some(Epoch::new(DENEB_FORK_EPOCH)); //spec.electra_fork_epoch = Some(Epoch::new(ELECTRA_FORK_EPOCH)); + //spec.fulu_fork_epoch = Some(Epoch::new(FULU_FORK_EPOCH)); let spec = Arc::new(spec); env.eth2_config.spec = spec.clone(); diff --git a/testing/simulator/src/fallback_sim.rs b/testing/simulator/src/fallback_sim.rs index b3b9a46001..7d4bdfa264 100644 --- a/testing/simulator/src/fallback_sim.rs +++ b/testing/simulator/src/fallback_sim.rs @@ -21,7 +21,8 @@ const ALTAIR_FORK_EPOCH: u64 = 0; const BELLATRIX_FORK_EPOCH: u64 = 0; const CAPELLA_FORK_EPOCH: u64 = 1; const DENEB_FORK_EPOCH: u64 = 2; -//const ELECTRA_FORK_EPOCH: u64 = 3; +// const ELECTRA_FORK_EPOCH: u64 = 3; +// const FULU_FORK_EPOCH: u64 = 4; // Since simulator tests are non-deterministic and there is a non-zero chance of missed // attestations, define an acceptable network-wide attestation performance. @@ -123,6 +124,7 @@ pub fn run_fallback_sim(matches: &ArgMatches) -> Result<(), String> { spec.capella_fork_epoch = Some(Epoch::new(CAPELLA_FORK_EPOCH)); spec.deneb_fork_epoch = Some(Epoch::new(DENEB_FORK_EPOCH)); //spec.electra_fork_epoch = Some(Epoch::new(ELECTRA_FORK_EPOCH)); + //spec.fulu_fork_epoch = Some(Epoch::new(FULU_FORK_EPOCH)); let spec = Arc::new(spec); env.eth2_config.spec = spec.clone(); diff --git a/testing/simulator/src/local_network.rs b/testing/simulator/src/local_network.rs index 59efc09baa..a95c15c231 100644 --- a/testing/simulator/src/local_network.rs +++ b/testing/simulator/src/local_network.rs @@ -88,6 +88,11 @@ fn default_mock_execution_config( + spec.seconds_per_slot * E::slots_per_epoch() * electra_fork_epoch.as_u64(), ) } + if let Some(fulu_fork_epoch) = spec.fulu_fork_epoch { + mock_execution_config.osaka_time = Some( + genesis_time + spec.seconds_per_slot * E::slots_per_epoch() * fulu_fork_epoch.as_u64(), + ) + } mock_execution_config } diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index daa0802f9b..526479d67f 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -361,6 +361,14 @@ impl CandidateBeaconNode { "endpoint_electra_fork_epoch" => ?beacon_node_spec.electra_fork_epoch, "hint" => UPDATE_REQUIRED_LOG_HINT, ); + } else if beacon_node_spec.fulu_fork_epoch != spec.fulu_fork_epoch { + warn!( + log, + "Beacon node has mismatched Fulu fork epoch"; + "endpoint" => %self.beacon_node, + "endpoint_fulu_fork_epoch" => ?beacon_node_spec.fulu_fork_epoch, + "hint" => UPDATE_REQUIRED_LOG_HINT, + ); } Ok(()) diff --git a/validator_client/doppelganger_service/src/lib.rs b/validator_client/doppelganger_service/src/lib.rs index 35228fe354..4a593c2700 100644 --- a/validator_client/doppelganger_service/src/lib.rs +++ b/validator_client/doppelganger_service/src/lib.rs @@ -162,7 +162,7 @@ impl DoppelgangerState { /// If the BN fails to respond to either of these requests, simply return an empty response. /// This behaviour is to help prevent spurious failures on the BN from needlessly preventing /// doppelganger progression. -async fn beacon_node_liveness<'a, T: 'static + SlotClock, E: EthSpec>( +async fn beacon_node_liveness( beacon_nodes: Arc>, log: Logger, current_epoch: Epoch, diff --git a/validator_client/http_api/Cargo.toml b/validator_client/http_api/Cargo.toml index 76a021ab8c..651e658a7a 100644 --- a/validator_client/http_api/Cargo.toml +++ b/validator_client/http_api/Cargo.toml @@ -21,6 +21,7 @@ eth2_keystore = { workspace = true } ethereum_serde_utils = { workspace = true } filesystem = { workspace = true } graffiti_file = { workspace = true } +health_metrics = { workspace = true } initialized_validators = { workspace = true } lighthouse_version = { workspace = true } logging = { workspace = true } diff --git a/validator_client/http_api/src/lib.rs b/validator_client/http_api/src/lib.rs index f3dab3780c..9c3e3da63d 100644 --- a/validator_client/http_api/src/lib.rs +++ b/validator_client/http_api/src/lib.rs @@ -32,6 +32,7 @@ use eth2::lighthouse_vc::{ PublicKeyBytes, SetGraffitiRequest, }, }; +use health_metrics::observe::Observe; use lighthouse_version::version_with_platform; use logging::SSELoggingComponents; use parking_lot::RwLock; @@ -106,6 +107,7 @@ pub struct Config { impl Default for Config { fn default() -> Self { + // This value is always overridden when building config from CLI. let http_token_path = dirs::home_dir() .unwrap_or_else(|| PathBuf::from(".")) .join(DEFAULT_ROOT_DIR) diff --git a/validator_client/http_api/src/test_utils.rs b/validator_client/http_api/src/test_utils.rs index 390095eec7..0531626846 100644 --- a/validator_client/http_api/src/test_utils.rs +++ b/validator_client/http_api/src/test_utils.rs @@ -251,9 +251,9 @@ impl ApiTester { pub async fn test_get_lighthouse_spec(self) -> Self { let result = self .client - .get_lighthouse_spec::() + .get_lighthouse_spec::() .await - .map(|res| ConfigAndPreset::Electra(res.data)) + .map(|res| ConfigAndPreset::Fulu(res.data)) .unwrap(); let expected = ConfigAndPreset::from_chain_spec::(&E::default_spec(), None); diff --git a/validator_client/http_api/src/tests.rs b/validator_client/http_api/src/tests.rs index 7ea3d7ebaa..4e9acc4237 100644 --- a/validator_client/http_api/src/tests.rs +++ b/validator_client/http_api/src/tests.rs @@ -214,9 +214,9 @@ impl ApiTester { pub async fn test_get_lighthouse_spec(self) -> Self { let result = self .client - .get_lighthouse_spec::() + .get_lighthouse_spec::() .await - .map(|res| ConfigAndPreset::Electra(res.data)) + .map(|res| ConfigAndPreset::Fulu(res.data)) .unwrap(); let expected = ConfigAndPreset::from_chain_spec::(&E::default_spec(), None); diff --git a/validator_client/http_metrics/Cargo.toml b/validator_client/http_metrics/Cargo.toml index c29a4d18fa..a3432410bc 100644 --- a/validator_client/http_metrics/Cargo.toml +++ b/validator_client/http_metrics/Cargo.toml @@ -5,6 +5,7 @@ edition = { workspace = true } authors = ["Sigma Prime "] [dependencies] +health_metrics = { workspace = true } lighthouse_version = { workspace = true } malloc_utils = { workspace = true } metrics = { workspace = true } diff --git a/validator_client/http_metrics/src/lib.rs b/validator_client/http_metrics/src/lib.rs index 984b752e5a..f1c6d4ed8a 100644 --- a/validator_client/http_metrics/src/lib.rs +++ b/validator_client/http_metrics/src/lib.rs @@ -206,7 +206,7 @@ pub fn gather_prometheus_metrics( scrape_allocator_metrics(); } - warp_utils::metrics::scrape_health_metrics(); + health_metrics::metrics::scrape_health_metrics(); encoder .encode(&metrics::gather(), &mut buffer) diff --git a/validator_client/signing_method/src/web3signer.rs b/validator_client/signing_method/src/web3signer.rs index a04922c8c0..885b53683f 100644 --- a/validator_client/signing_method/src/web3signer.rs +++ b/validator_client/signing_method/src/web3signer.rs @@ -30,6 +30,7 @@ pub enum ForkName { Capella, Deneb, Electra, + Fulu, } #[derive(Debug, PartialEq, Serialize)] @@ -109,6 +110,11 @@ impl<'a, E: EthSpec, Payload: AbstractExecPayload> Web3SignerObject<'a, E, Pa block: None, block_header: Some(block.block_header()), }), + BeaconBlock::Fulu(_) => Ok(Web3SignerObject::BeaconBlock { + version: ForkName::Fulu, + block: None, + block_header: Some(block.block_header()), + }), } } diff --git a/validator_client/slashing_protection/src/slashing_database.rs b/validator_client/slashing_protection/src/slashing_database.rs index baaf930c68..71611339f9 100644 --- a/validator_client/slashing_protection/src/slashing_database.rs +++ b/validator_client/slashing_protection/src/slashing_database.rs @@ -1113,9 +1113,7 @@ fn max_or(opt_x: Option, y: T) -> T { /// /// If prev is `None` and `new` is `Some` then `true` is returned. fn monotonic(new: Option, prev: Option) -> bool { - new.map_or(false, |new_val| { - prev.map_or(true, |prev_val| new_val >= prev_val) - }) + new.is_some_and(|new_val| prev.map_or(true, |prev_val| new_val >= prev_val)) } /// The result of importing a single entry from an interchange file. diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index 0fecb5202d..bb72ef81c8 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -314,10 +314,11 @@ impl Config { config.http_api.store_passwords_in_secrets_dir = true; } - if cli_args.get_one::("http-token-path").is_some() { - config.http_api.http_token_path = parse_required(cli_args, "http-token-path") - // For backward compatibility, default to the path under the validator dir if not provided. - .unwrap_or_else(|_| config.validator_dir.join(PK_FILENAME)); + if let Some(http_token_path) = cli_args.get_one::("http-token-path") { + config.http_api.http_token_path = PathBuf::from(http_token_path); + } else { + // For backward compatibility, default to the path under the validator dir if not provided. + config.http_api.http_token_path = config.validator_dir.join(PK_FILENAME); } /* diff --git a/validator_client/validator_services/Cargo.toml b/validator_client/validator_services/Cargo.toml index 21f0ae2d77..b4495a7c81 100644 --- a/validator_client/validator_services/Cargo.toml +++ b/validator_client/validator_services/Cargo.toml @@ -8,6 +8,7 @@ authors = ["Sigma Prime "] beacon_node_fallback = { workspace = true } bls = { workspace = true } doppelganger_service = { workspace = true } +either = { workspace = true } environment = { workspace = true } eth2 = { workspace = true } futures = { workspace = true } diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index e31ad4f661..9a6f94d52b 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -1,5 +1,6 @@ use crate::duties_service::{DutiesService, DutyAndProof}; use beacon_node_fallback::{ApiTopic, BeaconNodeFallback}; +use either::Either; use environment::RuntimeContext; use futures::future::join_all; use slog::{crit, debug, error, info, trace, warn}; @@ -457,8 +458,34 @@ impl AttestationService { &[validator_metrics::ATTESTATIONS_HTTP_POST], ); if fork_name.electra_enabled() { + let single_attestations = attestations + .iter() + .zip(validator_indices) + .filter_map(|(a, i)| { + match a.to_single_attestation_with_attester_index(*i) { + Ok(a) => Some(a), + Err(e) => { + // This shouldn't happen unless BN and VC are out of sync with + // respect to the Electra fork. + error!( + log, + "Unable to convert to SingleAttestation"; + "error" => ?e, + "committee_index" => attestation_data.index, + "slot" => slot.as_u64(), + "type" => "unaggregated", + ); + None + } + } + }) + .collect::>(); + beacon_node - .post_beacon_pool_attestations_v2(attestations, fork_name) + .post_beacon_pool_attestations_v2::( + Either::Right(single_attestations), + fork_name, + ) .await } else { beacon_node diff --git a/validator_client/validator_services/src/preparation_service.rs b/validator_client/validator_services/src/preparation_service.rs index 480f4af2b3..fe6eab3a8a 100644 --- a/validator_client/validator_services/src/preparation_service.rs +++ b/validator_client/validator_services/src/preparation_service.rs @@ -258,7 +258,7 @@ impl PreparationService { .slot_clock .now() .map_or(E::genesis_epoch(), |slot| slot.epoch(E::slots_per_epoch())); - spec.bellatrix_fork_epoch.map_or(false, |fork_epoch| { + spec.bellatrix_fork_epoch.is_some_and(|fork_epoch| { current_epoch + PROPOSER_PREPARATION_LOOKAHEAD_EPOCHS >= fork_epoch }) } diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index af501326f4..dd3e05088e 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -94,7 +94,7 @@ impl SyncDutiesMap { self.committees .read() .get(&committee_period) - .map_or(false, |committee_duties| { + .is_some_and(|committee_duties| { let validator_duties = committee_duties.validators.read(); validator_indices .iter() diff --git a/validator_manager/src/create_validators.rs b/validator_manager/src/create_validators.rs index d4403b4613..b40fe61a82 100644 --- a/validator_manager/src/create_validators.rs +++ b/validator_manager/src/create_validators.rs @@ -286,7 +286,7 @@ struct ValidatorsAndDeposits { } impl ValidatorsAndDeposits { - async fn new<'a, E: EthSpec>(config: CreateConfig, spec: &ChainSpec) -> Result { + async fn new(config: CreateConfig, spec: &ChainSpec) -> Result { let CreateConfig { // The output path is handled upstream. output_path: _, @@ -545,7 +545,7 @@ pub async fn cli_run( } } -async fn run<'a, E: EthSpec>(config: CreateConfig, spec: &ChainSpec) -> Result<(), String> { +async fn run(config: CreateConfig, spec: &ChainSpec) -> Result<(), String> { let output_path = config.output_path.clone(); if !output_path.exists() { diff --git a/validator_manager/src/delete_validators.rs b/validator_manager/src/delete_validators.rs index a2d6c062fa..5ef647c5af 100644 --- a/validator_manager/src/delete_validators.rs +++ b/validator_manager/src/delete_validators.rs @@ -86,7 +86,7 @@ pub async fn cli_run(matches: &ArgMatches, dump_config: DumpConfig) -> Result<() } } -async fn run<'a>(config: DeleteConfig) -> Result<(), String> { +async fn run(config: DeleteConfig) -> Result<(), String> { let DeleteConfig { vc_url, vc_token_path, diff --git a/validator_manager/src/import_validators.rs b/validator_manager/src/import_validators.rs index 3cebc10bb3..63c7ca4596 100644 --- a/validator_manager/src/import_validators.rs +++ b/validator_manager/src/import_validators.rs @@ -209,7 +209,7 @@ pub async fn cli_run(matches: &ArgMatches, dump_config: DumpConfig) -> Result<() } } -async fn run<'a>(config: ImportConfig) -> Result<(), String> { +async fn run(config: ImportConfig) -> Result<(), String> { let ImportConfig { validators_file_path, keystore_file_path, diff --git a/validator_manager/src/list_validators.rs b/validator_manager/src/list_validators.rs index e3deb0b21a..a0a1c5fb40 100644 --- a/validator_manager/src/list_validators.rs +++ b/validator_manager/src/list_validators.rs @@ -58,7 +58,7 @@ pub async fn cli_run(matches: &ArgMatches, dump_config: DumpConfig) -> Result<() } } -async fn run<'a>(config: ListConfig) -> Result, String> { +async fn run(config: ListConfig) -> Result, String> { let ListConfig { vc_url, vc_token_path, diff --git a/validator_manager/src/move_validators.rs b/validator_manager/src/move_validators.rs index 4d0820f5a8..abac071673 100644 --- a/validator_manager/src/move_validators.rs +++ b/validator_manager/src/move_validators.rs @@ -277,7 +277,7 @@ pub async fn cli_run(matches: &ArgMatches, dump_config: DumpConfig) -> Result<() } } -async fn run<'a>(config: MoveConfig) -> Result<(), String> { +async fn run(config: MoveConfig) -> Result<(), String> { let MoveConfig { src_vc_url, src_vc_token_path, diff --git a/wordlist.txt b/wordlist.txt index 6287366cbc..bb8b46b525 100644 --- a/wordlist.txt +++ b/wordlist.txt @@ -162,6 +162,7 @@ keypair keypairs keystore keystores +leveldb linter linux localhost @@ -191,6 +192,7 @@ pre pubkey pubkeys rc +redb reimport resync roadmap