From e4ed317b7681018e1a9f5d995df527ae1b74af4a Mon Sep 17 00:00:00 2001 From: Age Manning Date: Fri, 15 Sep 2023 03:07:24 +0000 Subject: [PATCH 01/19] Add Experimental QUIC support (#4577) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Issue Addressed #4402 ## Proposed Changes This PR adds QUIC support to Lighthouse. As this is not officially spec'd this will only work between lighthouse <-> lighthouse connections. We attempt a QUIC connection (if the node advertises it) and if it fails we fallback to TCP. This should be a backwards compatible modification. We want to test this functionality on live networks to observe any improvements in bandwidth/latency. NOTE: This also removes the websockets transport as I believe no one is really using it. It should be mentioned in our release however. Co-authored-by: João Oliveira --- Cargo.lock | 351 +++++++----------- beacon_node/client/src/lib.rs | 14 - beacon_node/http_api/src/lib.rs | 17 +- beacon_node/http_api/src/test_utils.rs | 2 - beacon_node/lighthouse_network/Cargo.toml | 6 +- beacon_node/lighthouse_network/src/config.rs | 86 +++-- .../lighthouse_network/src/discovery/enr.rs | 30 +- .../src/discovery/enr_ext.rs | 63 +++- .../lighthouse_network/src/discovery/mod.rs | 141 +++---- .../lighthouse_network/src/listen_addr.rs | 55 ++- beacon_node/lighthouse_network/src/metrics.rs | 10 + .../src/peer_manager/mod.rs | 45 +-- .../src/peer_manager/network_behaviour.rs | 96 ++++- .../src/peer_manager/peerdb.rs | 32 +- .../src/peer_manager/peerdb/peer_info.rs | 38 +- .../lighthouse_network/src/service/mod.rs | 90 ++--- .../lighthouse_network/src/service/utils.rs | 41 +- .../lighthouse_network/src/types/globals.rs | 20 - .../lighthouse_network/tests/common.rs | 57 ++- .../lighthouse_network/tests/rpc_tests.rs | 133 +++++-- beacon_node/network/src/nat.rs | 88 +++-- .../src/network_beacon_processor/tests.rs | 11 +- beacon_node/network/src/service.rs | 53 +-- beacon_node/network/src/service/tests.rs | 2 +- beacon_node/src/cli.rs | 71 +++- beacon_node/src/config.rs | 124 +++++-- book/src/SUMMARY.md | 1 + book/src/developers.md | 51 +++ boot_node/src/config.rs | 6 +- lighthouse/tests/beacon_node.rs | 127 +++++-- scripts/local_testnet/beacon_node.sh | 17 +- scripts/local_testnet/start_local_testnet.sh | 2 +- scripts/tests/doppelganger_protection.sh | 28 +- testing/node_test_rig/src/lib.rs | 2 +- testing/simulator/src/local_network.rs | 3 + 35 files changed, 1161 insertions(+), 752 deletions(-) create mode 100644 book/src/developers.md diff --git a/Cargo.lock b/Cargo.lock index ad7b6fe14c..9bde2d0ad2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -63,9 +63,9 @@ dependencies = [ [[package]] name = "addr2line" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" dependencies = [ "gimli", ] @@ -328,7 +328,7 @@ checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" dependencies = [ "async-stream-impl", "futures-core", - "pin-project-lite 0.2.12", + "pin-project-lite", ] [[package]] @@ -374,7 +374,7 @@ dependencies = [ "futures-sink", "futures-util", "memchr", - "pin-project-lite 0.2.12", + "pin-project-lite", ] [[package]] @@ -446,7 +446,7 @@ dependencies = [ "memchr", "mime", "percent-encoding", - "pin-project-lite 0.2.12", + "pin-project-lite", "rustversion", "serde", "serde_json", @@ -478,9 +478,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.68" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" dependencies = [ "addr2line", "cc", @@ -517,9 +517,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.2" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" +checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" [[package]] name = "base64ct" @@ -959,9 +959,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "305fe645edc1442a0fa8b6726ba61d422798d37a52e12eaecf4b022ebbb88f01" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" dependencies = [ "jobserver", "libc", @@ -1391,7 +1391,7 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a011bbe2c35ce9c1f143b7af6f94f29a167beb4cd1d29e6740ce836f723120e" dependencies = [ - "nix 0.26.2", + "nix 0.26.3", "windows-sys 0.48.0", ] @@ -1602,9 +1602,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7684a49fb1af197853ef7b2ee694bc1f5b4179556f1e5710e1760c5db6f5e929" +checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" [[package]] name = "derivative" @@ -1643,9 +1643,9 @@ dependencies = [ [[package]] name = "diesel" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7a532c1f99a0f596f6960a60d1e119e91582b24b39e2d83a190e61262c3ef0c" +checksum = "d98235fdc2f355d330a8244184ab6b4b33c28679c0b4158f63138e51d6cf7e88" dependencies = [ "bitflags 2.4.0", "byteorder", @@ -1657,9 +1657,9 @@ dependencies = [ [[package]] name = "diesel_derives" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74398b79d81e52e130d991afeed9c86034bb1b7735f46d2f5bf7deb261d80303" +checksum = "e054665eaf6d97d1e7125512bb2d35d07c73ac86cc6920174cb42d1ab697a554" dependencies = [ "diesel_table_macro_syntax", "proc-macro2", @@ -1943,9 +1943,9 @@ dependencies = [ [[package]] name = "encoding_rs" -version = "0.8.32" +version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" dependencies = [ "cfg-if", ] @@ -1975,7 +1975,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0be7b2ac146c1f99fe245c02d16af0696450d8e06c135db75e10eeb9e642c20d" dependencies = [ - "base64 0.21.2", + "base64 0.21.3", "bytes", "ed25519-dalek", "hex", @@ -2889,7 +2889,7 @@ dependencies = [ "futures-io", "memchr", "parking", - "pin-project-lite 0.2.12", + "pin-project-lite", "waker-fn", ] @@ -2956,7 +2956,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project-lite 0.2.12", + "pin-project-lite", "pin-utils", "slab", ] @@ -3040,9 +3040,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.27.3" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" [[package]] name = "git-version" @@ -3096,9 +3096,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.20" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049" +checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" dependencies = [ "bytes", "fnv", @@ -3335,7 +3335,7 @@ checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes", "http", - "pin-project-lite 0.2.12", + "pin-project-lite", ] [[package]] @@ -3441,7 +3441,7 @@ dependencies = [ "httparse", "httpdate", "itoa", - "pin-project-lite 0.2.12", + "pin-project-lite", "socket2 0.4.9", "tokio", "tower-service", @@ -3730,7 +3730,7 @@ dependencies = [ "socket2 0.5.3", "widestring 1.0.2", "windows-sys 0.48.0", - "winreg 0.50.0", + "winreg", ] [[package]] @@ -3809,7 +3809,7 @@ version = "8.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" dependencies = [ - "base64 0.21.2", + "base64 0.21.3", "pem", "ring", "serde", @@ -4021,7 +4021,6 @@ dependencies = [ "libp2p-quic", "libp2p-swarm", "libp2p-tcp", - "libp2p-websocket", "libp2p-yamux", "multiaddr 0.18.0", "pin-project", @@ -4101,7 +4100,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d157562dba6017193e5285acf6b1054759e83540bfd79f75b69d6ce774c88da" dependencies = [ "asynchronous-codec", - "base64 0.21.2", + "base64 0.21.3", "byteorder", "bytes", "either", @@ -4362,26 +4361,6 @@ dependencies = [ "yasna", ] -[[package]] -name = "libp2p-websocket" -version = "0.42.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3facf0691bab65f571bc97c6c65ffa836248ca631d631b7691ac91deb7fceb5f" -dependencies = [ - "either", - "futures", - "futures-rustls", - "libp2p-core", - "libp2p-identity", - "log", - "parking_lot 0.12.1", - "quicksink", - "rw-stream-sink", - "soketto", - "url", - "webpki-roots 0.25.2", -] - [[package]] name = "libp2p-yamux" version = "0.44.1" @@ -4534,6 +4513,7 @@ dependencies = [ "lazy_static", "libp2p", "libp2p-mplex", + "libp2p-quic", "lighthouse_metrics", "lighthouse_version", "lru 0.7.8", @@ -5240,14 +5220,13 @@ dependencies = [ [[package]] name = "nix" -version = "0.26.2" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" +checksum = "abbbc55ad7b13aac85f9401c796dcda1b864e07fcad40ad47792eaa8932ea502" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.0", "cfg-if", "libc", - "static_assertions", ] [[package]] @@ -5309,9 +5288,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" dependencies = [ "autocfg 1.1.0", "num-integer", @@ -5388,9 +5367,9 @@ dependencies = [ [[package]] name = "object" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +checksum = "77ac5bbd07aea88c60a577a1ce218075ffd59208b2d7ca97adf9bfc5aeb21ebe" dependencies = [ "memchr", ] @@ -5456,11 +5435,11 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.56" +version = "0.10.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "729b745ad4a5575dd06a3e1af1414bd330ee561c01b3899eb584baeaa8def17e" +checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.0", "cfg-if", "foreign-types", "libc", @@ -5497,9 +5476,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.91" +version = "0.9.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "866b5f16f90776b9bb8dc1e1802ac6f0513de3a7a7465867bfbc563dc737faac" +checksum = "db7e971c2c2bba161b2d2fdf37080177eff520b3bc044787c7f1f5f9e78d869b" dependencies = [ "cc", "libc", @@ -5653,7 +5632,7 @@ dependencies = [ "libc", "redox_syscall 0.3.5", "smallvec 1.11.0", - "windows-targets 0.48.3", + "windows-targets 0.48.5", ] [[package]] @@ -5783,15 +5762,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.1.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "257b64915a082f7811703966789728173279bdebb956b143dbcd23f6f970a777" - -[[package]] -name = "pin-project-lite" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pin-utils" @@ -5877,7 +5850,7 @@ dependencies = [ "concurrent-queue", "libc", "log", - "pin-project-lite 0.2.12", + "pin-project-lite", "windows-sys 0.48.0", ] @@ -5906,11 +5879,11 @@ dependencies = [ [[package]] name = "postgres-protocol" -version = "0.6.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b7fa9f396f51dffd61546fd8573ee20592287996568e6175ceb0f8699ad75d" +checksum = "49b6c5ef183cd3ab4ba005f1ca64c21e8bd97ce4699cfea9e8d9a2c4958ca520" dependencies = [ - "base64 0.21.2", + "base64 0.21.3", "byteorder", "bytes", "fallible-iterator", @@ -5924,9 +5897,9 @@ dependencies = [ [[package]] name = "postgres-types" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f028f05971fe20f512bcc679e2c10227e57809a3af86a7606304435bc8896cd6" +checksum = "8d2234cdee9408b523530a9b6d2d6b373d1db34f6a8e51dc03ded1828d7fb67c" dependencies = [ "bytes", "fallible-iterator", @@ -6195,17 +6168,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "quicksink" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77de3c815e5a160b1539c6592796801df2043ae35e123b46d73380cfa57af858" -dependencies = [ - "futures-core", - "futures-sink", - "pin-project-lite 0.1.12", -] - [[package]] name = "quinn" version = "0.10.2" @@ -6214,7 +6176,7 @@ checksum = "8cc2c5017e4b43d5995dcea317bc46c1e09404c0a9664d2908f7f02dfe943d75" dependencies = [ "bytes", "futures-io", - "pin-project-lite 0.2.12", + "pin-project-lite", "quinn-proto", "quinn-udp", "rustc-hash", @@ -6441,14 +6403,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.3" +version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" +checksum = "12de2eff854e5fa4b1295edd650e227e9d8fb0c9e90b12e7f36d6a6811791a29" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.6", - "regex-syntax 0.7.4", + "regex-automata 0.3.7", + "regex-syntax 0.7.5", ] [[package]] @@ -6462,13 +6424,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" +checksum = "49530408a136e16e5b486e883fbb6ba058e8e4e8ae6621a77b048b314336e629" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.4", + "regex-syntax 0.7.5", ] [[package]] @@ -6479,17 +6441,17 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "reqwest" -version = "0.11.18" +version = "0.11.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" +checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" dependencies = [ - "base64 0.21.2", + "base64 0.21.3", "bytes", "encoding_rs", "futures-core", @@ -6507,7 +6469,7 @@ dependencies = [ "native-tls", "once_cell", "percent-encoding", - "pin-project-lite 0.2.12", + "pin-project-lite", "rustls 0.21.6", "rustls-pemfile", "serde", @@ -6523,8 +6485,8 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 0.22.6", - "winreg 0.10.1", + "webpki-roots 0.25.2", + "winreg", ] [[package]] @@ -6705,9 +6667,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.8" +version = "0.38.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f" +checksum = "9bfe0f2582b4931a45d1fa608f8a8722e8b3c7ac54dd6d5f3b3212791fedef49" dependencies = [ "bitflags 2.4.0", "errno", @@ -6746,7 +6708,7 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ - "base64 0.21.2", + "base64 0.21.3", ] [[package]] @@ -6956,9 +6918,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.183" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" dependencies = [ "serde_derive", ] @@ -6996,9 +6958,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.183" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", @@ -7092,19 +7054,6 @@ dependencies = [ "yaml-rust", ] -[[package]] -name = "sha-1" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99cd6713db3cf16b6c84e06321e049a9b9f699826e16096d23bbcc44d15d51a6" -dependencies = [ - "block-buffer 0.9.0", - "cfg-if", - "cpufeatures", - "digest 0.9.0", - "opaque-debug", -] - [[package]] name = "sha-1" version = "0.10.1" @@ -7249,15 +7198,15 @@ dependencies = [ [[package]] name = "siphasher" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" [[package]] name = "slab" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ "autocfg 1.1.0", ] @@ -7339,9 +7288,9 @@ checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" [[package]] name = "slog-async" -version = "2.7.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "766c59b252e62a34651412870ff55d8c4e6d04df19b43eecb2703e417b097ffe" +checksum = "72c8038f898a2c79507940990f05386455b3a317d8f18d4caea7cbc3d5096b84" dependencies = [ "crossbeam-channel", "slog", @@ -7498,21 +7447,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "soketto" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d1c5305e39e09653383c2c7244f2f78b3bcae37cf50c64cb4789c9f5096ec2" -dependencies = [ - "base64 0.13.1", - "bytes", - "futures", - "httparse", - "log", - "rand 0.8.5", - "sha-1 0.9.8", -] - [[package]] name = "spin" version = "0.5.2" @@ -7874,14 +7808,14 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.7.1" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc02fddf48964c42031a0b3fe0428320ecf3a73c401040fc0096f97794310651" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ "cfg-if", "fastrand 2.0.0", "redox_syscall 0.3.5", - "rustix 0.38.8", + "rustix 0.38.9", "windows-sys 0.48.0", ] @@ -7987,9 +7921,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.25" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fdd63d58b18d663fbdf70e049f00a22c8e42be082203be7f26589213cd75ea" +checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48" dependencies = [ "deranged", "itoa", @@ -8008,9 +7942,9 @@ checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" -version = "0.2.11" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb71511c991639bb078fd5bf97757e03914361c48100d52878b8e52b46fb92cd" +checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572" dependencies = [ "time-core", ] @@ -8091,7 +8025,7 @@ dependencies = [ "mio", "num_cpus", "parking_lot 0.12.1", - "pin-project-lite 0.2.12", + "pin-project-lite", "signal-hook-registry", "socket2 0.5.3", "tokio-macros", @@ -8104,7 +8038,7 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" dependencies = [ - "pin-project-lite 0.2.12", + "pin-project-lite", "tokio", ] @@ -8131,9 +8065,9 @@ dependencies = [ [[package]] name = "tokio-postgres" -version = "0.7.8" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e89f6234aa8fd43779746012fcf53603cdb91fdd8399aa0de868c2d56b6dde1" +checksum = "d340244b32d920260ae7448cb72b6e238bddc3d4f7603394e7dd46ed8e48f5b8" dependencies = [ "async-trait", "byteorder", @@ -8145,12 +8079,14 @@ dependencies = [ "parking_lot 0.12.1", "percent-encoding", "phf", - "pin-project-lite 0.2.12", + "pin-project-lite", "postgres-protocol", "postgres-types", + "rand 0.8.5", "socket2 0.5.3", "tokio", "tokio-util 0.7.8", + "whoami", ] [[package]] @@ -8181,7 +8117,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" dependencies = [ "futures-core", - "pin-project-lite 0.2.12", + "pin-project-lite", "tokio", "tokio-util 0.7.8", ] @@ -8225,7 +8161,7 @@ dependencies = [ "futures-io", "futures-sink", "log", - "pin-project-lite 0.2.12", + "pin-project-lite", "slab", "tokio", ] @@ -8239,7 +8175,7 @@ dependencies = [ "bytes", "futures-core", "futures-sink", - "pin-project-lite 0.2.12", + "pin-project-lite", "slab", "tokio", "tracing", @@ -8297,7 +8233,7 @@ dependencies = [ "futures-core", "futures-util", "pin-project", - "pin-project-lite 0.2.12", + "pin-project-lite", "tokio", "tower-layer", "tower-service", @@ -8324,7 +8260,7 @@ checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if", "log", - "pin-project-lite 0.2.12", + "pin-project-lite", "tracing-attributes", "tracing-core", ] @@ -8506,7 +8442,7 @@ dependencies = [ "log", "rand 0.8.5", "rustls 0.20.8", - "sha-1 0.10.1", + "sha-1", "thiserror", "url", "utf-8", @@ -8612,9 +8548,9 @@ checksum = "ccb97dac3243214f8d8507998906ca3e2e0b900bf9bf4870477f125b82e68f6e" [[package]] name = "unicase" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" dependencies = [ "version_check", ] @@ -9007,9 +8943,9 @@ checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" [[package]] name = "wasm-streams" -version = "0.2.3" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078" +checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" dependencies = [ "futures-util", "js-sys", @@ -9130,6 +9066,16 @@ version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" +[[package]] +name = "whoami" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" +dependencies = [ + "wasm-bindgen", + "web-sys", +] + [[package]] name = "widestring" version = "0.4.3" @@ -9198,7 +9144,7 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" dependencies = [ - "windows-targets 0.48.3", + "windows-targets 0.48.5", ] [[package]] @@ -9228,7 +9174,7 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets 0.48.3", + "windows-targets 0.48.5", ] [[package]] @@ -9248,17 +9194,17 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.48.3" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27f51fb4c64f8b770a823c043c7fad036323e1c48f55287b7bbb7987b2fcdf3b" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm 0.48.3", - "windows_aarch64_msvc 0.48.3", - "windows_i686_gnu 0.48.3", - "windows_i686_msvc 0.48.3", - "windows_x86_64_gnu 0.48.3", - "windows_x86_64_gnullvm 0.48.3", - "windows_x86_64_msvc 0.48.3", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", ] [[package]] @@ -9269,9 +9215,9 @@ checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.3" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fde1bb55ae4ce76a597a8566d82c57432bc69c039449d61572a7a353da28f68c" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_msvc" @@ -9287,9 +9233,9 @@ checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" [[package]] name = "windows_aarch64_msvc" -version = "0.48.3" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1513e8d48365a78adad7322fd6b5e4c4e99d92a69db8df2d435b25b1f1f286d4" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_i686_gnu" @@ -9305,9 +9251,9 @@ checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" [[package]] name = "windows_i686_gnu" -version = "0.48.3" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60587c0265d2b842298f5858e1a5d79d146f9ee0c37be5782e92a6eb5e1d7a83" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_msvc" @@ -9323,9 +9269,9 @@ checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" [[package]] name = "windows_i686_msvc" -version = "0.48.3" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224fe0e0ffff5d2ea6a29f82026c8f43870038a0ffc247aa95a52b47df381ac4" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_x86_64_gnu" @@ -9341,9 +9287,9 @@ checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" [[package]] name = "windows_x86_64_gnu" -version = "0.48.3" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62fc52a0f50a088de499712cbc012df7ebd94e2d6eb948435449d76a6287e7ad" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnullvm" @@ -9353,9 +9299,9 @@ checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.3" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2093925509d91ea3d69bcd20238f4c2ecdb1a29d3c281d026a09705d0dd35f3d" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_msvc" @@ -9371,28 +9317,19 @@ checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" [[package]] name = "windows_x86_64_msvc" -version = "0.48.3" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6ade45bc8bf02ae2aa34a9d54ba660a1a58204da34ba793c00d83ca3730b5f1" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "winnow" -version = "0.5.14" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d09770118a7eb1ccaf4a594a221334119a44a814fcb0d31c5b85e83e97227a97" +checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" dependencies = [ "memchr", ] -[[package]] -name = "winreg" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" -dependencies = [ - "winapi", -] - [[package]] name = "winreg" version = "0.50.0" diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index 584a0d736d..399aa06511 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -46,20 +46,6 @@ impl Client { self.http_metrics_listen_addr } - /// Returns the ipv4 port of the client's libp2p stack, if it was started. - pub fn libp2p_listen_ipv4_port(&self) -> Option { - self.network_globals - .as_ref() - .and_then(|n| n.listen_port_tcp4()) - } - - /// Returns the ipv6 port of the client's libp2p stack, if it was started. - pub fn libp2p_listen_ipv6_port(&self) -> Option { - self.network_globals - .as_ref() - .and_then(|n| n.listen_port_tcp6()) - } - /// Returns the list of libp2p addresses the client is listening to. pub fn libp2p_listen_addresses(&self) -> Option> { self.network_globals.as_ref().map(|n| n.listen_multiaddrs()) diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 19d62c060d..08b18e86d0 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -2825,12 +2825,8 @@ pub fn serve( })?; if let Some(peer_info) = network_globals.peers.read().peer_info(&peer_id) { - let address = if let Some(socket_addr) = peer_info.seen_addresses().next() { - let mut addr = lighthouse_network::Multiaddr::from(socket_addr.ip()); - addr.push(lighthouse_network::multiaddr::Protocol::Tcp( - socket_addr.port(), - )); - addr.to_string() + let address = if let Some(multiaddr) = peer_info.seen_multiaddrs().next() { + multiaddr.to_string() } else if let Some(addr) = peer_info.listening_addresses().first() { addr.to_string() } else { @@ -2878,13 +2874,8 @@ pub fn serve( .peers() .for_each(|(peer_id, peer_info)| { let address = - if let Some(socket_addr) = peer_info.seen_addresses().next() { - let mut addr = - lighthouse_network::Multiaddr::from(socket_addr.ip()); - addr.push(lighthouse_network::multiaddr::Protocol::Tcp( - socket_addr.port(), - )); - addr.to_string() + if let Some(multiaddr) = peer_info.seen_multiaddrs().next() { + multiaddr.to_string() } else if let Some(addr) = peer_info.listening_addresses().first() { addr.to_string() } else { diff --git a/beacon_node/http_api/src/test_utils.rs b/beacon_node/http_api/src/test_utils.rs index 33834d58ca..fb3497ff8b 100644 --- a/beacon_node/http_api/src/test_utils.rs +++ b/beacon_node/http_api/src/test_utils.rs @@ -152,8 +152,6 @@ pub async fn create_api_server_on_port( let enr = EnrBuilder::new("v4").build(&enr_key).unwrap(); let network_globals = Arc::new(NetworkGlobals::new( enr.clone(), - Some(TCP_PORT), - None, meta_data, vec![], false, diff --git a/beacon_node/lighthouse_network/Cargo.toml b/beacon_node/lighthouse_network/Cargo.toml index 925d278ad6..3e399f7480 100644 --- a/beacon_node/lighthouse_network/Cargo.toml +++ b/beacon_node/lighthouse_network/Cargo.toml @@ -44,12 +44,13 @@ prometheus-client = "0.21.0" unused_port = { path = "../../common/unused_port" } delay_map = "0.3.0" void = "1" +libp2p-quic= { version = "0.9.2", features=["tokio"]} libp2p-mplex = "0.40.0" [dependencies.libp2p] version = "0.52" default-features = false -features = ["websocket", "identify", "yamux", "noise", "gossipsub", "dns", "tcp", "tokio", "plaintext", "secp256k1", "macros", "ecdsa"] +features = ["identify", "yamux", "noise", "gossipsub", "dns", "tcp", "tokio", "plaintext", "secp256k1", "macros", "ecdsa"] [dev-dependencies] slog-term = "2.6.0" @@ -59,6 +60,3 @@ exit-future = "0.2.0" void = "1" quickcheck = "0.9.2" quickcheck_macros = "0.9.1" - -[features] -libp2p-websocket = [] \ No newline at end of file diff --git a/beacon_node/lighthouse_network/src/config.rs b/beacon_node/lighthouse_network/src/config.rs index 0ab7c03e7f..d8c3f619dc 100644 --- a/beacon_node/lighthouse_network/src/config.rs +++ b/beacon_node/lighthouse_network/src/config.rs @@ -58,18 +58,24 @@ pub struct Config { /// that no discovery address has been set in the CLI args. pub enr_address: (Option, Option), - /// The udp4 port to broadcast to peers in order to reach back for discovery. + /// The udp ipv4 port to broadcast to peers in order to reach back for discovery. pub enr_udp4_port: Option, - /// The tcp4 port to broadcast to peers in order to reach back for libp2p services. + /// The quic ipv4 port to broadcast to peers in order to reach back for libp2p services. + pub enr_quic4_port: Option, + + /// The tcp ipv4 port to broadcast to peers in order to reach back for libp2p services. pub enr_tcp4_port: Option, - /// The udp6 port to broadcast to peers in order to reach back for discovery. + /// The udp ipv6 port to broadcast to peers in order to reach back for discovery. pub enr_udp6_port: Option, - /// The tcp6 port to broadcast to peers in order to reach back for libp2p services. + /// The tcp ipv6 port to broadcast to peers in order to reach back for libp2p services. pub enr_tcp6_port: Option, + /// The quic ipv6 port to broadcast to peers in order to reach back for libp2p services. + pub enr_quic6_port: Option, + /// Target number of connected peers. pub target_peers: usize, @@ -102,6 +108,9 @@ pub struct Config { /// Disables the discovery protocol from starting. pub disable_discovery: bool, + /// Disables quic support. + pub disable_quic_support: bool, + /// Attempt to construct external port mappings with UPnP. pub upnp_enabled: bool, @@ -149,57 +158,76 @@ impl Config { /// Sets the listening address to use an ipv4 address. The discv5 ip_mode and table filter are /// adjusted accordingly to ensure addresses that are present in the enr are globally /// reachable. - pub fn set_ipv4_listening_address(&mut self, addr: Ipv4Addr, tcp_port: u16, udp_port: u16) { + pub fn set_ipv4_listening_address( + &mut self, + addr: Ipv4Addr, + tcp_port: u16, + disc_port: u16, + quic_port: u16, + ) { self.listen_addresses = ListenAddress::V4(ListenAddr { addr, - udp_port, + disc_port, + quic_port, tcp_port, }); - self.discv5_config.listen_config = discv5::ListenConfig::from_ip(addr.into(), udp_port); + self.discv5_config.listen_config = discv5::ListenConfig::from_ip(addr.into(), disc_port); self.discv5_config.table_filter = |enr| enr.ip4().as_ref().map_or(false, is_global_ipv4) } /// Sets the listening address to use an ipv6 address. The discv5 ip_mode and table filter is /// adjusted accordingly to ensure addresses that are present in the enr are globally /// reachable. - pub fn set_ipv6_listening_address(&mut self, addr: Ipv6Addr, tcp_port: u16, udp_port: u16) { + pub fn set_ipv6_listening_address( + &mut self, + addr: Ipv6Addr, + tcp_port: u16, + disc_port: u16, + quic_port: u16, + ) { self.listen_addresses = ListenAddress::V6(ListenAddr { addr, - udp_port, + disc_port, + quic_port, tcp_port, }); - self.discv5_config.listen_config = discv5::ListenConfig::from_ip(addr.into(), udp_port); + self.discv5_config.listen_config = discv5::ListenConfig::from_ip(addr.into(), disc_port); self.discv5_config.table_filter = |enr| enr.ip6().as_ref().map_or(false, is_global_ipv6) } /// Sets the listening address to use both an ipv4 and ipv6 address. The discv5 ip_mode and /// table filter is adjusted accordingly to ensure addresses that are present in the enr are /// globally reachable. + #[allow(clippy::too_many_arguments)] pub fn set_ipv4_ipv6_listening_addresses( &mut self, v4_addr: Ipv4Addr, tcp4_port: u16, - udp4_port: u16, + disc4_port: u16, + quic4_port: u16, v6_addr: Ipv6Addr, tcp6_port: u16, - udp6_port: u16, + disc6_port: u16, + quic6_port: u16, ) { self.listen_addresses = ListenAddress::DualStack( ListenAddr { addr: v4_addr, - udp_port: udp4_port, + disc_port: disc4_port, + quic_port: quic4_port, tcp_port: tcp4_port, }, ListenAddr { addr: v6_addr, - udp_port: udp6_port, + disc_port: disc6_port, + quic_port: quic6_port, tcp_port: tcp6_port, }, ); self.discv5_config.listen_config = discv5::ListenConfig::default() - .with_ipv4(v4_addr, udp4_port) - .with_ipv6(v6_addr, udp6_port); + .with_ipv4(v4_addr, disc4_port) + .with_ipv6(v6_addr, disc6_port); self.discv5_config.table_filter = |enr| match (&enr.ip4(), &enr.ip6()) { (None, None) => false, @@ -213,27 +241,32 @@ impl Config { match listen_addr { ListenAddress::V4(ListenAddr { addr, - udp_port, + disc_port, + quic_port, tcp_port, - }) => self.set_ipv4_listening_address(addr, tcp_port, udp_port), + }) => self.set_ipv4_listening_address(addr, tcp_port, disc_port, quic_port), ListenAddress::V6(ListenAddr { addr, - udp_port, + disc_port, + quic_port, tcp_port, - }) => self.set_ipv6_listening_address(addr, tcp_port, udp_port), + }) => self.set_ipv6_listening_address(addr, tcp_port, disc_port, quic_port), ListenAddress::DualStack( ListenAddr { addr: ip4addr, - udp_port: udp4_port, + disc_port: disc4_port, + quic_port: quic4_port, tcp_port: tcp4_port, }, ListenAddr { addr: ip6addr, - udp_port: udp6_port, + disc_port: disc6_port, + quic_port: quic6_port, tcp_port: tcp6_port, }, ) => self.set_ipv4_ipv6_listening_addresses( - ip4addr, tcp4_port, udp4_port, ip6addr, tcp6_port, udp6_port, + ip4addr, tcp4_port, disc4_port, quic4_port, ip6addr, tcp6_port, disc6_port, + quic6_port, ), } } @@ -272,7 +305,8 @@ impl Default for Config { ); let listen_addresses = ListenAddress::V4(ListenAddr { addr: Ipv4Addr::UNSPECIFIED, - udp_port: 9000, + disc_port: 9000, + quic_port: 9001, tcp_port: 9000, }); @@ -305,10 +339,11 @@ impl Default for Config { network_dir, listen_addresses, enr_address: (None, None), - enr_udp4_port: None, + enr_quic4_port: None, enr_tcp4_port: None, enr_udp6_port: None, + enr_quic6_port: None, enr_tcp6_port: None, target_peers: 50, gs_config, @@ -320,6 +355,7 @@ impl Default for Config { disable_peer_scoring: false, client_version: lighthouse_version::version_with_platform(), disable_discovery: false, + disable_quic_support: false, upnp_enabled: true, network_load: 3, private: false, diff --git a/beacon_node/lighthouse_network/src/discovery/enr.rs b/beacon_node/lighthouse_network/src/discovery/enr.rs index ef22f816a7..3f46285a80 100644 --- a/beacon_node/lighthouse_network/src/discovery/enr.rs +++ b/beacon_node/lighthouse_network/src/discovery/enr.rs @@ -17,6 +17,8 @@ use std::path::Path; use std::str::FromStr; use types::{EnrForkId, EthSpec}; +use super::enr_ext::{EnrExt, QUIC6_ENR_KEY, QUIC_ENR_KEY}; + /// The ENR field specifying the fork id. pub const ETH2_ENR_KEY: &str = "eth2"; /// The ENR field specifying the attestation subnet bitfield. @@ -142,7 +144,7 @@ pub fn build_or_load_enr( pub fn create_enr_builder_from_config( config: &NetworkConfig, - enable_tcp: bool, + enable_libp2p: bool, ) -> EnrBuilder { let mut builder = EnrBuilder::new("v4"); let (maybe_ipv4_address, maybe_ipv6_address) = &config.enr_address; @@ -163,7 +165,28 @@ pub fn create_enr_builder_from_config( builder.udp6(udp6_port); } - if enable_tcp { + if enable_libp2p { + // Add QUIC fields to the ENR. + // Since QUIC is used as an alternative transport for the libp2p protocols, + // the related fields should only be added when both QUIC and libp2p are enabled + if !config.disable_quic_support { + // If we are listening on ipv4, add the quic ipv4 port. + if let Some(quic4_port) = config + .enr_quic4_port + .or_else(|| config.listen_addrs().v4().map(|v4_addr| v4_addr.quic_port)) + { + builder.add_value(QUIC_ENR_KEY, &quic4_port); + } + + // If we are listening on ipv6, add the quic ipv6 port. + if let Some(quic6_port) = config + .enr_quic6_port + .or_else(|| config.listen_addrs().v6().map(|v6_addr| v6_addr.quic_port)) + { + builder.add_value(QUIC6_ENR_KEY, &quic6_port); + } + } + // If the ENR port is not set, and we are listening over that ip version, use the listening port instead. let tcp4_port = config .enr_tcp4_port @@ -218,6 +241,9 @@ fn compare_enr(local_enr: &Enr, disk_enr: &Enr) -> bool { // tcp ports must match && local_enr.tcp4() == disk_enr.tcp4() && local_enr.tcp6() == disk_enr.tcp6() + // quic ports must match + && local_enr.quic4() == disk_enr.quic4() + && local_enr.quic6() == disk_enr.quic6() // must match on the same fork && local_enr.get(ETH2_ENR_KEY) == disk_enr.get(ETH2_ENR_KEY) // take preference over disk udp port if one is not specified diff --git a/beacon_node/lighthouse_network/src/discovery/enr_ext.rs b/beacon_node/lighthouse_network/src/discovery/enr_ext.rs index 753da6292c..2efaa76ac3 100644 --- a/beacon_node/lighthouse_network/src/discovery/enr_ext.rs +++ b/beacon_node/lighthouse_network/src/discovery/enr_ext.rs @@ -6,12 +6,15 @@ use libp2p::core::multiaddr::Protocol; use libp2p::identity::{ed25519, secp256k1, KeyType, Keypair, PublicKey}; use tiny_keccak::{Hasher, Keccak}; +pub const QUIC_ENR_KEY: &str = "quic"; +pub const QUIC6_ENR_KEY: &str = "quic6"; + /// Extend ENR for libp2p types. pub trait EnrExt { /// The libp2p `PeerId` for the record. fn peer_id(&self) -> PeerId; - /// Returns a list of multiaddrs if the ENR has an `ip` and either a `tcp` or `udp` key **or** an `ip6` and either a `tcp6` or `udp6`. + /// Returns a list of multiaddrs if the ENR has an `ip` and one of [`tcp`,`udp`,`quic`] key **or** an `ip6` and one of [`tcp6`,`udp6`,`quic6`]. /// The vector remains empty if these fields are not defined. fn multiaddr(&self) -> Vec; @@ -26,6 +29,15 @@ pub trait EnrExt { /// Returns any multiaddrs that contain the TCP protocol. fn multiaddr_tcp(&self) -> Vec; + + /// Returns any QUIC multiaddrs that are registered in this ENR. + fn multiaddr_quic(&self) -> Vec; + + /// Returns the quic port if one is set. + fn quic4(&self) -> Option; + + /// Returns the quic6 port if one is set. + fn quic6(&self) -> Option; } /// Extend ENR CombinedPublicKey for libp2p types. @@ -49,7 +61,17 @@ impl EnrExt for Enr { self.public_key().as_peer_id() } - /// Returns a list of multiaddrs if the ENR has an `ip` and either a `tcp` or `udp` key **or** an `ip6` and either a `tcp6` or `udp6`. + /// Returns the quic port if one is set. + fn quic4(&self) -> Option { + self.get_decodable(QUIC_ENR_KEY).and_then(Result::ok) + } + + /// Returns the quic6 port if one is set. + fn quic6(&self) -> Option { + self.get_decodable(QUIC6_ENR_KEY).and_then(Result::ok) + } + + /// Returns a list of multiaddrs if the ENR has an `ip` and either a `tcp`, `quic` or `udp` key **or** an `ip6` and either a `tcp6` `quic6` or `udp6`. /// The vector remains empty if these fields are not defined. fn multiaddr(&self) -> Vec { let mut multiaddrs: Vec = Vec::new(); @@ -59,6 +81,12 @@ impl EnrExt for Enr { multiaddr.push(Protocol::Udp(udp)); multiaddrs.push(multiaddr); } + if let Some(quic) = self.quic4() { + let mut multiaddr: Multiaddr = ip.into(); + multiaddr.push(Protocol::Udp(quic)); + multiaddr.push(Protocol::QuicV1); + multiaddrs.push(multiaddr); + } if let Some(tcp) = self.tcp4() { let mut multiaddr: Multiaddr = ip.into(); @@ -73,6 +101,13 @@ impl EnrExt for Enr { multiaddrs.push(multiaddr); } + if let Some(quic6) = self.quic6() { + let mut multiaddr: Multiaddr = ip6.into(); + multiaddr.push(Protocol::Udp(quic6)); + multiaddr.push(Protocol::QuicV1); + multiaddrs.push(multiaddr); + } + if let Some(tcp6) = self.tcp6() { let mut multiaddr: Multiaddr = ip6.into(); multiaddr.push(Protocol::Tcp(tcp6)); @@ -174,8 +209,30 @@ impl EnrExt for Enr { multiaddrs } + /// Returns a list of multiaddrs if the ENR has an `ip` and a `quic` key **or** an `ip6` and a `quic6`. + fn multiaddr_quic(&self) -> Vec { + let mut multiaddrs: Vec = Vec::new(); + if let Some(quic_port) = self.quic4() { + if let Some(ip) = self.ip4() { + let mut multiaddr: Multiaddr = ip.into(); + multiaddr.push(Protocol::Udp(quic_port)); + multiaddr.push(Protocol::QuicV1); + multiaddrs.push(multiaddr); + } + } + + if let Some(quic6_port) = self.quic6() { + if let Some(ip6) = self.ip6() { + let mut multiaddr: Multiaddr = ip6.into(); + multiaddr.push(Protocol::Udp(quic6_port)); + multiaddr.push(Protocol::QuicV1); + multiaddrs.push(multiaddr); + } + } + multiaddrs + } + /// Returns a list of multiaddrs if the ENR has an `ip` and either a `tcp` or `udp` key **or** an `ip6` and either a `tcp6` or `udp6`. - /// The vector remains empty if these fields are not defined. fn multiaddr_tcp(&self) -> Vec { let mut multiaddrs: Vec = Vec::new(); if let Some(ip) = self.ip4() { diff --git a/beacon_node/lighthouse_network/src/discovery/mod.rs b/beacon_node/lighthouse_network/src/discovery/mod.rs index c3819f1eb9..4d8807336b 100644 --- a/beacon_node/lighthouse_network/src/discovery/mod.rs +++ b/beacon_node/lighthouse_network/src/discovery/mod.rs @@ -21,7 +21,6 @@ pub use libp2p::identity::{Keypair, PublicKey}; use enr::{ATTESTATION_BITFIELD_ENR_KEY, ETH2_ENR_KEY, SYNC_COMMITTEE_BITFIELD_ENR_KEY}; use futures::prelude::*; use futures::stream::FuturesUnordered; -use libp2p::multiaddr::Protocol; use libp2p::swarm::behaviour::{DialFailure, FromSwarm}; use libp2p::swarm::THandlerInEvent; pub use libp2p::{ @@ -75,7 +74,7 @@ const DURATION_DIFFERENCE: Duration = Duration::from_millis(1); /// of the peer if it is specified. #[derive(Debug)] pub struct DiscoveredPeers { - pub peers: HashMap>, + pub peers: HashMap>, } #[derive(Clone, PartialEq)] @@ -208,7 +207,8 @@ impl Discovery { let local_node_id = local_enr.node_id(); info!(log, "ENR Initialised"; "enr" => local_enr.to_base64(), "seq" => local_enr.seq(), "id"=> %local_enr.node_id(), - "ip4" => ?local_enr.ip4(), "udp4"=> ?local_enr.udp4(), "tcp4" => ?local_enr.tcp4(), "tcp6" => ?local_enr.tcp6(), "udp6" => ?local_enr.udp6() + "ip4" => ?local_enr.ip4(), "udp4"=> ?local_enr.udp4(), "tcp4" => ?local_enr.tcp4(), "tcp6" => ?local_enr.tcp6(), "udp6" => ?local_enr.udp6(), + "quic4" => ?local_enr.quic4(), "quic6" => ?local_enr.quic6() ); // convert the keypair into an ENR key @@ -230,7 +230,8 @@ impl Discovery { "peer_id" => %bootnode_enr.peer_id(), "ip" => ?bootnode_enr.ip4(), "udp" => ?bootnode_enr.udp4(), - "tcp" => ?bootnode_enr.tcp4() + "tcp" => ?bootnode_enr.tcp4(), + "quic" => ?bootnode_enr.quic4() ); let repr = bootnode_enr.to_string(); let _ = discv5.add_enr(bootnode_enr).map_err(|e| { @@ -281,7 +282,8 @@ impl Discovery { "peer_id" => %enr.peer_id(), "ip" => ?enr.ip4(), "udp" => ?enr.udp4(), - "tcp" => ?enr.tcp4() + "tcp" => ?enr.tcp4(), + "quic" => ?enr.quic4() ); let _ = discv5.add_enr(enr).map_err(|e| { error!( @@ -383,20 +385,6 @@ impl Discovery { self.discv5.table_entries_enr() } - /// Returns the ENR of a known peer if it exists. - pub fn enr_of_peer(&mut self, peer_id: &PeerId) -> Option { - // first search the local cache - if let Some(enr) = self.cached_enrs.get(peer_id) { - return Some(enr.clone()); - } - // not in the local cache, look in the routing table - if let Ok(node_id) = enr_ext::peer_id_to_node_id(peer_id) { - self.discv5.find_enr(&node_id) - } else { - None - } - } - /// Updates the local ENR TCP port. /// There currently isn't a case to update the address here. We opt for discovery to /// automatically update the external address. @@ -414,6 +402,23 @@ impl Discovery { Ok(()) } + // TODO: Group these functions here once the ENR is shared across discv5 and lighthouse and + // Lighthouse can modify the ENR directly. + // This currently doesn't support ipv6. All of these functions should be removed and + // addressed properly in the following issue. + // https://github.com/sigp/lighthouse/issues/4706 + pub fn update_enr_quic_port(&mut self, port: u16) -> Result<(), String> { + self.discv5 + .enr_insert("quic", &port) + .map_err(|e| format!("{:?}", e))?; + + // replace the global version + *self.network_globals.local_enr.write() = self.discv5.local_enr(); + // persist modified enr to disk + enr::save_enr_to_disk(Path::new(&self.enr_dir), &self.local_enr(), &self.log); + Ok(()) + } + /// Updates the local ENR UDP socket. /// /// This is with caution. Discovery should automatically maintain this. This should only be @@ -733,23 +738,6 @@ impl Discovery { target_peers: usize, additional_predicate: impl Fn(&Enr) -> bool + Send + 'static, ) { - // Make sure there are subnet queries included - let contains_queries = match &query { - QueryType::Subnet(queries) => !queries.is_empty(), - QueryType::FindPeers => true, - }; - - if !contains_queries { - debug!( - self.log, - "No subnets included in this request. Skipping discovery request." - ); - return; - } - - // Generate a random target node id. - let random_node = NodeId::random(); - let enr_fork_id = match self.local_enr().eth2() { Ok(v) => v, Err(e) => { @@ -773,7 +761,8 @@ impl Discovery { // Build the future let query_future = self .discv5 - .find_node_predicate(random_node, predicate, target_peers) + // Generate a random target node id. + .find_node_predicate(NodeId::random(), predicate, target_peers) .map(|v| QueryResult { query_type: query, result: v, @@ -787,7 +776,7 @@ impl Discovery { fn process_completed_queries( &mut self, query: QueryResult, - ) -> Option>> { + ) -> Option>> { match query.query_type { QueryType::FindPeers => { self.find_peer_active = false; @@ -797,12 +786,14 @@ impl Discovery { } Ok(r) => { debug!(self.log, "Discovery query completed"; "peers_found" => r.len()); - let mut results: HashMap<_, Option> = HashMap::new(); - r.iter().for_each(|enr| { - // cache the found ENR's - self.cached_enrs.put(enr.peer_id(), enr.clone()); - results.insert(enr.peer_id(), None); - }); + let results = r + .into_iter() + .map(|enr| { + // cache the found ENR's + self.cached_enrs.put(enr.peer_id(), enr.clone()); + (enr, None) + }) + .collect(); return Some(results); } Err(e) => { @@ -850,17 +841,17 @@ impl Discovery { let subnet_predicate = subnet_predicate::(vec![query.subnet], &self.log); - r.iter() + r.clone() + .into_iter() .filter(|enr| subnet_predicate(enr)) - .map(|enr| enr.peer_id()) - .for_each(|peer_id| { + .for_each(|enr| { if let Some(v) = metrics::get_int_counter( &metrics::SUBNET_PEERS_FOUND, &[query_str], ) { v.inc(); } - let other_min_ttl = mapped_results.get_mut(&peer_id); + let other_min_ttl = mapped_results.get_mut(&enr); // map peer IDs to the min_ttl furthest in the future match (query.min_ttl, other_min_ttl) { @@ -878,15 +869,11 @@ impl Discovery { } // update the mapping if we have a specified min_ttl (Some(min_ttl), Some(None)) => { - mapped_results.insert(peer_id, Some(min_ttl)); + mapped_results.insert(enr, Some(min_ttl)); } // first seen min_ttl for this enr - (Some(min_ttl), None) => { - mapped_results.insert(peer_id, Some(min_ttl)); - } - // first seen min_ttl for this enr - (None, None) => { - mapped_results.insert(peer_id, None); + (min_ttl, None) => { + mapped_results.insert(enr, min_ttl); } (None, Some(Some(_))) => {} // Don't replace the existing specific min_ttl (None, Some(None)) => {} // No-op because this is a duplicate @@ -910,7 +897,7 @@ impl Discovery { } /// Drives the queries returning any results from completed queries. - fn poll_queries(&mut self, cx: &mut Context) -> Option>> { + fn poll_queries(&mut self, cx: &mut Context) -> Option>> { while let Poll::Ready(Some(query_result)) = self.active_queries.poll_next_unpin(cx) { let result = self.process_completed_queries(query_result); if result.is_some() { @@ -957,23 +944,6 @@ impl NetworkBehaviour for Discovery { ) { } - fn handle_pending_outbound_connection( - &mut self, - _connection_id: ConnectionId, - maybe_peer: Option, - _addresses: &[Multiaddr], - _effective_role: libp2p::core::Endpoint, - ) -> Result, libp2p::swarm::ConnectionDenied> { - if let Some(enr) = maybe_peer.and_then(|peer_id| self.enr_of_peer(&peer_id)) { - // ENR's may have multiple Multiaddrs. The multi-addr associated with the UDP - // port is removed, which is assumed to be associated with the discv5 protocol (and - // therefore irrelevant for other libp2p components). - Ok(enr.multiaddr_tcp()) - } else { - Ok(vec![]) - } - } - // Main execution loop to drive the behaviour fn poll( &mut self, @@ -1047,25 +1017,8 @@ impl NetworkBehaviour for Discovery { // update network globals *self.network_globals.local_enr.write() = enr; // A new UDP socket has been detected. - // Build a multiaddr to report to libp2p - let addr = match socket_addr.ip() { - IpAddr::V4(v4_addr) => { - self.network_globals.listen_port_tcp4().map(|tcp4_port| { - Multiaddr::from(v4_addr).with(Protocol::Tcp(tcp4_port)) - }) - } - IpAddr::V6(v6_addr) => { - self.network_globals.listen_port_tcp6().map(|tcp6_port| { - Multiaddr::from(v6_addr).with(Protocol::Tcp(tcp6_port)) - }) - } - }; - - if let Some(address) = addr { - // NOTE: This doesn't actually track the external TCP port. More sophisticated NAT handling - // should handle this. - return Poll::Ready(ToSwarm::NewExternalAddrCandidate(address)); - } + // NOTE: We assume libp2p itself can keep track of IP changes and we do + // not inform it about IP changes found via discovery. } Discv5Event::EnrAdded { .. } | Discv5Event::TalkRequest(_) @@ -1152,8 +1105,6 @@ mod tests { let log = build_log(slog::Level::Debug, false); let globals = NetworkGlobals::new( enr, - Some(9000), - None, MetaData::V2(MetaDataV2 { seq_number: 0, attnets: Default::default(), @@ -1261,6 +1212,6 @@ mod tests { assert_eq!(results.len(), 2); // when a peer belongs to multiple subnet ids, we use the highest ttl. - assert_eq!(results.get(&enr1.peer_id()).unwrap(), &instant1); + assert_eq!(results.get(&enr1).unwrap(), &instant1); } } diff --git a/beacon_node/lighthouse_network/src/listen_addr.rs b/beacon_node/lighthouse_network/src/listen_addr.rs index 20d87d403c..53f7d9daca 100644 --- a/beacon_node/lighthouse_network/src/listen_addr.rs +++ b/beacon_node/lighthouse_network/src/listen_addr.rs @@ -6,14 +6,23 @@ use serde::{Deserialize, Serialize}; /// A listening address composed by an Ip, an UDP port and a TCP port. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct ListenAddr { + /// The IP address we will listen on. pub addr: Ip, - pub udp_port: u16, + /// The UDP port that discovery will listen on. + pub disc_port: u16, + /// The UDP port that QUIC will listen on. + pub quic_port: u16, + /// The TCP port that libp2p will listen on. pub tcp_port: u16, } impl + Clone> ListenAddr { - pub fn udp_socket_addr(&self) -> SocketAddr { - (self.addr.clone().into(), self.udp_port).into() + pub fn discovery_socket_addr(&self) -> SocketAddr { + (self.addr.clone().into(), self.disc_port).into() + } + + pub fn quic_socket_addr(&self) -> SocketAddr { + (self.addr.clone().into(), self.quic_port).into() } pub fn tcp_socket_addr(&self) -> SocketAddr { @@ -46,22 +55,41 @@ impl ListenAddress { } } - /// Returns the TCP addresses. - pub fn tcp_addresses(&self) -> impl Iterator + '_ { - let v4_multiaddr = self + /// Returns the addresses the Swarm will listen on, given the setup. + pub fn libp2p_addresses(&self) -> impl Iterator { + let v4_tcp_multiaddr = self .v4() .map(|v4_addr| Multiaddr::from(v4_addr.addr).with(Protocol::Tcp(v4_addr.tcp_port))); - let v6_multiaddr = self + + let v4_quic_multiaddr = self.v4().map(|v4_addr| { + Multiaddr::from(v4_addr.addr) + .with(Protocol::Udp(v4_addr.quic_port)) + .with(Protocol::QuicV1) + }); + + let v6_quic_multiaddr = self.v6().map(|v6_addr| { + Multiaddr::from(v6_addr.addr) + .with(Protocol::Udp(v6_addr.quic_port)) + .with(Protocol::QuicV1) + }); + + let v6_tcp_multiaddr = self .v6() .map(|v6_addr| Multiaddr::from(v6_addr.addr).with(Protocol::Tcp(v6_addr.tcp_port))); - v4_multiaddr.into_iter().chain(v6_multiaddr) + + v4_tcp_multiaddr + .into_iter() + .chain(v4_quic_multiaddr) + .chain(v6_quic_multiaddr) + .chain(v6_tcp_multiaddr) } #[cfg(test)] pub fn unused_v4_ports() -> Self { ListenAddress::V4(ListenAddr { addr: Ipv4Addr::UNSPECIFIED, - udp_port: unused_port::unused_udp4_port().unwrap(), + disc_port: unused_port::unused_udp4_port().unwrap(), + quic_port: unused_port::unused_udp4_port().unwrap(), tcp_port: unused_port::unused_tcp4_port().unwrap(), }) } @@ -70,7 +98,8 @@ impl ListenAddress { pub fn unused_v6_ports() -> Self { ListenAddress::V6(ListenAddr { addr: Ipv6Addr::UNSPECIFIED, - udp_port: unused_port::unused_udp6_port().unwrap(), + disc_port: unused_port::unused_udp6_port().unwrap(), + quic_port: unused_port::unused_udp6_port().unwrap(), tcp_port: unused_port::unused_tcp6_port().unwrap(), }) } @@ -84,12 +113,14 @@ impl slog::KV for ListenAddress { ) -> slog::Result { if let Some(v4_addr) = self.v4() { serializer.emit_arguments("ip4_address", &format_args!("{}", v4_addr.addr))?; - serializer.emit_u16("udp4_port", v4_addr.udp_port)?; + serializer.emit_u16("disc4_port", v4_addr.disc_port)?; + serializer.emit_u16("quic4_port", v4_addr.quic_port)?; serializer.emit_u16("tcp4_port", v4_addr.tcp_port)?; } if let Some(v6_addr) = self.v6() { serializer.emit_arguments("ip6_address", &format_args!("{}", v6_addr.addr))?; - serializer.emit_u16("udp6_port", v6_addr.udp_port)?; + serializer.emit_u16("disc6_port", v6_addr.disc_port)?; + serializer.emit_u16("quic6_port", v6_addr.quic_port)?; serializer.emit_u16("tcp6_port", v6_addr.tcp_port)?; } slog::Result::Ok(()) diff --git a/beacon_node/lighthouse_network/src/metrics.rs b/beacon_node/lighthouse_network/src/metrics.rs index 58cc992012..ae02b689d8 100644 --- a/beacon_node/lighthouse_network/src/metrics.rs +++ b/beacon_node/lighthouse_network/src/metrics.rs @@ -14,6 +14,16 @@ lazy_static! { "Count of libp2p peers currently connected" ); + pub static ref TCP_PEERS_CONNECTED: Result = try_create_int_gauge( + "libp2p_tcp_peers", + "Count of libp2p peers currently connected via TCP" + ); + + pub static ref QUIC_PEERS_CONNECTED: Result = try_create_int_gauge( + "libp2p_quic_peers", + "Count of libp2p peers currently connected via QUIC" + ); + pub static ref PEER_CONNECT_EVENT_COUNT: Result = try_create_int_counter( "libp2p_peer_connect_event_total", "Count of libp2p peer connect events (not the current number of connected peers)" diff --git a/beacon_node/lighthouse_network/src/peer_manager/mod.rs b/beacon_node/lighthouse_network/src/peer_manager/mod.rs index f6158ed3a0..9118dc1d0c 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/mod.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/mod.rs @@ -1,5 +1,6 @@ //! Implementation of Lighthouse's peer management system. +use crate::discovery::enr_ext::EnrExt; use crate::rpc::{GoodbyeReason, MetaData, Protocol, RPCError, RPCResponseErrorCode}; use crate::service::TARGET_SUBNET_PEERS; use crate::{error, metrics, Gossipsub}; @@ -13,7 +14,6 @@ use peerdb::{client::ClientKind, BanOperation, BanResult, ScoreUpdateResult}; use rand::seq::SliceRandom; use slog::{debug, error, trace, warn}; use smallvec::SmallVec; -use std::collections::BTreeMap; use std::{ sync::Arc, time::{Duration, Instant}, @@ -78,7 +78,7 @@ pub struct PeerManager { /// The target number of peers we would like to connect to. target_peers: usize, /// Peers queued to be dialed. - peers_to_dial: BTreeMap>, + peers_to_dial: Vec, /// The number of temporarily banned peers. This is used to prevent instantaneous /// reconnection. // NOTE: This just prevents re-connections. The state of the peer is otherwise unaffected. A @@ -312,16 +312,12 @@ impl PeerManager { /// Peers that have been returned by discovery requests that are suitable for dialing are /// returned here. /// - /// NOTE: By dialing `PeerId`s and not multiaddrs, libp2p requests the multiaddr associated - /// with a new `PeerId` which involves a discovery routing table lookup. We could dial the - /// multiaddr here, however this could relate to duplicate PeerId's etc. If the lookup - /// proves resource constraining, we should switch to multiaddr dialling here. + /// This function decides whether or not to dial these peers. #[allow(clippy::mutable_key_type)] - pub fn peers_discovered(&mut self, results: HashMap>) -> Vec { - let mut to_dial_peers = Vec::with_capacity(4); - + pub fn peers_discovered(&mut self, results: HashMap>) { + let mut to_dial_peers = 0; let connected_or_dialing = self.network_globals.connected_or_dialing_peers(); - for (peer_id, min_ttl) in results { + for (enr, min_ttl) in results { // There are two conditions in deciding whether to dial this peer. // 1. If we are less than our max connections. Discovery queries are executed to reach // our target peers, so its fine to dial up to our max peers (which will get pruned @@ -330,10 +326,8 @@ impl PeerManager { // considered a priority. We have pre-allocated some extra priority slots for these // peers as specified by PRIORITY_PEER_EXCESS. Therefore we dial these peers, even // if we are already at our max_peer limit. - if (min_ttl.is_some() - && connected_or_dialing + to_dial_peers.len() < self.max_priority_peers() - || connected_or_dialing + to_dial_peers.len() < self.max_peers()) - && self.network_globals.peers.read().should_dial(&peer_id) + if min_ttl.is_some() && connected_or_dialing + to_dial_peers < self.max_priority_peers() + || connected_or_dialing + to_dial_peers < self.max_peers() { // This should be updated with the peer dialing. In fact created once the peer is // dialed @@ -341,16 +335,16 @@ impl PeerManager { self.network_globals .peers .write() - .update_min_ttl(&peer_id, min_ttl); + .update_min_ttl(&enr.peer_id(), min_ttl); } - to_dial_peers.push(peer_id); + debug!(self.log, "Dialing discovered peer"; "peer_id" => %enr.peer_id()); + self.dial_peer(enr); + to_dial_peers += 1; } } // Queue another discovery if we need to - self.maintain_peer_count(to_dial_peers.len()); - - to_dial_peers + self.maintain_peer_count(to_dial_peers); } /// A STATUS message has been received from a peer. This resets the status timer. @@ -406,9 +400,16 @@ impl PeerManager { /* Notifications from the Swarm */ - // A peer is being dialed. - pub fn dial_peer(&mut self, peer_id: &PeerId, enr: Option) { - self.peers_to_dial.insert(*peer_id, enr); + /// A peer is being dialed. + pub fn dial_peer(&mut self, peer: Enr) { + if self + .network_globals + .peers + .read() + .should_dial(&peer.peer_id()) + { + self.peers_to_dial.push(peer); + } } /// Reports if a peer is banned or not. diff --git a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs index 70f421681a..fedb876bb2 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs @@ -3,7 +3,7 @@ use std::task::{Context, Poll}; use futures::StreamExt; -use libp2p::core::ConnectedPoint; +use libp2p::core::{multiaddr, ConnectedPoint}; use libp2p::identity::PeerId; use libp2p::swarm::behaviour::{ConnectionClosed, ConnectionEstablished, DialFailure, FromSwarm}; use libp2p::swarm::dial_opts::{DialOpts, PeerCondition}; @@ -12,6 +12,7 @@ use libp2p::swarm::{ConnectionId, NetworkBehaviour, PollParameters, ToSwarm}; use slog::{debug, error}; use types::EthSpec; +use crate::discovery::enr_ext::EnrExt; use crate::rpc::GoodbyeReason; use crate::types::SyncState; use crate::{metrics, ClearDialError}; @@ -95,11 +96,23 @@ impl NetworkBehaviour for PeerManager { self.events.shrink_to_fit(); } - if let Some((peer_id, maybe_enr)) = self.peers_to_dial.pop_first() { - self.inject_peer_connection(&peer_id, ConnectingType::Dialing, maybe_enr); + if let Some(enr) = self.peers_to_dial.pop() { + let peer_id = enr.peer_id(); + self.inject_peer_connection(&peer_id, ConnectingType::Dialing, Some(enr.clone())); + let quic_multiaddrs = enr.multiaddr_quic(); + if !quic_multiaddrs.is_empty() { + debug!(self.log, "Dialing QUIC supported peer"; "peer_id"=> %peer_id, "quic_multiaddrs" => ?quic_multiaddrs); + } + + // Prioritize Quic connections over Tcp ones. + let multiaddrs = quic_multiaddrs + .into_iter() + .chain(enr.multiaddr_tcp()) + .collect(); return Poll::Ready(ToSwarm::Dial { opts: DialOpts::peer_id(peer_id) .condition(PeerCondition::Disconnected) + .addresses(multiaddrs) .build(), }); } @@ -124,9 +137,11 @@ impl NetworkBehaviour for PeerManager { } FromSwarm::ConnectionClosed(ConnectionClosed { peer_id, + endpoint, + remaining_established, .. - }) => self.on_connection_closed(peer_id, remaining_established), + }) => self.on_connection_closed(peer_id, endpoint, remaining_established), FromSwarm::DialFailure(DialFailure { peer_id, error, @@ -184,7 +199,11 @@ impl PeerManager { endpoint: &ConnectedPoint, other_established: usize, ) { - debug!(self.log, "Connection established"; "peer_id" => %peer_id, "connection" => ?endpoint.to_endpoint()); + debug!(self.log, "Connection established"; "peer_id" => %peer_id, + "multiaddr" => %endpoint.get_remote_address(), + "connection" => ?endpoint.to_endpoint() + ); + if other_established == 0 { self.events.push(PeerManagerEvent::MetaData(peer_id)); } @@ -194,6 +213,34 @@ impl PeerManager { metrics::check_nat(); } + // increment prometheus metrics + if self.metrics_enabled { + let remote_addr = match endpoint { + ConnectedPoint::Dialer { address, .. } => address, + ConnectedPoint::Listener { send_back_addr, .. } => send_back_addr, + }; + match remote_addr.iter().find(|proto| { + matches!( + proto, + multiaddr::Protocol::QuicV1 | multiaddr::Protocol::Tcp(_) + ) + }) { + Some(multiaddr::Protocol::QuicV1) => { + metrics::inc_gauge(&metrics::QUIC_PEERS_CONNECTED); + } + Some(multiaddr::Protocol::Tcp(_)) => { + metrics::inc_gauge(&metrics::TCP_PEERS_CONNECTED); + } + Some(_) => unreachable!(), + None => { + error!(self.log, "Connection established via unknown transport"; "addr" => %remote_addr) + } + }; + + self.update_connected_peer_metrics(); + metrics::inc_counter(&metrics::PEER_CONNECT_EVENT_COUNT); + } + // Check to make sure the peer is not supposed to be banned match self.ban_status(&peer_id) { // TODO: directly emit the ban event? @@ -245,14 +292,15 @@ impl PeerManager { self.events .push(PeerManagerEvent::PeerConnectedOutgoing(peer_id)); } - } - - // increment prometheus metrics - self.update_connected_peer_metrics(); - metrics::inc_counter(&metrics::PEER_CONNECT_EVENT_COUNT); + }; } - fn on_connection_closed(&mut self, peer_id: PeerId, remaining_established: usize) { + fn on_connection_closed( + &mut self, + peer_id: PeerId, + endpoint: &ConnectedPoint, + remaining_established: usize, + ) { if remaining_established > 0 { return; } @@ -278,9 +326,31 @@ impl PeerManager { // reference so that peer manager can track this peer. self.inject_disconnect(&peer_id); + let remote_addr = match endpoint { + ConnectedPoint::Listener { send_back_addr, .. } => send_back_addr, + ConnectedPoint::Dialer { address, .. } => address, + }; + // Update the prometheus metrics - self.update_connected_peer_metrics(); - metrics::inc_counter(&metrics::PEER_DISCONNECT_EVENT_COUNT); + if self.metrics_enabled { + match remote_addr.iter().find(|proto| { + matches!( + proto, + multiaddr::Protocol::QuicV1 | multiaddr::Protocol::Tcp(_) + ) + }) { + Some(multiaddr::Protocol::QuicV1) => { + metrics::dec_gauge(&metrics::QUIC_PEERS_CONNECTED); + } + Some(multiaddr::Protocol::Tcp(_)) => { + metrics::dec_gauge(&metrics::TCP_PEERS_CONNECTED); + } + // If it's an unknown protocol we already logged when connection was established. + _ => {} + }; + self.update_connected_peer_metrics(); + metrics::inc_counter(&metrics::PEER_DISCONNECT_EVENT_COUNT); + } } /// A dial attempt has failed. diff --git a/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs b/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs index 52f0bbd9df..4a1efe8f2e 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs @@ -1,16 +1,11 @@ -use crate::{ - metrics, - multiaddr::{Multiaddr, Protocol}, - types::Subnet, - Enr, Gossipsub, PeerId, -}; +use crate::{metrics, multiaddr::Multiaddr, types::Subnet, Enr, Gossipsub, PeerId}; use peer_info::{ConnectionDirection, PeerConnectionStatus, PeerInfo}; use rand::seq::SliceRandom; use score::{PeerAction, ReportSource, Score, ScoreState}; use slog::{crit, debug, error, trace, warn}; use std::cmp::Ordering; use std::collections::{HashMap, HashSet}; -use std::net::{IpAddr, SocketAddr}; +use std::net::IpAddr; use std::time::Instant; use sync_status::SyncStatus; use types::EthSpec; @@ -764,28 +759,10 @@ impl PeerDB { | PeerConnectionStatus::Dialing { .. } => {} } - // Add the seen ip address and port to the peer's info - let socket_addr = match seen_address.iter().fold( - (None, None), - |(found_ip, found_port), protocol| match protocol { - Protocol::Ip4(ip) => (Some(ip.into()), found_port), - Protocol::Ip6(ip) => (Some(ip.into()), found_port), - Protocol::Tcp(port) => (found_ip, Some(port)), - _ => (found_ip, found_port), - }, - ) { - (Some(ip), Some(port)) => Some(SocketAddr::new(ip, port)), - (Some(_ip), None) => { - crit!(self.log, "Connected peer has an IP but no TCP port"; "peer_id" => %peer_id); - None - } - _ => None, - }; - // Update the connection state match direction { - ConnectionDirection::Incoming => info.connect_ingoing(socket_addr), - ConnectionDirection::Outgoing => info.connect_outgoing(socket_addr), + ConnectionDirection::Incoming => info.connect_ingoing(Some(seen_address)), + ConnectionDirection::Outgoing => info.connect_outgoing(Some(seen_address)), } } @@ -1274,6 +1251,7 @@ impl BannedPeersCount { #[cfg(test)] mod tests { use super::*; + use libp2p::core::multiaddr::Protocol; use libp2p::core::Multiaddr; use slog::{o, Drain}; use std::net::{Ipv4Addr, Ipv6Addr}; diff --git a/beacon_node/lighthouse_network/src/peer_manager/peerdb/peer_info.rs b/beacon_node/lighthouse_network/src/peer_manager/peerdb/peer_info.rs index 555266d0e2..44c54511dd 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/peerdb/peer_info.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/peerdb/peer_info.rs @@ -2,15 +2,15 @@ use super::client::Client; use super::score::{PeerAction, Score, ScoreState}; use super::sync_status::SyncStatus; use crate::discovery::Eth2Enr; -use crate::Multiaddr; use crate::{rpc::MetaData, types::Subnet}; use discv5::Enr; +use libp2p::core::multiaddr::{Multiaddr, Protocol}; use serde::{ ser::{SerializeStruct, Serializer}, Serialize, }; use std::collections::HashSet; -use std::net::{IpAddr, SocketAddr}; +use std::net::IpAddr; use std::time::Instant; use strum::AsRefStr; use types::EthSpec; @@ -29,9 +29,9 @@ pub struct PeerInfo { /// The known listening addresses of this peer. This is given by identify and can be arbitrary /// (including local IPs). listening_addresses: Vec, - /// This is addresses we have physically seen and this is what we use for banning/un-banning + /// These are the multiaddrs we have physically seen and is what we use for banning/un-banning /// peers. - seen_addresses: HashSet, + seen_multiaddrs: HashSet, /// The current syncing state of the peer. The state may be determined after it's initial /// connection. sync_status: SyncStatus, @@ -60,7 +60,7 @@ impl Default for PeerInfo { client: Client::default(), connection_status: Default::default(), listening_addresses: Vec::new(), - seen_addresses: HashSet::new(), + seen_multiaddrs: HashSet::new(), subnets: HashSet::new(), sync_status: SyncStatus::Unknown, meta_data: None, @@ -227,15 +227,21 @@ impl PeerInfo { } /// Returns the seen addresses of the peer. - pub fn seen_addresses(&self) -> impl Iterator + '_ { - self.seen_addresses.iter() + pub fn seen_multiaddrs(&self) -> impl Iterator + '_ { + self.seen_multiaddrs.iter() } /// Returns a list of seen IP addresses for the peer. pub fn seen_ip_addresses(&self) -> impl Iterator + '_ { - self.seen_addresses - .iter() - .map(|socket_addr| socket_addr.ip()) + self.seen_multiaddrs.iter().filter_map(|multiaddr| { + multiaddr.iter().find_map(|protocol| { + match protocol { + Protocol::Ip4(ip) => Some(ip.into()), + Protocol::Ip6(ip) => Some(ip.into()), + _ => None, // Only care for IP addresses + } + }) + }) } /// Returns the connection status of the peer. @@ -415,7 +421,7 @@ impl PeerInfo { /// Modifies the status to Connected and increases the number of ingoing /// connections by one - pub(super) fn connect_ingoing(&mut self, seen_address: Option) { + pub(super) fn connect_ingoing(&mut self, seen_multiaddr: Option) { match &mut self.connection_status { Connected { n_in, .. } => *n_in += 1, Disconnected { .. } @@ -428,14 +434,14 @@ impl PeerInfo { } } - if let Some(socket_addr) = seen_address { - self.seen_addresses.insert(socket_addr); + if let Some(multiaddr) = seen_multiaddr { + self.seen_multiaddrs.insert(multiaddr); } } /// Modifies the status to Connected and increases the number of outgoing /// connections by one - pub(super) fn connect_outgoing(&mut self, seen_address: Option) { + pub(super) fn connect_outgoing(&mut self, seen_multiaddr: Option) { match &mut self.connection_status { Connected { n_out, .. } => *n_out += 1, Disconnected { .. } @@ -447,8 +453,8 @@ impl PeerInfo { self.connection_direction = Some(ConnectionDirection::Outgoing); } } - if let Some(ip_addr) = seen_address { - self.seen_addresses.insert(ip_addr); + if let Some(multiaddr) = seen_multiaddr { + self.seen_multiaddrs.insert(multiaddr); } } diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 63e5bcbff6..e4e11f29c5 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -156,8 +156,6 @@ impl Network { let meta_data = utils::load_or_build_metadata(&config.network_dir, &log); let globals = NetworkGlobals::new( enr, - config.listen_addrs().v4().map(|v4_addr| v4_addr.tcp_port), - config.listen_addrs().v6().map(|v6_addr| v6_addr.tcp_port), meta_data, config .trusted_peers @@ -355,8 +353,9 @@ impl Network { let (swarm, bandwidth) = { // Set up the transport - tcp/ws with noise and mplex - let (transport, bandwidth) = build_transport(local_keypair.clone()) - .map_err(|e| format!("Failed to build transport: {:?}", e))?; + let (transport, bandwidth) = + build_transport(local_keypair.clone(), !config.disable_quic_support) + .map_err(|e| format!("Failed to build transport: {:?}", e))?; // use the executor for libp2p struct Executor(task_executor::TaskExecutor); @@ -411,9 +410,16 @@ impl Network { async fn start(&mut self, config: &crate::NetworkConfig) -> error::Result<()> { let enr = self.network_globals.local_enr(); info!(self.log, "Libp2p Starting"; "peer_id" => %enr.peer_id(), "bandwidth_config" => format!("{}-{}", config.network_load, NetworkLoad::from(config.network_load).name)); - debug!(self.log, "Attempting to open listening ports"; config.listen_addrs(), "discovery_enabled" => !config.disable_discovery); + debug!(self.log, "Attempting to open listening ports"; config.listen_addrs(), "discovery_enabled" => !config.disable_discovery, "quic_enabled" => !config.disable_quic_support); + + for listen_multiaddr in config.listen_addrs().libp2p_addresses() { + // If QUIC is disabled, ignore listening on QUIC ports + if config.disable_quic_support + && listen_multiaddr.iter().any(|v| v == MProtocol::QuicV1) + { + continue; + } - for listen_multiaddr in config.listen_addrs().tcp_addresses() { match self.swarm.listen_on(listen_multiaddr.clone()) { Ok(_) => { let mut log_address = listen_multiaddr; @@ -454,6 +460,20 @@ impl Network { boot_nodes.dedup(); for bootnode_enr in boot_nodes { + // If QUIC is enabled, attempt QUIC connections first + if !config.disable_quic_support { + for quic_multiaddr in &bootnode_enr.multiaddr_quic() { + if !self + .network_globals + .peers + .read() + .is_connected_or_dialing(&bootnode_enr.peer_id()) + { + dial(quic_multiaddr.clone()); + } + } + } + for multiaddr in &bootnode_enr.multiaddr() { // ignore udp multiaddr if it exists let components = multiaddr.iter().collect::>(); @@ -1036,30 +1056,27 @@ impl Network { } } - /// Dial cached enrs in discovery service that are in the given `subnet_id` and aren't + /// Dial cached Enrs in discovery service that are in the given `subnet_id` and aren't /// in Connected, Dialing or Banned state. fn dial_cached_enrs_in_subnet(&mut self, subnet: Subnet) { let predicate = subnet_predicate::(vec![subnet], &self.log); - let peers_to_dial: Vec = self + let peers_to_dial: Vec = self .discovery() .cached_enrs() - .filter_map(|(peer_id, enr)| { - let peers = self.network_globals.peers.read(); - if predicate(enr) && peers.should_dial(peer_id) { - Some(*peer_id) + .filter_map(|(_peer_id, enr)| { + if predicate(enr) { + Some(enr.clone()) } else { None } }) .collect(); - for peer_id in peers_to_dial { - debug!(self.log, "Dialing cached ENR peer"; "peer_id" => %peer_id); - // Remove the ENR from the cache to prevent continual re-dialing on disconnects - self.discovery_mut().remove_cached_enr(&peer_id); - // For any dial event, inform the peer manager - let enr = self.discovery_mut().enr_of_peer(&peer_id); - self.peer_manager_mut().dial_peer(&peer_id, enr); + // Remove the ENR from the cache to prevent continual re-dialing on disconnects + for enr in peers_to_dial { + debug!(self.log, "Dialing cached ENR peer"; "peer_id" => %enr.peer_id()); + self.discovery_mut().remove_cached_enr(&enr.peer_id()); + self.peer_manager_mut().dial_peer(enr); } } @@ -1345,22 +1362,6 @@ impl Network { } } - /// Handle a discovery event. - fn inject_discovery_event( - &mut self, - event: DiscoveredPeers, - ) -> Option> { - let DiscoveredPeers { peers } = event; - let to_dial_peers = self.peer_manager_mut().peers_discovered(peers); - for peer_id in to_dial_peers { - debug!(self.log, "Dialing discovered peer"; "peer_id" => %peer_id); - // For any dial event, inform the peer manager - let enr = self.discovery_mut().enr_of_peer(&peer_id); - self.peer_manager_mut().dial_peer(&peer_id, enr); - } - None - } - /// Handle an identify event. fn inject_identify_event( &mut self, @@ -1461,7 +1462,14 @@ impl Network { BehaviourEvent::BannedPeers(void) => void::unreachable(void), BehaviourEvent::Gossipsub(ge) => self.inject_gs_event(ge), BehaviourEvent::Eth2Rpc(re) => self.inject_rpc_event(re), - BehaviourEvent::Discovery(de) => self.inject_discovery_event(de), + // Inform the peer manager about discovered peers. + // + // The peer manager will subsequently decide which peers need to be dialed and then dial + // them. + BehaviourEvent::Discovery(DiscoveredPeers { peers }) => { + self.peer_manager_mut().peers_discovered(peers); + None + } BehaviourEvent::Identify(ie) => self.inject_identify_event(ie), BehaviourEvent::PeerManager(pe) => self.inject_pm_event(pe), BehaviourEvent::ConnectionLimits(le) => void::unreachable(le), @@ -1493,7 +1501,7 @@ impl Network { format!("Dialing local peer id {endpoint:?}") } libp2p::swarm::ListenError::Denied { cause } => { - format!("Connection was denied with cause {cause}") + format!("Connection was denied with cause: {cause:?}") } libp2p::swarm::ListenError::Transport(t) => match t { libp2p::TransportError::MultiaddrNotSupported(m) => { @@ -1543,13 +1551,7 @@ impl Network { None } } - SwarmEvent::Dialing { - peer_id, - connection_id: _, - } => { - debug!(self.log, "Swarm Dialing"; "peer_id" => ?peer_id); - None - } + SwarmEvent::Dialing { .. } => None, }; if let Some(ev) = maybe_event { diff --git a/beacon_node/lighthouse_network/src/service/utils.rs b/beacon_node/lighthouse_network/src/service/utils.rs index b8acc4ed6d..b02a47fefe 100644 --- a/beacon_node/lighthouse_network/src/service/utils.rs +++ b/beacon_node/lighthouse_network/src/service/utils.rs @@ -4,11 +4,13 @@ use crate::types::{ error, EnrAttestationBitfield, EnrSyncCommitteeBitfield, GossipEncoding, GossipKind, }; use crate::{GossipTopic, NetworkConfig}; +use futures::future::Either; use libp2p::bandwidth::BandwidthSinks; use libp2p::core::{multiaddr::Multiaddr, muxing::StreamMuxerBox, transport::Boxed}; use libp2p::gossipsub; use libp2p::identity::{secp256k1, Keypair}; use libp2p::{core, noise, yamux, PeerId, Transport, TransportExt}; +use libp2p_quic; use prometheus_client::registry::Registry; use slog::{debug, warn}; use ssz::Decode; @@ -37,19 +39,12 @@ pub struct Context<'a> { type BoxedTransport = Boxed<(PeerId, StreamMuxerBox)>; -/// The implementation supports TCP/IP, WebSockets over TCP/IP, noise as the encryption layer, and -/// mplex as the multiplexing layer. +/// The implementation supports TCP/IP, QUIC (experimental) over UDP, noise as the encryption layer, and +/// mplex/yamux as the multiplexing layer (when using TCP). pub fn build_transport( local_private_key: Keypair, + quic_support: bool, ) -> std::io::Result<(BoxedTransport, Arc)> { - let tcp = libp2p::tcp::tokio::Transport::new(libp2p::tcp::Config::default().nodelay(true)); - let transport = libp2p::dns::TokioDnsConfig::system(tcp)?; - #[cfg(feature = "libp2p-websocket")] - let transport = { - let trans_clone = transport.clone(); - transport.or_transport(libp2p::websocket::WsConfig::new(trans_clone)) - }; - // mplex config let mut mplex_config = libp2p_mplex::MplexConfig::new(); mplex_config.set_max_buffer_size(256); @@ -58,18 +53,34 @@ pub fn build_transport( // yamux config let mut yamux_config = yamux::Config::default(); yamux_config.set_window_update_mode(yamux::WindowUpdateMode::on_read()); - let (transport, bandwidth) = transport + + // Creates the TCP transport layer + let tcp = libp2p::tcp::tokio::Transport::new(libp2p::tcp::Config::default().nodelay(true)) .upgrade(core::upgrade::Version::V1) .authenticate(generate_noise_config(&local_private_key)) .multiplex(core::upgrade::SelectUpgrade::new( yamux_config, mplex_config, )) - .timeout(Duration::from_secs(10)) - .boxed() - .with_bandwidth_logging(); + .timeout(Duration::from_secs(10)); + + let (transport, bandwidth) = if quic_support { + // Enables Quic + // The default quic configuration suits us for now. + let quic_config = libp2p_quic::Config::new(&local_private_key); + tcp.or_transport(libp2p_quic::tokio::Transport::new(quic_config)) + .map(|either_output, _| match either_output { + Either::Left((peer_id, muxer)) => (peer_id, StreamMuxerBox::new(muxer)), + Either::Right((peer_id, muxer)) => (peer_id, StreamMuxerBox::new(muxer)), + }) + .with_bandwidth_logging() + } else { + tcp.with_bandwidth_logging() + }; + + // // Enables DNS over the transport. + let transport = libp2p::dns::TokioDnsConfig::system(transport)?.boxed(); - // Authentication Ok((transport, bandwidth)) } diff --git a/beacon_node/lighthouse_network/src/types/globals.rs b/beacon_node/lighthouse_network/src/types/globals.rs index 97eaaa0051..b2b605e8ae 100644 --- a/beacon_node/lighthouse_network/src/types/globals.rs +++ b/beacon_node/lighthouse_network/src/types/globals.rs @@ -16,10 +16,6 @@ pub struct NetworkGlobals { pub peer_id: RwLock, /// Listening multiaddrs. pub listen_multiaddrs: RwLock>, - /// The TCP port that the libp2p service is listening on over Ipv4. - listen_port_tcp4: Option, - /// The TCP port that the libp2p service is listening on over Ipv6. - listen_port_tcp6: Option, /// The collection of known peers. pub peers: RwLock>, // The local meta data of our node. @@ -35,8 +31,6 @@ pub struct NetworkGlobals { impl NetworkGlobals { pub fn new( enr: Enr, - listen_port_tcp4: Option, - listen_port_tcp6: Option, local_metadata: MetaData, trusted_peers: Vec, disable_peer_scoring: bool, @@ -46,8 +40,6 @@ impl NetworkGlobals { local_enr: RwLock::new(enr.clone()), peer_id: RwLock::new(enr.peer_id()), listen_multiaddrs: RwLock::new(Vec::new()), - listen_port_tcp4, - listen_port_tcp6, local_metadata: RwLock::new(local_metadata), peers: RwLock::new(PeerDB::new(trusted_peers, disable_peer_scoring, log)), gossipsub_subscriptions: RwLock::new(HashSet::new()), @@ -72,16 +64,6 @@ impl NetworkGlobals { self.listen_multiaddrs.read().clone() } - /// Returns the libp2p TCP port that this node has been configured to listen on. - pub fn listen_port_tcp4(&self) -> Option { - self.listen_port_tcp4 - } - - /// Returns the UDP discovery port that this node has been configured to listen on. - pub fn listen_port_tcp6(&self) -> Option { - self.listen_port_tcp6 - } - /// Returns the number of libp2p connected peers. pub fn connected_peers(&self) -> usize { self.peers.read().connected_peer_ids().count() @@ -139,8 +121,6 @@ impl NetworkGlobals { let enr = discv5::enr::EnrBuilder::new("v4").build(&enr_key).unwrap(); NetworkGlobals::new( enr, - Some(9000), - None, MetaData::V2(MetaDataV2 { seq_number: 0, attnets: Default::default(), diff --git a/beacon_node/lighthouse_network/tests/common.rs b/beacon_node/lighthouse_network/tests/common.rs index ffb4b0078b..7b437fe7a6 100644 --- a/beacon_node/lighthouse_network/tests/common.rs +++ b/beacon_node/lighthouse_network/tests/common.rs @@ -67,15 +67,19 @@ pub fn build_log(level: slog::Level, enabled: bool) -> slog::Logger { } } -pub fn build_config(port: u16, mut boot_nodes: Vec) -> NetworkConfig { +pub fn build_config(mut boot_nodes: Vec) -> NetworkConfig { let mut config = NetworkConfig::default(); + + // Find unused ports by using the 0 port. + let port = 0; + + let random_path: u16 = rand::random(); let path = TempBuilder::new() - .prefix(&format!("libp2p_test{}", port)) + .prefix(&format!("libp2p_test_{}", random_path)) .tempdir() .unwrap(); - config.set_ipv4_listening_address(std::net::Ipv4Addr::UNSPECIFIED, port, port); - config.enr_udp4_port = if port == 0 { None } else { Some(port) }; + config.set_ipv4_listening_address(std::net::Ipv4Addr::UNSPECIFIED, port, port, port); config.enr_address = (Some(std::net::Ipv4Addr::LOCALHOST), None); config.boot_nodes_enr.append(&mut boot_nodes); config.network_dir = path.into_path(); @@ -95,8 +99,7 @@ pub async fn build_libp2p_instance( fork_name: ForkName, spec: &ChainSpec, ) -> Libp2pInstance { - let port = 0; - let config = build_config(port, boot_nodes); + let config = build_config(boot_nodes); // launch libp2p service let (signal, exit) = exit_future::signal(); @@ -123,6 +126,12 @@ pub fn get_enr(node: &LibP2PService) -> Enr { node.local_enr() } +// Protocol for the node pair connection. +pub enum Protocol { + Tcp, + Quic, +} + // Constructs a pair of nodes with separate loggers. The sender dials the receiver. // This returns a (sender, receiver) pair. #[allow(dead_code)] @@ -131,6 +140,7 @@ pub async fn build_node_pair( log: &slog::Logger, fork_name: ForkName, spec: &ChainSpec, + protocol: Protocol, ) -> (Libp2pInstance, Libp2pInstance) { let sender_log = log.new(o!("who" => "sender")); let receiver_log = log.new(o!("who" => "receiver")); @@ -142,14 +152,45 @@ pub async fn build_node_pair( let sender_fut = async { loop { if let NetworkEvent::NewListenAddr(addr) = sender.next_event().await { - return addr; + // Only end once we've listened on the protocol we care about + match protocol { + Protocol::Tcp => { + if addr.iter().any(|multiaddr_proto| { + matches!(multiaddr_proto, libp2p::multiaddr::Protocol::Tcp(_)) + }) { + return addr; + } + } + Protocol::Quic => { + if addr.iter().any(|multiaddr_proto| { + matches!(multiaddr_proto, libp2p::multiaddr::Protocol::QuicV1) + }) { + return addr; + } + } + } } } }; let receiver_fut = async { loop { if let NetworkEvent::NewListenAddr(addr) = receiver.next_event().await { - return addr; + match protocol { + Protocol::Tcp => { + if addr.iter().any(|multiaddr_proto| { + matches!(multiaddr_proto, libp2p::multiaddr::Protocol::Tcp(_)) + }) { + return addr; + } + } + Protocol::Quic => { + if addr.iter().any(|multiaddr_proto| { + matches!(multiaddr_proto, libp2p::multiaddr::Protocol::QuicV1) + }) { + return addr; + } + } + } } } }; diff --git a/beacon_node/lighthouse_network/tests/rpc_tests.rs b/beacon_node/lighthouse_network/tests/rpc_tests.rs index 05fa5ab854..795afd06b9 100644 --- a/beacon_node/lighthouse_network/tests/rpc_tests.rs +++ b/beacon_node/lighthouse_network/tests/rpc_tests.rs @@ -1,4 +1,8 @@ #![cfg(test)] + +mod common; + +use common::Protocol; use lighthouse_network::rpc::methods::*; use lighthouse_network::{rpc::max_rpc_size, NetworkEvent, ReportSource, Request, Response}; use slog::{debug, warn, Level}; @@ -14,8 +18,6 @@ use types::{ Slot, }; -mod common; - type E = MinimalEthSpec; /// Merge block with length < max_rpc_size. @@ -49,7 +51,7 @@ fn merge_block_large(fork_context: &ForkContext, spec: &ChainSpec) -> BeaconBloc // Tests the STATUS RPC message #[test] #[allow(clippy::single_match)] -fn test_status_rpc() { +fn test_tcp_status_rpc() { // set up the logging. The level and enabled logging or not let log_level = Level::Debug; let enable_logging = false; @@ -62,8 +64,14 @@ fn test_status_rpc() { rt.block_on(async { // get sender/receiver - let (mut sender, mut receiver) = - common::build_node_pair(Arc::downgrade(&rt), &log, ForkName::Base, &spec).await; + let (mut sender, mut receiver) = common::build_node_pair( + Arc::downgrade(&rt), + &log, + ForkName::Base, + &spec, + Protocol::Tcp, + ) + .await; // Dummy STATUS RPC message let rpc_request = Request::Status(StatusMessage { @@ -141,7 +149,7 @@ fn test_status_rpc() { // Tests a streamed BlocksByRange RPC Message #[test] #[allow(clippy::single_match)] -fn test_blocks_by_range_chunked_rpc() { +fn test_tcp_blocks_by_range_chunked_rpc() { // set up the logging. The level and enabled logging or not let log_level = Level::Debug; let enable_logging = false; @@ -156,8 +164,14 @@ fn test_blocks_by_range_chunked_rpc() { rt.block_on(async { // get sender/receiver - let (mut sender, mut receiver) = - common::build_node_pair(Arc::downgrade(&rt), &log, ForkName::Merge, &spec).await; + let (mut sender, mut receiver) = common::build_node_pair( + Arc::downgrade(&rt), + &log, + ForkName::Merge, + &spec, + Protocol::Tcp, + ) + .await; // BlocksByRange Request let rpc_request = Request::BlocksByRange(BlocksByRangeRequest::new(0, messages_to_send)); @@ -267,7 +281,7 @@ fn test_blocks_by_range_chunked_rpc() { // Tests rejection of blocks over `MAX_RPC_SIZE`. #[test] #[allow(clippy::single_match)] -fn test_blocks_by_range_over_limit() { +fn test_tcp_blocks_by_range_over_limit() { // set up the logging. The level and enabled logging or not let log_level = Level::Debug; let enable_logging = false; @@ -282,8 +296,14 @@ fn test_blocks_by_range_over_limit() { rt.block_on(async { // get sender/receiver - let (mut sender, mut receiver) = - common::build_node_pair(Arc::downgrade(&rt), &log, ForkName::Merge, &spec).await; + let (mut sender, mut receiver) = common::build_node_pair( + Arc::downgrade(&rt), + &log, + ForkName::Merge, + &spec, + Protocol::Tcp, + ) + .await; // BlocksByRange Request let rpc_request = Request::BlocksByRange(BlocksByRangeRequest::new(0, messages_to_send)); @@ -350,7 +370,7 @@ fn test_blocks_by_range_over_limit() { // Tests that a streamed BlocksByRange RPC Message terminates when all expected chunks were received #[test] -fn test_blocks_by_range_chunked_rpc_terminates_correctly() { +fn test_tcp_blocks_by_range_chunked_rpc_terminates_correctly() { // set up the logging. The level and enabled logging or not let log_level = Level::Debug; let enable_logging = false; @@ -366,8 +386,14 @@ fn test_blocks_by_range_chunked_rpc_terminates_correctly() { rt.block_on(async { // get sender/receiver - let (mut sender, mut receiver) = - common::build_node_pair(Arc::downgrade(&rt), &log, ForkName::Base, &spec).await; + let (mut sender, mut receiver) = common::build_node_pair( + Arc::downgrade(&rt), + &log, + ForkName::Base, + &spec, + Protocol::Tcp, + ) + .await; // BlocksByRange Request let rpc_request = Request::BlocksByRange(BlocksByRangeRequest::new(0, messages_to_send)); @@ -476,7 +502,7 @@ fn test_blocks_by_range_chunked_rpc_terminates_correctly() { // Tests an empty response to a BlocksByRange RPC Message #[test] #[allow(clippy::single_match)] -fn test_blocks_by_range_single_empty_rpc() { +fn test_tcp_blocks_by_range_single_empty_rpc() { // set up the logging. The level and enabled logging or not let log_level = Level::Trace; let enable_logging = false; @@ -488,8 +514,14 @@ fn test_blocks_by_range_single_empty_rpc() { rt.block_on(async { // get sender/receiver - let (mut sender, mut receiver) = - common::build_node_pair(Arc::downgrade(&rt), &log, ForkName::Base, &spec).await; + let (mut sender, mut receiver) = common::build_node_pair( + Arc::downgrade(&rt), + &log, + ForkName::Base, + &spec, + Protocol::Tcp, + ) + .await; // BlocksByRange Request let rpc_request = Request::BlocksByRange(BlocksByRangeRequest::new(0, 10)); @@ -576,7 +608,7 @@ fn test_blocks_by_range_single_empty_rpc() { // serves to test the snappy framing format as well. #[test] #[allow(clippy::single_match)] -fn test_blocks_by_root_chunked_rpc() { +fn test_tcp_blocks_by_root_chunked_rpc() { // set up the logging. The level and enabled logging or not let log_level = Level::Debug; let enable_logging = false; @@ -589,8 +621,14 @@ fn test_blocks_by_root_chunked_rpc() { let rt = Arc::new(Runtime::new().unwrap()); // get sender/receiver rt.block_on(async { - let (mut sender, mut receiver) = - common::build_node_pair(Arc::downgrade(&rt), &log, ForkName::Merge, &spec).await; + let (mut sender, mut receiver) = common::build_node_pair( + Arc::downgrade(&rt), + &log, + ForkName::Merge, + &spec, + Protocol::Tcp, + ) + .await; // BlocksByRoot Request let rpc_request = @@ -702,7 +740,7 @@ fn test_blocks_by_root_chunked_rpc() { // Tests a streamed, chunked BlocksByRoot RPC Message terminates when all expected reponses have been received #[test] -fn test_blocks_by_root_chunked_rpc_terminates_correctly() { +fn test_tcp_blocks_by_root_chunked_rpc_terminates_correctly() { // set up the logging. The level and enabled logging or not let log_level = Level::Debug; let enable_logging = false; @@ -716,8 +754,14 @@ fn test_blocks_by_root_chunked_rpc_terminates_correctly() { let rt = Arc::new(Runtime::new().unwrap()); // get sender/receiver rt.block_on(async { - let (mut sender, mut receiver) = - common::build_node_pair(Arc::downgrade(&rt), &log, ForkName::Base, &spec).await; + let (mut sender, mut receiver) = common::build_node_pair( + Arc::downgrade(&rt), + &log, + ForkName::Base, + &spec, + Protocol::Tcp, + ) + .await; // BlocksByRoot Request let rpc_request = @@ -833,14 +877,9 @@ fn test_blocks_by_root_chunked_rpc_terminates_correctly() { }) } -// Tests a Goodbye RPC message -#[test] -#[allow(clippy::single_match)] -fn test_goodbye_rpc() { - // set up the logging. The level and enabled logging or not - let log_level = Level::Trace; - let enable_logging = false; - +/// Establishes a pair of nodes and disconnects the pair based on the selected protocol via an RPC +/// Goodbye message. +fn goodbye_test(log_level: Level, enable_logging: bool, protocol: Protocol) { let log = common::build_log(log_level, enable_logging); let rt = Arc::new(Runtime::new().unwrap()); @@ -850,7 +889,8 @@ fn test_goodbye_rpc() { // get sender/receiver rt.block_on(async { let (mut sender, mut receiver) = - common::build_node_pair(Arc::downgrade(&rt), &log, ForkName::Base, &spec).await; + common::build_node_pair(Arc::downgrade(&rt), &log, ForkName::Base, &spec, protocol) + .await; // build the sender future let sender_future = async { @@ -876,12 +916,9 @@ fn test_goodbye_rpc() { // build the receiver future let receiver_future = async { loop { - match receiver.next_event().await { - NetworkEvent::PeerDisconnected(_) => { - // Should receive sent RPC request - return; - } - _ => {} // Ignore other events + if let NetworkEvent::PeerDisconnected(_) = receiver.next_event().await { + // Should receive sent RPC request + return; } } }; @@ -896,3 +933,23 @@ fn test_goodbye_rpc() { } }) } + +// Tests a Goodbye RPC message +#[test] +#[allow(clippy::single_match)] +fn tcp_test_goodbye_rpc() { + // set up the logging. The level and enabled logging or not + let log_level = Level::Debug; + let enable_logging = true; + goodbye_test(log_level, enable_logging, Protocol::Tcp); +} + +// Tests a Goodbye RPC message +#[test] +#[allow(clippy::single_match)] +fn quic_test_goodbye_rpc() { + // set up the logging. The level and enabled logging or not + let log_level = Level::Debug; + let enable_logging = true; + goodbye_test(log_level, enable_logging, Protocol::Quic); +} diff --git a/beacon_node/network/src/nat.rs b/beacon_node/network/src/nat.rs index 9bf123e8de..d011ac42e8 100644 --- a/beacon_node/network/src/nat.rs +++ b/beacon_node/network/src/nat.rs @@ -12,20 +12,49 @@ use types::EthSpec; /// Configuration required to construct the UPnP port mappings. pub struct UPnPConfig { - /// The local tcp port. + /// The local TCP port. tcp_port: u16, - /// The local udp port. - udp_port: u16, + /// The local UDP discovery port. + disc_port: u16, + /// The local UDP quic port. + quic_port: u16, /// Whether discovery is enabled or not. disable_discovery: bool, + /// Whether quic is enabled or not. + disable_quic_support: bool, +} + +/// Contains mappings that managed to be established. +#[derive(Default, Debug)] +pub struct EstablishedUPnPMappings { + /// A TCP port mapping for libp2p. + pub tcp_port: Option, + /// A UDP port for the QUIC libp2p transport. + pub udp_quic_port: Option, + /// A UDP port for discv5. + pub udp_disc_port: Option, +} + +impl EstablishedUPnPMappings { + /// Returns true if at least one value is set. + pub fn is_some(&self) -> bool { + self.tcp_port.is_some() || self.udp_quic_port.is_some() || self.udp_disc_port.is_some() + } + + // Iterator over the UDP ports + pub fn udp_ports(&self) -> impl Iterator { + self.udp_quic_port.iter().chain(self.udp_disc_port.iter()) + } } impl UPnPConfig { pub fn from_config(config: &NetworkConfig) -> Option { config.listen_addrs().v4().map(|v4_addr| UPnPConfig { tcp_port: v4_addr.tcp_port, - udp_port: v4_addr.udp_port, + disc_port: v4_addr.disc_port, + quic_port: v4_addr.quic_port, disable_discovery: config.disable_discovery, + disable_quic_support: config.disable_quic_support, }) } } @@ -68,6 +97,8 @@ pub fn construct_upnp_mappings( debug!(log, "UPnP Local IP Discovered"; "ip" => ?local_ip); + let mut mappings = EstablishedUPnPMappings::default(); + match local_ip { IpAddr::V4(address) => { let libp2p_socket = SocketAddrV4::new(address, config.tcp_port); @@ -76,39 +107,46 @@ pub fn construct_upnp_mappings( // one. // I've found this to be more reliable. If multiple users are behind a single // router, they should ideally try to set different port numbers. - let tcp_socket = add_port_mapping( + mappings.tcp_port = add_port_mapping( &gateway, igd::PortMappingProtocol::TCP, libp2p_socket, "tcp", &log, - ).and_then(|_| { + ).map(|_| { let external_socket = external_ip.as_ref().map(|ip| SocketAddr::new((*ip).into(), config.tcp_port)).map_err(|_| ()); info!(log, "UPnP TCP route established"; "external_socket" => format!("{}:{}", external_socket.as_ref().map(|ip| ip.to_string()).unwrap_or_else(|_| "".into()), config.tcp_port)); - external_socket + config.tcp_port }).ok(); - let udp_socket = if !config.disable_discovery { - let discovery_socket = SocketAddrV4::new(address, config.udp_port); + let set_udp_mapping = |udp_port| { + let udp_socket = SocketAddrV4::new(address, udp_port); add_port_mapping( &gateway, igd::PortMappingProtocol::UDP, - discovery_socket, + udp_socket, "udp", &log, - ).and_then(|_| { - let external_socket = external_ip - .map(|ip| SocketAddr::new(ip.into(), config.udp_port)).map_err(|_| ()); - info!(log, "UPnP UDP route established"; "external_socket" => format!("{}:{}", external_socket.as_ref().map(|ip| ip.to_string()).unwrap_or_else(|_| "".into()), config.udp_port)); - external_socket - }).ok() - } else { - None + ).map(|_| { + info!(log, "UPnP UDP route established"; "external_socket" => format!("{}:{}", external_ip.as_ref().map(|ip| ip.to_string()).unwrap_or_else(|_| "".into()), udp_port)); + }) }; + // Set the discovery UDP port mapping + if !config.disable_discovery && set_udp_mapping(config.disc_port).is_ok() { + mappings.udp_disc_port = Some(config.disc_port); + } + + // Set the quic UDP port mapping + if !config.disable_quic_support && set_udp_mapping(config.quic_port).is_ok() { + mappings.udp_quic_port = Some(config.quic_port); + } + // report any updates to the network service. - network_send.send(NetworkMessage::UPnPMappingEstablished{ tcp_socket, udp_socket }) - .unwrap_or_else(|e| debug!(log, "Could not send message to the network service"; "error" => %e)); + if mappings.is_some() { + network_send.send(NetworkMessage::UPnPMappingEstablished{ mappings }) + .unwrap_or_else(|e| debug!(log, "Could not send message to the network service"; "error" => %e)); + } } _ => debug!(log, "UPnP no routes constructed. IPv6 not supported"), } @@ -161,12 +199,12 @@ fn add_port_mapping( } /// Removes the specified TCP and UDP port mappings. -pub fn remove_mappings(tcp_port: Option, udp_port: Option, log: &slog::Logger) { - if tcp_port.is_some() || udp_port.is_some() { +pub fn remove_mappings(mappings: &EstablishedUPnPMappings, log: &slog::Logger) { + if mappings.is_some() { debug!(log, "Removing UPnP port mappings"); match igd::search_gateway(Default::default()) { Ok(gateway) => { - if let Some(tcp_port) = tcp_port { + if let Some(tcp_port) = mappings.tcp_port { match gateway.remove_port(igd::PortMappingProtocol::TCP, tcp_port) { Ok(()) => debug!(log, "UPnP Removed TCP port mapping"; "port" => tcp_port), Err(e) => { @@ -174,8 +212,8 @@ pub fn remove_mappings(tcp_port: Option, udp_port: Option, log: &slog: } } } - if let Some(udp_port) = udp_port { - match gateway.remove_port(igd::PortMappingProtocol::UDP, udp_port) { + for udp_port in mappings.udp_ports() { + match gateway.remove_port(igd::PortMappingProtocol::UDP, *udp_port) { Ok(()) => debug!(log, "UPnP Removed UDP port mapping"; "port" => udp_port), Err(e) => { debug!(log, "UPnP Failed to remove UDP port mapping"; "port" => udp_port, "error" => %e) diff --git a/beacon_node/network/src/network_beacon_processor/tests.rs b/beacon_node/network/src/network_beacon_processor/tests.rs index a678edbf1f..ac5722a565 100644 --- a/beacon_node/network/src/network_beacon_processor/tests.rs +++ b/beacon_node/network/src/network_beacon_processor/tests.rs @@ -37,7 +37,6 @@ const VALIDATOR_COUNT: usize = SLOTS_PER_EPOCH as usize; const SMALL_CHAIN: u64 = 2; const LONG_CHAIN: u64 = SLOTS_PER_EPOCH * 2; -const TCP_PORT: u16 = 42; const SEQ_NUMBER: u64 = 0; /// The default time to wait for `BeaconProcessor` events. @@ -195,15 +194,7 @@ impl TestRig { }); let enr_key = CombinedKey::generate_secp256k1(); let enr = EnrBuilder::new("v4").build(&enr_key).unwrap(); - let network_globals = Arc::new(NetworkGlobals::new( - enr, - Some(TCP_PORT), - None, - meta_data, - vec![], - false, - &log, - )); + let network_globals = Arc::new(NetworkGlobals::new(enr, meta_data, vec![], false, &log)); let executor = harness.runtime.task_executor.clone(); diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index c355c671e8..174a0ec14c 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -1,4 +1,5 @@ use super::sync::manager::RequestId as SyncId; +use crate::nat::EstablishedUPnPMappings; use crate::network_beacon_processor::InvalidBlockStorage; use crate::persisted_dht::{clear_dht, load_dht, persist_dht}; use crate::router::{Router, RouterMessage}; @@ -26,7 +27,7 @@ use lighthouse_network::{ MessageId, NetworkEvent, NetworkGlobals, PeerId, }; use slog::{crit, debug, error, info, o, trace, warn}; -use std::{collections::HashSet, net::SocketAddr, pin::Pin, sync::Arc, time::Duration}; +use std::{collections::HashSet, pin::Pin, sync::Arc, time::Duration}; use store::HotColdDB; use strum::IntoStaticStr; use task_executor::ShutdownReason; @@ -93,12 +94,10 @@ pub enum NetworkMessage { /// The result of the validation validation_result: MessageAcceptance, }, - /// Called if a known external TCP socket address has been updated. + /// Called if UPnP managed to establish an external port mapping. UPnPMappingEstablished { - /// The external TCP address has been updated. - tcp_socket: Option, - /// The external UDP address has been updated. - udp_socket: Option, + /// The mappings that were established. + mappings: EstablishedUPnPMappings, }, /// Reports a peer to the peer manager for performing an action. ReportPeer { @@ -190,11 +189,8 @@ pub struct NetworkService { /// A collection of global variables, accessible outside of the network service. network_globals: Arc>, /// Stores potentially created UPnP mappings to be removed on shutdown. (TCP port and UDP - /// port). - upnp_mappings: (Option, Option), - /// Keeps track of if discovery is auto-updating or not. This is used to inform us if we should - /// update the UDP socket of discovery if the UPnP mappings get established. - discovery_auto_update: bool, + /// ports). + upnp_mappings: EstablishedUPnPMappings, /// A delay that expires when a new fork takes place. next_fork_update: Pin>>, /// A delay that expires when we need to subscribe to a new fork's topics. @@ -359,8 +355,7 @@ impl NetworkService { router_send, store, network_globals: network_globals.clone(), - upnp_mappings: (None, None), - discovery_auto_update: config.discv5_config.enr_update, + upnp_mappings: EstablishedUPnPMappings::default(), next_fork_update, next_fork_subscriptions, next_unsubscribe, @@ -616,32 +611,18 @@ impl NetworkService { } => { self.libp2p.send_error_reponse(peer_id, id, error, reason); } - NetworkMessage::UPnPMappingEstablished { - tcp_socket, - udp_socket, - } => { - self.upnp_mappings = (tcp_socket.map(|s| s.port()), udp_socket.map(|s| s.port())); + NetworkMessage::UPnPMappingEstablished { mappings } => { + self.upnp_mappings = mappings; // If there is an external TCP port update, modify our local ENR. - if let Some(tcp_socket) = tcp_socket { - if let Err(e) = self - .libp2p - .discovery_mut() - .update_enr_tcp_port(tcp_socket.port()) - { + if let Some(tcp_port) = self.upnp_mappings.tcp_port { + if let Err(e) = self.libp2p.discovery_mut().update_enr_tcp_port(tcp_port) { warn!(self.log, "Failed to update ENR"; "error" => e); } } - // if the discovery service is not auto-updating, update it with the - // UPnP mappings - if !self.discovery_auto_update { - if let Some(udp_socket) = udp_socket { - if let Err(e) = self - .libp2p - .discovery_mut() - .update_enr_udp_socket(udp_socket) - { - warn!(self.log, "Failed to update ENR"; "error" => e); - } + // If there is an external QUIC port update, modify our local ENR. + if let Some(quic_port) = self.upnp_mappings.udp_quic_port { + if let Err(e) = self.libp2p.discovery_mut().update_enr_quic_port(quic_port) { + warn!(self.log, "Failed to update ENR"; "error" => e); } } } @@ -994,7 +975,7 @@ impl Drop for NetworkService { } // attempt to remove port mappings - crate::nat::remove_mappings(self.upnp_mappings.0, self.upnp_mappings.1, &self.log); + crate::nat::remove_mappings(&self.upnp_mappings, &self.log); info!(self.log, "Network service shutdown"); } diff --git a/beacon_node/network/src/service/tests.rs b/beacon_node/network/src/service/tests.rs index 67f62ff90d..23bcf456de 100644 --- a/beacon_node/network/src/service/tests.rs +++ b/beacon_node/network/src/service/tests.rs @@ -60,7 +60,7 @@ mod tests { ); let mut config = NetworkConfig::default(); - config.set_ipv4_listening_address(std::net::Ipv4Addr::UNSPECIFIED, 21212, 21212); + config.set_ipv4_listening_address(std::net::Ipv4Addr::UNSPECIFIED, 21212, 21212, 21213); config.discv5_config.table_filter = |_| true; // Do not ignore local IPs config.upnp_enabled = false; config.boot_nodes_enr = enrs.clone(); diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index 837625e12a..ffc29365cb 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -75,11 +75,11 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .help("The address lighthouse will listen for UDP and TCP connections. To listen \ over IpV4 and IpV6 set this flag twice with the different values.\n\ Examples:\n\ - - --listen-address '0.0.0.0' will listen over Ipv4.\n\ - - --listen-address '::' will listen over Ipv6.\n\ + - --listen-address '0.0.0.0' will listen over IPv4.\n\ + - --listen-address '::' will listen over IPv6.\n\ - --listen-address '0.0.0.0' --listen-address '::' will listen over both \ - Ipv4 and Ipv6. The order of the given addresses is not relevant. However, \ - multiple Ipv4, or multiple Ipv6 addresses will not be accepted.") + IPv4 and IPv6. The order of the given addresses is not relevant. However, \ + multiple IPv4, or multiple IPv6 addresses will not be accepted.") .multiple(true) .max_values(2) .default_value("0.0.0.0") @@ -89,9 +89,10 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { Arg::with_name("port") .long("port") .value_name("PORT") - .help("The TCP/UDP port to listen on. The UDP port can be modified by the \ - --discovery-port flag. If listening over both Ipv4 and Ipv6 the --port flag \ - will apply to the Ipv4 address and --port6 to the Ipv6 address.") + .help("The TCP/UDP ports to listen on. There are two UDP ports. \ + The discovery UDP port will be set to this value and the Quic UDP port will be set to this value + 1. The discovery port can be modified by the \ + --discovery-port flag and the quic port can be modified by the --quic-port flag. If listening over both IPv4 and IPv6 the --port flag \ + will apply to the IPv4 address and --port6 to the IPv6 address.") .default_value("9000") .takes_value(true), ) @@ -99,8 +100,8 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { Arg::with_name("port6") .long("port6") .value_name("PORT") - .help("The TCP/UDP port to listen on over IpV6 when listening over both Ipv4 and \ - Ipv6. Defaults to 9090 when required.") + .help("The TCP/UDP ports to listen on over IPv6 when listening over both IPv4 and \ + IPv6. Defaults to 9090 when required. The Quic UDP port will be set to this value + 1.") .default_value("9090") .takes_value(true), ) @@ -111,12 +112,27 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .help("The UDP port that discovery will listen on. Defaults to `port`") .takes_value(true), ) + .arg( + Arg::with_name("quic-port") + .long("quic-port") + .value_name("PORT") + .help("The UDP port that quic will listen on. Defaults to `port` + 1") + .takes_value(true), + ) .arg( Arg::with_name("discovery-port6") .long("discovery-port6") .value_name("PORT") - .help("The UDP port that discovery will listen on over IpV6 if listening over \ - both Ipv4 and IpV6. Defaults to `port6`") + .help("The UDP port that discovery will listen on over IPv6 if listening over \ + both IPv4 and IPv6. Defaults to `port6`") + .takes_value(true), + ) + .arg( + Arg::with_name("quic-port6") + .long("quic-port6") + .value_name("PORT") + .help("The UDP port that quic will listen on over IPv6 if listening over \ + both IPv4 and IPv6. Defaults to `port6` + 1") .takes_value(true), ) .arg( @@ -159,7 +175,15 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .long("enr-udp-port") .value_name("PORT") .help("The UDP4 port of the local ENR. Set this only if you are sure other nodes \ - can connect to your local node on this port over IpV4.") + can connect to your local node on this port over IPv4.") + .takes_value(true), + ) + .arg( + Arg::with_name("enr-quic-port") + .long("enr-quic-port") + .value_name("PORT") + .help("The quic UDP4 port that will be set on the local ENR. Set this only if you are sure other nodes \ + can connect to your local node on this port over IPv4.") .takes_value(true), ) .arg( @@ -167,7 +191,15 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .long("enr-udp6-port") .value_name("PORT") .help("The UDP6 port of the local ENR. Set this only if you are sure other nodes \ - can connect to your local node on this port over IpV6.") + can connect to your local node on this port over IPv6.") + .takes_value(true), + ) + .arg( + Arg::with_name("enr-quic6-port") + .long("enr-quic6-port") + .value_name("PORT") + .help("The quic UDP6 port that will be set on the local ENR. Set this only if you are sure other nodes \ + can connect to your local node on this port over IPv6.") .takes_value(true), ) .arg( @@ -175,7 +207,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .long("enr-tcp-port") .value_name("PORT") .help("The TCP4 port of the local ENR. Set this only if you are sure other nodes \ - can connect to your local node on this port over IpV4. The --port flag is \ + can connect to your local node on this port over IPv4. The --port flag is \ used if this is not set.") .takes_value(true), ) @@ -184,7 +216,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .long("enr-tcp6-port") .value_name("PORT") .help("The TCP6 port of the local ENR. Set this only if you are sure other nodes \ - can connect to your local node on this port over IpV6. The --port6 flag is \ + can connect to your local node on this port over IPv6. The --port6 flag is \ used if this is not set.") .takes_value(true), ) @@ -225,11 +257,18 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { without an ENR.") .takes_value(true), ) + // NOTE: This is hidden because it is primarily a developer feature for testnets and + // debugging. We remove it from the list to avoid clutter. .arg( Arg::with_name("disable-discovery") .long("disable-discovery") .help("Disables the discv5 discovery protocol. The node will not search for new peers or participate in the discovery protocol.") - .takes_value(false), + .hidden(true) + ) + .arg( + Arg::with_name("disable-quic") + .long("disable-quic") + .help("Disables the quic transport. The node will rely solely on the TCP transport for libp2p connections.") ) .arg( Arg::with_name("disable-peer-scoring") diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 70495777e2..b6d0b75d9f 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -914,15 +914,15 @@ pub fn parse_listening_addresses( .map_err(|parse_error| format!("Failed to parse --port6 as an integer: {parse_error}"))? .unwrap_or(9090); - // parse the possible udp ports - let maybe_udp_port = cli_args + // parse the possible discovery ports. + let maybe_disc_port = cli_args .value_of("discovery-port") .map(str::parse::) .transpose() .map_err(|parse_error| { format!("Failed to parse --discovery-port as an integer: {parse_error}") })?; - let maybe_udp6_port = cli_args + let maybe_disc6_port = cli_args .value_of("discovery-port6") .map(str::parse::) .transpose() @@ -930,6 +930,24 @@ pub fn parse_listening_addresses( format!("Failed to parse --discovery-port6 as an integer: {parse_error}") })?; + // parse the possible quic port. + let maybe_quic_port = cli_args + .value_of("quic-port") + .map(str::parse::) + .transpose() + .map_err(|parse_error| { + format!("Failed to parse --quic-port as an integer: {parse_error}") + })?; + + // parse the possible quic port. + let maybe_quic6_port = cli_args + .value_of("quic-port6") + .map(str::parse::) + .transpose() + .map_err(|parse_error| { + format!("Failed to parse --quic6-port as an integer: {parse_error}") + })?; + // Now put everything together let listening_addresses = match (maybe_ipv4, maybe_ipv6) { (None, None) => { @@ -940,7 +958,7 @@ pub fn parse_listening_addresses( // A single ipv6 address was provided. Set the ports if cli_args.is_present("port6") { - warn!(log, "When listening only over IpV6, use the --port flag. The value of --port6 will be ignored.") + warn!(log, "When listening only over IPv6, use the --port flag. The value of --port6 will be ignored.") } // use zero ports if required. If not, use the given port. let tcp_port = use_zero_ports @@ -948,20 +966,32 @@ pub fn parse_listening_addresses( .transpose()? .unwrap_or(port); - if maybe_udp6_port.is_some() { - warn!(log, "When listening only over IpV6, use the --discovery-port flag. The value of --discovery-port6 will be ignored.") + if maybe_disc6_port.is_some() { + warn!(log, "When listening only over IPv6, use the --discovery-port flag. The value of --discovery-port6 will be ignored.") } + + if maybe_quic6_port.is_some() { + warn!(log, "When listening only over IPv6, use the --quic-port flag. The value of --quic-port6 will be ignored.") + } + // use zero ports if required. If not, use the specific udp port. If none given, use // the tcp port. - let udp_port = use_zero_ports + let disc_port = use_zero_ports .then(unused_port::unused_udp6_port) .transpose()? - .or(maybe_udp_port) + .or(maybe_disc_port) .unwrap_or(port); + let quic_port = use_zero_ports + .then(unused_port::unused_udp6_port) + .transpose()? + .or(maybe_quic_port) + .unwrap_or(port + 1); + ListenAddress::V6(lighthouse_network::ListenAddr { addr: ipv6, - udp_port, + quic_port, + disc_port, tcp_port, }) } @@ -973,16 +1003,25 @@ pub fn parse_listening_addresses( .then(unused_port::unused_tcp4_port) .transpose()? .unwrap_or(port); - // use zero ports if required. If not, use the specific udp port. If none given, use + // use zero ports if required. If not, use the specific discovery port. If none given, use // the tcp port. - let udp_port = use_zero_ports + let disc_port = use_zero_ports .then(unused_port::unused_udp4_port) .transpose()? - .or(maybe_udp_port) + .or(maybe_disc_port) .unwrap_or(port); + // use zero ports if required. If not, use the specific quic port. If none given, use + // the tcp port + 1. + let quic_port = use_zero_ports + .then(unused_port::unused_udp4_port) + .transpose()? + .or(maybe_quic_port) + .unwrap_or(port + 1); + ListenAddress::V4(lighthouse_network::ListenAddr { addr: ipv4, - udp_port, + disc_port, + quic_port, tcp_port, }) } @@ -991,31 +1030,44 @@ pub fn parse_listening_addresses( .then(unused_port::unused_tcp4_port) .transpose()? .unwrap_or(port); - let ipv4_udp_port = use_zero_ports + let ipv4_disc_port = use_zero_ports .then(unused_port::unused_udp4_port) .transpose()? - .or(maybe_udp_port) + .or(maybe_disc_port) .unwrap_or(ipv4_tcp_port); + let ipv4_quic_port = use_zero_ports + .then(unused_port::unused_udp4_port) + .transpose()? + .or(maybe_quic_port) + .unwrap_or(port + 1); // Defaults to 9090 when required let ipv6_tcp_port = use_zero_ports .then(unused_port::unused_tcp6_port) .transpose()? .unwrap_or(port6); - let ipv6_udp_port = use_zero_ports + let ipv6_disc_port = use_zero_ports .then(unused_port::unused_udp6_port) .transpose()? - .or(maybe_udp6_port) + .or(maybe_disc6_port) .unwrap_or(ipv6_tcp_port); + let ipv6_quic_port = use_zero_ports + .then(unused_port::unused_udp6_port) + .transpose()? + .or(maybe_quic6_port) + .unwrap_or(ipv6_tcp_port + 1); + ListenAddress::DualStack( lighthouse_network::ListenAddr { addr: ipv4, - udp_port: ipv4_udp_port, + disc_port: ipv4_disc_port, + quic_port: ipv4_quic_port, tcp_port: ipv4_tcp_port, }, lighthouse_network::ListenAddr { addr: ipv6, - udp_port: ipv6_udp_port, + disc_port: ipv6_disc_port, + quic_port: ipv6_quic_port, tcp_port: ipv6_tcp_port, }, ) @@ -1131,6 +1183,14 @@ pub fn set_network_config( ); } + if let Some(enr_quic_port_str) = cli_args.value_of("enr-quic-port") { + config.enr_quic4_port = Some( + enr_quic_port_str + .parse::() + .map_err(|_| format!("Invalid quic port: {}", enr_quic_port_str))?, + ); + } + if let Some(enr_tcp_port_str) = cli_args.value_of("enr-tcp-port") { config.enr_tcp4_port = Some( enr_tcp_port_str @@ -1147,6 +1207,14 @@ pub fn set_network_config( ); } + if let Some(enr_quic_port_str) = cli_args.value_of("enr-quic6-port") { + config.enr_quic6_port = Some( + enr_quic_port_str + .parse::() + .map_err(|_| format!("Invalid quic port: {}", enr_quic_port_str))?, + ); + } + if let Some(enr_tcp_port_str) = cli_args.value_of("enr-tcp6-port") { config.enr_tcp6_port = Some( enr_tcp_port_str @@ -1156,9 +1224,9 @@ pub fn set_network_config( } if cli_args.is_present("enr-match") { - // Match the Ip and UDP port in the enr. + // Match the IP and UDP port in the ENR. - // set the enr address to localhost if the address is unspecified + // Set the ENR address to localhost if the address is unspecified. if let Some(ipv4_addr) = config.listen_addrs().v4().cloned() { let ipv4_enr_addr = if ipv4_addr.addr == Ipv4Addr::UNSPECIFIED { Ipv4Addr::LOCALHOST @@ -1166,7 +1234,7 @@ pub fn set_network_config( ipv4_addr.addr }; config.enr_address.0 = Some(ipv4_enr_addr); - config.enr_udp4_port = Some(ipv4_addr.udp_port); + config.enr_udp4_port = Some(ipv4_addr.disc_port); } if let Some(ipv6_addr) = config.listen_addrs().v6().cloned() { @@ -1176,7 +1244,7 @@ pub fn set_network_config( ipv6_addr.addr }; config.enr_address.1 = Some(ipv6_enr_addr); - config.enr_udp6_port = Some(ipv6_addr.udp_port); + config.enr_udp6_port = Some(ipv6_addr.disc_port); } } @@ -1209,11 +1277,11 @@ pub fn set_network_config( // actually matters. Just use the udp port. let port = match config.listen_addrs() { - ListenAddress::V4(v4_addr) => v4_addr.udp_port, - ListenAddress::V6(v6_addr) => v6_addr.udp_port, + ListenAddress::V4(v4_addr) => v4_addr.disc_port, + ListenAddress::V6(v6_addr) => v6_addr.disc_port, ListenAddress::DualStack(v4_addr, _v6_addr) => { // NOTE: slight preference for ipv4 that I don't think is of importance. - v4_addr.udp_port + v4_addr.disc_port } }; @@ -1272,6 +1340,10 @@ pub fn set_network_config( warn!(log, "Discovery is disabled. New peers will not be found"); } + if cli_args.is_present("disable-quic") { + config.disable_quic_support = true; + } + if cli_args.is_present("disable-upnp") { config.upnp_enabled = false; } diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index 507896f431..3f58d8aa45 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -56,3 +56,4 @@ * [Contributing](./contributing.md) * [Development Environment](./setup.md) * [FAQs](./faq.md) +* [Protocol Developers](./developers.md) \ No newline at end of file diff --git a/book/src/developers.md b/book/src/developers.md new file mode 100644 index 0000000000..2ba09bd341 --- /dev/null +++ b/book/src/developers.md @@ -0,0 +1,51 @@ +# For Protocol Developers + +_Documentation for protocol developers._ + +This section lists Lighthouse-specific decisions that are not strictly spec'd and may be useful for +other protocol developers wishing to interact with lighthouse. + + +## Custom ENR Fields + +Lighthouse currently uses the following ENR fields: + +### Ethereum Consensus Specified + +| Field | Description | +| ---- | ---- | +| `eth2` | The `ENRForkId` in SSZ bytes specifying which fork the node is on | +| `attnets` | An SSZ bitfield which indicates which of the 64 subnets the node is subscribed to for an extended period of time | +| `syncnets` | An SSZ bitfield which indicates which of the sync committee subnets the node is subscribed to | + + +### Lighthouse Custom Fields + +Lighthouse is currently using the following custom ENR fields. +| Field | Description | +| ---- | ---- | +| `quic` | The UDP port on which the QUIC transport is listening on IPv4 | +| `quic6` | The UDP port on which the QUIC transport is listening on IPv6 | + + +## Custom RPC Messages + +The specification leaves room for implementation-specific errors. Lighthouse uses the following +custom RPC error messages. + +### Goodbye Reason Codes + +| Code | Message | Description | +| ---- | ---- | ---- | +| 128 | Unable to Verify Network | Teku uses this, so we adopted it. It relates to having a fork mismatch | +| 129 | Too Many Peers | Lighthouse can close a connection because it has reached its peer-limit and pruned excess peers | +| 250 | Bad Score | The node has been dropped due to having a bad peer score | +| 251 | Banned | The peer has been banned and disconnected | +| 252 | Banned IP | The IP the node is connected to us with has been banned | + + +### Error Codes + +| Code | Message | Description | +| ---- | ---- | ---- | +| 139 | Rate Limited | The peer has been rate limited so we return this error as a response | \ No newline at end of file diff --git a/boot_node/src/config.rs b/boot_node/src/config.rs index 779269921a..817bd2ab52 100644 --- a/boot_node/src/config.rs +++ b/boot_node/src/config.rs @@ -58,12 +58,12 @@ impl BootNodeConfig { set_network_config(&mut network_config, matches, &data_dir, &logger)?; - // Set the Enr UDP ports to the listening ports if not present. + // Set the Enr Discovery ports to the listening ports if not present. if let Some(listening_addr_v4) = network_config.listen_addrs().v4() { network_config.enr_udp4_port = Some( network_config .enr_udp4_port - .unwrap_or(listening_addr_v4.udp_port), + .unwrap_or(listening_addr_v4.disc_port), ) }; @@ -71,7 +71,7 @@ impl BootNodeConfig { network_config.enr_udp6_port = Some( network_config .enr_udp6_port - .unwrap_or(listening_addr_v6.udp_port), + .unwrap_or(listening_addr_v6.disc_port), ) }; diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 05b4358509..10fcb101a8 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -1010,12 +1010,12 @@ fn network_port_flag_over_ipv4() { .run() .with_config(|config| { assert_eq!( - config - .network - .listen_addrs() - .v4() - .map(|listen_addr| (listen_addr.udp_port, listen_addr.tcp_port)), - Some((port, port)) + config.network.listen_addrs().v4().map(|listen_addr| ( + listen_addr.disc_port, + listen_addr.quic_port, + listen_addr.tcp_port + )), + Some((port, port + 1, port)) ); }); } @@ -1028,22 +1028,22 @@ fn network_port_flag_over_ipv6() { .run() .with_config(|config| { assert_eq!( - config - .network - .listen_addrs() - .v6() - .map(|listen_addr| (listen_addr.udp_port, listen_addr.tcp_port)), - Some((port, port)) + config.network.listen_addrs().v6().map(|listen_addr| ( + listen_addr.disc_port, + listen_addr.quic_port, + listen_addr.tcp_port + )), + Some((port, port + 1, port)) ); }); } #[test] fn network_port_and_discovery_port_flags_over_ipv4() { let tcp4_port = unused_tcp4_port().expect("Unable to find unused port."); - let udp4_port = unused_udp4_port().expect("Unable to find unused port."); + let disc4_port = unused_udp4_port().expect("Unable to find unused port."); CommandLineTest::new() .flag("port", Some(tcp4_port.to_string().as_str())) - .flag("discovery-port", Some(udp4_port.to_string().as_str())) + .flag("discovery-port", Some(disc4_port.to_string().as_str())) .run() .with_config(|config| { assert_eq!( @@ -1051,19 +1051,19 @@ fn network_port_and_discovery_port_flags_over_ipv4() { .network .listen_addrs() .v4() - .map(|listen_addr| (listen_addr.tcp_port, listen_addr.udp_port)), - Some((tcp4_port, udp4_port)) + .map(|listen_addr| (listen_addr.tcp_port, listen_addr.disc_port)), + Some((tcp4_port, disc4_port)) ); }); } #[test] fn network_port_and_discovery_port_flags_over_ipv6() { let tcp6_port = unused_tcp6_port().expect("Unable to find unused port."); - let udp6_port = unused_udp6_port().expect("Unable to find unused port."); + let disc6_port = unused_udp6_port().expect("Unable to find unused port."); CommandLineTest::new() .flag("listen-address", Some("::1")) .flag("port", Some(tcp6_port.to_string().as_str())) - .flag("discovery-port", Some(udp6_port.to_string().as_str())) + .flag("discovery-port", Some(disc6_port.to_string().as_str())) .run() .with_config(|config| { assert_eq!( @@ -1071,24 +1071,24 @@ fn network_port_and_discovery_port_flags_over_ipv6() { .network .listen_addrs() .v6() - .map(|listen_addr| (listen_addr.tcp_port, listen_addr.udp_port)), - Some((tcp6_port, udp6_port)) + .map(|listen_addr| (listen_addr.tcp_port, listen_addr.disc_port)), + Some((tcp6_port, disc6_port)) ); }); } #[test] fn network_port_and_discovery_port_flags_over_ipv4_and_ipv6() { let tcp4_port = unused_tcp4_port().expect("Unable to find unused port."); - let udp4_port = unused_udp4_port().expect("Unable to find unused port."); + let disc4_port = unused_udp4_port().expect("Unable to find unused port."); let tcp6_port = unused_tcp6_port().expect("Unable to find unused port."); - let udp6_port = unused_udp6_port().expect("Unable to find unused port."); + let disc6_port = unused_udp6_port().expect("Unable to find unused port."); CommandLineTest::new() .flag("listen-address", Some("::1")) .flag("listen-address", Some("127.0.0.1")) .flag("port", Some(tcp4_port.to_string().as_str())) - .flag("discovery-port", Some(udp4_port.to_string().as_str())) + .flag("discovery-port", Some(disc4_port.to_string().as_str())) .flag("port6", Some(tcp6_port.to_string().as_str())) - .flag("discovery-port6", Some(udp6_port.to_string().as_str())) + .flag("discovery-port6", Some(disc6_port.to_string().as_str())) .run() .with_config(|config| { assert_eq!( @@ -1096,8 +1096,8 @@ fn network_port_and_discovery_port_flags_over_ipv4_and_ipv6() { .network .listen_addrs() .v4() - .map(|listen_addr| (listen_addr.tcp_port, listen_addr.udp_port)), - Some((tcp4_port, udp4_port)) + .map(|listen_addr| (listen_addr.tcp_port, listen_addr.disc_port)), + Some((tcp4_port, disc4_port)) ); assert_eq!( @@ -1105,8 +1105,47 @@ fn network_port_and_discovery_port_flags_over_ipv4_and_ipv6() { .network .listen_addrs() .v6() - .map(|listen_addr| (listen_addr.tcp_port, listen_addr.udp_port)), - Some((tcp6_port, udp6_port)) + .map(|listen_addr| (listen_addr.tcp_port, listen_addr.disc_port)), + Some((tcp6_port, disc6_port)) + ); + }); +} + +#[test] +fn network_port_discovery_quic_port_flags_over_ipv4_and_ipv6() { + let tcp4_port = unused_tcp4_port().expect("Unable to find unused port."); + let disc4_port = unused_udp4_port().expect("Unable to find unused port."); + let quic4_port = unused_udp4_port().expect("Unable to find unused port."); + let tcp6_port = unused_tcp6_port().expect("Unable to find unused port."); + let disc6_port = unused_udp6_port().expect("Unable to find unused port."); + let quic6_port = unused_udp6_port().expect("Unable to find unused port."); + CommandLineTest::new() + .flag("listen-address", Some("::1")) + .flag("listen-address", Some("127.0.0.1")) + .flag("port", Some(tcp4_port.to_string().as_str())) + .flag("discovery-port", Some(disc4_port.to_string().as_str())) + .flag("quic-port", Some(quic4_port.to_string().as_str())) + .flag("port6", Some(tcp6_port.to_string().as_str())) + .flag("discovery-port6", Some(disc6_port.to_string().as_str())) + .flag("quic-port6", Some(quic6_port.to_string().as_str())) + .run() + .with_config(|config| { + assert_eq!( + config.network.listen_addrs().v4().map(|listen_addr| ( + listen_addr.tcp_port, + listen_addr.disc_port, + listen_addr.quic_port + )), + Some((tcp4_port, disc4_port, quic4_port)) + ); + + assert_eq!( + config.network.listen_addrs().v6().map(|listen_addr| ( + listen_addr.tcp_port, + listen_addr.disc_port, + listen_addr.quic_port + )), + Some((tcp6_port, disc6_port, quic6_port)) ); }); } @@ -1118,6 +1157,14 @@ fn disable_discovery_flag() { .run_with_zero_port() .with_config(|config| assert!(config.network.disable_discovery)); } + +#[test] +fn disable_quic_flag() { + CommandLineTest::new() + .flag("disable-quic", None) + .run_with_zero_port() + .with_config(|config| assert!(config.network.disable_quic_support)); +} #[test] fn disable_peer_scoring_flag() { CommandLineTest::new() @@ -1224,6 +1271,14 @@ fn enr_udp_port_flag() { .with_config(|config| assert_eq!(config.network.enr_udp4_port, Some(port))); } #[test] +fn enr_quic_port_flag() { + let port = unused_udp4_port().expect("Unable to find unused port."); + CommandLineTest::new() + .flag("enr-quic-port", Some(port.to_string().as_str())) + .run_with_zero_port() + .with_config(|config| assert_eq!(config.network.enr_quic4_port, Some(port))); +} +#[test] fn enr_tcp_port_flag() { let port = unused_tcp4_port().expect("Unable to find unused port."); CommandLineTest::new() @@ -1240,6 +1295,14 @@ fn enr_udp6_port_flag() { .with_config(|config| assert_eq!(config.network.enr_udp6_port, Some(port))); } #[test] +fn enr_quic6_port_flag() { + let port = unused_udp6_port().expect("Unable to find unused port."); + CommandLineTest::new() + .flag("enr-quic6-port", Some(port.to_string().as_str())) + .run_with_zero_port() + .with_config(|config| assert_eq!(config.network.enr_quic6_port, Some(port))); +} +#[test] fn enr_tcp6_port_flag() { let port = unused_tcp6_port().expect("Unable to find unused port."); CommandLineTest::new() @@ -1262,7 +1325,7 @@ fn enr_match_flag_over_ipv4() { assert_eq!( config.network.listen_addrs().v4().map(|listen_addr| ( listen_addr.addr, - listen_addr.udp_port, + listen_addr.disc_port, listen_addr.tcp_port )), Some((addr, udp4_port, tcp4_port)) @@ -1287,7 +1350,7 @@ fn enr_match_flag_over_ipv6() { assert_eq!( config.network.listen_addrs().v6().map(|listen_addr| ( listen_addr.addr, - listen_addr.udp_port, + listen_addr.disc_port, listen_addr.tcp_port )), Some((addr, udp6_port, tcp6_port)) @@ -1319,7 +1382,7 @@ fn enr_match_flag_over_ipv4_and_ipv6() { assert_eq!( config.network.listen_addrs().v6().map(|listen_addr| ( listen_addr.addr, - listen_addr.udp_port, + listen_addr.disc_port, listen_addr.tcp_port )), Some((ipv6_addr, udp6_port, tcp6_port)) @@ -1327,7 +1390,7 @@ fn enr_match_flag_over_ipv4_and_ipv6() { assert_eq!( config.network.listen_addrs().v4().map(|listen_addr| ( listen_addr.addr, - listen_addr.udp_port, + listen_addr.disc_port, listen_addr.tcp_port )), Some((ipv4_addr, udp4_port, tcp4_port)) diff --git a/scripts/local_testnet/beacon_node.sh b/scripts/local_testnet/beacon_node.sh index 1a04d12d4a..70a36614c7 100755 --- a/scripts/local_testnet/beacon_node.sh +++ b/scripts/local_testnet/beacon_node.sh @@ -39,10 +39,11 @@ done # Get positional arguments data_dir=${@:$OPTIND+0:1} -network_port=${@:$OPTIND+1:1} -http_port=${@:$OPTIND+2:1} -execution_endpoint=${@:$OPTIND+3:1} -execution_jwt=${@:$OPTIND+4:1} +tcp_port=${@:$OPTIND+1:1} +quic_port=${@:$OPTIND+2:1} +http_port=${@:$OPTIND+3:1} +execution_endpoint=${@:$OPTIND+4:1} +execution_jwt=${@:$OPTIND+5:1} lighthouse_binary=lighthouse @@ -56,9 +57,11 @@ exec $lighthouse_binary \ --disable-peer-scoring \ --staking \ --enr-address 127.0.0.1 \ - --enr-udp-port $network_port \ - --enr-tcp-port $network_port \ - --port $network_port \ + --enr-udp-port $tcp_port \ + --enr-tcp-port $tcp_port \ + --enr-quic-port $quic_port \ + --port $tcp_port \ + --quic-port $quic_port \ --http-port $http_port \ --disable-packet-filter \ --target-peers $((BN_COUNT - 1)) \ diff --git a/scripts/local_testnet/start_local_testnet.sh b/scripts/local_testnet/start_local_testnet.sh index 4b8357b993..c641871ad9 100755 --- a/scripts/local_testnet/start_local_testnet.sh +++ b/scripts/local_testnet/start_local_testnet.sh @@ -139,7 +139,7 @@ sed -i 's/"shanghaiTime".*$/"shanghaiTime": 0,/g' $genesis_file for (( bn=1; bn<=$BN_COUNT; bn++ )); do secret=$DATADIR/geth_datadir$bn/geth/jwtsecret echo $secret - execute_command_add_PID beacon_node_$bn.log ./beacon_node.sh $SAS -d $DEBUG_LEVEL $DATADIR/node_$bn $((BN_udp_tcp_base + $bn)) $((BN_http_port_base + $bn)) http://localhost:$((EL_base_auth_http + $bn)) $secret + execute_command_add_PID beacon_node_$bn.log ./beacon_node.sh $SAS -d $DEBUG_LEVEL $DATADIR/node_$bn $((BN_udp_tcp_base + $bn)) $((BN_udp_tcp_base + $bn + 100)) $((BN_http_port_base + $bn)) http://localhost:$((EL_base_auth_http + $bn)) $secret done # Start requested number of validator clients diff --git a/scripts/tests/doppelganger_protection.sh b/scripts/tests/doppelganger_protection.sh index e9d3e39ce5..1eefa7cf52 100755 --- a/scripts/tests/doppelganger_protection.sh +++ b/scripts/tests/doppelganger_protection.sh @@ -43,23 +43,23 @@ sleep 10 echo "Starting local execution nodes" -exit_if_fails ../local_testnet/geth.sh $HOME/.lighthouse/local-testnet/geth_datadir1 7000 6000 5000 $genesis_file &> geth.log & -exit_if_fails ../local_testnet/geth.sh $HOME/.lighthouse/local-testnet/geth_datadir2 7100 6100 5100 $genesis_file &> /dev/null & -exit_if_fails ../local_testnet/geth.sh $HOME/.lighthouse/local-testnet/geth_datadir3 7200 6200 5200 $genesis_file &> /dev/null & +exit_if_fails ../local_testnet/geth.sh $HOME/.lighthouse/local-testnet/geth_datadir1 6000 5000 4000 $genesis_file &> geth.log & +exit_if_fails ../local_testnet/geth.sh $HOME/.lighthouse/local-testnet/geth_datadir2 6100 5100 4100 $genesis_file &> /dev/null & +exit_if_fails ../local_testnet/geth.sh $HOME/.lighthouse/local-testnet/geth_datadir3 6200 5200 4200 $genesis_file &> /dev/null & sleep 20 echo "Starting local beacon nodes" -exit_if_fails ../local_testnet/beacon_node.sh -d debug $HOME/.lighthouse/local-testnet/node_1 9000 8000 http://localhost:5000 $HOME/.lighthouse/local-testnet/geth_datadir1/geth/jwtsecret &> beacon1.log & -exit_if_fails ../local_testnet/beacon_node.sh $HOME/.lighthouse/local-testnet/node_2 9100 8100 http://localhost:5100 $HOME/.lighthouse/local-testnet/geth_datadir2/geth/jwtsecret &> /dev/null & -exit_if_fails ../local_testnet/beacon_node.sh $HOME/.lighthouse/local-testnet/node_3 9200 8200 http://localhost:5200 $HOME/.lighthouse/local-testnet/geth_datadir3/geth/jwtsecret &> /dev/null & +exit_if_fails ../local_testnet/beacon_node.sh -d debug $HOME/.lighthouse/local-testnet/node_1 8000 7000 9000 http://localhost:4000 $HOME/.lighthouse/local-testnet/geth_datadir1/geth/jwtsecret &> /dev/null & +exit_if_fails ../local_testnet/beacon_node.sh $HOME/.lighthouse/local-testnet/node_2 8100 7100 9100 http://localhost:4100 $HOME/.lighthouse/local-testnet/geth_datadir2/geth/jwtsecret &> /dev/null & +exit_if_fails ../local_testnet/beacon_node.sh $HOME/.lighthouse/local-testnet/node_3 8200 7200 9200 http://localhost:4200 $HOME/.lighthouse/local-testnet/geth_datadir3/geth/jwtsecret &> /dev/null & echo "Starting local validator clients" -exit_if_fails ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_1 http://localhost:8000 &> /dev/null & -exit_if_fails ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_2 http://localhost:8100 &> /dev/null & -exit_if_fails ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_3 http://localhost:8200 &> /dev/null & +exit_if_fails ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_1 http://localhost:9000 &> /dev/null & +exit_if_fails ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_2 http://localhost:9100 &> /dev/null & +exit_if_fails ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_3 http://localhost:9200 &> /dev/null & echo "Waiting an epoch before starting the next validator client" sleep $(( $SECONDS_PER_SLOT * 32 )) @@ -70,7 +70,7 @@ if [[ "$BEHAVIOR" == "failure" ]]; then # Use same keys as keys from VC1 and connect to BN2 # This process should not last longer than 2 epochs - timeout $(( $SECONDS_PER_SLOT * 32 * 2 )) ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_1_doppelganger http://localhost:8100 + timeout $(( $SECONDS_PER_SLOT * 32 * 2 )) ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_1_doppelganger http://localhost:9100 DOPPELGANGER_EXIT=$? echo "Shutting down" @@ -96,7 +96,7 @@ if [[ "$BEHAVIOR" == "success" ]]; then echo "Starting the last validator client" - ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_4 http://localhost:8100 & + ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_4 http://localhost:9100 & DOPPELGANGER_FAILURE=0 # Sleep three epochs, then make sure all validators were active in epoch 2. Use @@ -110,7 +110,7 @@ if [[ "$BEHAVIOR" == "success" ]]; then cd $HOME/.lighthouse/local-testnet/node_4/validators for val in 0x*; do [[ -e $val ]] || continue - curl -s localhost:8100/lighthouse/validator_inclusion/3/$val | jq | grep -q '"is_previous_epoch_target_attester": false' + curl -s localhost:9100/lighthouse/validator_inclusion/3/$val | jq | grep -q '"is_previous_epoch_target_attester": false' IS_ATTESTER=$? if [[ $IS_ATTESTER -eq 0 ]]; then echo "$val did not attest in epoch 2." @@ -128,7 +128,7 @@ if [[ "$BEHAVIOR" == "success" ]]; then sleep $(( $SECONDS_PER_SLOT * 32 * 2 )) for val in 0x*; do [[ -e $val ]] || continue - curl -s localhost:8100/lighthouse/validator_inclusion/5/$val | jq | grep -q '"is_previous_epoch_target_attester": true' + curl -s localhost:9100/lighthouse/validator_inclusion/5/$val | jq | grep -q '"is_previous_epoch_target_attester": true' IS_ATTESTER=$? if [[ $IS_ATTESTER -eq 0 ]]; then echo "$val attested in epoch 4." @@ -154,4 +154,4 @@ if [[ "$BEHAVIOR" == "success" ]]; then fi fi -exit 0 \ No newline at end of file +exit 0 diff --git a/testing/node_test_rig/src/lib.rs b/testing/node_test_rig/src/lib.rs index 3cd8205eb6..d842a72747 100644 --- a/testing/node_test_rig/src/lib.rs +++ b/testing/node_test_rig/src/lib.rs @@ -98,7 +98,7 @@ pub fn testing_client_config() -> ClientConfig { // Setting ports to `0` means that the OS will choose some available port. client_config .network - .set_ipv4_listening_address(std::net::Ipv4Addr::UNSPECIFIED, 0, 0); + .set_ipv4_listening_address(std::net::Ipv4Addr::UNSPECIFIED, 0, 0, 0); client_config.network.upnp_enabled = false; client_config.http_api.enabled = true; client_config.http_api.listen_port = 0; diff --git a/testing/simulator/src/local_network.rs b/testing/simulator/src/local_network.rs index e35870d126..69fa8ded02 100644 --- a/testing/simulator/src/local_network.rs +++ b/testing/simulator/src/local_network.rs @@ -14,6 +14,7 @@ use std::{sync::Arc, time::Duration}; use types::{Epoch, EthSpec}; const BOOTNODE_PORT: u16 = 42424; +const QUIC_PORT: u16 = 43424; pub const INVALID_ADDRESS: &str = "http://127.0.0.1:42423"; pub const EXECUTION_PORT: u16 = 4000; @@ -63,6 +64,7 @@ impl LocalNetwork { std::net::Ipv4Addr::UNSPECIFIED, BOOTNODE_PORT, BOOTNODE_PORT, + QUIC_PORT, ); beacon_config.network.enr_udp4_port = Some(BOOTNODE_PORT); beacon_config.network.enr_tcp4_port = Some(BOOTNODE_PORT); @@ -154,6 +156,7 @@ impl LocalNetwork { std::net::Ipv4Addr::UNSPECIFIED, BOOTNODE_PORT + count, BOOTNODE_PORT + count, + QUIC_PORT + count, ); beacon_config.network.enr_udp4_port = Some(BOOTNODE_PORT + count); beacon_config.network.enr_tcp4_port = Some(BOOTNODE_PORT + count); From b88e57c989e9fb1cd5ac7a374206e6a6a2fbf558 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Fri, 15 Sep 2023 08:49:14 +0000 Subject: [PATCH 02/19] Update Java runtime requirement to 17 for Web3Signer tests (#4681) Web3Signer now requires Java runtime v17, see [v23.8.0 release](https://github.com/Consensys/web3signer/releases/tag/23.8.0). We have some Web3Signer tests that requires a compatible Java runtime to be installed on dev machines. This PR updates `setup` documentation in Lighthouse book, and also fixes a small typo. --- book/src/setup.md | 2 +- testing/web3signer_tests/tls/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/book/src/setup.md b/book/src/setup.md index 1ae6e63540..d7eafbdf9f 100644 --- a/book/src/setup.md +++ b/book/src/setup.md @@ -14,7 +14,7 @@ The additional requirements for developers are: don't have `anvil` available on your `PATH`. - [`cmake`](https://cmake.org/cmake/help/latest/command/install.html). Used by some dependencies. See [`Installation Guide`](./installation.md) for more info. -- [`java 11 runtime`](https://openjdk.java.net/projects/jdk/). 11 is the minimum, +- [`java 17 runtime`](https://openjdk.java.net/projects/jdk/). 17 is the minimum, used by web3signer_tests. - [`libpq-dev`](https://www.postgresql.org/docs/devel/libpq.html). Also know as `libpq-devel` on some systems. diff --git a/testing/web3signer_tests/tls/README.md b/testing/web3signer_tests/tls/README.md index 53ff97c272..199e304375 100644 --- a/testing/web3signer_tests/tls/README.md +++ b/testing/web3signer_tests/tls/README.md @@ -1,6 +1,6 @@ ## TLS Testing Files The files in this directory are used for testing TLS with web3signer. We store them in this -repository since the are not sensitive and it's simpler than regenerating them for each test. +repository since they are not sensitive and it's simpler than regenerating them for each test. The files were generated using the `./generate.sh` script. From d386a07b0c90727c1ba1df296ca875312b1f6571 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Fri, 15 Sep 2023 10:08:30 +0000 Subject: [PATCH 03/19] validator client: start http api before genesis (#4714) ## Issue Addressed On a new network a user might require importing validators before waiting until genesis has occurred. ## Proposed Changes Starts the validator client http api before waiting for genesis ## Additional Info cc @antondlr --- lighthouse/src/main.rs | 3 +- testing/node_test_rig/src/lib.rs | 13 +++-- validator_client/src/lib.rs | 85 ++++++++++++++++---------------- 3 files changed, 51 insertions(+), 50 deletions(-) diff --git a/lighthouse/src/main.rs b/lighthouse/src/main.rs index 6384fc53cd..f98af96176 100644 --- a/lighthouse/src/main.rs +++ b/lighthouse/src/main.rs @@ -8,6 +8,7 @@ use env_logger::{Builder, Env}; use environment::{EnvironmentBuilder, LoggerConfig}; use eth2_network_config::{Eth2NetworkConfig, DEFAULT_HARDCODED_NETWORK, HARDCODED_NET_NAMES}; use ethereum_hashing::have_sha_extensions; +use futures::TryFutureExt; use lighthouse_version::VERSION; use malloc_utils::configure_memory_allocator; use slog::{crit, info, warn}; @@ -659,8 +660,8 @@ fn run( executor.clone().spawn( async move { if let Err(e) = ProductionValidatorClient::new(context, config) + .and_then(|mut vc| async move { vc.start_service().await }) .await - .and_then(|mut vc| vc.start_service()) { crit!(log, "Failed to start validator client"; "reason" => e); // Ignore the error since it always occurs during normal operation when diff --git a/testing/node_test_rig/src/lib.rs b/testing/node_test_rig/src/lib.rs index d842a72747..0fdc5cd669 100644 --- a/testing/node_test_rig/src/lib.rs +++ b/testing/node_test_rig/src/lib.rs @@ -220,14 +220,13 @@ impl LocalValidatorClient { config.validator_dir = files.validator_dir.path().into(); config.secrets_dir = files.secrets_dir.path().into(); - ProductionValidatorClient::new(context, config) + let mut client = ProductionValidatorClient::new(context, config).await?; + + client + .start_service() .await - .map(move |mut client| { - client - .start_service() - .expect("should start validator services"); - Self { client, files } - }) + .expect("should start validator services"); + Ok(Self { client, files }) } } diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 6f071055a4..473a67945f 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -98,6 +98,8 @@ pub struct ProductionValidatorClient { slot_clock: SystemTimeSlotClock, http_api_listen_addr: Option, config: Config, + beacon_nodes: Arc>, + genesis_time: u64, } impl ProductionValidatorClient { @@ -501,12 +503,6 @@ impl ProductionValidatorClient { context.service_context("sync_committee".into()), ); - // Wait until genesis has occurred. - // - // It seems most sensible to move this into the `start_service` function, but I'm caution - // of making too many changes this close to genesis (<1 week). - wait_for_genesis(&beacon_nodes, genesis_time, &context).await?; - Ok(Self { context, duties_service, @@ -519,10 +515,12 @@ impl ProductionValidatorClient { config, slot_clock, http_api_listen_addr: None, + genesis_time, + beacon_nodes, }) } - pub fn start_service(&mut self) -> Result<(), String> { + pub async fn start_service(&mut self) -> Result<(), String> { // We use `SLOTS_PER_EPOCH` as the capacity of the block notification channel, because // we don't expect notifications to be delayed by more than a single slot, let alone a // whole epoch! @@ -530,6 +528,44 @@ impl ProductionValidatorClient { let (block_service_tx, block_service_rx) = mpsc::channel(channel_capacity); let log = self.context.log(); + let api_secret = ApiSecret::create_or_open(&self.config.validator_dir)?; + + self.http_api_listen_addr = if self.config.http_api.enabled { + let ctx = Arc::new(http_api::Context { + task_executor: self.context.executor.clone(), + api_secret, + validator_store: Some(self.validator_store.clone()), + validator_dir: Some(self.config.validator_dir.clone()), + secrets_dir: Some(self.config.secrets_dir.clone()), + graffiti_file: self.config.graffiti_file.clone(), + graffiti_flag: self.config.graffiti, + spec: self.context.eth2_config.spec.clone(), + config: self.config.http_api.clone(), + sse_logging_components: self.context.sse_logging_components.clone(), + slot_clock: self.slot_clock.clone(), + log: log.clone(), + _phantom: PhantomData, + }); + + let exit = self.context.executor.exit(); + + let (listen_addr, server) = http_api::serve(ctx, exit) + .map_err(|e| format!("Unable to start HTTP API server: {:?}", e))?; + + self.context + .clone() + .executor + .spawn_without_exit(server, "http-api"); + + Some(listen_addr) + } else { + info!(log, "HTTP API server is disabled"); + None + }; + + // Wait until genesis has occurred. + wait_for_genesis(&self.beacon_nodes, self.genesis_time, &self.context).await?; + duties_service::start_update_service(self.duties_service.clone(), block_service_tx); self.block_service @@ -568,41 +604,6 @@ impl ProductionValidatorClient { spawn_notifier(self).map_err(|e| format!("Failed to start notifier: {}", e))?; - let api_secret = ApiSecret::create_or_open(&self.config.validator_dir)?; - - self.http_api_listen_addr = if self.config.http_api.enabled { - let ctx = Arc::new(http_api::Context { - task_executor: self.context.executor.clone(), - api_secret, - validator_store: Some(self.validator_store.clone()), - validator_dir: Some(self.config.validator_dir.clone()), - secrets_dir: Some(self.config.secrets_dir.clone()), - graffiti_file: self.config.graffiti_file.clone(), - graffiti_flag: self.config.graffiti, - spec: self.context.eth2_config.spec.clone(), - config: self.config.http_api.clone(), - sse_logging_components: self.context.sse_logging_components.clone(), - slot_clock: self.slot_clock.clone(), - log: log.clone(), - _phantom: PhantomData, - }); - - let exit = self.context.executor.exit(); - - let (listen_addr, server) = http_api::serve(ctx, exit) - .map_err(|e| format!("Unable to start HTTP API server: {:?}", e))?; - - self.context - .clone() - .executor - .spawn_without_exit(server, "http-api"); - - Some(listen_addr) - } else { - info!(log, "HTTP API server is disabled"); - None - }; - if self.config.enable_latency_measurement_service { latency::start_latency_service( self.context.clone(), From 4b6cb3db2ceaf60dda8b42c7fb2fa9017a1a055f Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Wed, 20 Sep 2023 01:19:03 +0000 Subject: [PATCH 04/19] Prevent port re-use in HTTP API tests (#4745) ## Issue Addressed CI is plagued by `AddrAlreadyInUse` failures, which are caused by race conditions in allocating free ports. This PR removes all usages of the `unused_port` crate for Lighthouse's HTTP API, in favour of passing `:0` as the listen address. As a result, the listen address isn't known ahead of time and must be read from the listening socket after it binds. This requires tying some self-referential knots, which is a little disruptive, but hopefully doesn't clash too much with Deneb :crossed_fingers: There are still a few usages of `unused_tcp4_port` left in cases where we start external processes, like the `watch` Postgres DB, Anvil, Geth, Nethermind, etc. Removing these usages is non-trivial because it's hard to read the port back from an external process after starting it with `--port 0`. We might be able to do something on Linux where we read from `/proc/`, but I'll leave that for future work. --- Cargo.lock | 5 +- beacon_node/beacon_chain/Cargo.toml | 1 - beacon_node/beacon_chain/src/test_utils.rs | 101 +++++++++--------- beacon_node/execution_layer/Cargo.toml | 3 + beacon_node/execution_layer/src/lib.rs | 61 ++++++----- .../src/test_utils/mock_builder.rs | 61 ++++------- .../src/test_utils/mock_execution_layer.rs | 3 - .../execution_layer/src/test_utils/mod.rs | 2 +- beacon_node/http_api/Cargo.toml | 1 - beacon_node/http_api/src/lib.rs | 5 +- beacon_node/http_api/src/test_utils.rs | 12 +-- beacon_node/http_api/tests/tests.rs | 101 +++++++++--------- 12 files changed, 172 insertions(+), 184 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9bde2d0ad2..2ed223459e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -601,7 +601,6 @@ dependencies = [ "tree_hash", "tree_hash_derive", "types", - "unused_port", ] [[package]] @@ -2583,7 +2582,9 @@ dependencies = [ name = "execution_layer" version = "0.1.0" dependencies = [ + "arc-swap", "async-trait", + "axum", "builder_client", "bytes", "environment", @@ -2598,6 +2599,7 @@ dependencies = [ "hash-db", "hash256-std-hasher", "hex", + "hyper", "jsonwebtoken", "keccak-hash", "lazy_static", @@ -3381,7 +3383,6 @@ dependencies = [ "tokio-stream", "tree_hash", "types", - "unused_port", "warp", "warp_utils", ] diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml index b537327fb3..58cb4dd0ad 100644 --- a/beacon_node/beacon_chain/Cargo.toml +++ b/beacon_node/beacon_chain/Cargo.toml @@ -65,7 +65,6 @@ sensitive_url = { path = "../../common/sensitive_url" } superstruct = "0.5.0" hex = "0.4.2" exit-future = "0.2.0" -unused_port = {path = "../../common/unused_port"} oneshot_broadcast = { path = "../../common/oneshot_broadcast" } [[test]] diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 5fc05c5551..5e54b1194d 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -17,8 +17,8 @@ use bls::get_withdrawal_credentials; use execution_layer::{ auth::JwtKey, test_utils::{ - ExecutionBlockGenerator, MockExecutionLayer, TestingBuilder, DEFAULT_JWT_SECRET, - DEFAULT_TERMINAL_BLOCK, + ExecutionBlockGenerator, MockBuilder, MockBuilderServer, MockExecutionLayer, + DEFAULT_JWT_SECRET, DEFAULT_TERMINAL_BLOCK, }, ExecutionLayer, }; @@ -167,7 +167,6 @@ pub struct Builder { store_mutator: Option>, execution_layer: Option>, mock_execution_layer: Option>, - mock_builder: Option>, testing_slot_clock: Option, runtime: TestRuntime, log: Logger, @@ -301,7 +300,6 @@ where store_mutator: None, execution_layer: None, mock_execution_layer: None, - mock_builder: None, testing_slot_clock: None, runtime, log, @@ -433,7 +431,11 @@ where self } - pub fn mock_execution_layer(mut self) -> Self { + pub fn mock_execution_layer(self) -> Self { + self.mock_execution_layer_with_config(None) + } + + pub fn mock_execution_layer_with_config(mut self, builder_threshold: Option) -> Self { let spec = self.spec.clone().expect("cannot build without spec"); let shanghai_time = spec.capella_fork_epoch.map(|epoch| { HARNESS_GENESIS_TIME + spec.seconds_per_slot * E::slots_per_epoch() * epoch.as_u64() @@ -442,55 +444,15 @@ where self.runtime.task_executor.clone(), DEFAULT_TERMINAL_BLOCK, shanghai_time, - None, + builder_threshold, Some(JwtKey::from_slice(&DEFAULT_JWT_SECRET).unwrap()), spec, - None, ); self.execution_layer = Some(mock.el.clone()); self.mock_execution_layer = Some(mock); self } - pub fn mock_execution_layer_with_builder( - mut self, - beacon_url: SensitiveUrl, - builder_threshold: Option, - ) -> Self { - // Get a random unused port - let port = unused_port::unused_tcp4_port().unwrap(); - let builder_url = SensitiveUrl::parse(format!("http://127.0.0.1:{port}").as_str()).unwrap(); - - let spec = self.spec.clone().expect("cannot build without spec"); - let shanghai_time = spec.capella_fork_epoch.map(|epoch| { - HARNESS_GENESIS_TIME + spec.seconds_per_slot * E::slots_per_epoch() * epoch.as_u64() - }); - let mock_el = MockExecutionLayer::new( - self.runtime.task_executor.clone(), - DEFAULT_TERMINAL_BLOCK, - shanghai_time, - builder_threshold, - Some(JwtKey::from_slice(&DEFAULT_JWT_SECRET).unwrap()), - spec.clone(), - Some(builder_url.clone()), - ) - .move_to_terminal_block(); - - let mock_el_url = SensitiveUrl::parse(mock_el.server.url().as_str()).unwrap(); - - self.mock_builder = Some(TestingBuilder::new( - mock_el_url, - builder_url, - beacon_url, - spec, - self.runtime.task_executor.clone(), - )); - self.execution_layer = Some(mock_el.el.clone()); - self.mock_execution_layer = Some(mock_el); - - self - } - /// Instruct the mock execution engine to always return a "valid" response to any payload it is /// asked to execute. pub fn mock_execution_layer_all_payloads_valid(self) -> Self { @@ -572,7 +534,7 @@ where shutdown_receiver: Arc::new(Mutex::new(shutdown_receiver)), runtime: self.runtime, mock_execution_layer: self.mock_execution_layer, - mock_builder: self.mock_builder.map(Arc::new), + mock_builder: None, rng: make_rng(), } } @@ -597,7 +559,7 @@ pub struct BeaconChainHarness { pub runtime: TestRuntime, pub mock_execution_layer: Option>, - pub mock_builder: Option>>, + pub mock_builder: Option>>, pub rng: Mutex, } @@ -633,6 +595,49 @@ where .execution_block_generator() } + pub fn set_mock_builder(&mut self, beacon_url: SensitiveUrl) -> MockBuilderServer { + let mock_el = self + .mock_execution_layer + .as_ref() + .expect("harness was not built with mock execution layer"); + + let mock_el_url = SensitiveUrl::parse(mock_el.server.url().as_str()).unwrap(); + + // Create the builder, listening on a free port. + let (mock_builder, mock_builder_server) = MockBuilder::new_for_testing( + mock_el_url, + beacon_url, + self.spec.clone(), + self.runtime.task_executor.clone(), + ); + + // Set the builder URL in the execution layer now that its port is known. + let builder_listen_addr = mock_builder_server.local_addr(); + let port = builder_listen_addr.port(); + mock_el + .el + .set_builder_url( + SensitiveUrl::parse(format!("http://127.0.0.1:{port}").as_str()).unwrap(), + None, + ) + .unwrap(); + + self.mock_builder = Some(Arc::new(mock_builder)); + + // Sanity check. + let el_builder = self + .chain + .execution_layer + .as_ref() + .unwrap() + .builder() + .unwrap(); + let mock_el_builder = mock_el.el.builder().unwrap(); + assert!(Arc::ptr_eq(&el_builder, &mock_el_builder)); + + mock_builder_server + } + pub fn get_all_validators(&self) -> Vec { (0..self.validator_keypairs.len()).collect() } diff --git a/beacon_node/execution_layer/Cargo.toml b/beacon_node/execution_layer/Cargo.toml index f2259a4813..80b4c06928 100644 --- a/beacon_node/execution_layer/Cargo.toml +++ b/beacon_node/execution_layer/Cargo.toml @@ -42,6 +42,8 @@ ethers-core = "1.0.2" builder_client = { path = "../builder_client" } fork_choice = { path = "../../consensus/fork_choice" } mev-rs = { git = "https://github.com/ralexstokes/mev-rs", rev = "216657016d5c0889b505857c89ae42c7aa2764af" } +axum = "0.6" +hyper = "0.14" ethereum-consensus = { git = "https://github.com/ralexstokes/ethereum-consensus", rev = "e380108" } ssz_rs = "0.9.0" tokio-stream = { version = "0.1.9", features = [ "sync" ] } @@ -51,3 +53,4 @@ hash256-std-hasher = "0.15.2" triehash = "0.8.4" hash-db = "0.15.2" pretty_reqwest_error = { path = "../../common/pretty_reqwest_error" } +arc-swap = "1.6.0" diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index b57bba7518..e036016de5 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -5,6 +5,7 @@ //! deposit-contract functionality that the `beacon_node/eth1` crate already provides. use crate::payload_cache::PayloadCache; +use arc_swap::ArcSwapOption; use auth::{strip_prefix, Auth, JwtKey}; use builder_client::BuilderHttpClient; pub use engine_api::EngineCapabilities; @@ -209,7 +210,7 @@ pub enum FailedCondition { struct Inner { engine: Arc, - builder: Option, + builder: ArcSwapOption, execution_engine_forkchoice_lock: Mutex<()>, suggested_fee_recipient: Option
, proposer_preparation_data: Mutex>, @@ -324,25 +325,9 @@ impl ExecutionLayer { Engine::new(api, executor.clone(), &log) }; - let builder = builder_url - .map(|url| { - let builder_client = BuilderHttpClient::new(url.clone(), builder_user_agent) - .map_err(Error::Builder)?; - - info!( - log, - "Using external block builder"; - "builder_url" => ?url, - "builder_profit_threshold" => builder_profit_threshold, - "local_user_agent" => builder_client.get_user_agent(), - ); - Ok::<_, Error>(builder_client) - }) - .transpose()?; - let inner = Inner { engine: Arc::new(engine), - builder, + builder: ArcSwapOption::empty(), execution_engine_forkchoice_lock: <_>::default(), suggested_fee_recipient, proposer_preparation_data: Mutex::new(HashMap::new()), @@ -356,19 +341,45 @@ impl ExecutionLayer { last_new_payload_errored: RwLock::new(false), }; - Ok(Self { + let el = Self { inner: Arc::new(inner), - }) - } -} + }; + + if let Some(builder_url) = builder_url { + el.set_builder_url(builder_url, builder_user_agent)?; + } + + Ok(el) + } -impl ExecutionLayer { fn engine(&self) -> &Arc { &self.inner.engine } - pub fn builder(&self) -> &Option { - &self.inner.builder + pub fn builder(&self) -> Option> { + self.inner.builder.load_full() + } + + /// Set the builder URL after initialization. + /// + /// This is useful for breaking circular dependencies between mock ELs and mock builders in + /// tests. + pub fn set_builder_url( + &self, + builder_url: SensitiveUrl, + builder_user_agent: Option, + ) -> Result<(), Error> { + let builder_client = BuilderHttpClient::new(builder_url.clone(), builder_user_agent) + .map_err(Error::Builder)?; + info!( + self.log(), + "Using external block builder"; + "builder_url" => ?builder_url, + "builder_profit_threshold" => self.inner.builder_profit_threshold.as_u128(), + "local_user_agent" => builder_client.get_user_agent(), + ); + self.inner.builder.swap(Some(Arc::new(builder_client))); + Ok(()) } /// Cache a full payload, keyed on the `tree_hash_root` of the payload diff --git a/beacon_node/execution_layer/src/test_utils/mock_builder.rs b/beacon_node/execution_layer/src/test_utils/mock_builder.rs index 9471be636d..c9dd196fab 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_builder.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_builder.rs @@ -40,6 +40,11 @@ use types::{ Uint256, }; +pub type MockBuilderServer = axum::Server< + hyper::server::conn::AddrIncoming, + axum::routing::IntoMakeService, +>; + #[derive(Clone)] pub enum Operation { FeeRecipient(Address), @@ -170,19 +175,25 @@ impl BidStuff for BuilderBid { } } -pub struct TestingBuilder { - server: BlindedBlockProviderServer>, - pub builder: MockBuilder, +#[derive(Clone)] +pub struct MockBuilder { + el: ExecutionLayer, + beacon_client: BeaconNodeHttpClient, + spec: ChainSpec, + context: Arc, + val_registration_cache: Arc>>, + builder_sk: SecretKey, + operations: Arc>>, + invalidate_signatures: Arc>, } -impl TestingBuilder { - pub fn new( +impl MockBuilder { + pub fn new_for_testing( mock_el_url: SensitiveUrl, - builder_url: SensitiveUrl, beacon_url: SensitiveUrl, spec: ChainSpec, executor: TaskExecutor, - ) -> Self { + ) -> (Self, MockBuilderServer) { let file = NamedTempFile::new().unwrap(); let path = file.path().into(); std::fs::write(&path, hex::encode(DEFAULT_JWT_SECRET)).unwrap(); @@ -211,39 +222,13 @@ impl TestingBuilder { spec, context, ); - let port = builder_url.full.port().unwrap(); - let host: Ipv4Addr = builder_url - .full - .host_str() - .unwrap() - .to_string() - .parse() - .unwrap(); - let server = BlindedBlockProviderServer::new(host, port, builder.clone()); - Self { server, builder } + let host: Ipv4Addr = Ipv4Addr::LOCALHOST; + let port = 0; + let provider = BlindedBlockProviderServer::new(host, port, builder.clone()); + let server = provider.serve(); + (builder, server) } - pub async fn run(&self) { - let server = self.server.serve(); - if let Err(err) = server.await { - println!("error while listening for incoming: {err}") - } - } -} - -#[derive(Clone)] -pub struct MockBuilder { - el: ExecutionLayer, - beacon_client: BeaconNodeHttpClient, - spec: ChainSpec, - context: Arc, - val_registration_cache: Arc>>, - builder_sk: SecretKey, - operations: Arc>>, - invalidate_signatures: Arc>, -} - -impl MockBuilder { pub fn new( el: ExecutionLayer, beacon_client: BeaconNodeHttpClient, diff --git a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs index 2b512d8b1c..05f6c137e8 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs @@ -31,7 +31,6 @@ impl MockExecutionLayer { None, Some(JwtKey::from_slice(&DEFAULT_JWT_SECRET).unwrap()), spec, - None, ) } @@ -43,7 +42,6 @@ impl MockExecutionLayer { builder_threshold: Option, jwt_key: Option, spec: ChainSpec, - builder_url: Option, ) -> Self { let handle = executor.handle().unwrap(); @@ -65,7 +63,6 @@ impl MockExecutionLayer { let config = Config { execution_endpoints: vec![url], - builder_url, secret_files: vec![path], suggested_fee_recipient: Some(Address::repeat_byte(42)), builder_profit_threshold: builder_threshold.unwrap_or(DEFAULT_BUILDER_THRESHOLD_WEI), diff --git a/beacon_node/execution_layer/src/test_utils/mod.rs b/beacon_node/execution_layer/src/test_utils/mod.rs index 99d264aa7b..059c0275bb 100644 --- a/beacon_node/execution_layer/src/test_utils/mod.rs +++ b/beacon_node/execution_layer/src/test_utils/mod.rs @@ -25,7 +25,7 @@ use warp::{http::StatusCode, Filter, Rejection}; use crate::EngineCapabilities; pub use execution_block_generator::{generate_pow_block, Block, ExecutionBlockGenerator}; pub use hook::Hook; -pub use mock_builder::{Context as MockBuilderContext, MockBuilder, Operation, TestingBuilder}; +pub use mock_builder::{Context as MockBuilderContext, MockBuilder, MockBuilderServer, Operation}; pub use mock_execution_layer::MockExecutionLayer; pub const DEFAULT_TERMINAL_DIFFICULTY: u64 = 6400; diff --git a/beacon_node/http_api/Cargo.toml b/beacon_node/http_api/Cargo.toml index 0fa286f165..cac2f85495 100644 --- a/beacon_node/http_api/Cargo.toml +++ b/beacon_node/http_api/Cargo.toml @@ -40,7 +40,6 @@ logging = { path = "../../common/logging" } ethereum_serde_utils = "0.5.0" operation_pool = { path = "../operation_pool" } sensitive_url = { path = "../../common/sensitive_url" } -unused_port = { path = "../../common/unused_port" } store = { path = "../store" } bytes = "1.1.0" beacon_processor = { path = "../beacon_processor" } diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 08b18e86d0..801d23dc75 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -3635,12 +3635,13 @@ pub fn serve( // send the response back to our original HTTP request // task via a channel. let builder_future = async move { - let builder = chain + let arc_builder = chain .execution_layer .as_ref() .ok_or(BeaconChainError::ExecutionLayerMissing) .map_err(warp_utils::reject::beacon_chain_error)? - .builder() + .builder(); + let builder = arc_builder .as_ref() .ok_or(BeaconChainError::BuilderMissing) .map_err(warp_utils::reject::beacon_chain_error)?; diff --git a/beacon_node/http_api/src/test_utils.rs b/beacon_node/http_api/src/test_utils.rs index fb3497ff8b..4ab7f91359 100644 --- a/beacon_node/http_api/src/test_utils.rs +++ b/beacon_node/http_api/src/test_utils.rs @@ -129,17 +129,9 @@ pub async fn create_api_server( test_runtime: &TestRuntime, log: Logger, ) -> ApiServer> { - // Get a random unused port. - let port = unused_port::unused_tcp4_port().unwrap(); - create_api_server_on_port(chain, test_runtime, log, port).await -} + // Use port 0 to allocate a new unused port. + let port = 0; -pub async fn create_api_server_on_port( - chain: Arc>, - test_runtime: &TestRuntime, - log: Logger, - port: u16, -) -> ApiServer> { let (network_senders, network_receivers) = NetworkSenders::new(); // Default metadata diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index adaf1a0f2d..c4040471a6 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -10,15 +10,14 @@ use eth2::{ types::{BlockId as CoreBlockId, ForkChoiceNode, StateId as CoreStateId, *}, BeaconNodeHttpClient, Error, StatusCode, Timeouts, }; -use execution_layer::test_utils::TestingBuilder; -use execution_layer::test_utils::DEFAULT_BUILDER_THRESHOLD_WEI; use execution_layer::test_utils::{ - Operation, DEFAULT_BUILDER_PAYLOAD_VALUE_WEI, DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI, + MockBuilder, Operation, DEFAULT_BUILDER_PAYLOAD_VALUE_WEI, DEFAULT_BUILDER_THRESHOLD_WEI, + DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI, }; use futures::stream::{Stream, StreamExt}; use futures::FutureExt; use http_api::{ - test_utils::{create_api_server, create_api_server_on_port, ApiServer}, + test_utils::{create_api_server, ApiServer}, BlockId, StateId, }; use lighthouse_network::{Enr, EnrExt, PeerId}; @@ -73,7 +72,7 @@ struct ApiTester { network_rx: NetworkReceivers, local_enr: Enr, external_peer_id: PeerId, - mock_builder: Option>>, + mock_builder: Option>>, } struct ApiTesterConfig { @@ -120,24 +119,28 @@ impl ApiTester { } pub async fn new_from_config(config: ApiTesterConfig) -> Self { - // Get a random unused port let spec = config.spec; - let port = unused_port::unused_tcp4_port().unwrap(); - let beacon_url = SensitiveUrl::parse(format!("http://127.0.0.1:{port}").as_str()).unwrap(); - let harness = Arc::new( - BeaconChainHarness::builder(MainnetEthSpec) - .spec(spec.clone()) - .chain_config(ChainConfig { - reconstruct_historic_states: config.retain_historic_states, - ..ChainConfig::default() - }) - .logger(logging::test_logger()) - .deterministic_keypairs(VALIDATOR_COUNT) - .fresh_ephemeral_store() - .mock_execution_layer_with_builder(beacon_url.clone(), config.builder_threshold) - .build(), - ); + let mut harness = BeaconChainHarness::builder(MainnetEthSpec) + .spec(spec.clone()) + .chain_config(ChainConfig { + reconstruct_historic_states: config.retain_historic_states, + ..ChainConfig::default() + }) + .logger(logging::test_logger()) + .deterministic_keypairs(VALIDATOR_COUNT) + .fresh_ephemeral_store() + .mock_execution_layer_with_config(config.builder_threshold) + .build(); + + harness + .mock_execution_layer + .as_ref() + .unwrap() + .server + .execution_block_generator() + .move_to_terminal_block() + .unwrap(); harness.advance_slot(); @@ -245,29 +248,40 @@ impl ApiTester { let ApiServer { server, - listening_socket: _, + listening_socket, network_rx, local_enr, external_peer_id, - } = create_api_server_on_port(chain.clone(), &harness.runtime, log, port).await; + } = create_api_server(chain.clone(), &harness.runtime, log).await; harness.runtime.task_executor.spawn(server, "api_server"); + // Late-initalize the mock builder now that the mock execution node and beacon API ports + // have been allocated. + let beacon_api_port = listening_socket.port(); + let beacon_url = + SensitiveUrl::parse(format!("http://127.0.0.1:{beacon_api_port}").as_str()).unwrap(); + let mock_builder_server = harness.set_mock_builder(beacon_url.clone()); + + // Start the mock builder service prior to building the chain out. + harness.runtime.task_executor.spawn( + async move { + if let Err(e) = mock_builder_server.await { + panic!("error in mock builder server: {e:?}"); + } + }, + "mock_builder_server", + ); + + let mock_builder = harness.mock_builder.clone(); + let client = BeaconNodeHttpClient::new( beacon_url, Timeouts::set_all(Duration::from_secs(SECONDS_PER_SLOT)), ); - let builder_ref = harness.mock_builder.as_ref().unwrap().clone(); - harness.runtime.task_executor.spawn( - async move { builder_ref.run().await }, - "mock_builder_server", - ); - - let mock_builder = harness.mock_builder.clone(); - Self { - harness, + harness: Arc::new(harness), chain, client, next_block, @@ -379,7 +393,6 @@ impl ApiTester { .mock_builder .as_ref() .unwrap() - .builder .add_operation(Operation::Value(Uint256::from( DEFAULT_BUILDER_THRESHOLD_WEI, ))); @@ -402,7 +415,6 @@ impl ApiTester { .mock_builder .as_ref() .unwrap() - .builder .add_operation(Operation::Value(Uint256::from( DEFAULT_BUILDER_PAYLOAD_VALUE_WEI, ))); @@ -3275,6 +3287,7 @@ impl ApiTester { .unwrap() .get_payload_by_root(&payload.tree_hash_root()) .is_none()); + self } @@ -3283,7 +3296,6 @@ impl ApiTester { self.mock_builder .as_ref() .unwrap() - .builder .add_operation(Operation::GasLimit(30_000_000)); let slot = self.chain.slot().unwrap(); @@ -3326,7 +3338,6 @@ impl ApiTester { self.mock_builder .as_ref() .unwrap() - .builder .add_operation(Operation::FeeRecipient(test_fee_recipient)); let slot = self.chain.slot().unwrap(); @@ -3368,7 +3379,6 @@ impl ApiTester { self.mock_builder .as_ref() .unwrap() - .builder .add_operation(Operation::ParentHash(invalid_parent_hash)); let slot = self.chain.slot().unwrap(); @@ -3417,7 +3427,6 @@ impl ApiTester { self.mock_builder .as_ref() .unwrap() - .builder .add_operation(Operation::PrevRandao(invalid_prev_randao)); let slot = self.chain.slot().unwrap(); @@ -3462,7 +3471,6 @@ impl ApiTester { self.mock_builder .as_ref() .unwrap() - .builder .add_operation(Operation::BlockNumber(invalid_block_number)); let slot = self.chain.slot().unwrap(); @@ -3509,7 +3517,6 @@ impl ApiTester { self.mock_builder .as_ref() .unwrap() - .builder .add_operation(Operation::Timestamp(invalid_timestamp)); let slot = self.chain.slot().unwrap(); @@ -3549,11 +3556,7 @@ impl ApiTester { } pub async fn test_payload_rejects_invalid_signature(self) -> Self { - self.mock_builder - .as_ref() - .unwrap() - .builder - .invalid_signatures(); + self.mock_builder.as_ref().unwrap().invalid_signatures(); let slot = self.chain.slot().unwrap(); let epoch = self.chain.epoch().unwrap(); @@ -3831,7 +3834,6 @@ impl ApiTester { self.mock_builder .as_ref() .unwrap() - .builder .add_operation(Operation::Value(Uint256::from( DEFAULT_BUILDER_THRESHOLD_WEI - 1, ))); @@ -3868,7 +3870,6 @@ impl ApiTester { self.mock_builder .as_ref() .unwrap() - .builder .add_operation(Operation::Value(Uint256::from( DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI + 1, ))); @@ -3905,7 +3906,6 @@ impl ApiTester { self.mock_builder .as_ref() .unwrap() - .builder .add_operation(Operation::Value(Uint256::from( DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI, ))); @@ -3942,7 +3942,6 @@ impl ApiTester { self.mock_builder .as_ref() .unwrap() - .builder .add_operation(Operation::Value(Uint256::from( DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI - 1, ))); @@ -3979,7 +3978,6 @@ impl ApiTester { self.mock_builder .as_ref() .unwrap() - .builder .add_operation(Operation::Value(Uint256::from( DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI + 1, ))); @@ -3996,7 +3994,6 @@ impl ApiTester { self.mock_builder .as_ref() .unwrap() - .builder .add_operation(Operation::WithdrawalsRoot(withdrawals_root)); let epoch = self.chain.epoch().unwrap(); @@ -4029,7 +4026,6 @@ impl ApiTester { self.mock_builder .as_ref() .unwrap() - .builder .add_operation(Operation::Value(Uint256::from( DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI + 1, ))); @@ -4037,7 +4033,6 @@ impl ApiTester { self.mock_builder .as_ref() .unwrap() - .builder .add_operation(Operation::WithdrawalsRoot(Hash256::repeat_byte(0x42))); let slot = self.chain.slot().unwrap(); From 1e9925435ec236a6dffe5fe3829d1e502de3c127 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Thu, 21 Sep 2023 00:26:52 +0000 Subject: [PATCH 05/19] Reuse fork choice read lock instead of re-acquiring it immediately (#4688) ## Issue Addressed I went through the code base and look for places where we acquire fork choice locks (after the deadlock bug was found and fixed in #4687), and discovered an instance where we re-acquire a lock immediately after dropping it. This shouldn't cause deadlock like the other issue, but is slightly less efficient. --- beacon_node/beacon_chain/src/block_verification.rs | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index 3654484e1f..ef7f1b3394 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -754,22 +754,16 @@ impl GossipVerifiedBlock { // reboot if the `observed_block_producers` cache is empty. In that case, without this // check, we will load the parent and state from disk only to find out later that we // already know this block. - if chain - .canonical_head - .fork_choice_read_lock() - .contains_block(&block_root) - { + let fork_choice_read_lock = chain.canonical_head.fork_choice_read_lock(); + if fork_choice_read_lock.contains_block(&block_root) { return Err(BlockError::BlockIsAlreadyKnown); } // Do not process a block that doesn't descend from the finalized root. // // We check this *before* we load the parent so that we can return a more detailed error. - check_block_is_finalized_checkpoint_or_descendant( - chain, - &chain.canonical_head.fork_choice_read_lock(), - &block, - )?; + check_block_is_finalized_checkpoint_or_descendant(chain, &fork_choice_read_lock, &block)?; + drop(fork_choice_read_lock); let block_epoch = block.slot().epoch(T::EthSpec::slots_per_epoch()); let (parent_block, block) = verify_parent_block_is_known(chain, block)?; From 5a35278aea7a8bcbdc9273e0363f3125589b4eef Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 21 Sep 2023 00:26:53 +0000 Subject: [PATCH 06/19] Add more checks and logging before genesis (#4730) ## Proposed Changes This PR adds more logging prior to genesis, particularly on networks that start with execution enabled. There are new checks using `eth_getBlockByHash/Number` to verify that the genesis state's `latest_execution_payload_header` matches the execution node's genesis block. The first commit also runs the merge-readiness/Capella-readiness checks prior to genesis. This has two effects: - Give more information on the execution node's status and its readiness for genesis. - Prevent the `el_offline` status from being set on `/eth/v1/node/syncing`, which previously caused the VC to complain loudly. I would like to include this for the Holesky reboot. It would have caught the misconfig that doomed the first Holesky. ## Additional Info - Geth doesn't serve payload bodies prior to genesis, which is why we use the legacy methods. I haven't checked with other ELs yet. - Currently this is logging errors with _Capella_ genesis states generated by `ethereum-genesis-generator` because the `withdrawals_root` is not set correctly (it is 0x0). This is not a blocker for Holesky, as it starts from Bellatrix (Pari is investigating). --- beacon_node/beacon_chain/src/errors.rs | 2 + .../beacon_chain/src/merge_readiness.rs | 112 +++++++++++++++++- beacon_node/client/src/notifier.rs | 80 ++++++++++++- beacon_node/execution_layer/src/lib.rs | 17 ++- beacon_node/http_api/src/lib.rs | 3 +- .../types/src/execution_payload_header.rs | 5 +- 6 files changed, 209 insertions(+), 10 deletions(-) diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index 8714131150..073761b0f8 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -146,6 +146,8 @@ pub enum BeaconChainError { BlockVariantLacksExecutionPayload(Hash256), ExecutionLayerErrorPayloadReconstruction(ExecutionBlockHash, Box), EngineGetCapabilititesFailed(Box), + ExecutionLayerGetBlockByNumberFailed(Box), + ExecutionLayerGetBlockByHashFailed(Box), BlockHashMissingFromExecutionLayer(ExecutionBlockHash), InconsistentPayloadReconstructed { slot: Slot, diff --git a/beacon_node/beacon_chain/src/merge_readiness.rs b/beacon_node/beacon_chain/src/merge_readiness.rs index bfc2b36fdb..52a5ea912e 100644 --- a/beacon_node/beacon_chain/src/merge_readiness.rs +++ b/beacon_node/beacon_chain/src/merge_readiness.rs @@ -1,8 +1,10 @@ //! Provides tools for checking if a node is ready for the Bellatrix upgrade and following merge //! transition. -use crate::{BeaconChain, BeaconChainTypes}; +use crate::{BeaconChain, BeaconChainError as Error, BeaconChainTypes}; +use execution_layer::BlockByNumberQuery; use serde::{Deserialize, Serialize, Serializer}; +use slog::debug; use std::fmt; use std::fmt::Write; use types::*; @@ -120,6 +122,25 @@ impl fmt::Display for MergeReadiness { } } +pub enum GenesisExecutionPayloadStatus { + Correct(ExecutionBlockHash), + BlockHashMismatch { + got: ExecutionBlockHash, + expected: ExecutionBlockHash, + }, + TransactionsRootMismatch { + got: Hash256, + expected: Hash256, + }, + WithdrawalsRootMismatch { + got: Hash256, + expected: Hash256, + }, + OtherMismatch, + Irrelevant, + AlreadyHappened, +} + impl BeaconChain { /// Returns `true` if user has an EL configured, or if the Bellatrix fork has occurred or will /// occur within `MERGE_READINESS_PREPARATION_SECONDS`. @@ -144,9 +165,9 @@ impl BeaconChain { } /// Attempts to connect to the EL and confirm that it is ready for the merge. - pub async fn check_merge_readiness(&self) -> MergeReadiness { + pub async fn check_merge_readiness(&self, current_slot: Slot) -> MergeReadiness { if let Some(el) = self.execution_layer.as_ref() { - if !el.is_synced_for_notifier().await { + if !el.is_synced_for_notifier(current_slot).await { // The EL is not synced. return MergeReadiness::NotSynced; } @@ -161,6 +182,91 @@ impl BeaconChain { MergeReadiness::NoExecutionEndpoint } } + + /// Check that the execution payload embedded in the genesis state matches the EL's genesis + /// block. + pub async fn check_genesis_execution_payload_is_correct( + &self, + ) -> Result { + let head_snapshot = self.head_snapshot(); + let genesis_state = &head_snapshot.beacon_state; + + if genesis_state.slot() != 0 { + return Ok(GenesisExecutionPayloadStatus::AlreadyHappened); + } + + let Ok(latest_execution_payload_header) = genesis_state.latest_execution_payload_header() + else { + return Ok(GenesisExecutionPayloadStatus::Irrelevant); + }; + let fork = self.spec.fork_name_at_epoch(Epoch::new(0)); + + let execution_layer = self + .execution_layer + .as_ref() + .ok_or(Error::ExecutionLayerMissing)?; + let exec_block_hash = latest_execution_payload_header.block_hash(); + + // Use getBlockByNumber(0) to check that the block hash matches. + // At present, Geth does not respond to engine_getPayloadBodiesByRange before genesis. + let execution_block = execution_layer + .get_block_by_number(BlockByNumberQuery::Tag("0x0")) + .await + .map_err(|e| Error::ExecutionLayerGetBlockByNumberFailed(Box::new(e)))? + .ok_or(Error::BlockHashMissingFromExecutionLayer(exec_block_hash))?; + + if execution_block.block_hash != exec_block_hash { + return Ok(GenesisExecutionPayloadStatus::BlockHashMismatch { + got: execution_block.block_hash, + expected: exec_block_hash, + }); + } + + // Double-check the block by reconstructing it. + let execution_payload = execution_layer + .get_payload_by_hash_legacy(exec_block_hash, fork) + .await + .map_err(|e| Error::ExecutionLayerGetBlockByHashFailed(Box::new(e)))? + .ok_or(Error::BlockHashMissingFromExecutionLayer(exec_block_hash))?; + + // Verify payload integrity. + let header_from_payload = ExecutionPayloadHeader::from(execution_payload.to_ref()); + + let got_transactions_root = header_from_payload.transactions_root(); + let expected_transactions_root = latest_execution_payload_header.transactions_root(); + let got_withdrawals_root = header_from_payload.withdrawals_root().ok(); + let expected_withdrawals_root = latest_execution_payload_header.withdrawals_root().ok(); + + if got_transactions_root != expected_transactions_root { + return Ok(GenesisExecutionPayloadStatus::TransactionsRootMismatch { + got: got_transactions_root, + expected: expected_transactions_root, + }); + } + + if let Some(&expected) = expected_withdrawals_root { + if let Some(&got) = got_withdrawals_root { + if got != expected { + return Ok(GenesisExecutionPayloadStatus::WithdrawalsRootMismatch { + got, + expected, + }); + } + } + } + + if header_from_payload.to_ref() != latest_execution_payload_header { + debug!( + self.log, + "Genesis execution payload reconstruction failure"; + "consensus_node_header" => ?latest_execution_payload_header, + "execution_node_header" => ?header_from_payload + ); + return Ok(GenesisExecutionPayloadStatus::OtherMismatch); + } + + Ok(GenesisExecutionPayloadStatus::Correct(exec_block_hash)) + } } /// Utility function to serialize a Uint256 as a decimal string. diff --git a/beacon_node/client/src/notifier.rs b/beacon_node/client/src/notifier.rs index 7d81594ee6..2c7738e8fa 100644 --- a/beacon_node/client/src/notifier.rs +++ b/beacon_node/client/src/notifier.rs @@ -1,7 +1,7 @@ use crate::metrics; use beacon_chain::{ capella_readiness::CapellaReadiness, - merge_readiness::{MergeConfig, MergeReadiness}, + merge_readiness::{GenesisExecutionPayloadStatus, MergeConfig, MergeReadiness}, BeaconChain, BeaconChainTypes, ExecutionStatus, }; use lighthouse_network::{types::SyncState, NetworkGlobals}; @@ -62,6 +62,9 @@ pub fn spawn_notifier( "wait_time" => estimated_time_pretty(Some(next_slot.as_secs() as f64)), ); eth1_logging(&beacon_chain, &log); + merge_readiness_logging(Slot::new(0), &beacon_chain, &log).await; + capella_readiness_logging(Slot::new(0), &beacon_chain, &log).await; + genesis_execution_payload_logging(&beacon_chain, &log).await; sleep(slot_duration).await; } _ => break, @@ -365,7 +368,7 @@ async fn merge_readiness_logging( return; } - match beacon_chain.check_merge_readiness().await { + match beacon_chain.check_merge_readiness(current_slot).await { MergeReadiness::Ready { config, current_difficulty, @@ -476,6 +479,79 @@ async fn capella_readiness_logging( } } +async fn genesis_execution_payload_logging( + beacon_chain: &BeaconChain, + log: &Logger, +) { + match beacon_chain + .check_genesis_execution_payload_is_correct() + .await + { + Ok(GenesisExecutionPayloadStatus::Correct(block_hash)) => { + info!( + log, + "Execution enabled from genesis"; + "genesis_payload_block_hash" => ?block_hash, + ); + } + Ok(GenesisExecutionPayloadStatus::BlockHashMismatch { got, expected }) => { + error!( + log, + "Genesis payload block hash mismatch"; + "info" => "genesis is misconfigured and likely to fail", + "consensus_node_block_hash" => ?expected, + "execution_node_block_hash" => ?got, + ); + } + Ok(GenesisExecutionPayloadStatus::TransactionsRootMismatch { got, expected }) => { + error!( + log, + "Genesis payload transactions root mismatch"; + "info" => "genesis is misconfigured and likely to fail", + "consensus_node_transactions_root" => ?expected, + "execution_node_transactions_root" => ?got, + ); + } + Ok(GenesisExecutionPayloadStatus::WithdrawalsRootMismatch { got, expected }) => { + error!( + log, + "Genesis payload withdrawals root mismatch"; + "info" => "genesis is misconfigured and likely to fail", + "consensus_node_withdrawals_root" => ?expected, + "execution_node_withdrawals_root" => ?got, + ); + } + Ok(GenesisExecutionPayloadStatus::OtherMismatch) => { + error!( + log, + "Genesis payload header mismatch"; + "info" => "genesis is misconfigured and likely to fail", + "detail" => "see debug logs for payload headers" + ); + } + Ok(GenesisExecutionPayloadStatus::Irrelevant) => { + info!( + log, + "Execution is not enabled from genesis"; + ); + } + Ok(GenesisExecutionPayloadStatus::AlreadyHappened) => { + warn!( + log, + "Unable to check genesis which has already occurred"; + "info" => "this is probably a race condition or a bug" + ); + } + Err(e) => { + error!( + log, + "Unable to check genesis execution payload"; + "error" => ?e + ); + } + } +} + fn eth1_logging(beacon_chain: &BeaconChain, log: &Logger) { let current_slot_opt = beacon_chain.slot().ok(); diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index e036016de5..32e255a166 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -520,9 +520,9 @@ impl ExecutionLayer { /// /// This function is a wrapper over `Self::is_synced` that makes an additional /// check for the execution layer sync status. Checks if the latest block has - /// a `block_number != 0`. + /// a `block_number != 0` *if* the `current_slot` is also `> 0`. /// Returns the `Self::is_synced` response if unable to get latest block. - pub async fn is_synced_for_notifier(&self) -> bool { + pub async fn is_synced_for_notifier(&self, current_slot: Slot) -> bool { let synced = self.is_synced().await; if synced { if let Ok(Some(block)) = self @@ -531,7 +531,7 @@ impl ExecutionLayer { .get_block_by_number(BlockByNumberQuery::Tag(LATEST_TAG)) .await { - if block.block_number == 0 { + if block.block_number == 0 && current_slot > 0 { return false; } } @@ -1605,6 +1605,17 @@ impl ExecutionLayer { } } + pub async fn get_block_by_number( + &self, + query: BlockByNumberQuery<'_>, + ) -> Result, Error> { + self.engine() + .request(|engine| async move { engine.api.get_block_by_number(query).await }) + .await + .map_err(Box::new) + .map_err(Error::EngineError) + } + pub async fn get_payload_by_hash_legacy( &self, hash: ExecutionBlockHash, diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 801d23dc75..fc1398d2b2 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -4327,7 +4327,8 @@ pub fn serve( .then( |task_spawner: TaskSpawner, chain: Arc>| { task_spawner.spawn_async_with_rejection(Priority::P1, async move { - let merge_readiness = chain.check_merge_readiness().await; + let current_slot = chain.slot_clock.now_or_genesis().unwrap_or(Slot::new(0)); + let merge_readiness = chain.check_merge_readiness(current_slot).await; Ok::<_, warp::reject::Rejection>( warp::reply::json(&api_types::GenericResponse::from(merge_readiness)) .into_response(), diff --git a/consensus/types/src/execution_payload_header.rs b/consensus/types/src/execution_payload_header.rs index 1fb29db9d3..77bea03dbf 100644 --- a/consensus/types/src/execution_payload_header.rs +++ b/consensus/types/src/execution_payload_header.rs @@ -28,7 +28,10 @@ use BeaconStateError; serde(bound = "T: EthSpec", deny_unknown_fields), arbitrary(bound = "T: EthSpec") ), - ref_attributes(derive(PartialEq, TreeHash), tree_hash(enum_behaviour = "transparent")), + ref_attributes( + derive(PartialEq, TreeHash, Debug), + tree_hash(enum_behaviour = "transparent") + ), cast_error(ty = "Error", expr = "BeaconStateError::IncorrectStateVariant"), partial_getter_error(ty = "Error", expr = "BeaconStateError::IncorrectStateVariant") )] From f9a3c00518480f0f341e31a96ee7981c4e476d97 Mon Sep 17 00:00:00 2001 From: chonghe Date: Thu, 21 Sep 2023 00:26:55 +0000 Subject: [PATCH 07/19] Update local testnet script (#4733) There is an issue with the file `scripts/local_testnet/start_local_testnet.sh` - when we use a non-default `$SPEC-PRESET` in `vars.env` it runs into an error: ``` executing: ./setup.sh >> /home/ck/.lighthouse/local-testnet/testnet/setup.log parse error: Invalid numeric literal at line 1, column 7 ``` @jimmygchen found the issue and the updated script includes the flag `--spec $SPEC-PRESET` --- scripts/local_testnet/start_local_testnet.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/local_testnet/start_local_testnet.sh b/scripts/local_testnet/start_local_testnet.sh index c641871ad9..fdf9ae1724 100755 --- a/scripts/local_testnet/start_local_testnet.sh +++ b/scripts/local_testnet/start_local_testnet.sh @@ -103,7 +103,7 @@ echo "executing: ./setup.sh >> $LOG_DIR/setup.log" ./setup.sh >> $LOG_DIR/setup.log 2>&1 # Update future hardforks time in the EL genesis file based on the CL genesis time -GENESIS_TIME=$(lcli pretty-ssz --testnet-dir $TESTNET_DIR BeaconState $TESTNET_DIR/genesis.ssz | jq | grep -Po 'genesis_time": "\K.*\d') +GENESIS_TIME=$(lcli pretty-ssz --spec $SPEC_PRESET --testnet-dir $TESTNET_DIR BeaconState $TESTNET_DIR/genesis.ssz | jq | grep -Po 'genesis_time": "\K.*\d') echo $GENESIS_TIME CAPELLA_TIME=$((GENESIS_TIME + (CAPELLA_FORK_EPOCH * 32 * SECONDS_PER_SLOT))) echo $CAPELLA_TIME From fe3bd03234de641a8a3504c46312c7eb6245cb6d Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Thu, 21 Sep 2023 00:26:56 +0000 Subject: [PATCH 08/19] Fix local testnet to generate keys in the correct folders (#4752) Fix local testnet to generate keys in the correct folders when `BN_COUNT` and `VC_COUNT` don't match. The current script place the generated validator keys in validator folders based on the `BN_COUNT` config, e.g. `node_1/validators`, `node_2/validators`..etc. We should be using `VC_COUNT` here instead, otherwise the number of validator clients may not match the number of directories generated, and would result in either: 1. a VC not having any keys (when `BN_COUNT` < `VC_COUNT`) 2. a validator key directory not being used (when `BN_COUNT` > `VC_COUNT`). --- scripts/local_testnet/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/local_testnet/setup.sh b/scripts/local_testnet/setup.sh index 283aa0c026..e026ba1c02 100755 --- a/scripts/local_testnet/setup.sh +++ b/scripts/local_testnet/setup.sh @@ -46,6 +46,6 @@ lcli \ insecure-validators \ --count $VALIDATOR_COUNT \ --base-dir $DATADIR \ - --node-count $BN_COUNT + --node-count $VC_COUNT echo Validators generated with keystore passwords at $DATADIR. From 082bb2d63802169344f7b4adf59ac1dcd5ff7308 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Thu, 21 Sep 2023 04:17:24 +0000 Subject: [PATCH 09/19] Self hosted docker builds (#4592) ## Issue Addressed We're OOM'ing on Docker builds on the Deneb branch https://github.com/sigp/lighthouse/issues/3929 Are we ok to self host automated docker builds? Co-authored-by: realbigsean Co-authored-by: realbigsean Co-authored-by: antondlr --- .github/workflows/docker.yml | 61 +++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 21ca4940d9..007070dbb5 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -17,6 +17,8 @@ env: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} IMAGE_NAME: ${{ github.repository_owner}}/lighthouse LCLI_IMAGE_NAME: ${{ github.repository_owner }}/lcli + # Enable self-hosted runners for the sigp repo only. + SELF_HOSTED_RUNNERS: ${{ github.repository == 'sigp/lighthouse' }} jobs: # Extract the VERSION which is either `latest` or `vX.Y.Z`, and the VERSION_SUFFIX @@ -48,7 +50,8 @@ jobs: VERSION_SUFFIX: ${{ env.VERSION_SUFFIX }} build-docker-single-arch: name: build-docker-${{ matrix.binary }}${{ matrix.features.version_suffix }} - runs-on: ubuntu-22.04 + # Use self-hosted runners only on the sigp repo. + runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release"]') || 'ubuntu-22.04' }} strategy: matrix: binary: [aarch64, @@ -64,14 +67,13 @@ jobs: needs: [extract-version] env: - # We need to enable experimental docker features in order to use `docker buildx` - DOCKER_CLI_EXPERIMENTAL: enabled VERSION: ${{ needs.extract-version.outputs.VERSION }} VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }} FEATURE_SUFFIX: ${{ matrix.features.version_suffix }} steps: - uses: actions/checkout@v3 - name: Update Rust + if: env.SELF_HOSTED_RUNNERS == 'false' run: rustup update stable - name: Dockerhub login run: | @@ -80,16 +82,14 @@ jobs: run: | cargo install cross env CROSS_PROFILE=${{ matrix.profile }} CROSS_FEATURES=${{ matrix.features.env }} make build-${{ matrix.binary }} + - name: Make bin dir + run: mkdir ./bin - name: Move cross-built binary into Docker scope (if ARM) if: startsWith(matrix.binary, 'aarch64') - run: | - mkdir ./bin; - mv ./target/aarch64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ./bin; + run: mv ./target/aarch64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ./bin - name: Move cross-built binary into Docker scope (if x86_64) if: startsWith(matrix.binary, 'x86_64') - run: | - mkdir ./bin; - mv ./target/x86_64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ./bin; + run: mv ./target/x86_64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ./bin - name: Map aarch64 to arm64 short arch if: startsWith(matrix.binary, 'aarch64') run: echo "SHORT_ARCH=arm64" >> $GITHUB_ENV @@ -99,17 +99,24 @@ jobs: - name: Set modernity suffix if: endsWith(matrix.binary, '-portable') != true run: echo "MODERNITY_SUFFIX=-modern" >> $GITHUB_ENV; - # Install dependencies for emulation. Have to create a new builder to pick up emulation support. - - name: Build Dockerfile and push - run: | - docker run --privileged --rm tonistiigi/binfmt --install ${SHORT_ARCH} - docker buildx create --use --name cross-builder - docker buildx build \ - --platform=linux/${SHORT_ARCH} \ - --file ./Dockerfile.cross . \ - --tag ${IMAGE_NAME}:${VERSION}-${SHORT_ARCH}${VERSION_SUFFIX}${MODERNITY_SUFFIX}${FEATURE_SUFFIX} \ - --provenance=false \ - --push + + - name: Install QEMU + if: env.SELF_HOSTED_RUNNERS == 'false' + run: sudo apt-get update && sudo apt-get install -y qemu-user-static + + - name: Set up Docker Buildx + if: env.SELF_HOSTED_RUNNERS == 'false' + uses: docker/setup-buildx-action@v2 + + - name: Build and push + uses: docker/build-push-action@v4 + with: + file: ./Dockerfile.cross + context: . + platforms: linux/${{ env.SHORT_ARCH }} + push: true + tags: ${{ env.IMAGE_NAME }}:${{ env.VERSION }}-${{ env.SHORT_ARCH }}${{ env.VERSION_SUFFIX }}${{ env.MODERNITY_SUFFIX }}${{ env.FEATURE_SUFFIX }} + build-docker-multiarch: name: build-docker-multiarch${{ matrix.modernity }} runs-on: ubuntu-22.04 @@ -118,20 +125,22 @@ jobs: matrix: modernity: ["", "-modern"] env: - # We need to enable experimental docker features in order to use `docker manifest` - DOCKER_CLI_EXPERIMENTAL: enabled VERSION: ${{ needs.extract-version.outputs.VERSION }} VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }} steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Dockerhub login run: | echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin + - name: Create and push multiarch manifest run: | - docker manifest create ${IMAGE_NAME}:${VERSION}${VERSION_SUFFIX}${{ matrix.modernity }} \ - --amend ${IMAGE_NAME}:${VERSION}-arm64${VERSION_SUFFIX}${{ matrix.modernity }} \ - --amend ${IMAGE_NAME}:${VERSION}-amd64${VERSION_SUFFIX}${{ matrix.modernity }}; - docker manifest push ${IMAGE_NAME}:${VERSION}${VERSION_SUFFIX}${{ matrix.modernity }} + docker buildx imagetools create -t ${IMAGE_NAME}:${VERSION}${VERSION_SUFFIX}${{ matrix.modernity }} \ + ${IMAGE_NAME}:${VERSION}-arm64${VERSION_SUFFIX}${{ matrix.modernity }} \ + ${IMAGE_NAME}:${VERSION}-amd64${VERSION_SUFFIX}${{ matrix.modernity }}; + build-docker-lcli: runs-on: ubuntu-22.04 needs: [extract-version] From a0478da99087e5f8f727864e055d179b9618c374 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Thu, 21 Sep 2023 04:17:25 +0000 Subject: [PATCH 10/19] Fix genesis state download panic when running in debug mode (#4753) ## Issue Addressed #4738 ## Proposed Changes See the above issue for details. Went with option #2 to use the async reqwest client in `Eth2NetworkConfig` and propagate the async-ness. --- Cargo.lock | 3 + Dockerfile | 2 +- account_manager/src/validator/exit.rs | 15 +--- .../src/validator/slashing_protection.rs | 15 +--- beacon_node/client/src/builder.rs | 11 +-- boot_node/src/config.rs | 4 +- boot_node/src/lib.rs | 23 +++--- boot_node/src/server.rs | 31 ++++++-- common/eth2_network_config/Cargo.toml | 3 + common/eth2_network_config/src/lib.rs | 72 ++++++++++--------- lcli/Dockerfile | 2 +- lighthouse/Cargo.toml | 2 +- 12 files changed, 91 insertions(+), 92 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2ed223459e..b3b8fc0541 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2220,9 +2220,11 @@ dependencies = [ name = "eth2_network_config" version = "0.2.0" dependencies = [ + "bytes", "discv5", "eth2_config", "ethereum_ssz", + "futures", "logging", "pretty_reqwest_error", "reqwest", @@ -2231,6 +2233,7 @@ dependencies = [ "sha2 0.10.7", "slog", "tempfile", + "tokio", "types", "url", "zip", diff --git a/Dockerfile b/Dockerfile index f07c42dd85..bcddef8a6f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.68.2-bullseye AS builder +FROM rust:1.69.0-bullseye AS builder RUN apt-get update && apt-get -y upgrade && apt-get install -y cmake libclang-dev COPY . lighthouse ARG FEATURES diff --git a/account_manager/src/validator/exit.rs b/account_manager/src/validator/exit.rs index 1ff61a7c01..602390556c 100644 --- a/account_manager/src/validator/exit.rs +++ b/account_manager/src/validator/exit.rs @@ -10,7 +10,6 @@ use eth2_keystore::Keystore; use eth2_network_config::Eth2NetworkConfig; use safe_arith::SafeArith; use sensitive_url::SensitiveUrl; -use slog::Logger; use slot_clock::{SlotClock, SystemTimeSlotClock}; use std::path::{Path, PathBuf}; use std::time::Duration; @@ -79,12 +78,6 @@ pub fn cli_run(matches: &ArgMatches, env: Environment) -> Result< let password_file_path: Option = clap_utils::parse_optional(matches, PASSWORD_FILE_FLAG)?; - let genesis_state_url: Option = - clap_utils::parse_optional(matches, "genesis-state-url")?; - let genesis_state_url_timeout = - clap_utils::parse_required(matches, "genesis-state-url-timeout") - .map(Duration::from_secs)?; - let stdin_inputs = cfg!(windows) || matches.is_present(STDIN_INPUTS_FLAG); let no_wait = matches.is_present(NO_WAIT); let no_confirmation = matches.is_present(NO_CONFIRMATION); @@ -111,9 +104,6 @@ pub fn cli_run(matches: &ArgMatches, env: Environment) -> Result< ð2_network_config, no_wait, no_confirmation, - genesis_state_url, - genesis_state_url_timeout, - env.core_context().log(), ))?; Ok(()) @@ -130,13 +120,10 @@ async fn publish_voluntary_exit( eth2_network_config: &Eth2NetworkConfig, no_wait: bool, no_confirmation: bool, - genesis_state_url: Option, - genesis_state_url_timeout: Duration, - log: &Logger, ) -> Result<(), String> { let genesis_data = get_geneisis_data(client).await?; let testnet_genesis_root = eth2_network_config - .genesis_validators_root::(genesis_state_url.as_deref(), genesis_state_url_timeout, log)? + .genesis_validators_root::()? .ok_or("Genesis state is unknown")?; // Verify that the beacon node and validator being exited are on the same network. diff --git a/account_manager/src/validator/slashing_protection.rs b/account_manager/src/validator/slashing_protection.rs index 570f29b4ad..c6d81275a5 100644 --- a/account_manager/src/validator/slashing_protection.rs +++ b/account_manager/src/validator/slashing_protection.rs @@ -7,7 +7,6 @@ use slashing_protection::{ use std::fs::File; use std::path::PathBuf; use std::str::FromStr; -use std::time::Duration; use types::{Epoch, EthSpec, PublicKeyBytes, Slot}; pub const CMD: &str = "slashing-protection"; @@ -82,24 +81,12 @@ pub fn cli_run( validator_base_dir: PathBuf, ) -> Result<(), String> { let slashing_protection_db_path = validator_base_dir.join(SLASHING_PROTECTION_FILENAME); - - let genesis_state_url: Option = - clap_utils::parse_optional(matches, "genesis-state-url")?; - let genesis_state_url_timeout = - clap_utils::parse_required(matches, "genesis-state-url-timeout") - .map(Duration::from_secs)?; - - let context = env.core_context(); let eth2_network_config = env .eth2_network_config .ok_or("Unable to get testnet configuration from the environment")?; let genesis_validators_root = eth2_network_config - .genesis_validators_root::( - genesis_state_url.as_deref(), - genesis_state_url_timeout, - context.log(), - )? + .genesis_validators_root::()? .ok_or_else(|| "Unable to get genesis state, has genesis occurred?".to_string())?; match matches.subcommand() { diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index 07990e3c1f..50e78aa458 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -256,7 +256,7 @@ where "Starting from known genesis state"; ); - let genesis_state = genesis_state(&runtime_context, &config, log)?; + let genesis_state = genesis_state(&runtime_context, &config, log).await?; builder.genesis_state(genesis_state).map(|v| (v, None))? } @@ -276,7 +276,7 @@ where .map_err(|e| format!("Unable to parse weak subj state SSZ: {:?}", e))?; let anchor_block = SignedBeaconBlock::from_ssz_bytes(&anchor_block_bytes, &spec) .map_err(|e| format!("Unable to parse weak subj block SSZ: {:?}", e))?; - let genesis_state = genesis_state(&runtime_context, &config, log)?; + let genesis_state = genesis_state(&runtime_context, &config, log).await?; builder .weak_subjectivity_state(anchor_state, anchor_block, genesis_state) @@ -377,7 +377,7 @@ where debug!(context.log(), "Downloaded finalized block"); - let genesis_state = genesis_state(&runtime_context, &config, log)?; + let genesis_state = genesis_state(&runtime_context, &config, log).await?; info!( context.log(), @@ -1083,7 +1083,7 @@ where } /// Obtain the genesis state from the `eth2_network_config` in `context`. -fn genesis_state( +async fn genesis_state( context: &RuntimeContext, config: &ClientConfig, log: &Logger, @@ -1097,6 +1097,7 @@ fn genesis_state( config.genesis_state_url.as_deref(), config.genesis_state_url_timeout, log, - )? + ) + .await? .ok_or_else(|| "Genesis state is unknown".to_string()) } diff --git a/boot_node/src/config.rs b/boot_node/src/config.rs index 817bd2ab52..5d7853bd24 100644 --- a/boot_node/src/config.rs +++ b/boot_node/src/config.rs @@ -25,7 +25,7 @@ pub struct BootNodeConfig { } impl BootNodeConfig { - pub fn new( + pub async fn new( matches: &ArgMatches<'_>, eth2_network_config: &Eth2NetworkConfig, ) -> Result { @@ -99,7 +99,7 @@ impl BootNodeConfig { if eth2_network_config.genesis_state_is_known() { let genesis_state = eth2_network_config - .genesis_state::(genesis_state_url.as_deref(), genesis_state_url_timeout, &logger)? + .genesis_state::(genesis_state_url.as_deref(), genesis_state_url_timeout, &logger).await? .ok_or_else(|| { "The genesis state for this network is not known, this is an unsupported mode" .to_string() diff --git a/boot_node/src/lib.rs b/boot_node/src/lib.rs index 3d9dada0fd..d76e7906b2 100644 --- a/boot_node/src/lib.rs +++ b/boot_node/src/lib.rs @@ -7,7 +7,7 @@ mod cli; pub mod config; mod server; pub use cli::cli_app; -use config::{BootNodeConfig, BootNodeConfigSerialization}; +use config::BootNodeConfig; use types::{EthSpec, EthSpecId}; const LOG_CHANNEL_SIZE: usize = 2048; @@ -81,20 +81,13 @@ fn main( .build() .map_err(|e| format!("Failed to build runtime: {}", e))?; - // parse the CLI args into a useable config - let config: BootNodeConfig = BootNodeConfig::new(bn_matches, eth2_network_config)?; - - // Dump configs if `dump-config` or `dump-chain-config` flags are set - let config_sz = BootNodeConfigSerialization::from_config_ref(&config); - clap_utils::check_dump_configs::<_, T>( - lh_matches, - &config_sz, - ð2_network_config.chain_spec::()?, - )?; - // Run the boot node - if !lh_matches.is_present("immediate-shutdown") { - runtime.block_on(server::run(config, log)); - } + runtime.block_on(server::run::( + lh_matches, + bn_matches, + eth2_network_config, + log, + ))?; + Ok(()) } diff --git a/boot_node/src/server.rs b/boot_node/src/server.rs index 3823b28726..5a5729dc04 100644 --- a/boot_node/src/server.rs +++ b/boot_node/src/server.rs @@ -1,6 +1,9 @@ //! The main bootnode server execution. use super::BootNodeConfig; +use crate::config::BootNodeConfigSerialization; +use clap::ArgMatches; +use eth2_network_config::Eth2NetworkConfig; use lighthouse_network::{ discv5::{enr::NodeId, Discv5, Discv5Event}, EnrExt, Eth2Enr, @@ -8,7 +11,27 @@ use lighthouse_network::{ use slog::info; use types::EthSpec; -pub async fn run(config: BootNodeConfig, log: slog::Logger) { +pub async fn run( + lh_matches: &ArgMatches<'_>, + bn_matches: &ArgMatches<'_>, + eth2_network_config: &Eth2NetworkConfig, + log: slog::Logger, +) -> Result<(), String> { + // parse the CLI args into a useable config + let config: BootNodeConfig = BootNodeConfig::new(bn_matches, eth2_network_config).await?; + + // Dump configs if `dump-config` or `dump-chain-config` flags are set + let config_sz = BootNodeConfigSerialization::from_config_ref(&config); + clap_utils::check_dump_configs::<_, T>( + lh_matches, + &config_sz, + ð2_network_config.chain_spec::()?, + )?; + + if lh_matches.is_present("immediate-shutdown") { + return Ok(()); + } + let BootNodeConfig { boot_nodes, local_enr, @@ -65,8 +88,7 @@ pub async fn run(config: BootNodeConfig, log: slog::Logger) { // start the server if let Err(e) = discv5.start().await { - slog::crit!(log, "Could not start discv5 server"; "error" => %e); - return; + return Err(format!("Could not start discv5 server: {e:?}")); } // if there are peers in the local routing table, establish a session by running a query @@ -82,8 +104,7 @@ pub async fn run(config: BootNodeConfig, log: slog::Logger) { let mut event_stream = match discv5.event_stream().await { Ok(stream) => stream, Err(e) => { - slog::crit!(log, "Failed to obtain event stream"; "error" => %e); - return; + return Err(format!("Failed to obtain event stream: {e:?}")); } }; diff --git a/common/eth2_network_config/Cargo.toml b/common/eth2_network_config/Cargo.toml index e73f64d5a8..76bf4fab0e 100644 --- a/common/eth2_network_config/Cargo.toml +++ b/common/eth2_network_config/Cargo.toml @@ -12,6 +12,7 @@ eth2_config = { path = "../eth2_config" } [dev-dependencies] tempfile = "3.1.0" +tokio = "1.14.0" [dependencies] serde_yaml = "0.8.13" @@ -26,3 +27,5 @@ url = "2.2.2" sensitive_url = { path = "../sensitive_url" } slog = "2.5.2" logging = { path = "../logging" } +futures = "0.3.7" +bytes = "1.1.0" \ No newline at end of file diff --git a/common/eth2_network_config/src/lib.rs b/common/eth2_network_config/src/lib.rs index 769f656e85..99093cf3b7 100644 --- a/common/eth2_network_config/src/lib.rs +++ b/common/eth2_network_config/src/lib.rs @@ -11,10 +11,11 @@ //! To add a new built-in testnet, add it to the `define_hardcoded_nets` invocation in the `eth2_config` //! crate. +use bytes::Bytes; use discv5::enr::{CombinedKey, Enr}; use eth2_config::{instantiate_hardcoded_nets, HardcodedNet}; use pretty_reqwest_error::PrettyReqwestError; -use reqwest::blocking::Client; +use reqwest::{Client, Error}; use sensitive_url::SensitiveUrl; use sha2::{Digest, Sha256}; use slog::{info, warn, Logger}; @@ -127,14 +128,8 @@ impl Eth2NetworkConfig { self.genesis_state_source != GenesisStateSource::Unknown } - /// The `genesis_validators_root` of the genesis state. May download the - /// genesis state if the value is not already available. - pub fn genesis_validators_root( - &self, - genesis_state_url: Option<&str>, - timeout: Duration, - log: &Logger, - ) -> Result, String> { + /// The `genesis_validators_root` of the genesis state. + pub fn genesis_validators_root(&self) -> Result, String> { if let GenesisStateSource::Url { genesis_validators_root, .. @@ -149,10 +144,8 @@ impl Eth2NetworkConfig { ) }) } else { - self.genesis_state::(genesis_state_url, timeout, log)? - .map(|state| state.genesis_validators_root()) - .map(Result::Ok) - .transpose() + self.get_genesis_state_from_bytes::() + .map(|state| Some(state.genesis_validators_root())) } } @@ -170,7 +163,7 @@ impl Eth2NetworkConfig { /// /// If the genesis state is configured to be downloaded from a URL, then the /// `genesis_state_url` will override the built-in list of download URLs. - pub fn genesis_state( + pub async fn genesis_state( &self, genesis_state_url: Option<&str>, timeout: Duration, @@ -180,15 +173,7 @@ impl Eth2NetworkConfig { match &self.genesis_state_source { GenesisStateSource::Unknown => Ok(None), GenesisStateSource::IncludedBytes => { - let state = self - .genesis_state_bytes - .as_ref() - .map(|bytes| { - BeaconState::from_ssz_bytes(bytes.as_ref(), &spec).map_err(|e| { - format!("Built-in genesis state SSZ bytes are invalid: {:?}", e) - }) - }) - .ok_or("Genesis state bytes missing from Eth2NetworkConfig")??; + let state = self.get_genesis_state_from_bytes()?; Ok(Some(state)) } GenesisStateSource::Url { @@ -200,9 +185,9 @@ impl Eth2NetworkConfig { format!("Unable to parse genesis state bytes checksum: {:?}", e) })?; let bytes = if let Some(specified_url) = genesis_state_url { - download_genesis_state(&[specified_url], timeout, checksum, log) + download_genesis_state(&[specified_url], timeout, checksum, log).await } else { - download_genesis_state(built_in_urls, timeout, checksum, log) + download_genesis_state(built_in_urls, timeout, checksum, log).await }?; let state = BeaconState::from_ssz_bytes(bytes.as_ref(), &spec).map_err(|e| { format!("Downloaded genesis state SSZ bytes are invalid: {:?}", e) @@ -228,6 +213,17 @@ impl Eth2NetworkConfig { } } + fn get_genesis_state_from_bytes(&self) -> Result, String> { + let spec = self.chain_spec::()?; + self.genesis_state_bytes + .as_ref() + .map(|bytes| { + BeaconState::from_ssz_bytes(bytes.as_ref(), &spec) + .map_err(|e| format!("Built-in genesis state SSZ bytes are invalid: {:?}", e)) + }) + .ok_or("Genesis state bytes missing from Eth2NetworkConfig")? + } + /// Write the files to the directory. /// /// Overwrites files if specified to do so. @@ -352,7 +348,7 @@ impl Eth2NetworkConfig { /// Try to download a genesis state from each of the `urls` in the order they /// are defined. Return `Ok` if any url returns a response that matches the /// given `checksum`. -fn download_genesis_state( +async fn download_genesis_state( urls: &[&str], timeout: Duration, checksum: Hash256, @@ -384,12 +380,7 @@ fn download_genesis_state( ); let client = Client::new(); - let response = client - .get(url) - .header("Accept", "application/octet-stream") - .timeout(timeout) - .send() - .and_then(|r| r.error_for_status().and_then(|r| r.bytes())); + let response = get_state_bytes(timeout, url, client).await; match response { Ok(bytes) => { @@ -419,6 +410,18 @@ fn download_genesis_state( )) } +async fn get_state_bytes(timeout: Duration, url: Url, client: Client) -> Result { + client + .get(url) + .header("Accept", "application/octet-stream") + .timeout(timeout) + .send() + .await? + .error_for_status()? + .bytes() + .await +} + /// Parses the `url` and joins the necessary state download path. fn parse_state_download_url(url: &str) -> Result { Url::parse(url) @@ -463,11 +466,12 @@ mod tests { assert_eq!(spec, config.chain_spec::().unwrap()); } - #[test] - fn mainnet_genesis_state() { + #[tokio::test] + async fn mainnet_genesis_state() { let config = Eth2NetworkConfig::from_hardcoded_net(&MAINNET).unwrap(); config .genesis_state::(None, Duration::from_secs(1), &logging::test_logger()) + .await .expect("beacon state can decode"); } diff --git a/lcli/Dockerfile b/lcli/Dockerfile index a50aa17027..1ee80e14fd 100644 --- a/lcli/Dockerfile +++ b/lcli/Dockerfile @@ -1,7 +1,7 @@ # `lcli` requires the full project to be in scope, so this should be built either: # - from the `lighthouse` dir with the command: `docker build -f ./lcli/Dockerflie .` # - from the current directory with the command: `docker build -f ./Dockerfile ../` -FROM rust:1.68.2-bullseye AS builder +FROM rust:1.69.0-bullseye AS builder RUN apt-get update && apt-get -y upgrade && apt-get install -y cmake libclang-dev COPY . lighthouse ARG PORTABLE diff --git a/lighthouse/Cargo.toml b/lighthouse/Cargo.toml index 012276f4ce..920a1f64e7 100644 --- a/lighthouse/Cargo.toml +++ b/lighthouse/Cargo.toml @@ -4,7 +4,7 @@ version = "4.4.1" authors = ["Sigma Prime "] edition = "2021" autotests = false -rust-version = "1.68.2" +rust-version = "1.69.0" [features] default = ["slasher-lmdb"] From 992b476eaced3dbce313dcbeed993f3336745e40 Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Thu, 21 Sep 2023 06:38:31 +0000 Subject: [PATCH 11/19] Add SSZ support to validator block production endpoints (#4534) ## Issue Addressed #4531 ## Proposed Changes add SSZ support to the following block production endpoints: GET /eth/v2/validator/blocks/{slot} GET /eth/v1/validator/blinded_blocks/{slot} ## Additional Info i updated a few existing tests to use ssz instead of writing completely new tests --- beacon_node/http_api/src/lib.rs | 48 ++++++++-- beacon_node/http_api/tests/tests.rs | 87 +++++++++++++++++- common/eth2/src/lib.rs | 136 ++++++++++++++++++++++++++-- validator_client/src/lib.rs | 3 + 4 files changed, 255 insertions(+), 19 deletions(-) diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index fc1398d2b2..cde05e6821 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -3002,6 +3002,7 @@ pub fn serve( .and(warp::path::end()) .and(not_while_syncing_filter.clone()) .and(warp::query::()) + .and(warp::header::optional::("accept")) .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and(log_filter.clone()) @@ -3009,6 +3010,7 @@ pub fn serve( |endpoint_version: EndpointVersion, slot: Slot, query: api_types::ValidatorBlocksQuery, + accept_header: Option, task_spawner: TaskSpawner, chain: Arc>, log: Logger| { @@ -3053,9 +3055,24 @@ pub fn serve( .fork_name(&chain.spec) .map_err(inconsistent_fork_rejection)?; - fork_versioned_response(endpoint_version, fork_name, block) - .map(|response| warp::reply::json(&response).into_response()) - .map(|res| add_consensus_version_header(res, fork_name)) + match accept_header { + Some(api_types::Accept::Ssz) => Response::builder() + .status(200) + .header("Content-Type", "application/octet-stream") + .body(block.as_ssz_bytes().into()) + .map(|res: Response| { + add_consensus_version_header(res, fork_name) + }) + .map_err(|e| { + warp_utils::reject::custom_server_error(format!( + "failed to create response: {}", + e + )) + }), + _ => fork_versioned_response(endpoint_version, fork_name, block) + .map(|response| warp::reply::json(&response).into_response()) + .map(|res| add_consensus_version_header(res, fork_name)), + } }) }, ); @@ -3072,11 +3089,13 @@ pub fn serve( .and(warp::path::end()) .and(not_while_syncing_filter.clone()) .and(warp::query::()) + .and(warp::header::optional::("accept")) .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .then( |slot: Slot, query: api_types::ValidatorBlocksQuery, + accept_header: Option, task_spawner: TaskSpawner, chain: Arc>| { task_spawner.spawn_async_with_rejection(Priority::P0, async move { @@ -3114,10 +3133,25 @@ pub fn serve( .fork_name(&chain.spec) .map_err(inconsistent_fork_rejection)?; - // Pose as a V2 endpoint so we return the fork `version`. - fork_versioned_response(V2, fork_name, block) - .map(|response| warp::reply::json(&response).into_response()) - .map(|res| add_consensus_version_header(res, fork_name)) + match accept_header { + Some(api_types::Accept::Ssz) => Response::builder() + .status(200) + .header("Content-Type", "application/octet-stream") + .body(block.as_ssz_bytes().into()) + .map(|res: Response| { + add_consensus_version_header(res, fork_name) + }) + .map_err(|e| { + warp_utils::reject::custom_server_error(format!( + "failed to create response: {}", + e + )) + }), + // Pose as a V2 endpoint so we return the fork `version`. + _ => fork_versioned_response(V2, fork_name, block) + .map(|response| warp::reply::json(&response).into_response()) + .map(|res| add_consensus_version_header(res, fork_name)), + } }) }, ); diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index c4040471a6..1fbdab07cc 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -2513,6 +2513,70 @@ impl ApiTester { self } + pub async fn test_block_production_ssz(self) -> Self { + let fork = self.chain.canonical_head.cached_head().head_fork(); + let genesis_validators_root = self.chain.genesis_validators_root; + + for _ in 0..E::slots_per_epoch() * 3 { + let slot = self.chain.slot().unwrap(); + let epoch = self.chain.epoch().unwrap(); + + let proposer_pubkey_bytes = self + .client + .get_validator_duties_proposer(epoch) + .await + .unwrap() + .data + .into_iter() + .find(|duty| duty.slot == slot) + .map(|duty| duty.pubkey) + .unwrap(); + let proposer_pubkey = (&proposer_pubkey_bytes).try_into().unwrap(); + + let sk = self + .validator_keypairs() + .iter() + .find(|kp| kp.pk == proposer_pubkey) + .map(|kp| kp.sk.clone()) + .unwrap(); + + let randao_reveal = { + let domain = self.chain.spec.get_domain( + epoch, + Domain::Randao, + &fork, + genesis_validators_root, + ); + let message = epoch.signing_root(domain); + sk.sign(message).into() + }; + + let block_bytes = self + .client + .get_validator_blocks_ssz::>(slot, &randao_reveal, None) + .await + .unwrap() + .expect("block bytes"); + + let block = + BeaconBlock::>::from_ssz_bytes(&block_bytes, &self.chain.spec) + .expect("block bytes can be decoded"); + + let signed_block = block.sign(&sk, &fork, genesis_validators_root, &self.chain.spec); + + self.client + .post_beacon_blocks_ssz(&signed_block) + .await + .unwrap(); + + assert_eq!(self.chain.head_beacon_block().as_ref(), &signed_block); + + self.chain.slot_clock.set_slot(slot.as_u64() + 1); + } + + self + } + pub async fn test_block_production_no_verify_randao(self) -> Self { for _ in 0..E::slots_per_epoch() { let slot = self.chain.slot().unwrap(); @@ -2694,12 +2758,15 @@ impl ApiTester { sk.sign(message).into() }; - let block = self + let block_bytes = self .client - .get_validator_blinded_blocks::(slot, &randao_reveal, None) + .get_validator_blinded_blocks_ssz::(slot, &randao_reveal, None) .await .unwrap() - .data; + .expect("block bytes"); + + let block = BeaconBlock::::from_ssz_bytes(&block_bytes, &self.chain.spec) + .expect("block bytes can be decoded"); let signed_block = block.sign(&sk, &fork, genesis_validators_root, &self.chain.spec); @@ -4891,6 +4958,20 @@ async fn block_production_verify_randao_invalid() { .await; } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn block_production_ssz_full_payload() { + ApiTester::new().await.test_block_production_ssz().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn block_production_ssz_with_skip_slots() { + ApiTester::new() + .await + .skip_slots(E::slots_per_epoch() * 2) + .test_block_production_ssz() + .await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn blinded_block_production_full_payload_premerge() { ApiTester::new() diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 74c2f3802f..5b43ee3a7a 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -120,6 +120,7 @@ pub struct Timeouts { pub get_beacon_blocks_ssz: Duration, pub get_debug_beacon_states: Duration, pub get_deposit_snapshot: Duration, + pub get_validator_block_ssz: Duration, } impl Timeouts { @@ -135,6 +136,7 @@ impl Timeouts { get_beacon_blocks_ssz: timeout, get_debug_beacon_states: timeout, get_deposit_snapshot: timeout, + get_validator_block_ssz: timeout, } } } @@ -1598,14 +1600,14 @@ impl BeaconNodeHttpClient { .await } - /// `GET v2/validator/blocks/{slot}` - pub async fn get_validator_blocks_modular>( + /// returns `GET v2/validator/blocks/{slot}` URL path + pub async fn get_validator_blocks_path>( &self, slot: Slot, randao_reveal: &SignatureBytes, graffiti: Option<&Graffiti>, skip_randao_verification: SkipRandaoVerification, - ) -> Result>, Error> { + ) -> Result { let mut path = self.eth_path(V2)?; path.path_segments_mut() @@ -1627,9 +1629,66 @@ impl BeaconNodeHttpClient { .append_pair("skip_randao_verification", ""); } + Ok(path) + } + + /// `GET v2/validator/blocks/{slot}` + pub async fn get_validator_blocks_modular>( + &self, + slot: Slot, + randao_reveal: &SignatureBytes, + graffiti: Option<&Graffiti>, + skip_randao_verification: SkipRandaoVerification, + ) -> Result>, Error> { + let path = self + .get_validator_blocks_path::( + slot, + randao_reveal, + graffiti, + skip_randao_verification, + ) + .await?; + self.get(path).await } + /// `GET v2/validator/blocks/{slot}` in ssz format + pub async fn get_validator_blocks_ssz>( + &self, + slot: Slot, + randao_reveal: &SignatureBytes, + graffiti: Option<&Graffiti>, + ) -> Result>, Error> { + self.get_validator_blocks_modular_ssz::( + slot, + randao_reveal, + graffiti, + SkipRandaoVerification::No, + ) + .await + } + + /// `GET v2/validator/blocks/{slot}` in ssz format + pub async fn get_validator_blocks_modular_ssz>( + &self, + slot: Slot, + randao_reveal: &SignatureBytes, + graffiti: Option<&Graffiti>, + skip_randao_verification: SkipRandaoVerification, + ) -> Result>, Error> { + let path = self + .get_validator_blocks_path::( + slot, + randao_reveal, + graffiti, + skip_randao_verification, + ) + .await?; + + self.get_bytes_opt_accept_header(path, Accept::Ssz, self.timeouts.get_validator_block_ssz) + .await + } + /// `GET v2/validator/blinded_blocks/{slot}` pub async fn get_validator_blinded_blocks>( &self, @@ -1646,17 +1705,14 @@ impl BeaconNodeHttpClient { .await } - /// `GET v1/validator/blinded_blocks/{slot}` - pub async fn get_validator_blinded_blocks_modular< - T: EthSpec, - Payload: AbstractExecPayload, - >( + /// returns `GET v1/validator/blinded_blocks/{slot}` URL path + pub async fn get_validator_blinded_blocks_path>( &self, slot: Slot, randao_reveal: &SignatureBytes, graffiti: Option<&Graffiti>, skip_randao_verification: SkipRandaoVerification, - ) -> Result>, Error> { + ) -> Result { let mut path = self.eth_path(V1)?; path.path_segments_mut() @@ -1678,9 +1734,71 @@ impl BeaconNodeHttpClient { .append_key_only("skip_randao_verification"); } + Ok(path) + } + + /// `GET v1/validator/blinded_blocks/{slot}` + pub async fn get_validator_blinded_blocks_modular< + T: EthSpec, + Payload: AbstractExecPayload, + >( + &self, + slot: Slot, + randao_reveal: &SignatureBytes, + graffiti: Option<&Graffiti>, + skip_randao_verification: SkipRandaoVerification, + ) -> Result>, Error> { + let path = self + .get_validator_blinded_blocks_path::( + slot, + randao_reveal, + graffiti, + skip_randao_verification, + ) + .await?; + self.get(path).await } + /// `GET v2/validator/blinded_blocks/{slot}` in ssz format + pub async fn get_validator_blinded_blocks_ssz>( + &self, + slot: Slot, + randao_reveal: &SignatureBytes, + graffiti: Option<&Graffiti>, + ) -> Result>, Error> { + self.get_validator_blinded_blocks_modular_ssz::( + slot, + randao_reveal, + graffiti, + SkipRandaoVerification::No, + ) + .await + } + + pub async fn get_validator_blinded_blocks_modular_ssz< + T: EthSpec, + Payload: AbstractExecPayload, + >( + &self, + slot: Slot, + randao_reveal: &SignatureBytes, + graffiti: Option<&Graffiti>, + skip_randao_verification: SkipRandaoVerification, + ) -> Result>, Error> { + let path = self + .get_validator_blinded_blocks_path::( + slot, + randao_reveal, + graffiti, + skip_randao_verification, + ) + .await?; + + self.get_bytes_opt_accept_header(path, Accept::Ssz, self.timeouts.get_validator_block_ssz) + .await + } + /// `GET validator/attestation_data?slot,committee_index` pub async fn get_validator_attestation_data( &self, diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 473a67945f..6925e285fc 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -82,6 +82,7 @@ const HTTP_SYNC_DUTIES_TIMEOUT_QUOTIENT: u32 = 4; const HTTP_GET_BEACON_BLOCK_SSZ_TIMEOUT_QUOTIENT: u32 = 4; const HTTP_GET_DEBUG_BEACON_STATE_QUOTIENT: u32 = 4; const HTTP_GET_DEPOSIT_SNAPSHOT_QUOTIENT: u32 = 4; +const HTTP_GET_VALIDATOR_BLOCK_SSZ_TIMEOUT_QUOTIENT: u32 = 4; const DOPPELGANGER_SERVICE_NAME: &str = "doppelganger"; @@ -309,6 +310,8 @@ impl ProductionValidatorClient { / HTTP_GET_BEACON_BLOCK_SSZ_TIMEOUT_QUOTIENT, get_debug_beacon_states: slot_duration / HTTP_GET_DEBUG_BEACON_STATE_QUOTIENT, get_deposit_snapshot: slot_duration / HTTP_GET_DEPOSIT_SNAPSHOT_QUOTIENT, + get_validator_block_ssz: slot_duration + / HTTP_GET_VALIDATOR_BLOCK_SSZ_TIMEOUT_QUOTIENT, } } else { Timeouts::set_all(slot_duration) From d3fe3ad33792944aa53d846ec3ec9c7c83674ed5 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Thu, 21 Sep 2023 06:38:32 +0000 Subject: [PATCH 12/19] Update holesky config for relaunch (#4760) ## Issue Addressed #4759 Note: Sigma Prime ENR hasn't been updated, tracking it in #4759 --- common/eth2_config/src/lib.rs | 2 +- .../holesky/boot_enr.yaml | 9 +++++---- .../built_in_network_configs/holesky/config.yaml | 16 ++++++---------- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/common/eth2_config/src/lib.rs b/common/eth2_config/src/lib.rs index 6487151a92..bf707c4d17 100644 --- a/common/eth2_config/src/lib.rs +++ b/common/eth2_config/src/lib.rs @@ -29,7 +29,7 @@ const HOLESKY_GENESIS_STATE_SOURCE: GenesisStateSource = GenesisStateSource::Url // more details. "https://sigp-public-genesis-states.s3.ap-southeast-2.amazonaws.com/holesky/", ], - checksum: "0x76631cd0b9ddc5b2c766b496e23f16759ce1181446a4efb40e5540cd15b78a07", + checksum: "0xd750639607c337bbb192b15c27f447732267bf72d1650180a0e44c2d93a80741", genesis_validators_root: "0x9143aa7c615a7f7115e2b6aac319c03529df8242ae705fba9df39b79c59fa8b1", }; diff --git a/common/eth2_network_config/built_in_network_configs/holesky/boot_enr.yaml b/common/eth2_network_config/built_in_network_configs/holesky/boot_enr.yaml index 616d41d672..53e8e525fe 100644 --- a/common/eth2_network_config/built_in_network_configs/holesky/boot_enr.yaml +++ b/common/eth2_network_config/built_in_network_configs/holesky/boot_enr.yaml @@ -1,8 +1,9 @@ # EF -- enr:-Iq4QJk4WqRkjsX5c2CXtOra6HnxN-BMXnWhmhEQO9Bn9iABTJGdjUOurM7Btj1ouKaFkvTRoju5vz2GPmVON2dffQKGAX53x8JigmlkgnY0gmlwhLKAlv6Jc2VjcDI1NmsxoQK6S-Cii_KmfFdUJL2TANL3ksaKUnNXvTCv1tLwXs0QgIN1ZHCCIyk -- enr:-KG4QF6d6vMSboSujAXTI4vYqArccm0eIlXfcxf2Lx_VE1q6IkQo_2D5LAO3ZSBVUs0w5rrVDmABJZuMzISe_pZundADhGV0aDKQqX6DZjABcAAAAQAAAAAAAIJpZIJ2NIJpcISygIjpiXNlY3AyNTZrMaEDF3aSa7QSCvdqLpANNd8GML4PLEZVg45fKQwMWhDZjd2DdGNwgiMog3VkcIIjKA -- enr:-Ly4QJLXSSAj3ggPBIcodvBU6IyfpU_yW7E9J-5syoJorBuvcYj_Fokcjr303bQoTdWXADf8po0ssh75Mr5wVGzZZsMBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpCpfoNmMAFwAAABAAAAAAAAgmlkgnY0gmlwhJK-DYCJc2VjcDI1NmsxoQJrIlXIQDvQ6t9yDySqJYDXgZgLXzTvq8W7OI51jfmxJohzeW5jbmV0cwCDdGNwgiMog3VkcIIjKA +- enr:-Ku4QFo-9q73SspYI8cac_4kTX7yF800VXqJW4Lj3HkIkb5CMqFLxciNHePmMt4XdJzHvhrCC5ADI4D_GkAsxGJRLnQBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpAhnTT-AQFwAP__________gmlkgnY0gmlwhLKAiOmJc2VjcDI1NmsxoQORcM6e19T1T9gi7jxEZjk_sjVLGFscUNqAY9obgZaxbIN1ZHCCIyk +- enr:-Ku4QPG7F72mbKx3gEQEx07wpYYusGDh-ni6SNkLvOS-hhN-BxIggN7tKlmalb0L5JPoAfqD-akTZ-gX06hFeBEz4WoBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpAhnTT-AQFwAP__________gmlkgnY0gmlwhJK-DYCJc2VjcDI1NmsxoQKLVXFOhp2uX6jeT0DvvDpPcU8FWMjQdR4wMuORMhpX24N1ZHCCIyk +- enr:-LK4QPxe-mDiSOtEB_Y82ozvxn9aQM07Ui8A-vQHNgYGMMthfsfOabaaTHhhJHFCBQQVRjBww_A5bM1rf8MlkJU_l68Eh2F0dG5ldHOIAADAAAAAAACEZXRoMpBpt9l0BAFwAAABAAAAAAAAgmlkgnY0gmlwhLKAiOmJc2VjcDI1NmsxoQJu6T9pclPObAzEVQ53DpVQqjadmVxdTLL-J3h9NFoCeIN0Y3CCIyiDdWRwgiMo +- enr:-Ly4QGbOw4xNel5EhmDsJJ-QhC9XycWtsetnWoZ0uRy381GHdHsNHJiCwDTOkb3S1Ade0SFQkWJX_pgb3g8Jfh93rvMBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpBpt9l0BAFwAAABAAAAAAAAgmlkgnY0gmlwhJK-DYCJc2VjcDI1NmsxoQOxKv9sv3zKF8GDewgFGGHKP5HCZZpPpTrwl9eXKAWGxIhzeW5jbmV0cwCDdGNwgiMog3VkcIIjKA # Teku -- enr:-LK4QMlzEff6d-M0A1pSFG5lJ2c56i_I-ZftdojZbW3ehkGNM4pkQuHQqzVvF1BG9aDjIakjnmO23mCBFFZ2w5zOsugEh2F0dG5ldHOIAAAAAAYAAACEZXRoMpCpfoNmMAFwAAABAAAAAAAAgmlkgnY0gmlwhKyuI_mJc2VjcDI1NmsxoQIH1kQRCZW-4AIVyAeXj5o49m_IqNFKRHp6tSpfXMUrSYN0Y3CCIyiDdWRwgiMo +- enr:-LS4QG0uV4qvcpJ-HFDJRGBmnlD3TJo7yc4jwK8iP7iKaTlfQ5kZvIDspLMJhk7j9KapuL9yyHaZmwTEZqr10k9XumyCEcmHYXR0bmV0c4gAAAAABgAAAIRldGgykGm32XQEAXAAAAEAAAAAAACCaWSCdjSCaXCErK4j-YlzZWNwMjU2azGhAgfWRBEJlb7gAhXIB5ePmjj2b8io0UpEenq1Kl9cxStJg3RjcIIjKIN1ZHCCIyg # Sigma Prime - enr:-Le4QI88slOwzz66Ksq8Vnz324DPb1BzSiY-WYPvnoJIl-lceW9bmSJnwDzgNbCjp5wsBigg76x4tValvGgQPxxSjrMBhGV0aDKQqX6DZjABcAAAAQAAAAAAAIJpZIJ2NIJpcIQ5gR6Wg2lwNpAgAUHQBwEQAAAAAAAAADR-iXNlY3AyNTZrMaEDPMSNdcL92uNIyCsS177Z6KTXlbZakQqxv3aQcWawNXeDdWRwgiMohHVkcDaCI4I diff --git a/common/eth2_network_config/built_in_network_configs/holesky/config.yaml b/common/eth2_network_config/built_in_network_configs/holesky/config.yaml index a6bfd87ade..845d208307 100644 --- a/common/eth2_network_config/built_in_network_configs/holesky/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/holesky/config.yaml @@ -6,9 +6,9 @@ CONFIG_NAME: holesky # --------------------------------------------------------------- # `2**14` (= 16,384) MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 16384 -# Sep-15-2023 13:55:00 +UTC -MIN_GENESIS_TIME: 1694786100 -GENESIS_FORK_VERSION: 0x00017000 +# Sep-28-2023 11:55:00 +UTC +MIN_GENESIS_TIME: 1695902100 +GENESIS_FORK_VERSION: 0x01017000 # Genesis delay 5 mins GENESIS_DELAY: 300 @@ -20,23 +20,19 @@ GENESIS_DELAY: 300 # - Temporarily set to max uint64 value: 2**64 - 1 # Altair -ALTAIR_FORK_VERSION: 0x10017000 +ALTAIR_FORK_VERSION: 0x02017000 ALTAIR_FORK_EPOCH: 0 # Merge -BELLATRIX_FORK_VERSION: 0x20017000 +BELLATRIX_FORK_VERSION: 0x03017000 BELLATRIX_FORK_EPOCH: 0 TERMINAL_TOTAL_DIFFICULTY: 0 TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000 TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615 # Capella -CAPELLA_FORK_VERSION: 0x30017000 +CAPELLA_FORK_VERSION: 0x04017000 CAPELLA_FORK_EPOCH: 256 -# DENEB -DENEB_FORK_VERSION: 0x40017000 -DENEB_FORK_EPOCH: 18446744073709551615 - # Time parameters # --------------------------------------------------------------- # 12 seconds From 0074a3b5f5d5c6076ece630fa680d29d58eaea56 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 21 Sep 2023 06:38:33 +0000 Subject: [PATCH 13/19] Fix block & state queries prior to genesis (#4761) ## Issue Addressed Closes #4751 ## Proposed Changes Prevent `state_root_at_slot` and `block_root_at_slot` from erroring out due to a call to `self.slot()?` that fails before genesis. This fixes pre-genesis queries for: - block at slot 0 - block by genesis block root - state at slot 0 - state by genesis state root - state at `finalized` tag - state at `justified` tag --- beacon_node/beacon_chain/src/beacon_chain.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index d5b86d63ff..9587783508 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -809,10 +809,10 @@ impl BeaconChain { /// /// May return a database error. pub fn state_root_at_slot(&self, request_slot: Slot) -> Result, Error> { - if request_slot > self.slot()? { - return Ok(None); - } else if request_slot == self.spec.genesis_slot { + if request_slot == self.spec.genesis_slot { return Ok(Some(self.genesis_state_root)); + } else if request_slot > self.slot()? { + return Ok(None); } // Check limits w.r.t historic state bounds. @@ -889,10 +889,10 @@ impl BeaconChain { /// /// May return a database error. fn block_root_at_slot_skips_none(&self, request_slot: Slot) -> Result, Error> { - if request_slot > self.slot()? { - return Ok(None); - } else if request_slot == self.spec.genesis_slot { + if request_slot == self.spec.genesis_slot { return Ok(Some(self.genesis_block_root)); + } else if request_slot > self.slot()? { + return Ok(None); } let prev_slot = request_slot.saturating_sub(1_u64); @@ -952,10 +952,10 @@ impl BeaconChain { /// /// May return a database error. fn block_root_at_slot_skips_prev(&self, request_slot: Slot) -> Result, Error> { - if request_slot > self.slot()? { - return Ok(None); - } else if request_slot == self.spec.genesis_slot { + if request_slot == self.spec.genesis_slot { return Ok(Some(self.genesis_block_root)); + } else if request_slot > self.slot()? { + return Ok(None); } // Try an optimized path of reading the root directly from the head state. From d0b1abc6fa3b2a3d3e19c6ac0704459bda721ed3 Mon Sep 17 00:00:00 2001 From: antondlr Date: Thu, 21 Sep 2023 22:36:59 +0000 Subject: [PATCH 14/19] Update Holesky boot ENR (#4763) ## Issue Addressed update boot ENR for Holesky relaunch --- .../built_in_network_configs/holesky/boot_enr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/eth2_network_config/built_in_network_configs/holesky/boot_enr.yaml b/common/eth2_network_config/built_in_network_configs/holesky/boot_enr.yaml index 53e8e525fe..4dd2808102 100644 --- a/common/eth2_network_config/built_in_network_configs/holesky/boot_enr.yaml +++ b/common/eth2_network_config/built_in_network_configs/holesky/boot_enr.yaml @@ -6,4 +6,4 @@ # Teku - enr:-LS4QG0uV4qvcpJ-HFDJRGBmnlD3TJo7yc4jwK8iP7iKaTlfQ5kZvIDspLMJhk7j9KapuL9yyHaZmwTEZqr10k9XumyCEcmHYXR0bmV0c4gAAAAABgAAAIRldGgykGm32XQEAXAAAAEAAAAAAACCaWSCdjSCaXCErK4j-YlzZWNwMjU2azGhAgfWRBEJlb7gAhXIB5ePmjj2b8io0UpEenq1Kl9cxStJg3RjcIIjKIN1ZHCCIyg # Sigma Prime -- enr:-Le4QI88slOwzz66Ksq8Vnz324DPb1BzSiY-WYPvnoJIl-lceW9bmSJnwDzgNbCjp5wsBigg76x4tValvGgQPxxSjrMBhGV0aDKQqX6DZjABcAAAAQAAAAAAAIJpZIJ2NIJpcIQ5gR6Wg2lwNpAgAUHQBwEQAAAAAAAAADR-iXNlY3AyNTZrMaEDPMSNdcL92uNIyCsS177Z6KTXlbZakQqxv3aQcWawNXeDdWRwgiMohHVkcDaCI4I +- enr:-Le4QLoE1wFHSlGcm48a9ZESb_MRLqPPu6G0vHqu4MaUcQNDHS69tsy-zkN0K6pglyzX8m24mkb-LtBcbjAYdP1uxm4BhGV0aDKQabfZdAQBcAAAAQAAAAAAAIJpZIJ2NIJpcIQ5gR6Wg2lwNpAgAUHQBwEQAAAAAAAAADR-iXNlY3AyNTZrMaEDPMSNdcL92uNIyCsS177Z6KTXlbZakQqxv3aQcWawNXeDdWRwgiMohHVkcDaCI4I From 2441a247abec9a3b821af936f5fe0aee9b5533d2 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 21 Sep 2023 22:37:00 +0000 Subject: [PATCH 15/19] Bump `quinn-proto` to address rustsec vuln (#4767) ## Issue Addressed NA ## Proposed Changes Bumps `quinn-proto` to address a QUIC-related vulnerability: https://rustsec.org/advisories/RUSTSEC-2023-0063 Fixes a `cargo audit` failure. ## Additional Info NA --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b3b8fc0541..40c4c516b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6192,9 +6192,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.10.4" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13f81c9a9d574310b8351f8666f5a93ac3b0069c45c28ad52c10291389a7cf9" +checksum = "2c78e758510582acc40acb90458401172d41f1016f8c9dde89e49677afb7eec1" dependencies = [ "bytes", "rand 0.8.5", From c5588eb66e26272bb99b590492c6bd7bb5a5ff81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Fri, 22 Sep 2023 02:33:10 +0000 Subject: [PATCH 16/19] require http and metrics for respective flags (#4674) ## Issue Addressed following discussion on https://github.com/sigp/lighthouse/pull/4639#discussion_r1305183750 this PR makes the `http` and `metrics` sub-flags to require those main flags enabled --- beacon_node/src/cli.rs | 31 +++++--- beacon_node/src/config.rs | 122 ++++++++++++++++---------------- lighthouse/tests/beacon_node.rs | 5 ++ validator_client/src/cli.rs | 16 +++-- 4 files changed, 100 insertions(+), 74 deletions(-) diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index ffc29365cb..dacff79f6c 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -1,4 +1,4 @@ -use clap::{App, Arg}; +use clap::{App, Arg, ArgGroup}; use strum::VariantNames; use types::ProgressiveBalancesMode; @@ -355,22 +355,25 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .arg( Arg::with_name("http-address") .long("http-address") + .requires("enable_http") .value_name("ADDRESS") .help("Set the listen address for the RESTful HTTP API server.") - .default_value("127.0.0.1") + .default_value_if("enable_http", None, "127.0.0.1") .takes_value(true), ) .arg( Arg::with_name("http-port") .long("http-port") + .requires("enable_http") .value_name("PORT") .help("Set the listen TCP port for the RESTful HTTP API server.") - .default_value("5052") + .default_value_if("enable_http", None, "5052") .takes_value(true), ) .arg( Arg::with_name("http-allow-origin") .long("http-allow-origin") + .requires("enable_http") .value_name("ORIGIN") .help("Set the value of the Access-Control-Allow-Origin response HTTP header. \ Use * to allow any origin (not recommended in production). \ @@ -381,11 +384,13 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .arg( Arg::with_name("http-disable-legacy-spec") .long("http-disable-legacy-spec") + .requires("enable_http") .hidden(true) ) .arg( Arg::with_name("http-spec-fork") .long("http-spec-fork") + .requires("enable_http") .value_name("FORK") .help("Serve the spec for a specific hard fork on /eth/v1/config/spec. It should \ not be necessary to set this flag.") @@ -403,6 +408,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .arg( Arg::with_name("http-tls-cert") .long("http-tls-cert") + .requires("enable_http") .help("The path of the certificate to be used when serving the HTTP API server \ over TLS.") .takes_value(true) @@ -410,6 +416,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .arg( Arg::with_name("http-tls-key") .long("http-tls-key") + .requires("enable_http") .help("The path of the private key to be used when serving the HTTP API server \ over TLS. Must not be password-protected.") .takes_value(true) @@ -417,6 +424,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .arg( Arg::with_name("http-allow-sync-stalled") .long("http-allow-sync-stalled") + .requires("enable_http") .help("Forces the HTTP to indicate that the node is synced when sync is actually \ stalled. This is useful for very small testnets. TESTING ONLY. DO NOT USE ON \ MAINNET.") @@ -424,8 +432,9 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .arg( Arg::with_name("http-sse-capacity-multiplier") .long("http-sse-capacity-multiplier") + .requires("enable_http") .takes_value(true) - .default_value("1") + .default_value_if("enable_http", None, "1") .value_name("N") .help("Multiplier to apply to the length of HTTP server-sent-event (SSE) channels. \ Increasing this value can prevent messages from being dropped.") @@ -433,8 +442,9 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .arg( Arg::with_name("http-duplicate-block-status") .long("http-duplicate-block-status") + .requires("enable_http") .takes_value(true) - .default_value("202") + .default_value_if("enable_http", None, "202") .value_name("STATUS_CODE") .help("Status code to send when a block that is already known is POSTed to the \ HTTP API.") @@ -442,13 +452,14 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .arg( Arg::with_name("http-enable-beacon-processor") .long("http-enable-beacon-processor") + .requires("enable_http") .value_name("BOOLEAN") .help("The beacon processor is a scheduler which provides quality-of-service and \ DoS protection. When set to \"true\", HTTP API requests will be queued and scheduled \ alongside other tasks. When set to \"false\", HTTP API responses will be executed \ immediately.") .takes_value(true) - .default_value("true") + .default_value_if("enable_http", None, "true") ) /* Prometheus metrics HTTP server related arguments */ .arg( @@ -461,22 +472,25 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { Arg::with_name("metrics-address") .long("metrics-address") .value_name("ADDRESS") + .requires("metrics") .help("Set the listen address for the Prometheus metrics HTTP server.") - .default_value("127.0.0.1") + .default_value_if("metrics", None, "127.0.0.1") .takes_value(true), ) .arg( Arg::with_name("metrics-port") .long("metrics-port") + .requires("metrics") .value_name("PORT") .help("Set the listen TCP port for the Prometheus metrics HTTP server.") - .default_value("5054") + .default_value_if("metrics", None, "5054") .takes_value(true), ) .arg( Arg::with_name("metrics-allow-origin") .long("metrics-allow-origin") .value_name("ORIGIN") + .requires("metrics") .help("Set the value of the Access-Control-Allow-Origin response HTTP header. \ Use * to allow any origin (not recommended in production). \ If no value is supplied, the CORS allowed origin is set to the listen \ @@ -1259,4 +1273,5 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .default_value("64") .takes_value(true) ) + .group(ArgGroup::with_name("enable_http").args(&["http", "gui", "staking"])) } diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index b6d0b75d9f..4ab92a7fd4 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -94,70 +94,70 @@ pub fn get_config( * Http API server */ - if cli_args.is_present("http") { + if cli_args.is_present("enable_http") { client_config.http_api.enabled = true; + + if let Some(address) = cli_args.value_of("http-address") { + client_config.http_api.listen_addr = address + .parse::() + .map_err(|_| "http-address is not a valid IP address.")?; + } + + if let Some(port) = cli_args.value_of("http-port") { + client_config.http_api.listen_port = port + .parse::() + .map_err(|_| "http-port is not a valid u16.")?; + } + + if let Some(allow_origin) = cli_args.value_of("http-allow-origin") { + // Pre-validate the config value to give feedback to the user on node startup, instead of + // as late as when the first API response is produced. + hyper::header::HeaderValue::from_str(allow_origin) + .map_err(|_| "Invalid allow-origin value")?; + + client_config.http_api.allow_origin = Some(allow_origin.to_string()); + } + + if cli_args.is_present("http-disable-legacy-spec") { + warn!( + log, + "The flag --http-disable-legacy-spec is deprecated and will be removed" + ); + } + + if let Some(fork_name) = clap_utils::parse_optional(cli_args, "http-spec-fork")? { + client_config.http_api.spec_fork_name = Some(fork_name); + } + + if cli_args.is_present("http-enable-tls") { + client_config.http_api.tls_config = Some(TlsConfig { + cert: cli_args + .value_of("http-tls-cert") + .ok_or("--http-tls-cert was not provided.")? + .parse::() + .map_err(|_| "http-tls-cert is not a valid path name.")?, + key: cli_args + .value_of("http-tls-key") + .ok_or("--http-tls-key was not provided.")? + .parse::() + .map_err(|_| "http-tls-key is not a valid path name.")?, + }); + } + + if cli_args.is_present("http-allow-sync-stalled") { + client_config.http_api.allow_sync_stalled = true; + } + + client_config.http_api.sse_capacity_multiplier = + parse_required(cli_args, "http-sse-capacity-multiplier")?; + + client_config.http_api.enable_beacon_processor = + parse_required(cli_args, "http-enable-beacon-processor")?; + + client_config.http_api.duplicate_block_status_code = + parse_required(cli_args, "http-duplicate-block-status")?; } - if let Some(address) = cli_args.value_of("http-address") { - client_config.http_api.listen_addr = address - .parse::() - .map_err(|_| "http-address is not a valid IP address.")?; - } - - if let Some(port) = cli_args.value_of("http-port") { - client_config.http_api.listen_port = port - .parse::() - .map_err(|_| "http-port is not a valid u16.")?; - } - - if let Some(allow_origin) = cli_args.value_of("http-allow-origin") { - // Pre-validate the config value to give feedback to the user on node startup, instead of - // as late as when the first API response is produced. - hyper::header::HeaderValue::from_str(allow_origin) - .map_err(|_| "Invalid allow-origin value")?; - - client_config.http_api.allow_origin = Some(allow_origin.to_string()); - } - - if cli_args.is_present("http-disable-legacy-spec") { - warn!( - log, - "The flag --http-disable-legacy-spec is deprecated and will be removed" - ); - } - - if let Some(fork_name) = clap_utils::parse_optional(cli_args, "http-spec-fork")? { - client_config.http_api.spec_fork_name = Some(fork_name); - } - - if cli_args.is_present("http-enable-tls") { - client_config.http_api.tls_config = Some(TlsConfig { - cert: cli_args - .value_of("http-tls-cert") - .ok_or("--http-tls-cert was not provided.")? - .parse::() - .map_err(|_| "http-tls-cert is not a valid path name.")?, - key: cli_args - .value_of("http-tls-key") - .ok_or("--http-tls-key was not provided.")? - .parse::() - .map_err(|_| "http-tls-key is not a valid path name.")?, - }); - } - - if cli_args.is_present("http-allow-sync-stalled") { - client_config.http_api.allow_sync_stalled = true; - } - - client_config.http_api.sse_capacity_multiplier = - parse_required(cli_args, "http-sse-capacity-multiplier")?; - - client_config.http_api.enable_beacon_processor = - parse_required(cli_args, "http-enable-beacon-processor")?; - - client_config.http_api.duplicate_block_status_code = - parse_required(cli_args, "http-duplicate-block-status")?; - if let Some(cache_size) = clap_utils::parse_optional(cli_args, "shuffling-cache-size")? { client_config.chain.shuffling_cache_size = cache_size; } diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 10fcb101a8..f5fe7334df 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -1466,6 +1466,7 @@ fn http_flag() { fn http_address_flag() { let addr = "127.0.0.99".parse::().unwrap(); CommandLineTest::new() + .flag("http", None) .flag("http-address", Some("127.0.0.99")) .run_with_zero_port() .with_config(|config| assert_eq!(config.http_api.listen_addr, addr)); @@ -1474,6 +1475,7 @@ fn http_address_flag() { fn http_address_ipv6_flag() { let addr = "::1".parse::().unwrap(); CommandLineTest::new() + .flag("http", None) .flag("http-address", Some("::1")) .run_with_zero_port() .with_config(|config| assert_eq!(config.http_api.listen_addr, addr)); @@ -1483,6 +1485,7 @@ fn http_port_flag() { let port1 = unused_tcp4_port().expect("Unable to find unused port."); let port2 = unused_tcp4_port().expect("Unable to find unused port."); CommandLineTest::new() + .flag("http", None) .flag("http-port", Some(port1.to_string().as_str())) .flag("port", Some(port2.to_string().as_str())) .run() @@ -2446,6 +2449,7 @@ fn http_sse_capacity_multiplier_default() { #[test] fn http_sse_capacity_multiplier_override() { CommandLineTest::new() + .flag("http", None) .flag("http-sse-capacity-multiplier", Some("10")) .run_with_zero_port() .with_config(|config| assert_eq!(config.http_api.sse_capacity_multiplier, 10)); @@ -2463,6 +2467,7 @@ fn http_duplicate_block_status_default() { #[test] fn http_duplicate_block_status_override() { CommandLineTest::new() + .flag("http", None) .flag("http-duplicate-block-status", Some("301")) .run_with_zero_port() .with_config(|config| { diff --git a/validator_client/src/cli.rs b/validator_client/src/cli.rs index 0789ac78a0..0af92a9e39 100644 --- a/validator_client/src/cli.rs +++ b/validator_client/src/cli.rs @@ -170,6 +170,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .arg( Arg::with_name("http-address") .long("http-address") + .requires("http") .value_name("ADDRESS") .help("Set the address for the HTTP address. The HTTP server is not encrypted \ and therefore it is unsafe to publish on a public network. When this \ @@ -189,14 +190,16 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .arg( Arg::with_name("http-port") .long("http-port") + .requires("http") .value_name("PORT") .help("Set the listen TCP port for the RESTful HTTP API server.") - .default_value("5062") + .default_value_if("http", None, "5062") .takes_value(true), ) .arg( Arg::with_name("http-allow-origin") .long("http-allow-origin") + .requires("http") .value_name("ORIGIN") .help("Set the value of the Access-Control-Allow-Origin response HTTP header. \ Use * to allow any origin (not recommended in production). \ @@ -207,21 +210,21 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .arg( Arg::with_name("http-allow-keystore-export") .long("http-allow-keystore-export") + .requires("http") .help("If present, allow access to the DELETE /lighthouse/keystores HTTP \ API method, which allows exporting keystores and passwords to HTTP API \ consumers who have access to the API token. This method is useful for \ exporting validators, however it should be used with caution since it \ exposes private key data to authorized users.") - .required(false) .takes_value(false), ) .arg( Arg::with_name("http-store-passwords-in-secrets-dir") .long("http-store-passwords-in-secrets-dir") + .requires("http") .help("If present, any validators created via the HTTP will have keystore \ passwords stored in the secrets-dir rather than the validator \ definitions file.") - .required(false) .takes_value(false), ) /* Prometheus metrics HTTP server related arguments */ @@ -234,22 +237,25 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .arg( Arg::with_name("metrics-address") .long("metrics-address") + .requires("metrics") .value_name("ADDRESS") .help("Set the listen address for the Prometheus metrics HTTP server.") - .default_value("127.0.0.1") + .default_value_if("metrics", None, "127.0.0.1") .takes_value(true), ) .arg( Arg::with_name("metrics-port") .long("metrics-port") + .requires("metrics") .value_name("PORT") .help("Set the listen TCP port for the Prometheus metrics HTTP server.") - .default_value("5064") + .default_value_if("metrics", None, "5064") .takes_value(true), ) .arg( Arg::with_name("metrics-allow-origin") .long("metrics-allow-origin") + .requires("metrics") .value_name("ORIGIN") .help("Set the value of the Access-Control-Allow-Origin response HTTP header. \ Use * to allow any origin (not recommended in production). \ From c4e907de9f531c8bd18d9547515c1918a5e2b0e1 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Fri, 22 Sep 2023 02:33:11 +0000 Subject: [PATCH 17/19] Update the voluntary exit endpoint to comply with the key manager specification (#4679) ## Issue Addressed #4635 ## Proposed Changes Wrap the `SignedVoluntaryExit` object in a `GenericResponse` container, adding an additional `data` layer, to ensure compliance with the key manager API specification. The new response would look like this: ```json {"data":{"message":{"epoch":"196868","validator_index":"505597"},"signature":"0xhexsig"}} ``` This is a backward incompatible change and will affect Siren as well. --- common/eth2/src/lighthouse_vc/http_client.rs | 2 +- .../src/http_api/create_signed_voluntary_exit.rs | 5 +++-- validator_client/src/http_api/tests.rs | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/common/eth2/src/lighthouse_vc/http_client.rs b/common/eth2/src/lighthouse_vc/http_client.rs index 7bf4cf5b19..b2d53c5e08 100644 --- a/common/eth2/src/lighthouse_vc/http_client.rs +++ b/common/eth2/src/lighthouse_vc/http_client.rs @@ -666,7 +666,7 @@ impl ValidatorClientHttpClient { &self, pubkey: &PublicKeyBytes, epoch: Option, - ) -> Result { + ) -> Result, Error> { let mut path = self.server.full.clone(); path.path_segments_mut() diff --git a/validator_client/src/http_api/create_signed_voluntary_exit.rs b/validator_client/src/http_api/create_signed_voluntary_exit.rs index b777d15806..a9586da57e 100644 --- a/validator_client/src/http_api/create_signed_voluntary_exit.rs +++ b/validator_client/src/http_api/create_signed_voluntary_exit.rs @@ -1,5 +1,6 @@ use crate::validator_store::ValidatorStore; use bls::{PublicKey, PublicKeyBytes}; +use eth2::types::GenericResponse; use slog::{info, Logger}; use slot_clock::SlotClock; use std::sync::Arc; @@ -11,7 +12,7 @@ pub async fn create_signed_voluntary_exit>, slot_clock: T, log: Logger, -) -> Result { +) -> Result, warp::Rejection> { let epoch = match maybe_epoch { Some(epoch) => epoch, None => get_current_epoch::(slot_clock).ok_or_else(|| { @@ -60,7 +61,7 @@ pub async fn create_signed_voluntary_exit Date: Fri, 22 Sep 2023 02:33:12 +0000 Subject: [PATCH 18/19] Add new teku bootnodes (#4724) Adds new Teku bootnodes Co-authored-by: Paul Hauner --- .../built_in_network_configs/mainnet/boot_enr.yaml | 6 ++++-- lighthouse/tests/beacon_node.rs | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/common/eth2_network_config/built_in_network_configs/mainnet/boot_enr.yaml b/common/eth2_network_config/built_in_network_configs/mainnet/boot_enr.yaml index 428a082cc0..7442f6576e 100644 --- a/common/eth2_network_config/built_in_network_configs/mainnet/boot_enr.yaml +++ b/common/eth2_network_config/built_in_network_configs/mainnet/boot_enr.yaml @@ -10,11 +10,13 @@ - enr:-Ku4QEWzdnVtXc2Q0ZVigfCGggOVB2Vc1ZCPEc6j21NIFLODSJbvNaef1g4PxhPwl_3kax86YPheFUSLXPRs98vvYsoBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpC1MD8qAAAAAP__________gmlkgnY0gmlwhDZBrP2Jc2VjcDI1NmsxoQM6jr8Rb1ktLEsVcKAPa08wCsKUmvoQ8khiOl_SLozf9IN1ZHCCIyg # Teku team (Consensys) - enr:-KG4QOtcP9X1FbIMOe17QNMKqDxCpm14jcX5tiOE4_TyMrFqbmhPZHK_ZPG2Gxb1GE2xdtodOfx9-cgvNtxnRyHEmC0ghGV0aDKQ9aX9QgAAAAD__________4JpZIJ2NIJpcIQDE8KdiXNlY3AyNTZrMaEDhpehBDbZjM_L9ek699Y7vhUJ-eAdMyQW_Fil522Y0fODdGNwgiMog3VkcIIjKA -- enr:-KG4QDyytgmE4f7AnvW-ZaUOIi9i79qX4JwjRAiXBZCU65wOfBu-3Nb5I7b_Rmg3KCOcZM_C3y5pg7EBU5XGrcLTduQEhGV0aDKQ9aX9QgAAAAD__________4JpZIJ2NIJpcIQ2_DUbiXNlY3AyNTZrMaEDKnz_-ps3UUOfHWVYaskI5kWYO_vtYMGYCQRAR3gHDouDdGNwgiMog3VkcIIjKA +- enr:-KG4QL-eqFoHy0cI31THvtZjpYUu_Jdw_MO7skQRJxY1g5HTN1A0epPCU6vi0gLGUgrzpU-ygeMSS8ewVxDpKfYmxMMGhGV0aDKQtTA_KgAAAAD__________4JpZIJ2NIJpcIQ2_DUbiXNlY3AyNTZrMaED8GJ2vzUqgL6-KD1xalo1CsmY4X1HaDnyl6Y_WayCo9GDdGNwgiMog3VkcIIjKA +- enr:-KG4QMOEswP62yzDjSwWS4YEjtTZ5PO6r65CPqYBkgTTkrpaedQ8uEUo1uMALtJIvb2w_WWEVmg5yt1UAuK1ftxUU7QDhGV0aDKQu6TalgMAAAD__________4JpZIJ2NIJpcIQEnfA2iXNlY3AyNTZrMaEDfol8oLr6XJ7FsdAYE7lpJhKMls4G_v6qQOGKJUWGb_uDdGNwgiMog3VkcIIjKA +- enr:-KG4QF4B5WrlFcRhUU6dZETwY5ZzAXnA0vGC__L1Kdw602nDZwXSTs5RFXFIFUnbQJmhNGVU6OIX7KVrCSTODsz1tK4DhGV0aDKQu6TalgMAAAD__________4JpZIJ2NIJpcIQExNYEiXNlY3AyNTZrMaECQmM9vp7KhaXhI-nqL_R0ovULLCFSFTa9CPPSdb1zPX6DdGNwgiMog3VkcIIjKA # Prysm team (Prysmatic Labs) - enr:-Ku4QImhMc1z8yCiNJ1TyUxdcfNucje3BGwEHzodEZUan8PherEo4sF7pPHPSIB1NNuSg5fZy7qFsjmUKs2ea1Whi0EBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpD1pf1CAAAAAP__________gmlkgnY0gmlwhBLf22SJc2VjcDI1NmsxoQOVphkDqal4QzPMksc5wnpuC3gvSC8AfbFOnZY_On34wIN1ZHCCIyg - enr:-Ku4QP2xDnEtUXIjzJ_DhlCRN9SN99RYQPJL92TMlSv7U5C1YnYLjwOQHgZIUXw6c-BvRg2Yc2QsZxxoS_pPRVe0yK8Bh2F0dG5ldHOIAAAAAAAAAACEZXRoMpD1pf1CAAAAAP__________gmlkgnY0gmlwhBLf22SJc2VjcDI1NmsxoQMeFF5GrS7UZpAH2Ly84aLK-TyvH-dRo0JM1i8yygH50YN1ZHCCJxA - enr:-Ku4QPp9z1W4tAO8Ber_NQierYaOStqhDqQdOPY3bB3jDgkjcbk6YrEnVYIiCBbTxuar3CzS528d2iE7TdJsrL-dEKoBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpD1pf1CAAAAAP__________gmlkgnY0gmlwhBLf22SJc2VjcDI1NmsxoQMw5fqqkw2hHC4F5HZZDPsNmPdB1Gi8JPQK7pRc9XHh-oN1ZHCCKvg # Nimbus team - enr:-LK4QA8FfhaAjlb_BXsXxSfiysR7R52Nhi9JBt4F8SPssu8hdE1BXQQEtVDC3qStCW60LSO7hEsVHv5zm8_6Vnjhcn0Bh2F0dG5ldHOIAAAAAAAAAACEZXRoMpC1MD8qAAAAAP__________gmlkgnY0gmlwhAN4aBKJc2VjcDI1NmsxoQJerDhsJ-KxZ8sHySMOCmTO6sHM3iCFQ6VMvLTe948MyYN0Y3CCI4yDdWRwgiOM -- enr:-LK4QKWrXTpV9T78hNG6s8AM6IO4XH9kFT91uZtFg1GcsJ6dKovDOr1jtAAFPnS2lvNltkOGA9k29BUN7lFh_sjuc9QBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpC1MD8qAAAAAP__________gmlkgnY0gmlwhANAdd-Jc2VjcDI1NmsxoQLQa6ai7y9PMN5hpLe5HmiJSlYzMuzP7ZhwRiwHvqNXdoN0Y3CCI4yDdWRwgiOM \ No newline at end of file +- enr:-LK4QKWrXTpV9T78hNG6s8AM6IO4XH9kFT91uZtFg1GcsJ6dKovDOr1jtAAFPnS2lvNltkOGA9k29BUN7lFh_sjuc9QBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpC1MD8qAAAAAP__________gmlkgnY0gmlwhANAdd-Jc2VjcDI1NmsxoQLQa6ai7y9PMN5hpLe5HmiJSlYzMuzP7ZhwRiwHvqNXdoN0Y3CCI4yDdWRwgiOM diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index f5fe7334df..cb51dffdce 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -1194,7 +1194,7 @@ fn default_backfill_rate_limiting_flag() { } #[test] fn default_boot_nodes() { - let number_of_boot_nodes = 15; + let number_of_boot_nodes = 17; CommandLineTest::new() .run_with_zero_port() From 69c39ad1e5510222248a105b3d911e9643145e3f Mon Sep 17 00:00:00 2001 From: antondlr Date: Fri, 22 Sep 2023 02:33:13 +0000 Subject: [PATCH 19/19] Use release workflow runners (#4765) ## Issue Addressed Build releases on self-hosted hardware to speed the process up --- .github/workflows/release.yml | 18 +++++++++--------- .github/workflows/test-suite.yml | 10 +++++----- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e38b03daf7..62eadd0451 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -38,31 +38,31 @@ jobs: x86_64-windows-portable] include: - arch: aarch64-unknown-linux-gnu - platform: ubuntu-latest + runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest' }} profile: maxperf - arch: aarch64-unknown-linux-gnu-portable - platform: ubuntu-latest + runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest' }} profile: maxperf - arch: x86_64-unknown-linux-gnu - platform: ubuntu-latest + runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest' }} profile: maxperf - arch: x86_64-unknown-linux-gnu-portable - platform: ubuntu-latest + runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest' }} profile: maxperf - arch: x86_64-apple-darwin - platform: macos-latest + runner: macos-latest profile: maxperf - arch: x86_64-apple-darwin-portable - platform: macos-latest + runner: macos-latest profile: maxperf - arch: x86_64-windows - platform: windows-2019 + runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "windows", "release"]') || 'windows-2019' }} profile: maxperf - arch: x86_64-windows-portable - platform: windows-2019 + runner: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "windows", "release"]') || 'windows-2019' }} profile: maxperf - runs-on: ${{ matrix.platform }} + runs-on: ${{ matrix.runner }} needs: extract-version steps: - name: Checkout sources diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 1d9856b93c..fd9b77ae2f 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -58,7 +58,7 @@ jobs: release-tests-ubuntu: name: release-tests-ubuntu # Use self-hosted runners only on the sigp repo. - runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "large"]') || 'ubuntu-latest' }} + runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "CI", "large"]') || 'ubuntu-latest' }} needs: cargo-fmt steps: - uses: actions/checkout@v3 @@ -73,7 +73,7 @@ jobs: run: make test-release release-tests-windows: name: release-tests-windows - runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "windows"]') || 'windows-2019' }} + runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "windows", "CI"]') || 'windows-2019' }} needs: cargo-fmt steps: - uses: actions/checkout@v3 @@ -98,7 +98,7 @@ jobs: beacon-chain-tests: name: beacon-chain-tests # Use self-hosted runners only on the sigp repo. - runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "large"]') || 'ubuntu-latest' }} + runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "CI", "large"]') || 'ubuntu-latest' }} needs: cargo-fmt steps: - uses: actions/checkout@v3 @@ -130,7 +130,7 @@ jobs: debug-tests-ubuntu: name: debug-tests-ubuntu # Use self-hosted runners only on the sigp repo. - runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "large"]') || 'ubuntu-latest' }} + runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "CI", "large"]') || 'ubuntu-latest' }} needs: cargo-fmt steps: - uses: actions/checkout@v3 @@ -156,7 +156,7 @@ jobs: ef-tests-ubuntu: name: ef-tests-ubuntu # Use self-hosted runners only on the sigp repo. - runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "small"]') || 'ubuntu-latest' }} + runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "CI", "small"]') || 'ubuntu-latest' }} needs: cargo-fmt steps: - uses: actions/checkout@v3