Merge branch 'unstable' of https://github.com/sigp/lighthouse into single_attestation

common/account_utils/Cargo.toml

@@ -3,20 +3,19 @@ name = "account_utils"
 version = "0.1.0"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = { workspace = true }
-
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-rand = { workspace = true }
-eth2_wallet = { workspace = true }
+directory = { workspace = true }
 eth2_keystore = { workspace = true }
+eth2_wallet = { workspace = true }
 filesystem = { workspace = true }
-zeroize = { workspace = true }
+rand = { workspace = true }
+regex = { workspace = true }
+rpassword = "5.0.0"
 serde = { workspace = true }
 serde_yaml = { workspace = true }
 slog = { workspace = true }
 types = { workspace = true }
 validator_dir = { workspace = true }
-regex = { workspace = true }
-rpassword = "5.0.0"
-directory = { workspace = true }
+zeroize = { workspace = true }

common/clap_utils/Cargo.toml

@@ -3,16 +3,15 @@ name = "clap_utils"
 version = "0.1.0"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = { workspace = true }
-
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
 alloy-primitives = { workspace = true }
 clap = { workspace = true }
-hex = { workspace = true }
 dirs = { workspace = true }
 eth2_network_config = { workspace = true }
 ethereum_ssz = { workspace = true }
+hex = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 serde_yaml = { workspace = true }

common/compare_fields_derive/Cargo.toml

@@ -8,5 +8,5 @@ edition = { workspace = true }
 proc-macro = true
 
 [dependencies]
-syn = { workspace = true }
 quote = { workspace = true }
+syn = { workspace = true }

common/deposit_contract/Cargo.toml

@@ -7,13 +7,13 @@ edition = { workspace = true }
 build = "build.rs"
 
 [build-dependencies]
+hex = { workspace = true }
 reqwest = { workspace = true }
 serde_json = { workspace = true }
 sha2 = { workspace = true }
-hex = { workspace = true }
 
 [dependencies]
-types = { workspace = true }
+ethabi = "16.0.0"
 ethereum_ssz = { workspace = true }
 tree_hash = { workspace = true }
-ethabi = "16.0.0"
+types = { workspace = true }

common/directory/Cargo.toml

@@ -3,7 +3,6 @@ name = "directory"
 version = "0.1.0"
 authors = ["pawan <pawandhananjay@gmail.com>"]
 edition = { workspace = true }
-
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]

common/eth2/Cargo.toml

@@ -3,30 +3,30 @@ name = "eth2"
 version = "0.1.0"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = { workspace = true }
-
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-serde = { workspace = true }
-serde_json = { workspace = true }
-ssz_types = { workspace = true }
-types = { workspace = true }
-reqwest = { workspace = true }
-lighthouse_network = { workspace = true }
-proto_array = { workspace = true }
-ethereum_serde_utils = { workspace = true }
+derivative = { workspace = true }
 eth2_keystore = { workspace = true }
-zeroize = { workspace = true }
-sensitive_url = { workspace = true }
+ethereum_serde_utils = { workspace = true }
 ethereum_ssz = { workspace = true }
 ethereum_ssz_derive = { workspace = true }
-futures-util = "0.3.8"
 futures = { workspace = true }
-store = { workspace = true }
-slashing_protection = { workspace = true }
+futures-util = "0.3.8"
+lighthouse_network = { workspace = true }
 mediatype = "0.19.13"
 pretty_reqwest_error = { workspace = true }
-derivative = { workspace = true }
+proto_array = { workspace = true }
+reqwest = { workspace = true }
+reqwest-eventsource = "0.5.0"
+sensitive_url = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+slashing_protection = { workspace = true }
+ssz_types = { workspace = true }
+store = { workspace = true }
+types = { workspace = true }
+zeroize = { workspace = true }
 
 [dev-dependencies]
 tokio = { workspace = true }

common/eth2/src/lib.rs

@@ -27,6 +27,7 @@ use reqwest::{
     Body, IntoUrl, RequestBuilder, Response,
 };
 pub use reqwest::{StatusCode, Url};
+use reqwest_eventsource::{Event, EventSource};
 pub use sensitive_url::{SensitiveError, SensitiveUrl};
 use serde::{de::DeserializeOwned, Serialize};
 use ssz::Encode;
@@ -53,6 +54,8 @@ pub const SSZ_CONTENT_TYPE_HEADER: &str = "application/octet-stream";
 pub enum Error {
     /// The `reqwest` client raised an error.
     HttpClient(PrettyReqwestError),
+    /// The `reqwest_eventsource` client raised an error.
+    SseClient(reqwest_eventsource::Error),
     /// The server returned an error message where the body was able to be parsed.
     ServerMessage(ErrorMessage),
     /// The server returned an error message with an array of errors.
@@ -94,6 +97,13 @@ impl Error {
     pub fn status(&self) -> Option<StatusCode> {
         match self {
             Error::HttpClient(error) => error.inner().status(),
+            Error::SseClient(error) => {
+                if let reqwest_eventsource::Error::InvalidStatusCode(status, _) = error {
+                    Some(*status)
+                } else {
+                    None
+                }
+            }
             Error::ServerMessage(msg) => StatusCode::try_from(msg.code).ok(),
             Error::ServerIndexedMessage(msg) => StatusCode::try_from(msg.code).ok(),
             Error::StatusCode(status) => Some(*status),
@@ -2593,16 +2603,29 @@ impl BeaconNodeHttpClient {
             .join(",");
         path.query_pairs_mut().append_pair("topics", &topic_string);
 
-        Ok(self
-            .client
-            .get(path)
-            .send()
-            .await?
-            .bytes_stream()
-            .map(|next| match next {
-                Ok(bytes) => EventKind::from_sse_bytes(bytes.as_ref()),
-                Err(e) => Err(Error::HttpClient(e.into())),
-            }))
+        let mut es = EventSource::get(path);
+        // If we don't await `Event::Open` here, then the consumer
+        // will not get any Message events until they start awaiting the stream.
+        // This is a way to register the stream with the sse server before
+        // message events start getting emitted.
+        while let Some(event) = es.next().await {
+            match event {
+                Ok(Event::Open) => break,
+                Err(err) => return Err(Error::SseClient(err)),
+                // This should never happen as we are guaranteed to get the
+                // Open event before any message starts coming through.
+                Ok(Event::Message(_)) => continue,
+            }
+        }
+        Ok(Box::pin(es.filter_map(|event| async move {
+            match event {
+                Ok(Event::Open) => None,
+                Ok(Event::Message(message)) => {
+                    Some(EventKind::from_sse_bytes(&message.event, &message.data))
+                }
+                Err(err) => Some(Err(Error::SseClient(err))),
+            }
+        })))
     }
 
     /// `POST validator/duties/sync/{epoch}`

common/eth2/src/types.rs

@@ -14,7 +14,7 @@ use serde_json::Value;
 use ssz::{Decode, DecodeError};
 use ssz_derive::{Decode, Encode};
 use std::fmt::{self, Display};
-use std::str::{from_utf8, FromStr};
+use std::str::FromStr;
 use std::sync::Arc;
 use std::time::Duration;
 use types::beacon_block_body::KzgCommitments;
@@ -1156,24 +1156,7 @@ impl<E: EthSpec> EventKind<E> {
         }
     }
 
-    pub fn from_sse_bytes(message: &[u8]) -> Result<Self, ServerError> {
-        let s = from_utf8(message)
-            .map_err(|e| ServerError::InvalidServerSentEvent(format!("{:?}", e)))?;
-
-        let mut split = s.split('\n');
-        let event = split
-            .next()
-            .ok_or_else(|| {
-                ServerError::InvalidServerSentEvent("Could not parse event tag".to_string())
-            })?
-            .trim_start_matches("event:");
-        let data = split
-            .next()
-            .ok_or_else(|| {
-                ServerError::InvalidServerSentEvent("Could not parse data tag".to_string())
-            })?
-            .trim_start_matches("data:");
-
+    pub fn from_sse_bytes(event: &str, data: &str) -> Result<Self, ServerError> {
         match event {
             "attestation" => Ok(EventKind::Attestation(serde_json::from_str(data).map_err(
                 |e| ServerError::InvalidServerSentEvent(format!("Attestation: {:?}", e)),

common/eth2_config/Cargo.toml

@@ -5,5 +5,5 @@ authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = { workspace = true }
 
 [dependencies]
-types = { workspace = true }
 paste = { workspace = true }
+types = { workspace = true }

common/eth2_interop_keypairs/Cargo.toml

@@ -3,16 +3,15 @@ name = "eth2_interop_keypairs"
 version = "0.2.0"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = { workspace = true }
-
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-num-bigint = "0.4.2"
+bls = { workspace = true }
 ethereum_hashing = { workspace = true }
 hex = { workspace = true }
-serde_yaml = { workspace = true }
+num-bigint = "0.4.2"
 serde = { workspace = true }
-bls = { workspace = true }
+serde_yaml = { workspace = true }
 
 [dev-dependencies]
 base64 = "0.13.0"

common/eth2_network_config/Cargo.toml

@@ -7,25 +7,25 @@ edition = { workspace = true }
 build = "build.rs"
 
 [build-dependencies]
-zip = { workspace = true }
 eth2_config = { workspace = true }
+zip = { workspace = true }
 
 [dev-dependencies]
+ethereum_ssz = { workspace = true }
 tempfile = { workspace = true }
 tokio = { workspace = true }
-ethereum_ssz = { workspace = true }
 
 [dependencies]
-serde_yaml = { workspace = true }
-types = { workspace = true }
-eth2_config = { workspace = true }
-discv5 = { workspace = true }
-reqwest = { workspace = true }
-pretty_reqwest_error = { workspace = true }
-sha2 = { workspace = true }
-url = { workspace = true }
-sensitive_url = { workspace = true }
-slog = { workspace = true }
-logging = { workspace = true }
 bytes = { workspace = true }
+discv5 = { workspace = true }
+eth2_config = { workspace = true }
 kzg = { workspace = true }
+logging = { workspace = true }
+pretty_reqwest_error = { workspace = true }
+reqwest = { workspace = true }
+sensitive_url = { workspace = true }
+serde_yaml = { workspace = true }
+sha2 = { workspace = true }
+slog = { workspace = true }
+types = { workspace = true }
+url = { workspace = true }

common/eth2_wallet_manager/Cargo.toml

@@ -3,7 +3,6 @@ name = "eth2_wallet_manager"
 version = "0.1.0"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = { workspace = true }
-
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]

common/eth2_wallet_manager/src/wallet_manager.rs

@@ -296,10 +296,10 @@ mod tests {
             )
             .expect("should create first wallet");
 
-        let uuid = w.wallet().uuid().clone();
+        let uuid = *w.wallet().uuid();
 
         assert_eq!(
-            load_wallet_raw(&base_dir, &uuid).nextaccount(),
+            load_wallet_raw(base_dir, &uuid).nextaccount(),
             0,
             "should start wallet with nextaccount 0"
         );
@@ -308,7 +308,7 @@ mod tests {
             w.next_validator(WALLET_PASSWORD, &[50; 32], &[51; 32])
                 .expect("should create validator");
             assert_eq!(
-                load_wallet_raw(&base_dir, &uuid).nextaccount(),
+                load_wallet_raw(base_dir, &uuid).nextaccount(),
                 i,
                 "should update wallet with nextaccount {}",
                 i
@@ -333,54 +333,54 @@ mod tests {
         let base_dir = dir.path();
         let mgr = WalletManager::open(base_dir).unwrap();
 
-        let uuid_a = create_wallet(&mgr, 0).wallet().uuid().clone();
-        let uuid_b = create_wallet(&mgr, 1).wallet().uuid().clone();
+        let uuid_a = *create_wallet(&mgr, 0).wallet().uuid();
+        let uuid_b = *create_wallet(&mgr, 1).wallet().uuid();
 
-        let locked_a = LockedWallet::open(&base_dir, &uuid_a).expect("should open wallet a");
+        let locked_a = LockedWallet::open(base_dir, &uuid_a).expect("should open wallet a");
 
         assert!(
-            lockfile_path(&base_dir, &uuid_a).exists(),
+            lockfile_path(base_dir, &uuid_a).exists(),
             "lockfile should exist"
         );
 
         drop(locked_a);
 
         assert!(
-            !lockfile_path(&base_dir, &uuid_a).exists(),
+            !lockfile_path(base_dir, &uuid_a).exists(),
             "lockfile have been cleaned up"
         );
 
-        let locked_a = LockedWallet::open(&base_dir, &uuid_a).expect("should open wallet a");
-        let locked_b = LockedWallet::open(&base_dir, &uuid_b).expect("should open wallet b");
+        let locked_a = LockedWallet::open(base_dir, &uuid_a).expect("should open wallet a");
+        let locked_b = LockedWallet::open(base_dir, &uuid_b).expect("should open wallet b");
 
         assert!(
-            lockfile_path(&base_dir, &uuid_a).exists(),
+            lockfile_path(base_dir, &uuid_a).exists(),
             "lockfile a should exist"
         );
 
         assert!(
-            lockfile_path(&base_dir, &uuid_b).exists(),
+            lockfile_path(base_dir, &uuid_b).exists(),
             "lockfile b should exist"
         );
 
-        match LockedWallet::open(&base_dir, &uuid_a) {
+        match LockedWallet::open(base_dir, &uuid_a) {
             Err(Error::LockfileError(_)) => {}
             _ => panic!("did not get locked error"),
         };
 
         drop(locked_a);
 
-        LockedWallet::open(&base_dir, &uuid_a)
+        LockedWallet::open(base_dir, &uuid_a)
             .expect("should open wallet a after previous instance is dropped");
 
-        match LockedWallet::open(&base_dir, &uuid_b) {
+        match LockedWallet::open(base_dir, &uuid_b) {
             Err(Error::LockfileError(_)) => {}
             _ => panic!("did not get locked error"),
         };
 
         drop(locked_b);
 
-        LockedWallet::open(&base_dir, &uuid_b)
+        LockedWallet::open(base_dir, &uuid_b)
             .expect("should open wallet a after previous instance is dropped");
     }
 }

common/lighthouse_version/Cargo.toml

@@ -3,7 +3,6 @@ name = "lighthouse_version"
 version = "0.1.0"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = { workspace = true }
-
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]

common/lighthouse_version/src/lib.rs

@@ -17,8 +17,8 @@ pub const VERSION: &str = git_version!(
         // NOTE: using --match instead of --exclude for compatibility with old Git
         "--match=thiswillnevermatchlol"
     ],
-    prefix = "Lighthouse/v6.0.0-",
-    fallback = "Lighthouse/v6.0.0"
+    prefix = "Lighthouse/v6.0.1-",
+    fallback = "Lighthouse/v6.0.1"
 );
 
 /// Returns the first eight characters of the latest commit hash for this build.

common/logging/Cargo.toml

@@ -19,7 +19,7 @@ sloggers = { workspace = true }
 take_mut = "0.2.2"
 tokio = { workspace = true, features = [ "time" ] }
 tracing = "0.1"
+tracing-appender = { workspace = true }
 tracing-core = { workspace = true }
 tracing-log = { workspace = true }
 tracing-subscriber = { workspace = true }
-tracing-appender = { workspace = true }

common/malloc_utils/Cargo.toml

@@ -5,8 +5,8 @@ authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = { workspace = true }
 
 [dependencies]
-metrics = { workspace = true }
 libc = "0.2.79"
+metrics = { workspace = true }
 parking_lot = { workspace = true }
 tikv-jemalloc-ctl = { version = "0.6.0", optional = true, features = ["stats"] }
 

common/monitoring_api/Cargo.toml

@@ -3,19 +3,18 @@ name = "monitoring_api"
 version = "0.1.0"
 authors = ["pawan <pawandhananjay@gmail.com>"]
 edition = { workspace = true }
-
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-reqwest = { workspace = true }
-task_executor = { workspace = true }
-tokio = { workspace = true }
 eth2 = { workspace = true }
-serde_json = { workspace = true }
-serde = { workspace = true }
 lighthouse_version = { workspace = true }
 metrics = { workspace = true }
+regex = { workspace = true }
+reqwest = { workspace = true }
+sensitive_url = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
 slog = { workspace = true }
 store = { workspace = true }
-regex = { workspace = true }
-sensitive_url = { workspace = true }
+task_executor = { workspace = true }
+tokio = { workspace = true }

common/oneshot_broadcast/Cargo.toml

@@ -2,7 +2,6 @@
 name = "oneshot_broadcast"
 version = "0.1.0"
 edition = { workspace = true }
-
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]

common/pretty_reqwest_error/Cargo.toml

@@ -2,7 +2,6 @@
 name = "pretty_reqwest_error"
 version = "0.1.0"
 edition = { workspace = true }
-
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]

common/sensitive_url/Cargo.toml

@@ -3,9 +3,8 @@ name = "sensitive_url"
 version = "0.1.0"
 authors = ["Mac L <mjladson@pm.me>"]
 edition = { workspace = true }
-
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-url = { workspace = true }
 serde = { workspace = true }
+url = { workspace = true }

common/slot_clock/Cargo.toml

@@ -5,6 +5,6 @@ authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = { workspace = true }
 
 [dependencies]
-types = { workspace = true }
 metrics = { workspace = true }
 parking_lot = { workspace = true }
+types = { workspace = true }

common/system_health/Cargo.toml

@@ -5,7 +5,7 @@ edition = { workspace = true }
 
 [dependencies]
 lighthouse_network = { workspace = true }
-types = { workspace = true }
-sysinfo = { workspace = true }
-serde = { workspace = true }
 parking_lot = { workspace = true }
+serde = { workspace = true }
+sysinfo = { workspace = true }
+types = { workspace = true }

common/system_health/src/lib.rs

@@ -235,14 +235,14 @@ pub fn observe_nat() -> NatState {
 
     let libp2p_ipv4 = lighthouse_network::metrics::get_int_gauge(
         &lighthouse_network::metrics::NAT_OPEN,
-        &["libp2p"],
+        &["libp2p_ipv4"],
     )
     .map(|g| g.get() == 1)
     .unwrap_or_default();
 
     let libp2p_ipv6 = lighthouse_network::metrics::get_int_gauge(
         &lighthouse_network::metrics::NAT_OPEN,
-        &["libp2p"],
+        &["libp2p_ipv6"],
     )
     .map(|g| g.get() == 1)
     .unwrap_or_default();

common/task_executor/Cargo.toml

@@ -11,10 +11,10 @@ tracing = ["dep:tracing"]
 
 [dependencies]
 async-channel = { workspace = true }
-tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
-slog = { workspace = true, optional = true }
 futures = { workspace = true }
-metrics = { workspace = true }
-sloggers = { workspace = true, optional = true }
 logging = { workspace = true, optional = true }
+metrics = { workspace = true }
+slog = { workspace = true, optional = true }
+sloggers = { workspace = true, optional = true }
+tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
 tracing = { workspace = true, optional = true }

common/test_random_derive/Cargo.toml

@@ -9,5 +9,5 @@ description = "Procedural derive macros for implementation of TestRandom trait"
 proc-macro = true
 
 [dependencies]
-syn = { workspace = true }
 quote = { workspace = true }
+syn = { workspace = true }

common/unused_port/Cargo.toml

@@ -2,7 +2,6 @@
 name = "unused_port"
 version = "0.1.0"
 edition = { workspace = true }
-
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]

common/validator_dir/Cargo.toml

@@ -6,21 +6,20 @@ edition = { workspace = true }
 
 [features]
 insecure_keys = []
-
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
 bls = { workspace = true }
+deposit_contract = { workspace = true }
+derivative = { workspace = true }
+directory = { workspace = true }
 eth2_keystore = { workspace = true }
 filesystem = { workspace = true }
-types = { workspace = true }
-rand = { workspace = true }
-deposit_contract = { workspace = true }
-tree_hash = { workspace = true }
 hex = { workspace = true }
-derivative = { workspace = true }
 lockfile = { workspace = true }
-directory = { workspace = true }
+rand = { workspace = true }
+tree_hash = { workspace = true }
+types = { workspace = true }
 
 [dev-dependencies]
 tempfile = { workspace = true }

common/warp_utils/Cargo.toml

@@ -3,20 +3,19 @@ name = "warp_utils"
 version = "0.1.0"
 authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = { workspace = true }
-
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-warp = { workspace = true }
-eth2 = { workspace = true }
-types = { workspace = true }
 beacon_chain = { workspace = true }
-state_processing = { workspace = true }
-safe_arith = { workspace = true }
-serde = { workspace = true }
-serde_json = { workspace = true }
-tokio = { workspace = true }
+bytes = { workspace = true }
+eth2 = { workspace = true }
 headers = "0.3.2"
 metrics = { workspace = true }
+safe_arith = { workspace = true }
+serde = { workspace = true }
 serde_array_query = "0.1.0"
-bytes = { workspace = true }
+serde_json = { workspace = true }
+state_processing = { workspace = true }
+tokio = { workspace = true }
+types = { workspace = true }
+warp = { workspace = true }