From 70d6e6705e86b1aed18e31e7887760845a62df93 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Fri, 23 Dec 2022 12:42:00 +1100 Subject: [PATCH 1/8] Add Beacon API endpoint to download blobs by block ID --- beacon_node/http_api/src/block_id.rs | 47 +++++++++++++++++++++++++++- beacon_node/http_api/src/lib.rs | 35 +++++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/beacon_node/http_api/src/block_id.rs b/beacon_node/http_api/src/block_id.rs index 5c785fe651..2578b4a67c 100644 --- a/beacon_node/http_api/src/block_id.rs +++ b/beacon_node/http_api/src/block_id.rs @@ -4,7 +4,7 @@ use eth2::types::BlockId as CoreBlockId; use std::fmt; use std::str::FromStr; use std::sync::Arc; -use types::{Hash256, SignedBeaconBlock, SignedBlindedBeaconBlock, Slot}; +use types::{BlobsSidecar, Hash256, SignedBeaconBlock, SignedBlindedBeaconBlock, Slot}; /// Wraps `eth2::types::BlockId` and provides a simple way to obtain a block or root for a given /// `BlockId`. @@ -211,6 +211,51 @@ impl BlockId { } } } + + /// Return the `BlobsSidecar` identified by `self`. + pub async fn blobs_sidecar( + &self, + chain: &BeaconChain, + ) -> Result<(Arc>), warp::Rejection> { + let root = match &self.0 { + CoreBlockId::Head => { + let (cached_head, _execution_status) = chain + .canonical_head + .head_and_execution_status() + .map_err(warp_utils::reject::beacon_chain_error)?; + cached_head.head_block_root() + } + CoreBlockId::Slot(slot) => { + let maybe_block_root = chain + .block_root_at_slot(*slot, WhenSlotSkipped::None) + .ok() + .flatten(); + match maybe_block_root { + Some(block_root) => block_root, + None => { + return Err(warp_utils::reject::custom_not_found(format!( + "Block root for slot {} not found", + slot + ))) + } + } + } + _ => self.root(chain)?.0, + }; + + match chain.store.get_blobs(&root) { + Ok(Some(blob)) => Ok((Arc::new(blob))), + Ok(None) => Err(warp_utils::reject::custom_not_found(format!( + "Blob with block root {} is not in the store", + root + ))), + // should we use `warp_utils::reject::beacon_chain_error` instead? + Err(e) => Err(warp_utils::reject::custom_not_found(format!( + "Error fetching blob with block root {}: {:?}", + root, e + ))), + } + } } impl FromStr for BlockId { diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index e5336ef6ad..6caf99a07f 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -3324,6 +3324,41 @@ pub fn serve( ))) }); + // GET lighthouse/beacon/blob_sidecars/{block_id} + let get_blob_sidecars = warp::path("lighthouse") + .and(warp::path("beacon")) + .and(warp::path("blob_sidecars")) + .and(block_id_or_err) + .and(warp::path::end()) + .and(chain_filter.clone()) + .and(warp::header::optional::("accept")) + .and_then( + |block_id: BlockId, + chain: Arc>, + accept_header: Option| { + async move { + let blobs_sidecar = block_id.blobs_sidecar(&chain).await?; + + match accept_header { + Some(api_types::Accept::Ssz) => Response::builder() + .status(200) + .header("Content-Type", "application/octet-stream") + .body(blobs_sidecar.as_ssz_bytes().into()) + .map_err(|e| { + warp_utils::reject::custom_server_error(format!( + "failed to create response: {}", + e + )) + }), + _ => Ok(warp::reply::json(&api_types::GenericResponse::from( + blobs_sidecar, + )) + .into_response()), + } + } + }, + ); + let get_events = eth_v1 .and(warp::path("events")) .and(warp::path::end()) From 0155c94f86b9cb5fce501ddc2bdcfcf997c567a9 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Fri, 23 Dec 2022 14:47:15 +1100 Subject: [PATCH 2/8] Fix code formatting --- beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs b/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs index d69b42c0be..df5350d949 100644 --- a/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs +++ b/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs @@ -719,7 +719,10 @@ fn context_bytes_to_fork_name( let encoded = hex::encode(context_bytes); RPCError::ErrorResponse( RPCResponseErrorCode::InvalidRequest, - format!("Context bytes {} do not correspond to a valid fork", encoded), + format!( + "Context bytes {} do not correspond to a valid fork", + encoded + ), ) }) } From bf7f709b51927aae74402736d5ba04f6383012c0 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Fri, 23 Dec 2022 14:52:03 +1100 Subject: [PATCH 3/8] Add missing route --- beacon_node/http_api/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 6caf99a07f..8d3f3cac33 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -3494,6 +3494,7 @@ pub fn serve( .or(get_lighthouse_attestation_performance.boxed()) .or(get_lighthouse_block_packing_efficiency.boxed()) .or(get_lighthouse_merge_readiness.boxed()) + .or(get_blob_sidecars.boxed()) .or(get_events.boxed()), ) .boxed() From 3b9041047a2a0bc71fb0c68684a39e51913cab4d Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Fri, 23 Dec 2022 15:28:08 +1100 Subject: [PATCH 4/8] Fix typoe and add blobs endpoint to eth2 lib. --- beacon_node/http_api/src/lib.rs | 8 ++++---- common/eth2/src/lib.rs | 30 ++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 8d3f3cac33..0eb261d96a 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -3324,10 +3324,10 @@ pub fn serve( ))) }); - // GET lighthouse/beacon/blob_sidecars/{block_id} - let get_blob_sidecars = warp::path("lighthouse") + // GET lighthouse/beacon/blobs_sidecars/{block_id} + let get_blobs_sidecars = warp::path("lighthouse") .and(warp::path("beacon")) - .and(warp::path("blob_sidecars")) + .and(warp::path("blobs_sidecars")) .and(block_id_or_err) .and(warp::path::end()) .and(chain_filter.clone()) @@ -3494,7 +3494,7 @@ pub fn serve( .or(get_lighthouse_attestation_performance.boxed()) .or(get_lighthouse_block_packing_efficiency.boxed()) .or(get_lighthouse_merge_readiness.boxed()) - .or(get_blob_sidecars.boxed()) + .or(get_blobs_sidecars.boxed()) .or(get_events.boxed()), ) .boxed() diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index fcfff7284a..f08c4eb30e 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -680,6 +680,19 @@ impl BeaconNodeHttpClient { Ok(path) } + /// Path for `lighthouse/beacon/blobs_sidecars/{block_id}` + pub fn get_blobs_sidecar_path(&self, block_id: BlockId) -> Result { + let mut path = self.server.full.clone(); + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("lighthouse") + .push("beacon") + .push("blobs_sidecars") + .push(&block_id.to_string()); + Ok(path) + } + /// Path for `v1/beacon/blinded_blocks/{block_id}` pub fn get_beacon_blinded_blocks_path(&self, block_id: BlockId) -> Result { let mut path = self.eth_path(V1)?; @@ -735,6 +748,23 @@ impl BeaconNodeHttpClient { })) } + /// `GET lighthouse/beacon/blobs_sidecars/{block_id}` + /// + /// Returns `Ok(None)` on a 404 error. + pub async fn get_blobs_sidecar( + &self, + block_id: BlockId, + ) -> Result>>, Error> { + let path = self.get_blobs_sidecar_path(block_id)?; + let response = match self.get_response(path, |b| b).await.optional()? { + Some(res) => res, + None => return Ok(None), + }; + + let GenericResponse { data } = response.json().await?; + Ok(Some(GenericResponse { data })) + } + /// `GET v1/beacon/blinded_blocks/{block_id}` /// /// Returns `Ok(None)` on a 404 error. From f6d5e8fea347842e72f8244a21d58596a375a9de Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Fri, 23 Dec 2022 16:14:53 +1100 Subject: [PATCH 5/8] Rename variable only --- beacon_node/http_api/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 0eb261d96a..e8ec77f87b 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -3325,7 +3325,7 @@ pub fn serve( }); // GET lighthouse/beacon/blobs_sidecars/{block_id} - let get_blobs_sidecars = warp::path("lighthouse") + let get_lighthouse_blobs_sidecars = warp::path("lighthouse") .and(warp::path("beacon")) .and(warp::path("blobs_sidecars")) .and(block_id_or_err) @@ -3494,7 +3494,7 @@ pub fn serve( .or(get_lighthouse_attestation_performance.boxed()) .or(get_lighthouse_block_packing_efficiency.boxed()) .or(get_lighthouse_merge_readiness.boxed()) - .or(get_blobs_sidecars.boxed()) + .or(get_lighthouse_blobs_sidecars.boxed()) .or(get_events.boxed()), ) .boxed() From 11736b68d3139df2cca465f7ea9aafe2365b89be Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Wed, 4 Jan 2023 01:23:31 +1100 Subject: [PATCH 6/8] Return `beacon_chain_error` if blob query from BeaconStore returns an error --- beacon_node/http_api/src/block_id.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/beacon_node/http_api/src/block_id.rs b/beacon_node/http_api/src/block_id.rs index 2578b4a67c..f08e4c200d 100644 --- a/beacon_node/http_api/src/block_id.rs +++ b/beacon_node/http_api/src/block_id.rs @@ -249,11 +249,7 @@ impl BlockId { "Blob with block root {} is not in the store", root ))), - // should we use `warp_utils::reject::beacon_chain_error` instead? - Err(e) => Err(warp_utils::reject::custom_not_found(format!( - "Error fetching blob with block root {}: {:?}", - root, e - ))), + Err(e) => Err(warp_utils::reject::beacon_chain_error(e.into())), } } } From 355cab87040d4beaa37d3140cf999f275b708460 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Wed, 4 Jan 2023 01:57:15 +1100 Subject: [PATCH 7/8] Simplify `blob_sidecar` query and remove override for `Head` and `Slot` --- beacon_node/http_api/src/block_id.rs | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/beacon_node/http_api/src/block_id.rs b/beacon_node/http_api/src/block_id.rs index f08e4c200d..fd45e14faa 100644 --- a/beacon_node/http_api/src/block_id.rs +++ b/beacon_node/http_api/src/block_id.rs @@ -217,32 +217,7 @@ impl BlockId { &self, chain: &BeaconChain, ) -> Result<(Arc>), warp::Rejection> { - let root = match &self.0 { - CoreBlockId::Head => { - let (cached_head, _execution_status) = chain - .canonical_head - .head_and_execution_status() - .map_err(warp_utils::reject::beacon_chain_error)?; - cached_head.head_block_root() - } - CoreBlockId::Slot(slot) => { - let maybe_block_root = chain - .block_root_at_slot(*slot, WhenSlotSkipped::None) - .ok() - .flatten(); - match maybe_block_root { - Some(block_root) => block_root, - None => { - return Err(warp_utils::reject::custom_not_found(format!( - "Block root for slot {} not found", - slot - ))) - } - } - } - _ => self.root(chain)?.0, - }; - + let root = self.root(chain)?.0; match chain.store.get_blobs(&root) { Ok(Some(blob)) => Ok((Arc::new(blob))), Ok(None) => Err(warp_utils::reject::custom_not_found(format!( From 11e3902dc8ad1e2f5c7d6e3cde6ffba722d6fc03 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Wed, 4 Jan 2023 18:51:28 +1100 Subject: [PATCH 8/8] Add support for blobs_sidecar ssz parsing --- lcli/src/parse_ssz.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/lcli/src/parse_ssz.rs b/lcli/src/parse_ssz.rs index fff6f7de58..0e87e330b1 100644 --- a/lcli/src/parse_ssz.rs +++ b/lcli/src/parse_ssz.rs @@ -63,6 +63,7 @@ pub fn run_parse_ssz(matches: &ArgMatches) -> Result<(), String> { "state_merge" => decode_and_print::>(&bytes, format)?, "state_capella" => decode_and_print::>(&bytes, format)?, "state_eip4844" => decode_and_print::>(&bytes, format)?, + "blobs_sidecar" => decode_and_print::>(&bytes, format)?, other => return Err(format!("Unknown type: {}", other)), };