From c26ce824a061bfb9377600927f9d39615522b78c Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 15 Mar 2023 17:48:22 -0400 Subject: [PATCH] general chaos --- .../src/engine_api/json_structures.rs | 4 +- beacon_node/execution_layer/src/lib.rs | 4 +- .../lighthouse_network/src/rpc/protocol.rs | 13 +++---- beacon_node/store/src/hot_cold_store.rs | 16 ++++---- .../store/src/impls/execution_payload.rs | 6 +-- beacon_node/store/src/lib.rs | 2 +- common/eth2/src/lib.rs | 2 +- consensus/types/src/blob_sidecar.rs | 1 + consensus/types/src/lib.rs | 4 -- consensus/types/src/signed_block_and_blobs.rs | 39 ------------------- 10 files changed, 26 insertions(+), 65 deletions(-) delete mode 100644 consensus/types/src/signed_block_and_blobs.rs diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index 658022e119..b90d314737 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -3,10 +3,12 @@ use serde::{Deserialize, Serialize}; use strum::EnumString; use superstruct::superstruct; use types::{ - Blobs, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, + EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadEip4844, ExecutionPayloadMerge, FixedVector, Transaction, Unsigned, VariableList, Withdrawal, }; +use types::beacon_block_body::KzgCommitments; +use types::blob_sidecar::Blobs; #[derive(Debug, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index a7a8bdfe26..5e2039ebad 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -46,13 +46,15 @@ use types::transaction::{AccessTuple, BlobTransaction, EcdsaSignature, SignedBlo use types::Withdrawals; use types::{AbstractExecPayload, BeaconStateError, ExecPayload, VersionedHash}; use types::{ - BlindedPayload, Blobs, BlockType, ChainSpec, Epoch, ExecutionBlockHash, ExecutionPayload, + BlindedPayload, BlockType, ChainSpec, Epoch, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadEip4844, ExecutionPayloadMerge, ForkName, }; use types::{ ProposerPreparationData, PublicKeyBytes, Signature, SignedBeaconBlock, Slot, Transaction, Uint256, }; +use types::beacon_block_body::KzgCommitments; +use types::blob_sidecar::Blobs; mod block_hash; mod engine_api; diff --git a/beacon_node/lighthouse_network/src/rpc/protocol.rs b/beacon_node/lighthouse_network/src/rpc/protocol.rs index bf7d9ca2a8..a2acc7976a 100644 --- a/beacon_node/lighthouse_network/src/rpc/protocol.rs +++ b/beacon_node/lighthouse_network/src/rpc/protocol.rs @@ -20,11 +20,10 @@ use tokio_util::{ codec::Framed, compat::{Compat, FuturesAsyncReadCompatExt}, }; -use types::BlobsSidecar; use types::{ BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockCapella, BeaconBlockMerge, EmptyBlock, EthSpec, ForkContext, ForkName, Hash256, MainnetEthSpec, Signature, - SignedBeaconBlock, + SignedBeaconBlock, BlobSidecar }; lazy_static! { @@ -115,12 +114,12 @@ lazy_static! { .as_ssz_bytes() .len(); - pub static ref BLOBS_SIDECAR_MIN: usize = BlobsSidecar::::empty().as_ssz_bytes().len(); - pub static ref BLOBS_SIDECAR_MAX: usize = BlobsSidecar::::max_size(); + pub static ref BLOB_SIDECAR_MIN: usize = BlobSidecar::::empty().as_ssz_bytes().len(); + pub static ref BLOB_SIDECAR_MAX: usize = BlobSidecar::::max_size(); //FIXME(sean) these are underestimates - pub static ref SIGNED_BLOCK_AND_BLOBS_MIN: usize = *BLOBS_SIDECAR_MIN + *SIGNED_BEACON_BLOCK_BASE_MIN; - pub static ref SIGNED_BLOCK_AND_BLOBS_MAX: usize =*BLOBS_SIDECAR_MAX + *SIGNED_BEACON_BLOCK_EIP4844_MAX; + pub static ref SIGNED_BLOCK_AND_BLOBS_MIN: usize = *BLOB_SIDECAR_MIN + *SIGNED_BEACON_BLOCK_BASE_MIN; + pub static ref SIGNED_BLOCK_AND_BLOBS_MAX: usize =*BLOB_SIDECAR_MAX + *SIGNED_BEACON_BLOCK_EIP4844_MAX; } /// The maximum bytes that can be sent across the RPC pre-merge. @@ -385,7 +384,7 @@ impl ProtocolId { Protocol::Goodbye => RpcLimits::new(0, 0), // Goodbye request has no response Protocol::BlocksByRange => rpc_block_limits_by_fork(fork_context.current_fork()), Protocol::BlocksByRoot => rpc_block_limits_by_fork(fork_context.current_fork()), - Protocol::BlobsByRange => RpcLimits::new(*BLOBS_SIDECAR_MIN, *BLOBS_SIDECAR_MAX), + Protocol::BlobsByRange => RpcLimits::new(*BLOB_SIDECAR_MIN, *BLOB_SIDECAR_MAX), Protocol::BlobsByRoot => { // TODO: wrong too RpcLimits::new(*SIGNED_BLOCK_AND_BLOBS_MIN, *SIGNED_BLOCK_AND_BLOBS_MAX) diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index 60e2f77595..718ff7f722 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -66,7 +66,7 @@ pub struct HotColdDB, Cold: ItemStore> { /// The hot database also contains all blocks. pub hot_db: Hot, /// LRU cache of deserialized blobs. Updated whenever a blob is loaded. - blob_cache: Mutex>>, + blob_cache: Mutex>>, /// LRU cache of deserialized blocks. Updated whenever a block is loaded. block_cache: Mutex>>, /// Chain spec. @@ -547,7 +547,7 @@ impl, Cold: ItemStore> HotColdDB /// Check if the blobs sidecar for a block exists on disk. pub fn blobs_sidecar_exists(&self, block_root: &Hash256) -> Result { - self.get_item::>(block_root) + self.get_item::>(block_root) .map(|blobs| blobs.is_some()) } @@ -568,7 +568,7 @@ impl, Cold: ItemStore> HotColdDB blobs_db.key_delete(DBColumn::BeaconBlob.into(), block_root.as_bytes()) } - pub fn put_blobs(&self, block_root: &Hash256, blobs: BlobsSidecar) -> Result<(), Error> { + pub fn put_blobs(&self, block_root: &Hash256, blobs: BlobSidecarList) -> Result<(), Error> { let blobs_db = self.blobs_db.as_ref().unwrap_or(&self.cold_db); blobs_db.put_bytes( DBColumn::BeaconBlob.into(), @@ -582,7 +582,7 @@ impl, Cold: ItemStore> HotColdDB pub fn blobs_as_kv_store_ops( &self, key: &Hash256, - blobs: &BlobsSidecar, + blobs: &BlobSidecarList, ops: &mut Vec, ) { let db_key = get_key_for_col(DBColumn::BeaconBlob.into(), key.as_bytes()); @@ -925,8 +925,8 @@ impl, Cold: ItemStore> HotColdDB for op in blob_cache_ops.iter_mut() { let reverse_op = match op { StoreOp::PutBlobs(block_root, _) => StoreOp::DeleteBlobs(*block_root), - StoreOp::DeleteBlobs(_) => match blobs_to_delete.pop() { - Some(blobs) => StoreOp::PutBlobs(blobs.beacon_block_root, Arc::new(blobs)), + StoreOp::DeleteBlobs(block_root) => match blobs_to_delete.pop() { + Some(blobs) => StoreOp::PutBlobs(*block_root, Arc::new(blobs)), None => return Err(HotColdDBError::Rollback.into()), }, _ => return Err(HotColdDBError::Rollback.into()), @@ -1320,12 +1320,12 @@ impl, Cold: ItemStore> HotColdDB } /// Fetch a blobs sidecar from the store. - pub fn get_blobs(&self, block_root: &Hash256) -> Result>, Error> { + pub fn get_blobs(&self, block_root: &Hash256) -> Result>, Error> { let blobs_db = self.blobs_db.as_ref().unwrap_or(&self.cold_db); match blobs_db.get_bytes(DBColumn::BeaconBlob.into(), block_root.as_bytes())? { Some(ref blobs_bytes) => { - let blobs = BlobsSidecar::from_ssz_bytes(blobs_bytes)?; + let blobs = BlobSidecarList::from_ssz_bytes(blobs_bytes)?; // FIXME(sean) I was attempting to use a blob cache here but was getting deadlocks, // may want to attempt to use one again self.blob_cache.lock().put(*block_root, blobs.clone()); diff --git a/beacon_node/store/src/impls/execution_payload.rs b/beacon_node/store/src/impls/execution_payload.rs index 01a2dba0b0..1f5c9debc0 100644 --- a/beacon_node/store/src/impls/execution_payload.rs +++ b/beacon_node/store/src/impls/execution_payload.rs @@ -1,8 +1,8 @@ use crate::{DBColumn, Error, StoreItem}; use ssz::{Decode, Encode}; use types::{ - BlobsSidecar, EthSpec, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadEip4844, - ExecutionPayloadMerge, + EthSpec, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadEip4844, + ExecutionPayloadMerge,BlobSidecarList }; macro_rules! impl_store_item { @@ -25,7 +25,7 @@ macro_rules! impl_store_item { impl_store_item!(ExecutionPayloadMerge); impl_store_item!(ExecutionPayloadCapella); impl_store_item!(ExecutionPayloadEip4844); -impl_store_item!(BlobsSidecar); +impl_store_item!(BlobSidecarList); /// This fork-agnostic implementation should be only used for writing. /// diff --git a/beacon_node/store/src/lib.rs b/beacon_node/store/src/lib.rs index 3056c29292..ea382a934e 100644 --- a/beacon_node/store/src/lib.rs +++ b/beacon_node/store/src/lib.rs @@ -159,7 +159,7 @@ pub trait ItemStore: KeyValueStore + Sync + Send + Sized + 'stati pub enum StoreOp<'a, E: EthSpec> { PutBlock(Hash256, Arc>), PutState(Hash256, &'a BeaconState), - PutBlobs(Hash256, Arc>), + PutBlobs(Hash256, Arc>), PutOrphanedBlobsKey(Hash256), PutStateSummary(Hash256, HotStateSummary), PutStateTemporaryFlag(Hash256), diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 2a27d31da9..fec41b8bca 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -704,7 +704,7 @@ impl BeaconNodeHttpClient { pub async fn get_blobs_sidecar( &self, block_id: BlockId, - ) -> Result>>, Error> { + ) -> Result>>, Error> { let path = self.get_blobs_sidecar_path(block_id)?; let response = match self.get_response(path, |b| b).await.optional()? { Some(res) => res, diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index 3484b5cdba..ad6509cb0f 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -48,6 +48,7 @@ pub struct BlobSidecar { } pub type BlobSidecarList = VariableList, ::MaxBlobsPerBlock>; +pub type Blobs = VariableList, ::MaxExtraDataBytes>; impl SignedRoot for BlobSidecar {} diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 592b6cc758..c866442605 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -100,7 +100,6 @@ pub mod sqlite; pub mod beacon_block_and_blob_sidecars; pub mod blob_sidecar; pub mod signed_blob; -pub mod signed_block_and_blobs; pub mod transaction; use ethereum_types::{H160, H256}; @@ -181,9 +180,6 @@ pub use crate::signed_beacon_block::{ }; pub use crate::signed_beacon_block_header::SignedBeaconBlockHeader; pub use crate::signed_blob::*; -pub use crate::signed_block_and_blobs::{ - SignedBeaconBlockAndBlobsSidecar, SignedBeaconBlockAndBlobsSidecarDecode, -}; pub use crate::signed_bls_to_execution_change::SignedBlsToExecutionChange; pub use crate::signed_contribution_and_proof::SignedContributionAndProof; pub use crate::signed_voluntary_exit::SignedVoluntaryExit; diff --git a/consensus/types/src/signed_block_and_blobs.rs b/consensus/types/src/signed_block_and_blobs.rs deleted file mode 100644 index 9bfdfc4576..0000000000 --- a/consensus/types/src/signed_block_and_blobs.rs +++ /dev/null @@ -1,39 +0,0 @@ -use crate::{ - AbstractExecPayload, EthSpec, SignedBeaconBlock, SignedBeaconBlockEip4844, - SignedBlobSidecar, -}; -use crate::{BlobsSidecar, EthSpec, SignedBeaconBlock, SignedBeaconBlockEip4844}; -use derivative::Derivative; -use serde_derive::{Deserialize, Serialize}; -use ssz::{Decode, DecodeError}; -use ssz_derive::{Decode, Encode}; -use std::sync::Arc; -use tree_hash_derive::TreeHash; - -#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, PartialEq)] -#[serde(bound = "T: EthSpec")] -pub struct SignedBeaconBlockAndBlobsSidecarDecode { - pub beacon_block: SignedBeaconBlockEip4844, - pub blobs_sidecar: BlobsSidecar, -} - -// TODO: will be removed once we decouple blobs in Gossip -#[derive(Debug, Clone, Serialize, Deserialize, Encode, TreeHash, Derivative)] -#[derivative(PartialEq, Hash(bound = "T: EthSpec"))] -pub struct SignedBeaconBlockAndBlobsSidecar { - pub beacon_block: Arc>, - pub blobs_sidecar: Arc>, -} - -impl SignedBeaconBlockAndBlobsSidecar { - pub fn from_ssz_bytes(bytes: &[u8]) -> Result { - let SignedBeaconBlockAndBlobsSidecarDecode { - beacon_block, - blobs_sidecar, - } = SignedBeaconBlockAndBlobsSidecarDecode::from_ssz_bytes(bytes)?; - Ok(SignedBeaconBlockAndBlobsSidecar { - beacon_block: Arc::new(SignedBeaconBlock::Eip4844(beacon_block)), - blobs_sidecar: Arc::new(blobs_sidecar), - }) - } -}