From 8e4e499b5159c60fbde9ae0b8c03d8c910c402a0 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Sat, 17 Sep 2022 09:49:12 +0200 Subject: [PATCH 01/22] start adding types --- consensus/types/src/beacon_block.rs | 7 +- consensus/types/src/beacon_block_body.rs | 8 +- consensus/types/src/eth_spec.rs | 14 ++- consensus/types/src/kzg_commitment.rs | 112 +++++++++++++++++++++++ consensus/types/src/kzg_proof.rs | 112 +++++++++++++++++++++++ consensus/types/src/lib.rs | 5 + 6 files changed, 252 insertions(+), 6 deletions(-) create mode 100644 consensus/types/src/kzg_commitment.rs create mode 100644 consensus/types/src/kzg_proof.rs diff --git a/consensus/types/src/beacon_block.rs b/consensus/types/src/beacon_block.rs index da8566dcb2..cbfbd250f9 100644 --- a/consensus/types/src/beacon_block.rs +++ b/consensus/types/src/beacon_block.rs @@ -1,6 +1,6 @@ use crate::beacon_block_body::{ BeaconBlockBodyAltair, BeaconBlockBodyBase, BeaconBlockBodyMerge, BeaconBlockBodyRef, - BeaconBlockBodyRefMut, + BeaconBlockBodyRefMut, BeaconBlockBobyEip4844 }; use crate::test_utils::TestRandom; use crate::*; @@ -17,7 +17,7 @@ use tree_hash_derive::TreeHash; /// A block of the `BeaconChain`. #[superstruct( - variants(Base, Altair, Merge), + variants(Base, Altair, Merge, Eip4844), variant_attributes( derive( Debug, @@ -64,6 +64,8 @@ pub struct BeaconBlock = FullPayload> { pub body: BeaconBlockBodyAltair, #[superstruct(only(Merge), partial_getter(rename = "body_merge"))] pub body: BeaconBlockBodyMerge, + #[superstruct(only(Eip4844), partial_getter(rename = "body_eip4844"))] + pub body: BeaconBlockBodyEip4844, } impl> SignedRoot for BeaconBlock {} @@ -540,6 +542,7 @@ macro_rules! impl_from { impl_from!(BeaconBlockBase, >, >, |body: BeaconBlockBodyBase<_, _>| body.into()); impl_from!(BeaconBlockAltair, >, >, |body: BeaconBlockBodyAltair<_, _>| body.into()); impl_from!(BeaconBlockMerge, >, >, |body: BeaconBlockBodyMerge<_, _>| body.into()); +impl_from!(BeaconBlockEip4844, >, >, |body: BeaconBlockBodyEip4844<_, _>| body.into()); // We can clone blocks with payloads to blocks without payloads, without cloning the payload. macro_rules! impl_clone_as_blinded { diff --git a/consensus/types/src/beacon_block_body.rs b/consensus/types/src/beacon_block_body.rs index 381a9bd43e..a4fdbf6896 100644 --- a/consensus/types/src/beacon_block_body.rs +++ b/consensus/types/src/beacon_block_body.rs @@ -13,7 +13,7 @@ use tree_hash_derive::TreeHash; /// /// This *superstruct* abstracts over the hard-fork. #[superstruct( - variants(Base, Altair, Merge), + variants(Base, Altair, Merge, Eip4844), variant_attributes( derive( Debug, @@ -47,14 +47,16 @@ pub struct BeaconBlockBody = FullPayload> pub attestations: VariableList, T::MaxAttestations>, pub deposits: VariableList, pub voluntary_exits: VariableList, - #[superstruct(only(Altair, Merge))] + #[superstruct(only(Altair, Merge, Eip4844))] pub sync_aggregate: SyncAggregate, // We flatten the execution payload so that serde can use the name of the inner type, // either `execution_payload` for full payloads, or `execution_payload_header` for blinded // payloads. - #[superstruct(only(Merge))] + #[superstruct(only(Merge, Eip4844))] #[serde(flatten)] pub execution_payload: Payload, + #[superstruct(only(Eip4844))] + pub blob_kzg_commitments: VariableList, #[superstruct(only(Base, Altair))] #[ssz(skip_serializing, skip_deserializing)] #[tree_hash(skip_hashing)] diff --git a/consensus/types/src/eth_spec.rs b/consensus/types/src/eth_spec.rs index e616976026..aafb3d236b 100644 --- a/consensus/types/src/eth_spec.rs +++ b/consensus/types/src/eth_spec.rs @@ -95,6 +95,10 @@ pub trait EthSpec: 'static + Default + Sync + Send + Clone + Debug + PartialEq + type GasLimitDenominator: Unsigned + Clone + Sync + Send + Debug + PartialEq; type MinGasLimit: Unsigned + Clone + Sync + Send + Debug + PartialEq; type MaxExtraDataBytes: Unsigned + Clone + Sync + Send + Debug + PartialEq; + /* + * New in Eip4844 + */ + type MaxBlobsPerBlock: Unsigned + Clone + Sync + Send + Debug + PartialEq; /* * Derived values (set these CAREFULLY) */ @@ -222,6 +226,11 @@ pub trait EthSpec: 'static + Default + Sync + Send + Clone + Debug + PartialEq + fn bytes_per_logs_bloom() -> usize { Self::BytesPerLogsBloom::to_usize() } + + /// Returns the `MAX_BLOBS_PER_BLOCK` constant for this specification. + fn max_blobs_per_block() -> usize { + Self::MaxBlobsPerBlock::to_usize() + } } /// Macro to inherit some type values from another EthSpec. @@ -265,6 +274,7 @@ impl EthSpec for MainnetEthSpec { type SyncSubcommitteeSize = U128; // 512 committee size / 4 sync committee subnet count type MaxPendingAttestations = U4096; // 128 max attestations * 32 slots per epoch type SlotsPerEth1VotingPeriod = U2048; // 64 epochs * 32 slots per epoch + type MaxBlobsPerBlock = U16; fn default_spec() -> ChainSpec { ChainSpec::mainnet() @@ -309,7 +319,8 @@ impl EthSpec for MinimalEthSpec { BytesPerLogsBloom, GasLimitDenominator, MinGasLimit, - MaxExtraDataBytes + MaxExtraDataBytes, + MaxBlobsPerBlock }); fn default_spec() -> ChainSpec { @@ -354,6 +365,7 @@ impl EthSpec for GnosisEthSpec { type SyncSubcommitteeSize = U128; // 512 committee size / 4 sync committee subnet count type MaxPendingAttestations = U2048; // 128 max attestations * 16 slots per epoch type SlotsPerEth1VotingPeriod = U1024; // 64 epochs * 16 slots per epoch + type MaxBlobsPerBlock = U16; fn default_spec() -> ChainSpec { ChainSpec::gnosis() diff --git a/consensus/types/src/kzg_commitment.rs b/consensus/types/src/kzg_commitment.rs new file mode 100644 index 0000000000..688f96e050 --- /dev/null +++ b/consensus/types/src/kzg_commitment.rs @@ -0,0 +1,112 @@ +use std::fmt; +use serde::{Deserialize, Deserializer, Serializer}; +use ssz::{Decode, DecodeError, Encode}; +use tree_hash::TreeHash; + +const KZG_COMMITMENT_BYTES_LEN: usize = 48; + +#[derive(Default, Debug, PartialEq, Hash, Clone, Copy, Serialize, Deserialize)] +#[serde(transparent)] +pub struct KzgCommitment(#[serde(with = "serde_kzg_commitment")] pub [u8; KZG_COMMITMENT_BYTES_LEN]); + +impl fmt::Display for KzgCommitment { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", eth2_serde_utils::hex::encode(&self.0)) + } +} + +impl From<[u8; KZG_COMMITMENT_BYTES_LEN]> for KzgCommitment { + fn from(bytes: [u8; KZG_COMMITMENT_BYTES_LEN]) -> Self { + Self(bytes) + } +} + +impl Into<[u8; KZG_COMMITMENT_BYTES_LEN]> for KzgCommitment { + fn into(self) -> [u8; KZG_COMMITMENT_BYTES_LEN] { + self.0 + } +} + +pub mod serde_kzg_commitment { + use serde::de::Error; + use super::*; + + pub fn serialize(bytes: &[u8; KZG_COMMITMENT_BYTES_LEN], serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(ð2_serde_utils::hex::encode(bytes)) + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result<[u8; KZG_COMMITMENT_BYTES_LEN], D::Error> + where + D: Deserializer<'de>, + { + let s: String = Deserialize::deserialize(deserializer)?; + + let bytes = eth2_serde_utils::hex::decode(&s).map_err(D::Error::custom)?; + + if bytes.len() != KZG_COMMITMENT_BYTES_LEN { + return Err(D::Error::custom(format!( + "incorrect byte length {}, expected {}", + bytes.len(), + KZG_COMMITMENT_BYTES_LEN + ))); + } + + let mut array = [0; KZG_COMMITMENT_BYTES_LEN]; + array[..].copy_from_slice(&bytes); + + Ok(array) + } +} + +impl Encode for KzgCommitment { + fn is_ssz_fixed_len() -> bool { + <[u8; KZG_COMMITMENT_BYTES_LEN] as Encode>::is_ssz_fixed_len() + } + + fn ssz_fixed_len() -> usize { + <[u8; KZG_COMMITMENT_BYTES_LEN] as Encode>::ssz_fixed_len() + } + + fn ssz_bytes_len(&self) -> usize { + self.0.ssz_bytes_len() + } + + fn ssz_append(&self, buf: &mut Vec) { + self.0.ssz_append(buf) + } +} + +impl Decode for KzgCommitment { + fn is_ssz_fixed_len() -> bool { + <[u8; KZG_COMMITMENT_BYTES_LEN] as Decode>::is_ssz_fixed_len() + } + + fn ssz_fixed_len() -> usize { + <[u8; KZG_COMMITMENT_BYTES_LEN] as Decode>::ssz_fixed_len() + } + + fn from_ssz_bytes(bytes: &[u8]) -> Result { + <[u8; KZG_COMMITMENT_BYTES_LEN]>::from_ssz_bytes(bytes).map(Self) + } +} + +impl TreeHash for KzgCommitment { + fn tree_hash_type() -> tree_hash::TreeHashType { + <[u8; KZG_COMMITMENT_BYTES_LEN]>::tree_hash_type() + } + + fn tree_hash_packed_encoding(&self) -> Vec { + self.0.tree_hash_packed_encoding() + } + + fn tree_hash_packing_factor() -> usize { + <[u8; KZG_COMMITMENT_BYTES_LEN]>::tree_hash_packing_factor() + } + + fn tree_hash_root(&self) -> tree_hash::Hash256 { + self.0.tree_hash_root() + } +} diff --git a/consensus/types/src/kzg_proof.rs b/consensus/types/src/kzg_proof.rs new file mode 100644 index 0000000000..50ee1266da --- /dev/null +++ b/consensus/types/src/kzg_proof.rs @@ -0,0 +1,112 @@ +use std::fmt; +use serde::{Deserialize, Deserializer, Serializer}; +use ssz::{Decode, DecodeError, Encode}; +use tree_hash::TreeHash; + +const KZG_PROOF_BYTES_LEN: usize = 48; + +#[derive(Default, Debug, PartialEq, Hash, Clone, Copy, Serialize, Deserialize)] +#[serde(transparent)] +pub struct KzgProof(#[serde(with = "serde_kzg_proof")] pub [u8; KZG_PROOF_BYTES_LEN]); + +impl fmt::Display for KzgProof { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", eth2_serde_utils::hex::encode(&self.0)) + } +} + +impl From<[u8; KZG_PROOF_BYTES_LEN]> for KzgProof { + fn from(bytes: [u8; KZG_PROOF_BYTES_LEN]) -> Self { + Self(bytes) + } +} + +impl Into<[u8; KZG_PROOF_BYTES_LEN]> for KzgProof { + fn into(self) -> [u8; KZG_PROOF_BYTES_LEN] { + self.0 + } +} + +pub mod serde_kzg_proof { + use serde::de::Error; + use super::*; + + pub fn serialize(bytes: &[u8; KZG_PROOF_BYTES_LEN], serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(ð2_serde_utils::hex::encode(bytes)) + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result<[u8; KZG_PROOF_BYTES_LEN], D::Error> + where + D: Deserializer<'de>, + { + let s: String = Deserialize::deserialize(deserializer)?; + + let bytes = eth2_serde_utils::hex::decode(&s).map_err(D::Error::custom)?; + + if bytes.len() != KZG_PROOF_BYTES_LEN { + return Err(D::Error::custom(format!( + "incorrect byte length {}, expected {}", + bytes.len(), + KZG_PROOF_BYTES_LEN + ))); + } + + let mut array = [0; KZG_PROOF_BYTES_LEN]; + array[..].copy_from_slice(&bytes); + + Ok(array) + } +} + +impl Encode for KzgProof { + fn is_ssz_fixed_len() -> bool { + <[u8; KZG_PROOF_BYTES_LEN] as Encode>::is_ssz_fixed_len() + } + + fn ssz_fixed_len() -> usize { + <[u8; KZG_PROOF_BYTES_LEN] as Encode>::ssz_fixed_len() + } + + fn ssz_bytes_len(&self) -> usize { + self.0.ssz_bytes_len() + } + + fn ssz_append(&self, buf: &mut Vec) { + self.0.ssz_append(buf) + } +} + +impl Decode for KzgProof { + fn is_ssz_fixed_len() -> bool { + <[u8; KZG_PROOF_BYTES_LEN] as Decode>::is_ssz_fixed_len() + } + + fn ssz_fixed_len() -> usize { + <[u8; KZG_PROOF_BYTES_LEN] as Decode>::ssz_fixed_len() + } + + fn from_ssz_bytes(bytes: &[u8]) -> Result { + <[u8; KZG_PROOF_BYTES_LEN]>::from_ssz_bytes(bytes).map(Self) + } +} + +impl TreeHash for KzgProof { + fn tree_hash_type() -> tree_hash::TreeHashType { + <[u8; KZG_PROOF_BYTES_LEN]>::tree_hash_type() + } + + fn tree_hash_packed_encoding(&self) -> Vec { + self.0.tree_hash_packed_encoding() + } + + fn tree_hash_packing_factor() -> usize { + <[u8; KZG_PROOF_BYTES_LEN]>::tree_hash_packing_factor() + } + + fn tree_hash_root(&self) -> tree_hash::Hash256 { + self.0.tree_hash_root() + } +} diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index f05012c0b7..c5a0e6ba07 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -14,6 +14,8 @@ #[macro_use] extern crate lazy_static; +extern crate core; + #[macro_use] pub mod test_utils; @@ -90,6 +92,9 @@ pub mod slot_data; #[cfg(feature = "sqlite")] pub mod sqlite; +pub mod kzg_commitment; +pub mod kzg_proof; + use ethereum_types::{H160, H256}; pub use crate::aggregate_and_proof::AggregateAndProof; From 95203c51d41b89cdccc0228d2ba7050b892f727d Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Sat, 17 Sep 2022 11:26:18 +0200 Subject: [PATCH 02/22] fix some bugx, adjust stucts --- beacon_node/store/src/partial_beacon_state.rs | 2 +- consensus/fork_choice/src/fork_choice.rs | 2 +- consensus/ssz/src/decode/impls.rs | 1 + consensus/ssz/src/encode/impls.rs | 1 + .../src/common/slash_validator.rs | 2 +- .../process_operations.rs | 2 +- .../src/per_epoch_processing.rs | 2 +- consensus/tree_hash/src/impls.rs | 1 + consensus/types/src/beacon_block.rs | 4 +- consensus/types/src/beacon_block_body.rs | 80 ++++++++++++++++++- consensus/types/src/beacon_state.rs | 11 ++- consensus/types/src/blob.rs | 9 +++ consensus/types/src/bls_field_element.rs | 7 ++ consensus/types/src/chain_spec.rs | 20 +++++ consensus/types/src/eth_spec.rs | 6 +- consensus/types/src/fork_name.rs | 18 ++++- consensus/types/src/kzg_commitment.rs | 19 ++++- consensus/types/src/kzg_proof.rs | 19 ++++- consensus/types/src/lib.rs | 6 +- consensus/types/src/signed_beacon_block.rs | 64 ++++++++++++++- 20 files changed, 256 insertions(+), 20 deletions(-) create mode 100644 consensus/types/src/blob.rs create mode 100644 consensus/types/src/bls_field_element.rs diff --git a/beacon_node/store/src/partial_beacon_state.rs b/beacon_node/store/src/partial_beacon_state.rs index 010796afd5..2d3d9cc7bf 100644 --- a/beacon_node/store/src/partial_beacon_state.rs +++ b/beacon_node/store/src/partial_beacon_state.rs @@ -14,7 +14,7 @@ use types::*; /// /// Utilises lazy-loading from separate storage for its vector fields. #[superstruct( - variants(Base, Altair, Merge), + variants(Base, Altair, Merge, Eip4844), variant_attributes(derive(Debug, PartialEq, Clone, Encode, Decode)) )] #[derive(Debug, PartialEq, Clone, Encode)] diff --git a/consensus/fork_choice/src/fork_choice.rs b/consensus/fork_choice/src/fork_choice.rs index f55a283ed1..e2780e41a1 100644 --- a/consensus/fork_choice/src/fork_choice.rs +++ b/consensus/fork_choice/src/fork_choice.rs @@ -773,7 +773,7 @@ where (parent_justified, parent_finalized) } else { let justification_and_finalization_state = match block { - BeaconBlockRef::Merge(_) | BeaconBlockRef::Altair(_) => { + BeaconBlockRef::Eip4844(_) | BeaconBlockRef::Merge(_) | BeaconBlockRef::Altair(_) => { let participation_cache = per_epoch_processing::altair::ParticipationCache::new(state, spec) .map_err(Error::ParticipationCacheBuild)?; diff --git a/consensus/ssz/src/decode/impls.rs b/consensus/ssz/src/decode/impls.rs index d91ddabe02..99f3185851 100644 --- a/consensus/ssz/src/decode/impls.rs +++ b/consensus/ssz/src/decode/impls.rs @@ -374,6 +374,7 @@ macro_rules! impl_decodable_for_u8_array { impl_decodable_for_u8_array!(4); impl_decodable_for_u8_array!(32); +impl_decodable_for_u8_array!(48); macro_rules! impl_for_vec { ($type: ty, $max_len: expr) => { diff --git a/consensus/ssz/src/encode/impls.rs b/consensus/ssz/src/encode/impls.rs index cfd95ba40d..1faf9123f5 100644 --- a/consensus/ssz/src/encode/impls.rs +++ b/consensus/ssz/src/encode/impls.rs @@ -483,6 +483,7 @@ macro_rules! impl_encodable_for_u8_array { impl_encodable_for_u8_array!(4); impl_encodable_for_u8_array!(32); +impl_encodable_for_u8_array!(48); #[cfg(test)] mod tests { diff --git a/consensus/state_processing/src/common/slash_validator.rs b/consensus/state_processing/src/common/slash_validator.rs index e9d94a1062..966a45860f 100644 --- a/consensus/state_processing/src/common/slash_validator.rs +++ b/consensus/state_processing/src/common/slash_validator.rs @@ -45,7 +45,7 @@ pub fn slash_validator( validator_effective_balance.safe_div(spec.whistleblower_reward_quotient)?; let proposer_reward = match state { BeaconState::Base(_) => whistleblower_reward.safe_div(spec.proposer_reward_quotient)?, - BeaconState::Altair(_) | BeaconState::Merge(_) => whistleblower_reward + BeaconState::Altair(_) | BeaconState::Merge(_) | BeaconState::Eip4844(_) => whistleblower_reward .safe_mul(PROPOSER_WEIGHT)? .safe_div(WEIGHT_DENOMINATOR)?, }; diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 31a4ac1fb4..6f2f7831db 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -230,7 +230,7 @@ pub fn process_attestations<'a, T: EthSpec, Payload: ExecPayload>( BeaconBlockBodyRef::Base(_) => { base::process_attestations(state, block_body.attestations(), verify_signatures, spec)?; } - BeaconBlockBodyRef::Altair(_) | BeaconBlockBodyRef::Merge(_) => { + BeaconBlockBodyRef::Altair(_) | BeaconBlockBodyRef::Merge(_) | BeaconBlockBodyRef::Eip4844(_) => { altair::process_attestations( state, block_body.attestations(), diff --git a/consensus/state_processing/src/per_epoch_processing.rs b/consensus/state_processing/src/per_epoch_processing.rs index cb90c67b56..fa71017167 100644 --- a/consensus/state_processing/src/per_epoch_processing.rs +++ b/consensus/state_processing/src/per_epoch_processing.rs @@ -37,7 +37,7 @@ pub fn process_epoch( match state { BeaconState::Base(_) => base::process_epoch(state, spec), - BeaconState::Altair(_) | BeaconState::Merge(_) => altair::process_epoch(state, spec), + BeaconState::Altair(_) | BeaconState::Merge(_) | BeaconState::Eip4844(_) => altair::process_epoch(state, spec), } } diff --git a/consensus/tree_hash/src/impls.rs b/consensus/tree_hash/src/impls.rs index 00fed489c7..263a93334e 100644 --- a/consensus/tree_hash/src/impls.rs +++ b/consensus/tree_hash/src/impls.rs @@ -81,6 +81,7 @@ macro_rules! impl_for_lt_32byte_u8_array { impl_for_lt_32byte_u8_array!(4); impl_for_lt_32byte_u8_array!(32); +impl_for_lt_32byte_u8_array!(48); impl TreeHash for U128 { fn tree_hash_type() -> TreeHashType { diff --git a/consensus/types/src/beacon_block.rs b/consensus/types/src/beacon_block.rs index cbfbd250f9..6984c25611 100644 --- a/consensus/types/src/beacon_block.rs +++ b/consensus/types/src/beacon_block.rs @@ -1,6 +1,6 @@ use crate::beacon_block_body::{ BeaconBlockBodyAltair, BeaconBlockBodyBase, BeaconBlockBodyMerge, BeaconBlockBodyRef, - BeaconBlockBodyRefMut, BeaconBlockBobyEip4844 + BeaconBlockBodyRefMut, BeaconBlockBodyEip4844 }; use crate::test_utils::TestRandom; use crate::*; @@ -189,6 +189,7 @@ impl<'a, T: EthSpec, Payload: ExecPayload> BeaconBlockRef<'a, T, Payload> { BeaconBlockRef::Base { .. } => ForkName::Base, BeaconBlockRef::Altair { .. } => ForkName::Altair, BeaconBlockRef::Merge { .. } => ForkName::Merge, + BeaconBlockRef::Eip4844 { .. } => ForkName::Eip4844, }; if fork_at_slot == object_fork { @@ -573,6 +574,7 @@ macro_rules! impl_clone_as_blinded { impl_clone_as_blinded!(BeaconBlockBase, >, >); impl_clone_as_blinded!(BeaconBlockAltair, >, >); impl_clone_as_blinded!(BeaconBlockMerge, >, >); +impl_clone_as_blinded!(BeaconBlockEip4844, >, >); // A reference to a full beacon block can be cloned into a blinded beacon block, without cloning the // execution payload. diff --git a/consensus/types/src/beacon_block_body.rs b/consensus/types/src/beacon_block_body.rs index a4fdbf6896..eaa99718cb 100644 --- a/consensus/types/src/beacon_block_body.rs +++ b/consensus/types/src/beacon_block_body.rs @@ -8,6 +8,7 @@ use std::marker::PhantomData; use superstruct::superstruct; use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; +use crate::kzg_commitment::KzgCommitment; /// The body of a `BeaconChain` block, containing operations. /// @@ -56,7 +57,7 @@ pub struct BeaconBlockBody = FullPayload> #[serde(flatten)] pub execution_payload: Payload, #[superstruct(only(Eip4844))] - pub blob_kzg_commitments: VariableList, + pub blob_kzg_commitments: VariableList, #[superstruct(only(Base, Altair))] #[ssz(skip_serializing, skip_deserializing)] #[tree_hash(skip_hashing)] @@ -71,6 +72,7 @@ impl<'a, T: EthSpec> BeaconBlockBodyRef<'a, T> { BeaconBlockBodyRef::Base { .. } => ForkName::Base, BeaconBlockBodyRef::Altair { .. } => ForkName::Altair, BeaconBlockBodyRef::Merge { .. } => ForkName::Merge, + BeaconBlockBodyRef::Eip4844 { .. } => ForkName::Eip4844, } } } @@ -253,6 +255,48 @@ impl From>> } } +impl From>> +for ( + BeaconBlockBodyEip4844>, + Option>, +) +{ + fn from(body: BeaconBlockBodyEip4844>) -> Self { + let BeaconBlockBodyEip4844 { + randao_reveal, + eth1_data, + graffiti, + proposer_slashings, + attester_slashings, + attestations, + deposits, + voluntary_exits, + sync_aggregate, + execution_payload: FullPayload { execution_payload }, + blob_kzg_commitments, + } = body; + + ( + BeaconBlockBodyEip4844 { + randao_reveal, + eth1_data, + graffiti, + proposer_slashings, + attester_slashings, + attestations, + deposits, + voluntary_exits, + sync_aggregate, + execution_payload: BlindedPayload { + execution_payload_header: From::from(&execution_payload), + }, + blob_kzg_commitments, + }, + None, + ) + } +} + // We can clone a full block into a blinded block, without cloning the payload. impl BeaconBlockBodyBase> { pub fn clone_as_blinded(&self) -> BeaconBlockBodyBase> { @@ -300,6 +344,40 @@ impl BeaconBlockBodyMerge> { } } +impl BeaconBlockBodyEip4844> { + pub fn clone_as_blinded(&self) -> BeaconBlockBodyEip4844> { + let BeaconBlockBodyEip4844 { + randao_reveal, + eth1_data, + graffiti, + proposer_slashings, + attester_slashings, + attestations, + deposits, + voluntary_exits, + sync_aggregate, + execution_payload: FullPayload { execution_payload }, + blob_kzg_commitments, + } = self; + + BeaconBlockBodyEip4844 { + randao_reveal: randao_reveal.clone(), + eth1_data: eth1_data.clone(), + graffiti: *graffiti, + proposer_slashings: proposer_slashings.clone(), + attester_slashings: attester_slashings.clone(), + attestations: attestations.clone(), + deposits: deposits.clone(), + voluntary_exits: voluntary_exits.clone(), + sync_aggregate: sync_aggregate.clone(), + execution_payload: BlindedPayload { + execution_payload_header: From::from(execution_payload), + }, + blob_kzg_commitments: blob_kzg_commitments.clone(), + } + } +} + impl From>> for ( BeaconBlockBody>, diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index fca200312f..0d9920ec96 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -172,7 +172,7 @@ impl From for Hash256 { /// The state of the `BeaconChain` at some slot. #[superstruct( - variants(Base, Altair, Merge), + variants(Base, Altair, Merge, Eip4844), variant_attributes( derive( Derivative, @@ -250,9 +250,9 @@ where pub current_epoch_attestations: VariableList, T::MaxPendingAttestations>, // Participation (Altair and later) - #[superstruct(only(Altair, Merge))] + #[superstruct(only(Altair, Merge, Eip4844))] pub previous_epoch_participation: VariableList, - #[superstruct(only(Altair, Merge))] + #[superstruct(only(Altair, Merge, Eip4844))] pub current_epoch_participation: VariableList, // Finality @@ -389,6 +389,7 @@ impl BeaconState { BeaconState::Base { .. } => ForkName::Base, BeaconState::Altair { .. } => ForkName::Altair, BeaconState::Merge { .. } => ForkName::Merge, + BeaconState::Eip4844 { .. } => ForkName::Eip4844, }; if fork_at_slot == object_fork { @@ -1102,6 +1103,7 @@ impl BeaconState { BeaconState::Base(state) => (&mut state.validators, &mut state.balances), BeaconState::Altair(state) => (&mut state.validators, &mut state.balances), BeaconState::Merge(state) => (&mut state.validators, &mut state.balances), + BeaconState::Eip4844(state) => (&mut state.validators, &mut state.balances), } } @@ -1298,12 +1300,14 @@ impl BeaconState { BeaconState::Base(_) => Err(BeaconStateError::IncorrectStateVariant), BeaconState::Altair(state) => Ok(&mut state.current_epoch_participation), BeaconState::Merge(state) => Ok(&mut state.current_epoch_participation), + BeaconState::Eip4844(state) => Ok(&mut state.current_epoch_participation), } } else if epoch == self.previous_epoch() { match self { BeaconState::Base(_) => Err(BeaconStateError::IncorrectStateVariant), BeaconState::Altair(state) => Ok(&mut state.previous_epoch_participation), BeaconState::Merge(state) => Ok(&mut state.previous_epoch_participation), + BeaconState::Eip4844(state) => Ok(&mut state.previous_epoch_participation), } } else { Err(BeaconStateError::EpochOutOfBounds) @@ -1588,6 +1592,7 @@ impl BeaconState { BeaconState::Base(inner) => BeaconState::Base(inner.clone()), BeaconState::Altair(inner) => BeaconState::Altair(inner.clone()), BeaconState::Merge(inner) => BeaconState::Merge(inner.clone()), + BeaconState::Eip4844(inner) => BeaconState::Eip4844(inner.clone()), }; if config.committee_caches { *res.committee_caches_mut() = self.committee_caches().clone(); diff --git a/consensus/types/src/blob.rs b/consensus/types/src/blob.rs new file mode 100644 index 0000000000..8ea3468ff8 --- /dev/null +++ b/consensus/types/src/blob.rs @@ -0,0 +1,9 @@ +use ssz_types::VariableList; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use ssz::{Decode, DecodeError, Encode}; +use crate::bls_field_element::BlsFieldElement; +use crate::EthSpec; + +#[derive(Default, Debug, PartialEq, Hash, Clone, Serialize, Deserialize)] +#[serde(transparent)] +pub struct Blob(pub VariableList); \ No newline at end of file diff --git a/consensus/types/src/bls_field_element.rs b/consensus/types/src/bls_field_element.rs new file mode 100644 index 0000000000..6693b5765b --- /dev/null +++ b/consensus/types/src/bls_field_element.rs @@ -0,0 +1,7 @@ +use crate::Uint256; +use serde::{Deserialize, Serialize}; +use ssz::{Decode, DecodeError, Encode}; + +#[derive(Default, Debug, PartialEq, Hash, Clone, Copy, Serialize, Deserialize)] +#[serde(transparent)] +pub struct BlsFieldElement(pub Uint256); \ No newline at end of file diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index b2ba24ac3e..b894bef12e 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -150,6 +150,12 @@ pub struct ChainSpec { pub terminal_block_hash_activation_epoch: Epoch, pub safe_slots_to_import_optimistically: u64, + /* + * Eip4844 hard fork params + */ + pub eip4844_fork_epoch: Option, + pub eip4844_fork_version: [u8; 4], + /* * Networking */ @@ -245,6 +251,7 @@ impl ChainSpec { ForkName::Base => self.genesis_fork_version, ForkName::Altair => self.altair_fork_version, ForkName::Merge => self.bellatrix_fork_version, + ForkName::Eip4844 => self.eip4844_fork_version, } } @@ -254,6 +261,7 @@ impl ChainSpec { ForkName::Base => Some(Epoch::new(0)), ForkName::Altair => self.altair_fork_epoch, ForkName::Merge => self.bellatrix_fork_epoch, + ForkName::Eip4844 => self.eip4844_fork_epoch, } } @@ -263,6 +271,7 @@ impl ChainSpec { BeaconState::Base(_) => self.inactivity_penalty_quotient, BeaconState::Altair(_) => self.inactivity_penalty_quotient_altair, BeaconState::Merge(_) => self.inactivity_penalty_quotient_bellatrix, + BeaconState::Eip4844(_) => self.inactivity_penalty_quotient_bellatrix, } } @@ -275,6 +284,7 @@ impl ChainSpec { BeaconState::Base(_) => self.proportional_slashing_multiplier, BeaconState::Altair(_) => self.proportional_slashing_multiplier_altair, BeaconState::Merge(_) => self.proportional_slashing_multiplier_bellatrix, + BeaconState::Eip4844(_) => self.proportional_slashing_multiplier_bellatrix, } } @@ -287,6 +297,7 @@ impl ChainSpec { BeaconState::Base(_) => self.min_slashing_penalty_quotient, BeaconState::Altair(_) => self.min_slashing_penalty_quotient_altair, BeaconState::Merge(_) => self.min_slashing_penalty_quotient_bellatrix, + BeaconState::Eip4844(_) => self.min_slashing_penalty_quotient_bellatrix, } } @@ -568,6 +579,12 @@ impl ChainSpec { terminal_block_hash_activation_epoch: Epoch::new(u64::MAX), safe_slots_to_import_optimistically: 128u64, + /* + * Eip4844 hard fork params + */ + eip4844_fork_epoch: None, + eip4844_fork_version: [0x03, 0x00, 0x00, 0x00], + /* * Network specific */ @@ -778,6 +795,9 @@ impl ChainSpec { terminal_block_hash_activation_epoch: Epoch::new(u64::MAX), safe_slots_to_import_optimistically: 128u64, + eip4844_fork_epoch: None, + eip4844_fork_version: [0x03, 0x00, 0x00, 0x64], + /* * Network specific */ diff --git a/consensus/types/src/eth_spec.rs b/consensus/types/src/eth_spec.rs index aafb3d236b..9d8a765c20 100644 --- a/consensus/types/src/eth_spec.rs +++ b/consensus/types/src/eth_spec.rs @@ -99,6 +99,7 @@ pub trait EthSpec: 'static + Default + Sync + Send + Clone + Debug + PartialEq + * New in Eip4844 */ type MaxBlobsPerBlock: Unsigned + Clone + Sync + Send + Debug + PartialEq; + type FieldElementsPerBlob: Unsigned + Clone + Sync + Send + Debug + PartialEq; /* * Derived values (set these CAREFULLY) */ @@ -275,6 +276,7 @@ impl EthSpec for MainnetEthSpec { type MaxPendingAttestations = U4096; // 128 max attestations * 32 slots per epoch type SlotsPerEth1VotingPeriod = U2048; // 64 epochs * 32 slots per epoch type MaxBlobsPerBlock = U16; + type FieldElementsPerBlob = U4096; fn default_spec() -> ChainSpec { ChainSpec::mainnet() @@ -320,7 +322,8 @@ impl EthSpec for MinimalEthSpec { GasLimitDenominator, MinGasLimit, MaxExtraDataBytes, - MaxBlobsPerBlock + MaxBlobsPerBlock, + FieldElementsPerBlob }); fn default_spec() -> ChainSpec { @@ -366,6 +369,7 @@ impl EthSpec for GnosisEthSpec { type MaxPendingAttestations = U2048; // 128 max attestations * 16 slots per epoch type SlotsPerEth1VotingPeriod = U1024; // 64 epochs * 16 slots per epoch type MaxBlobsPerBlock = U16; + type FieldElementsPerBlob = U4096; fn default_spec() -> ChainSpec { ChainSpec::gnosis() diff --git a/consensus/types/src/fork_name.rs b/consensus/types/src/fork_name.rs index e97b08309b..7afe0d7530 100644 --- a/consensus/types/src/fork_name.rs +++ b/consensus/types/src/fork_name.rs @@ -11,11 +11,12 @@ pub enum ForkName { Base, Altair, Merge, + Eip4844 } impl ForkName { pub fn list_all() -> Vec { - vec![ForkName::Base, ForkName::Altair, ForkName::Merge] + vec![ForkName::Base, ForkName::Altair, ForkName::Merge, ForkName::Eip4844] } /// Set the activation slots in the given `ChainSpec` so that the fork named by `self` @@ -38,6 +39,12 @@ impl ForkName { spec.bellatrix_fork_epoch = Some(Epoch::new(0)); spec } + ForkName::Eip4844 => { + spec.altair_fork_epoch = Some(Epoch::new(0)); + spec.bellatrix_fork_epoch = Some(Epoch::new(0)); + spec.eip4844_fork_epoch = Some(Epoch::new(0)); + spec + } } } @@ -49,6 +56,7 @@ impl ForkName { ForkName::Base => None, ForkName::Altair => Some(ForkName::Base), ForkName::Merge => Some(ForkName::Altair), + ForkName::Eip4844 => Some(ForkName::Merge), } } @@ -59,7 +67,8 @@ impl ForkName { match self { ForkName::Base => Some(ForkName::Altair), ForkName::Altair => Some(ForkName::Merge), - ForkName::Merge => None, + ForkName::Merge => Some(ForkName::Eip4844), + ForkName::Eip4844 => None, } } } @@ -101,6 +110,10 @@ macro_rules! map_fork_name_with { let (value, extra_data) = $body; ($t::Merge(value), extra_data) } + ForkName::Eip4844 => { + let (value, extra_data) = $body; + ($t::Eip4844(value), extra_data) + } } }; } @@ -124,6 +137,7 @@ impl Display for ForkName { ForkName::Base => "phase0".fmt(f), ForkName::Altair => "altair".fmt(f), ForkName::Merge => "bellatrix".fmt(f), + ForkName::Eip4844 => "eip4844".fmt(f), } } } diff --git a/consensus/types/src/kzg_commitment.rs b/consensus/types/src/kzg_commitment.rs index 688f96e050..cfddc95b3d 100644 --- a/consensus/types/src/kzg_commitment.rs +++ b/consensus/types/src/kzg_commitment.rs @@ -1,14 +1,21 @@ use std::fmt; -use serde::{Deserialize, Deserializer, Serializer}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use ssz::{Decode, DecodeError, Encode}; use tree_hash::TreeHash; +use crate::test_utils::{RngCore, TestRandom}; const KZG_COMMITMENT_BYTES_LEN: usize = 48; -#[derive(Default, Debug, PartialEq, Hash, Clone, Copy, Serialize, Deserialize)] +#[derive(Debug, PartialEq, Hash, Clone, Copy, Serialize, Deserialize)] #[serde(transparent)] pub struct KzgCommitment(#[serde(with = "serde_kzg_commitment")] pub [u8; KZG_COMMITMENT_BYTES_LEN]); +impl Default for KzgCommitment { + fn default() -> Self { + KzgCommitment([0; 48]) + } +} + impl fmt::Display for KzgCommitment { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", eth2_serde_utils::hex::encode(&self.0)) @@ -110,3 +117,11 @@ impl TreeHash for KzgCommitment { self.0.tree_hash_root() } } + +impl TestRandom for KzgCommitment { + fn random_for_test(rng: &mut impl RngCore) -> Self { + let mut bytes = [0; KZG_COMMITMENT_BYTES_LEN]; + rng.fill_bytes(&mut bytes); + Self(bytes) + } +} diff --git a/consensus/types/src/kzg_proof.rs b/consensus/types/src/kzg_proof.rs index 50ee1266da..5bec8e2f86 100644 --- a/consensus/types/src/kzg_proof.rs +++ b/consensus/types/src/kzg_proof.rs @@ -1,11 +1,12 @@ use std::fmt; -use serde::{Deserialize, Deserializer, Serializer}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use ssz::{Decode, DecodeError, Encode}; use tree_hash::TreeHash; +use crate::test_utils::{RngCore, TestRandom}; const KZG_PROOF_BYTES_LEN: usize = 48; -#[derive(Default, Debug, PartialEq, Hash, Clone, Copy, Serialize, Deserialize)] +#[derive(Debug, PartialEq, Hash, Clone, Copy, Serialize, Deserialize)] #[serde(transparent)] pub struct KzgProof(#[serde(with = "serde_kzg_proof")] pub [u8; KZG_PROOF_BYTES_LEN]); @@ -15,6 +16,12 @@ impl fmt::Display for KzgProof { } } +impl Default for KzgProof { + fn default() -> Self { + KzgProof([0; 48]) + } +} + impl From<[u8; KZG_PROOF_BYTES_LEN]> for KzgProof { fn from(bytes: [u8; KZG_PROOF_BYTES_LEN]) -> Self { Self(bytes) @@ -110,3 +117,11 @@ impl TreeHash for KzgProof { self.0.tree_hash_root() } } + +impl TestRandom for KzgProof { + fn random_for_test(rng: &mut impl RngCore) -> Self { + let mut bytes = [0; KZG_PROOF_BYTES_LEN]; + rng.fill_bytes(&mut bytes); + Self(bytes) + } +} diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index c5a0e6ba07..8085b1460f 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -94,6 +94,8 @@ pub mod sqlite; pub mod kzg_commitment; pub mod kzg_proof; +pub mod bls_field_element; +pub mod blob; use ethereum_types::{H160, H256}; @@ -104,11 +106,11 @@ pub use crate::attestation_duty::AttestationDuty; pub use crate::attester_slashing::AttesterSlashing; pub use crate::beacon_block::{ BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockMerge, BeaconBlockRef, - BeaconBlockRefMut, + BeaconBlockRefMut, BeaconBlockEip4844 }; pub use crate::beacon_block_body::{ BeaconBlockBody, BeaconBlockBodyAltair, BeaconBlockBodyBase, BeaconBlockBodyMerge, - BeaconBlockBodyRef, BeaconBlockBodyRefMut, + BeaconBlockBodyRef, BeaconBlockBodyRefMut, BeaconBlockBodyEip4844 }; pub use crate::beacon_block_header::BeaconBlockHeader; pub use crate::beacon_committee::{BeaconCommittee, OwnedBeaconCommittee}; diff --git a/consensus/types/src/signed_beacon_block.rs b/consensus/types/src/signed_beacon_block.rs index 5c40c4685c..4ab74ac211 100644 --- a/consensus/types/src/signed_beacon_block.rs +++ b/consensus/types/src/signed_beacon_block.rs @@ -38,7 +38,7 @@ impl From for Hash256 { /// A `BeaconBlock` and a signature from its proposer. #[superstruct( - variants(Base, Altair, Merge), + variants(Base, Altair, Merge, Eip4844), variant_attributes( derive( Debug, @@ -72,6 +72,8 @@ pub struct SignedBeaconBlock = FullPayload, #[superstruct(only(Merge), partial_getter(rename = "message_merge"))] pub message: BeaconBlockMerge, + #[superstruct(only(Eip4844), partial_getter(rename = "message_eip4844"))] + pub message: BeaconBlockEip4844, pub signature: Signature, } @@ -129,6 +131,9 @@ impl> SignedBeaconBlock { BeaconBlock::Merge(message) => { SignedBeaconBlock::Merge(SignedBeaconBlockMerge { message, signature }) } + BeaconBlock::Eip4844(message) => { + SignedBeaconBlock::Eip4844(SignedBeaconBlockEip4844 { message, signature }) + } } } @@ -307,6 +312,60 @@ impl SignedBeaconBlockMerge> { } } +impl SignedBeaconBlockEip4844> { + pub fn into_full_block( + self, + execution_payload: ExecutionPayload, + ) -> SignedBeaconBlockEip4844> { + let SignedBeaconBlockEip4844 { + message: + BeaconBlockEip4844 { + slot, + proposer_index, + parent_root, + state_root, + body: + BeaconBlockBodyEip4844 { + randao_reveal, + eth1_data, + graffiti, + proposer_slashings, + attester_slashings, + attestations, + deposits, + voluntary_exits, + sync_aggregate, + execution_payload: BlindedPayload { .. }, + blob_kzg_commitments, + }, + }, + signature, + } = self; + SignedBeaconBlockEip4844 { + message: BeaconBlockEip4844 { + slot, + proposer_index, + parent_root, + state_root, + body: BeaconBlockBodyEip4844 { + randao_reveal, + eth1_data, + graffiti, + proposer_slashings, + attester_slashings, + attestations, + deposits, + voluntary_exits, + sync_aggregate, + execution_payload: FullPayload { execution_payload }, + blob_kzg_commitments, + }, + }, + signature, + } + } +} + impl SignedBeaconBlock> { pub fn try_into_full_block( self, @@ -318,6 +377,9 @@ impl SignedBeaconBlock> { SignedBeaconBlock::Merge(block) => { SignedBeaconBlock::Merge(block.into_full_block(execution_payload?)) } + SignedBeaconBlock::Eip4844(block) => { + SignedBeaconBlock::Eip4844(block.into_full_block(execution_payload?)) + } }; Some(full_block) } From ca1e17b386d225ae1430db02377536801fc906af Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Sat, 17 Sep 2022 12:23:03 +0200 Subject: [PATCH 03/22] it compiles! --- beacon_node/beacon_chain/src/beacon_chain.rs | 23 ++++++++++- beacon_node/lighthouse_network/src/config.rs | 2 +- .../src/rpc/codec/ssz_snappy.rs | 16 +++++++- .../lighthouse_network/src/rpc/protocol.rs | 10 +++++ .../lighthouse_network/src/types/pubsub.rs | 6 ++- beacon_node/store/src/partial_beacon_state.rs | 39 ++++++++++++++++--- consensus/types/src/beacon_state.rs | 8 ++-- consensus/types/src/fork_context.rs | 10 +++++ consensus/types/src/lib.rs | 2 +- .../src/signing_method/web3signer.rs | 6 +++ 10 files changed, 107 insertions(+), 15 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 6637b8fd53..39b2279c47 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -3353,7 +3353,7 @@ impl BeaconChain { // allows it to run concurrently with things like attestation packing. let prepare_payload_handle = match &state { BeaconState::Base(_) | BeaconState::Altair(_) => None, - BeaconState::Merge(_) => { + BeaconState::Merge(_) | BeaconState::Eip4844(_) => { let prepare_payload_handle = get_execution_payload(self.clone(), &state, proposer_index, builder_params)?; Some(prepare_payload_handle) @@ -3609,6 +3609,27 @@ impl BeaconChain { .ok_or(BlockProductionError::MissingExecutionPayload)?, }, }), + BeaconState::Eip4844(_) => BeaconBlock::Eip4844(BeaconBlockEip4844 { + slot, + proposer_index, + parent_root, + state_root: Hash256::zero(), + body: BeaconBlockBodyEip4844 { + randao_reveal, + eth1_data, + graffiti, + proposer_slashings: proposer_slashings.into(), + attester_slashings: attester_slashings.into(), + attestations: attestations.into(), + deposits: deposits.into(), + voluntary_exits: voluntary_exits.into(), + sync_aggregate: sync_aggregate + .ok_or(BlockProductionError::MissingSyncAggregate)?, + execution_payload: execution_payload + .ok_or(BlockProductionError::MissingExecutionPayload)?, + blob_kzg_commitments: todo!(), // part of partial block or?? + }, + }), }; let block = SignedBeaconBlock::from_block( diff --git a/beacon_node/lighthouse_network/src/config.rs b/beacon_node/lighthouse_network/src/config.rs index 263ef0c7cb..4e8b9bb85c 100644 --- a/beacon_node/lighthouse_network/src/config.rs +++ b/beacon_node/lighthouse_network/src/config.rs @@ -296,7 +296,7 @@ pub fn gossipsub_config(network_load: u8, fork_context: Arc) -> Gos match fork_context.current_fork() { // according to: https://github.com/ethereum/consensus-specs/blob/dev/specs/merge/p2p-interface.md#the-gossip-domain-gossipsub // the derivation of the message-id remains the same in the merge - ForkName::Altair | ForkName::Merge => { + ForkName::Altair | ForkName::Merge | ForkName::Eip4844 => { let topic_len_bytes = topic_bytes.len().to_le_bytes(); let mut vec = Vec::with_capacity( prefix.len() + topic_len_bytes.len() + topic_bytes.len() + message.data.len(), diff --git a/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs b/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs index a46a05a8ce..e952eece30 100644 --- a/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs +++ b/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs @@ -17,7 +17,7 @@ use std::sync::Arc; use tokio_util::codec::{Decoder, Encoder}; use types::{ EthSpec, ForkContext, ForkName, SignedBeaconBlock, SignedBeaconBlockAltair, - SignedBeaconBlockBase, SignedBeaconBlockMerge, + SignedBeaconBlockBase, SignedBeaconBlockMerge, SignedBeaconBlockEip4844 }; use unsigned_varint::codec::Uvi; @@ -407,6 +407,10 @@ fn context_bytes( return match **ref_box_block { // NOTE: If you are adding another fork type here, be sure to modify the // `fork_context.to_context_bytes()` function to support it as well! + SignedBeaconBlock::Eip4844 { .. } => { + // Merge context being `None` implies that "merge never happened". + fork_context.to_context_bytes(ForkName::Eip4844) + } SignedBeaconBlock::Merge { .. } => { // Merge context being `None` implies that "merge never happened". fork_context.to_context_bytes(ForkName::Merge) @@ -586,6 +590,11 @@ fn handle_v2_response( decoded_buffer, )?), )))), + ForkName::Eip4844 => Ok(Some(RPCResponse::BlocksByRange(Arc::new( + SignedBeaconBlock::Eip4844(SignedBeaconBlockEip4844::from_ssz_bytes( + decoded_buffer, + )?), + )))), }, Protocol::BlocksByRoot => match fork_name { ForkName::Altair => Ok(Some(RPCResponse::BlocksByRoot(Arc::new( @@ -601,6 +610,11 @@ fn handle_v2_response( decoded_buffer, )?), )))), + ForkName::Eip4844 => Ok(Some(RPCResponse::BlocksByRange(Arc::new( + SignedBeaconBlock::Eip4844(SignedBeaconBlockEip4844::from_ssz_bytes( + decoded_buffer, + )?), + )))), }, _ => Err(RPCError::ErrorResponse( RPCResponseErrorCode::InvalidRequest, diff --git a/beacon_node/lighthouse_network/src/rpc/protocol.rs b/beacon_node/lighthouse_network/src/rpc/protocol.rs index 81960214b1..352348f74f 100644 --- a/beacon_node/lighthouse_network/src/rpc/protocol.rs +++ b/beacon_node/lighthouse_network/src/rpc/protocol.rs @@ -71,6 +71,10 @@ lazy_static! { + types::ExecutionPayload::::max_execution_payload_size() // adding max size of execution payload (~16gb) + ssz::BYTES_PER_LENGTH_OFFSET; // Adding the additional ssz offset for the `ExecutionPayload` field + pub static ref SIGNED_BEACON_BLOCK_EIP4844_MAX: usize = + // Size of a full altair block + *SIGNED_BEACON_BLOCK_MERGE_MAX + 999999999; //TODO + pub static ref BLOCKS_BY_ROOT_REQUEST_MIN: usize = VariableList::::from(Vec::::new()) .as_ssz_bytes() @@ -102,6 +106,7 @@ lazy_static! { pub(crate) const MAX_RPC_SIZE: usize = 1_048_576; // 1M /// The maximum bytes that can be sent across the RPC post-merge. pub(crate) const MAX_RPC_SIZE_POST_MERGE: usize = 10 * 1_048_576; // 10M +pub(crate) const MAX_RPC_SIZE_POST_EIP4844: usize = 20 * 1_048_576; // 10M /// The protocol prefix the RPC protocol id. const PROTOCOL_PREFIX: &str = "/eth2/beacon_chain/req"; /// Time allowed for the first byte of a request to arrive before we time out (Time To First Byte). @@ -113,6 +118,7 @@ const REQUEST_TIMEOUT: u64 = 15; /// Returns the maximum bytes that can be sent across the RPC. pub fn max_rpc_size(fork_context: &ForkContext) -> usize { match fork_context.current_fork() { + ForkName::Eip4844 => MAX_RPC_SIZE_POST_EIP4844, ForkName::Merge => MAX_RPC_SIZE_POST_MERGE, ForkName::Altair | ForkName::Base => MAX_RPC_SIZE, } @@ -135,6 +141,10 @@ pub fn rpc_block_limits_by_fork(current_fork: ForkName) -> RpcLimits { *SIGNED_BEACON_BLOCK_BASE_MIN, // Base block is smaller than altair and merge blocks *SIGNED_BEACON_BLOCK_MERGE_MAX, // Merge block is larger than base and altair blocks ), + ForkName::Eip4844 => RpcLimits::new( + *SIGNED_BEACON_BLOCK_BASE_MIN, // Base block is smaller than altair and merge blocks + *SIGNED_BEACON_BLOCK_EIP4844_MAX, // Merge block is larger than base and altair blocks + ), } } diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index a01072f8e4..ef3b53abfb 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -11,7 +11,7 @@ use std::sync::Arc; use types::{ Attestation, AttesterSlashing, EthSpec, ForkContext, ForkName, ProposerSlashing, SignedAggregateAndProof, SignedBeaconBlock, SignedBeaconBlockAltair, SignedBeaconBlockBase, - SignedBeaconBlockMerge, SignedContributionAndProof, SignedVoluntaryExit, SubnetId, + SignedBeaconBlockMerge, SignedBeaconBlockEip4844, SignedContributionAndProof, SignedVoluntaryExit, SubnetId, SyncCommitteeMessage, SyncSubnetId, }; @@ -167,6 +167,10 @@ impl PubsubMessage { SignedBeaconBlockMerge::from_ssz_bytes(data) .map_err(|e| format!("{:?}", e))?, ), + Some(ForkName::Eip4844) => SignedBeaconBlock::::Eip4844( + SignedBeaconBlockEip4844::from_ssz_bytes(data) + .map_err(|e| format!("{:?}", e))?, + ), None => { return Err(format!( "Unknown gossipsub fork digest: {:?}", diff --git a/beacon_node/store/src/partial_beacon_state.rs b/beacon_node/store/src/partial_beacon_state.rs index 2d3d9cc7bf..46bc0274f4 100644 --- a/beacon_node/store/src/partial_beacon_state.rs +++ b/beacon_node/store/src/partial_beacon_state.rs @@ -66,9 +66,9 @@ where pub current_epoch_attestations: VariableList, T::MaxPendingAttestations>, // Participation (Altair and later) - #[superstruct(only(Altair, Merge))] + #[superstruct(only(Altair, Merge, Eip4844))] pub previous_epoch_participation: VariableList, - #[superstruct(only(Altair, Merge))] + #[superstruct(only(Altair, Merge, Eip4844))] pub current_epoch_participation: VariableList, // Finality @@ -78,17 +78,17 @@ where pub finalized_checkpoint: Checkpoint, // Inactivity - #[superstruct(only(Altair, Merge))] + #[superstruct(only(Altair, Merge, Eip4844))] pub inactivity_scores: VariableList, // Light-client sync committees - #[superstruct(only(Altair, Merge))] + #[superstruct(only(Altair, Merge, Eip4844))] pub current_sync_committee: Arc>, - #[superstruct(only(Altair, Merge))] + #[superstruct(only(Altair, Merge, Eip4844))] pub next_sync_committee: Arc>, // Execution - #[superstruct(only(Merge))] + #[superstruct(only(Merge, Eip4844))] pub latest_execution_payload_header: ExecutionPayloadHeader, } @@ -178,6 +178,20 @@ impl PartialBeaconState { latest_execution_payload_header ] ), + BeaconState::Eip4844(s) => impl_from_state_forgetful!( + s, + outer, + Eip4844, + PartialBeaconStateEip4844, + [ + previous_epoch_participation, + current_epoch_participation, + current_sync_committee, + next_sync_committee, + inactivity_scores, + latest_execution_payload_header + ] + ), } } @@ -365,6 +379,19 @@ impl TryInto> for PartialBeaconState { latest_execution_payload_header ] ), + PartialBeaconState::Eip4844(inner) => impl_try_into_beacon_state!( + inner, + Eip4844, + BeaconStateEip4844, + [ + previous_epoch_participation, + current_epoch_participation, + current_sync_committee, + next_sync_committee, + inactivity_scores, + latest_execution_payload_header + ] + ), }; Ok(state) } diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index 0d9920ec96..dd0ac3b71c 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -267,17 +267,17 @@ where // Inactivity #[serde(with = "ssz_types::serde_utils::quoted_u64_var_list")] - #[superstruct(only(Altair, Merge))] + #[superstruct(only(Altair, Merge, Eip4844))] pub inactivity_scores: VariableList, // Light-client sync committees - #[superstruct(only(Altair, Merge))] + #[superstruct(only(Altair, Merge, Eip4844))] pub current_sync_committee: Arc>, - #[superstruct(only(Altair, Merge))] + #[superstruct(only(Altair, Merge, Eip4844))] pub next_sync_committee: Arc>, // Execution - #[superstruct(only(Merge))] + #[superstruct(only(Merge, Eip4844))] pub latest_execution_payload_header: ExecutionPayloadHeader, // Caching (not in the spec) diff --git a/consensus/types/src/fork_context.rs b/consensus/types/src/fork_context.rs index 52b9294c8c..c5316a6567 100644 --- a/consensus/types/src/fork_context.rs +++ b/consensus/types/src/fork_context.rs @@ -47,6 +47,16 @@ impl ForkContext { )); } + if spec.bellatrix_fork_epoch.is_some() { + fork_to_digest.push(( + ForkName::Merge, + ChainSpec::compute_fork_digest( + spec.bellatrix_fork_version, + genesis_validators_root, + ), + )); + } + let fork_to_digest: HashMap = fork_to_digest.into_iter().collect(); let digest_to_fork = fork_to_digest diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 8085b1460f..93c9b39afb 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -151,7 +151,7 @@ pub use crate::shuffling_id::AttestationShufflingId; pub use crate::signed_aggregate_and_proof::SignedAggregateAndProof; pub use crate::signed_beacon_block::{ SignedBeaconBlock, SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockHash, - SignedBeaconBlockMerge, SignedBlindedBeaconBlock, + SignedBeaconBlockMerge, SignedBlindedBeaconBlock, SignedBeaconBlockEip4844, }; pub use crate::signed_beacon_block_header::SignedBeaconBlockHeader; pub use crate::signed_contribution_and_proof::SignedContributionAndProof; diff --git a/validator_client/src/signing_method/web3signer.rs b/validator_client/src/signing_method/web3signer.rs index cf02ae0c32..0de260ecfc 100644 --- a/validator_client/src/signing_method/web3signer.rs +++ b/validator_client/src/signing_method/web3signer.rs @@ -26,6 +26,7 @@ pub enum ForkName { Phase0, Altair, Bellatrix, + Eip4844, } #[derive(Debug, PartialEq, Serialize)] @@ -90,6 +91,11 @@ impl<'a, T: EthSpec, Payload: ExecPayload> Web3SignerObject<'a, T, Payload> { block: None, block_header: Some(block.block_header()), }), + BeaconBlock::Eip4844(_) => Ok(Web3SignerObject::BeaconBlock { + version: ForkName::Eip4844, + block: None, + block_header: Some(block.block_header()), + }), } } From fe6be28e6bf3a134d7b1d7789348b259e48e3734 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Sat, 17 Sep 2022 13:20:18 +0200 Subject: [PATCH 04/22] beacon: consensus: implement engine api getBlobs --- beacon_node/execution_layer/src/engine_api.rs | 3 +- .../execution_layer/src/engine_api/http.rs | 32 +++++++++++- .../src/engine_api/json_structures.rs | 50 ++++++++++++++++++- beacon_node/execution_layer/src/lib.rs | 8 +-- consensus/types/src/beacon_block_body.rs | 8 +-- consensus/types/src/blob.rs | 18 ++++++- consensus/types/src/execution_payload.rs | 20 +++++++- consensus/types/src/payload.rs | 32 ++++++++++-- consensus/types/src/signed_beacon_block.rs | 4 +- 9 files changed, 155 insertions(+), 20 deletions(-) diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index ba0a37736b..86829e5f68 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -7,9 +7,10 @@ use serde::{Deserialize, Serialize}; use strum::IntoStaticStr; pub use types::{ Address, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadHeader, FixedVector, - Hash256, Uint256, VariableList, + Hash256, Uint256, VariableList, kzg_proof::KzgProof, kzg_commitment::KzgCommitment, blob::Blob, }; + pub mod auth; pub mod http; pub mod json_structures; diff --git a/beacon_node/execution_layer/src/engine_api/http.rs b/beacon_node/execution_layer/src/engine_api/http.rs index 0f848a7716..e37aef72d7 100644 --- a/beacon_node/execution_layer/src/engine_api/http.rs +++ b/beacon_node/execution_layer/src/engine_api/http.rs @@ -3,13 +3,14 @@ use super::*; use crate::auth::Auth; use crate::json_structures::*; +use eth2::lighthouse::Eth1Block; use reqwest::header::CONTENT_TYPE; use sensitive_url::SensitiveUrl; use serde::de::DeserializeOwned; use serde_json::json; use std::time::Duration; -use types::EthSpec; +use types::{EthSpec, FullPayload, execution_payload::BlobsBundle}; pub use deposit_log::{DepositLog, Log}; pub use reqwest::Client; @@ -34,6 +35,9 @@ pub const ENGINE_NEW_PAYLOAD_TIMEOUT: Duration = Duration::from_secs(8); pub const ENGINE_GET_PAYLOAD_V1: &str = "engine_getPayloadV1"; pub const ENGINE_GET_PAYLOAD_TIMEOUT: Duration = Duration::from_secs(2); +pub const ENGINE_GET_BLOBS_BUNDLE_V1: &str = "engine_getBlobsBundleV1"; +pub const ENGINE_GET_BLOBS_BUNDLE_TIMEOUT: Duration = Duration::from_secs(2); + pub const ENGINE_FORKCHOICE_UPDATED_V1: &str = "engine_forkchoiceUpdatedV1"; pub const ENGINE_FORKCHOICE_UPDATED_TIMEOUT: Duration = Duration::from_secs(8); @@ -664,6 +668,32 @@ impl HttpJsonRpc { Ok(response.into()) } + pub async fn get_blobs_bundle_v1( + &self, + payload_id: PayloadId, + ) -> Result, Error> { + let params = json!([JsonPayloadIdRequest::from(payload_id)]); + + let response: JsonBlobBundlesV1 = self + .rpc_request(ENGINE_GET_BLOBS_BUNDLE_V1, params, ENGINE_GET_BLOBS_BUNDLE_TIMEOUT) + .await?; + + Ok(response.into()) + } + + pub async fn get_full_payload( + &self, + payload_id: PayloadId, + ) -> Result, Error> { + let payload = self.get_payload_v1(payload_id).await; + let blobs = self.get_blobs_bundle_v1(payload_id).await; + + Ok(FullPayload{ + execution_payload: payload?, + blobs_bundle: blobs?.into(), + }) + } + pub async fn forkchoice_updated_v1( &self, forkchoice_state: ForkChoiceState, diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index 31aa79f055..4907acee3e 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -1,6 +1,6 @@ use super::*; use serde::{Deserialize, Serialize}; -use types::{EthSpec, ExecutionBlockHash, FixedVector, Transaction, Unsigned, VariableList}; +use types::{EthSpec, ExecutionBlockHash, FixedVector, Transaction, Unsigned, VariableList, execution_payload::BlobsBundle}; #[derive(Debug, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -269,6 +269,54 @@ impl From for PayloadAttributes { } } +#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[serde(bound = "T: EthSpec", rename_all = "camelCase")] +pub struct JsonBlobBundlesV1 { + pub block_hash: Hash256, + pub kzgs: Vec, + pub blobs: Vec>, + pub aggregated_proof: KzgProof, +} + +impl From> for JsonBlobBundlesV1 { + fn from(p: BlobsBundle) -> Self { + // Use this verbose deconstruction pattern to ensure no field is left unused. + let BlobsBundle { + block_hash, + aggregated_proof, + blobs, + kzgs, + } = p; + + Self { + block_hash, + aggregated_proof, + blobs, + kzgs, + } + } +} + +impl From> for BlobsBundle { + fn from(j: JsonBlobBundlesV1) -> Self { + // Use this verbose deconstruction pattern to ensure no field is left unused. + let JsonBlobBundlesV1 { + block_hash, + aggregated_proof, + blobs, + kzgs, + } = j; + + Self { + block_hash, + aggregated_proof, + blobs, + kzgs, + } + } +} + + #[derive(Debug, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct JsonForkChoiceStateV1 { diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 3bdca82ad0..5d1190f81c 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -835,10 +835,10 @@ impl ExecutionLayer { engine .api - .get_payload_v1::(payload_id) + .get_full_payload::(payload_id) .await .map(|full_payload| { - if full_payload.fee_recipient != suggested_fee_recipient { + if full_payload.execution_payload.fee_recipient != suggested_fee_recipient { error!( self.log(), "Inconsistent fee recipient"; @@ -847,11 +847,11 @@ impl ExecutionLayer { indicate that fees are being diverted to another address. Please \ ensure that the value of suggested_fee_recipient is set correctly and \ that the Execution Engine is trusted.", - "fee_recipient" => ?full_payload.fee_recipient, + "fee_recipient" => ?full_payload.execution_payload.fee_recipient, "suggested_fee_recipient" => ?suggested_fee_recipient, ); } - if f(self, &full_payload).is_some() { + if f(self, &full_payload.execution_payload).is_some() { warn!( self.log(), "Duplicate payload cached, this might indicate redundant proposal \ diff --git a/consensus/types/src/beacon_block_body.rs b/consensus/types/src/beacon_block_body.rs index eaa99718cb..d4f49d3773 100644 --- a/consensus/types/src/beacon_block_body.rs +++ b/consensus/types/src/beacon_block_body.rs @@ -232,7 +232,7 @@ impl From>> deposits, voluntary_exits, sync_aggregate, - execution_payload: FullPayload { execution_payload }, + execution_payload: FullPayload { execution_payload, blobs_bundle }, } = body; ( @@ -272,7 +272,7 @@ for ( deposits, voluntary_exits, sync_aggregate, - execution_payload: FullPayload { execution_payload }, + execution_payload: FullPayload { execution_payload, blobs_bundle }, blob_kzg_commitments, } = body; @@ -324,7 +324,7 @@ impl BeaconBlockBodyMerge> { deposits, voluntary_exits, sync_aggregate, - execution_payload: FullPayload { execution_payload }, + execution_payload: FullPayload { execution_payload, blobs_bundle }, } = self; BeaconBlockBodyMerge { @@ -356,7 +356,7 @@ impl BeaconBlockBodyEip4844> { deposits, voluntary_exits, sync_aggregate, - execution_payload: FullPayload { execution_payload }, + execution_payload: FullPayload { execution_payload, blobs_bundle }, blob_kzg_commitments, } = self; diff --git a/consensus/types/src/blob.rs b/consensus/types/src/blob.rs index 8ea3468ff8..efe243859d 100644 --- a/consensus/types/src/blob.rs +++ b/consensus/types/src/blob.rs @@ -1,9 +1,23 @@ use ssz_types::VariableList; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use ssz::{Decode, DecodeError, Encode}; +use crate::test_utils::RngCore; use crate::bls_field_element::BlsFieldElement; -use crate::EthSpec; +use crate::{EthSpec, Uint256}; +use crate::test_utils::TestRandom; #[derive(Default, Debug, PartialEq, Hash, Clone, Serialize, Deserialize)] #[serde(transparent)] -pub struct Blob(pub VariableList); \ No newline at end of file +pub struct Blob(pub VariableList); + +impl TestRandom for Blob { + fn random_for_test(rng: &mut impl RngCore) -> Self { + let mut res = Blob(VariableList::empty()); + for i in 0..4096 { + let slice = ethereum_types::U256([rng.next_u64(), rng.next_u64(), rng.next_u64(), rng.next_u64()]); + let elem =BlsFieldElement(slice); + res.0.push(elem); + } + res + } +} \ No newline at end of file diff --git a/consensus/types/src/execution_payload.rs b/consensus/types/src/execution_payload.rs index 412e5a8df3..9d6c42c916 100644 --- a/consensus/types/src/execution_payload.rs +++ b/consensus/types/src/execution_payload.rs @@ -1,4 +1,4 @@ -use crate::{test_utils::TestRandom, *}; +use crate::{test_utils::TestRandom, test_utils::RngCore, *, kzg_commitment::KzgCommitment, kzg_proof::KzgProof, blob::Blob}; use derivative::Derivative; use serde_derive::{Deserialize, Serialize}; use ssz::Encode; @@ -43,6 +43,24 @@ pub struct ExecutionPayload { pub transactions: Transactions, } +#[derive( + Default, Debug, Clone, Serialize, Deserialize, Derivative, +)] +#[serde(bound = "T: EthSpec")] +pub struct BlobsBundle { + pub block_hash: Hash256, + pub kzgs: Vec, + pub blobs: Vec>, + pub aggregated_proof: KzgProof, +} + + +impl TestRandom for BlobsBundle { + fn random_for_test(rng: &mut impl RngCore) -> Self { + todo!() + } +} + impl ExecutionPayload { pub fn empty() -> Self { Self::default() diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index 114ca02ecf..134fe38152 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -1,4 +1,4 @@ -use crate::{test_utils::TestRandom, *}; +use crate::{test_utils::TestRandom, test_utils::RngCore, *}; use derivative::Derivative; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; @@ -8,6 +8,8 @@ use std::fmt::Debug; use std::hash::Hash; use test_random_derive::TestRandom; use tree_hash::TreeHash; +use execution_payload::BlobsBundle; +use core::hash::Hasher; #[derive(Debug)] pub enum BlockType { @@ -218,16 +220,37 @@ impl Encode for BlindedPayload { } } -#[derive(Default, Debug, Clone, Serialize, Deserialize, TestRandom, Derivative)] -#[derivative(PartialEq, Hash(bound = "T: EthSpec"))] +#[derive(Default, Debug, Clone, Serialize, Deserialize)] #[serde(bound = "T: EthSpec")] pub struct FullPayload { pub execution_payload: ExecutionPayload, + pub blobs_bundle: Option>, +} + +impl TestRandom for FullPayload { + fn random_for_test(rng: &mut impl RngCore) -> Self { + todo!() + } +} + +impl PartialEq for FullPayload { + fn eq(&self, other: &FullPayload) -> bool { + todo!() + } +} + +impl Hash for FullPayload { + fn hash(&self, into: &mut H) { + todo!() + } } impl From> for FullPayload { fn from(execution_payload: ExecutionPayload) -> Self { - Self { execution_payload } + Self { + execution_payload, + blobs_bundle: None, + } } } @@ -265,6 +288,7 @@ impl Decode for FullPayload { fn from_ssz_bytes(bytes: &[u8]) -> Result { Ok(FullPayload { execution_payload: Decode::from_ssz_bytes(bytes)?, + blobs_bundle: None, }) } } diff --git a/consensus/types/src/signed_beacon_block.rs b/consensus/types/src/signed_beacon_block.rs index 4ab74ac211..55e0e8afd2 100644 --- a/consensus/types/src/signed_beacon_block.rs +++ b/consensus/types/src/signed_beacon_block.rs @@ -304,7 +304,7 @@ impl SignedBeaconBlockMerge> { deposits, voluntary_exits, sync_aggregate, - execution_payload: FullPayload { execution_payload }, + execution_payload: FullPayload { execution_payload: execution_payload, blobs_bundle: None }, }, }, signature, @@ -357,7 +357,7 @@ impl SignedBeaconBlockEip4844> { deposits, voluntary_exits, sync_aggregate, - execution_payload: FullPayload { execution_payload }, + execution_payload: FullPayload { execution_payload: execution_payload, blobs_bundle: None }, blob_kzg_commitments, }, }, From dcfae6c5cfc0ac8a1e4ce2ad04fc1d34da759bcd Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Sat, 17 Sep 2022 13:29:20 +0200 Subject: [PATCH 05/22] implement From for Payload --- beacon_node/beacon_chain/src/beacon_chain.rs | 3 ++- consensus/types/src/payload.rs | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 39b2279c47..7418a66c17 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -99,6 +99,7 @@ use types::*; pub use crate::canonical_head::{CanonicalHead, CanonicalHeadRwLock}; pub use fork_choice::CountUnrealized; +use types::kzg_commitment::KzgCommitment; pub type ForkChoiceError = fork_choice::Error; @@ -3627,7 +3628,7 @@ impl BeaconChain { .ok_or(BlockProductionError::MissingSyncAggregate)?, execution_payload: execution_payload .ok_or(BlockProductionError::MissingExecutionPayload)?, - blob_kzg_commitments: todo!(), // part of partial block or?? + blob_kzg_commitments: blob_kzg_commitments.into(), }, }), }; diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index 134fe38152..7974c5ef36 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -32,6 +32,7 @@ pub trait ExecPayload: + Hash + TryFrom> + From> + + From> + Send + 'static { @@ -172,6 +173,12 @@ impl From> for BlindedPayload { } } +impl From> for BlindedPayload { + fn from(full_payload: FullPayload) -> Self { + full_payload.execution_payload.into() + } +} + impl TreeHash for BlindedPayload { fn tree_hash_type() -> tree_hash::TreeHashType { >::tree_hash_type() From 8473f08d10844c3dd1547ccd01f83d7e7d9caa47 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Sat, 17 Sep 2022 14:10:15 +0200 Subject: [PATCH 06/22] beacon: consensus: implement engine api getBlobs --- .../beacon_chain/src/execution_payload.rs | 10 +++- .../execution_layer/src/engine_api/http.rs | 13 ----- beacon_node/execution_layer/src/lib.rs | 58 +++++++++++++++++-- consensus/types/src/beacon_block_body.rs | 8 +-- consensus/types/src/payload.rs | 9 +-- consensus/types/src/signed_beacon_block.rs | 4 +- 6 files changed, 72 insertions(+), 30 deletions(-) diff --git a/beacon_node/beacon_chain/src/execution_payload.rs b/beacon_node/beacon_chain/src/execution_payload.rs index 2221d1fc7c..84c6902270 100644 --- a/beacon_node/beacon_chain/src/execution_payload.rs +++ b/beacon_node/beacon_chain/src/execution_payload.rs @@ -24,7 +24,7 @@ use state_processing::per_block_processing::{ use std::sync::Arc; use tokio::task::JoinHandle; use tree_hash::TreeHash; -use types::*; +use types::{*, execution_payload::BlobsBundle}; pub type PreparePayloadResult = Result; pub type PreparePayloadHandle = JoinHandle>>; @@ -483,5 +483,13 @@ where .await .map_err(BlockProductionError::GetPayloadFailed)?; + /* + TODO: fetch blob bundles from el engine for block building + let suggested_fee_recipient = execution_layer.get_suggested_fee_recipient(proposer_index).await; + let blobs = execution_layer.get_blob_bundles(parent_hash, timestamp, random, suggested_fee_recipient) + .await + .map_err(BlockProductionError::GetPayloadFailed)?; + */ + Ok(execution_payload) } diff --git a/beacon_node/execution_layer/src/engine_api/http.rs b/beacon_node/execution_layer/src/engine_api/http.rs index e37aef72d7..bc4d790d8e 100644 --- a/beacon_node/execution_layer/src/engine_api/http.rs +++ b/beacon_node/execution_layer/src/engine_api/http.rs @@ -681,19 +681,6 @@ impl HttpJsonRpc { Ok(response.into()) } - pub async fn get_full_payload( - &self, - payload_id: PayloadId, - ) -> Result, Error> { - let payload = self.get_payload_v1(payload_id).await; - let blobs = self.get_blobs_bundle_v1(payload_id).await; - - Ok(FullPayload{ - execution_payload: payload?, - blobs_bundle: blobs?.into(), - }) - } - pub async fn forkchoice_updated_v1( &self, forkchoice_state: ForkChoiceState, diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 5d1190f81c..7eabee8199 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -20,6 +20,7 @@ use sensitive_url::SensitiveUrl; use serde::{Deserialize, Serialize}; use slog::{crit, debug, error, info, trace, warn, Logger}; use slot_clock::SlotClock; +use types::execution_payload::BlobsBundle; use std::collections::HashMap; use std::future::Future; use std::io::Write; @@ -759,6 +760,55 @@ impl ExecutionLayer { .await } + pub async fn get_blob_bundles( + &self, + parent_hash: ExecutionBlockHash, + timestamp: u64, + prev_randao: Hash256, + suggested_fee_recipient: Address, + ) -> Result, Error> { + debug!( + self.log(), + "Issuing engine_getPayload"; + "suggested_fee_recipient" => ?suggested_fee_recipient, + "prev_randao" => ?prev_randao, + "timestamp" => timestamp, + "parent_hash" => ?parent_hash, + ); + self.engine() + .request(|engine| async move { + let payload_id = if let Some(id) = engine + .get_payload_id(parent_hash, timestamp, prev_randao, suggested_fee_recipient) + .await + { + // The payload id has been cached for this engine. + metrics::inc_counter_vec( + &metrics::EXECUTION_LAYER_PRE_PREPARED_PAYLOAD_ID, + &[metrics::HIT], + ); + id + } else { + error!( + self.log(), + "Exec engine unable to produce blobs, did you call get_payload before?", + ); + return Err(ApiError::PayloadIdUnavailable); + }; + + engine + .api + .get_blobs_bundle_v1::(payload_id) + .await + .map(|bundle| { + // TODO verify the blob bundle here? + bundle.into() + }) + }) + .await + .map_err(Box::new) + .map_err(Error::EngineError) + } + async fn get_full_payload_with>( &self, parent_hash: ExecutionBlockHash, @@ -835,10 +885,10 @@ impl ExecutionLayer { engine .api - .get_full_payload::(payload_id) + .get_payload_v1::(payload_id) .await .map(|full_payload| { - if full_payload.execution_payload.fee_recipient != suggested_fee_recipient { + if full_payload.fee_recipient != suggested_fee_recipient { error!( self.log(), "Inconsistent fee recipient"; @@ -847,11 +897,11 @@ impl ExecutionLayer { indicate that fees are being diverted to another address. Please \ ensure that the value of suggested_fee_recipient is set correctly and \ that the Execution Engine is trusted.", - "fee_recipient" => ?full_payload.execution_payload.fee_recipient, + "fee_recipient" => ?full_payload.fee_recipient, "suggested_fee_recipient" => ?suggested_fee_recipient, ); } - if f(self, &full_payload.execution_payload).is_some() { + if f(self, &full_payload).is_some() { warn!( self.log(), "Duplicate payload cached, this might indicate redundant proposal \ diff --git a/consensus/types/src/beacon_block_body.rs b/consensus/types/src/beacon_block_body.rs index d4f49d3773..ec973b9f80 100644 --- a/consensus/types/src/beacon_block_body.rs +++ b/consensus/types/src/beacon_block_body.rs @@ -232,7 +232,7 @@ impl From>> deposits, voluntary_exits, sync_aggregate, - execution_payload: FullPayload { execution_payload, blobs_bundle }, + execution_payload: FullPayload { execution_payload }, } = body; ( @@ -272,7 +272,7 @@ for ( deposits, voluntary_exits, sync_aggregate, - execution_payload: FullPayload { execution_payload, blobs_bundle }, + execution_payload: FullPayload { execution_payload}, blob_kzg_commitments, } = body; @@ -324,7 +324,7 @@ impl BeaconBlockBodyMerge> { deposits, voluntary_exits, sync_aggregate, - execution_payload: FullPayload { execution_payload, blobs_bundle }, + execution_payload: FullPayload { execution_payload }, } = self; BeaconBlockBodyMerge { @@ -356,7 +356,7 @@ impl BeaconBlockBodyEip4844> { deposits, voluntary_exits, sync_aggregate, - execution_payload: FullPayload { execution_payload, blobs_bundle }, + execution_payload: FullPayload { execution_payload }, blob_kzg_commitments, } = self; diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index 7974c5ef36..23db2d961f 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -230,8 +230,7 @@ impl Encode for BlindedPayload { #[derive(Default, Debug, Clone, Serialize, Deserialize)] #[serde(bound = "T: EthSpec")] pub struct FullPayload { - pub execution_payload: ExecutionPayload, - pub blobs_bundle: Option>, + pub execution_payload: ExecutionPayload } impl TestRandom for FullPayload { @@ -255,8 +254,7 @@ impl Hash for FullPayload { impl From> for FullPayload { fn from(execution_payload: ExecutionPayload) -> Self { Self { - execution_payload, - blobs_bundle: None, + execution_payload } } } @@ -294,8 +292,7 @@ impl Decode for FullPayload { fn from_ssz_bytes(bytes: &[u8]) -> Result { Ok(FullPayload { - execution_payload: Decode::from_ssz_bytes(bytes)?, - blobs_bundle: None, + execution_payload: Decode::from_ssz_bytes(bytes)? }) } } diff --git a/consensus/types/src/signed_beacon_block.rs b/consensus/types/src/signed_beacon_block.rs index 55e0e8afd2..4ab74ac211 100644 --- a/consensus/types/src/signed_beacon_block.rs +++ b/consensus/types/src/signed_beacon_block.rs @@ -304,7 +304,7 @@ impl SignedBeaconBlockMerge> { deposits, voluntary_exits, sync_aggregate, - execution_payload: FullPayload { execution_payload: execution_payload, blobs_bundle: None }, + execution_payload: FullPayload { execution_payload }, }, }, signature, @@ -357,7 +357,7 @@ impl SignedBeaconBlockEip4844> { deposits, voluntary_exits, sync_aggregate, - execution_payload: FullPayload { execution_payload: execution_payload, blobs_bundle: None }, + execution_payload: FullPayload { execution_payload }, blob_kzg_commitments, }, }, From bcc738cb9d1e7049c74d2d5110a104cf7433da65 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Sat, 17 Sep 2022 14:31:57 +0200 Subject: [PATCH 07/22] progress on gossip stuff --- .../src/behaviour/gossip_cache.rs | 7 +++ .../lighthouse_network/src/types/pubsub.rs | 4 ++ .../lighthouse_network/src/types/topics.rs | 6 +- consensus/types/src/blob.rs | 53 +++++++++++++++++- consensus/types/src/blobs_sidecar.rs | 16 ++++++ consensus/types/src/bls_field_element.rs | 56 ++++++++++++++++++- consensus/types/src/lib.rs | 2 + consensus/types/src/signed_blobs_sidecar.rs | 13 +++++ 8 files changed, 153 insertions(+), 4 deletions(-) create mode 100644 consensus/types/src/blobs_sidecar.rs create mode 100644 consensus/types/src/signed_blobs_sidecar.rs diff --git a/beacon_node/lighthouse_network/src/behaviour/gossip_cache.rs b/beacon_node/lighthouse_network/src/behaviour/gossip_cache.rs index 4842605f7a..1c6ffd022d 100644 --- a/beacon_node/lighthouse_network/src/behaviour/gossip_cache.rs +++ b/beacon_node/lighthouse_network/src/behaviour/gossip_cache.rs @@ -20,6 +20,8 @@ pub struct GossipCache { topic_msgs: HashMap, Key>>, /// Timeout for blocks. beacon_block: Option, + /// Timeout for blobs. + blobs_sidecar: Option, /// Timeout for aggregate attestations. aggregates: Option, /// Timeout for attestations. @@ -41,6 +43,8 @@ pub struct GossipCacheBuilder { default_timeout: Option, /// Timeout for blocks. beacon_block: Option, + /// Timeout for blob sidecars. + blobs_sidecar: Option, /// Timeout for aggregate attestations. aggregates: Option, /// Timeout for attestations. @@ -117,6 +121,7 @@ impl GossipCacheBuilder { let GossipCacheBuilder { default_timeout, beacon_block, + blobs_sidecar, aggregates, attestation, voluntary_exit, @@ -129,6 +134,7 @@ impl GossipCacheBuilder { expirations: DelayQueue::default(), topic_msgs: HashMap::default(), beacon_block: beacon_block.or(default_timeout), + blobs_sidecar: blobs_sidecar.or(default_timeout), aggregates: aggregates.or(default_timeout), attestation: attestation.or(default_timeout), voluntary_exit: voluntary_exit.or(default_timeout), @@ -151,6 +157,7 @@ impl GossipCache { pub fn insert(&mut self, topic: GossipTopic, data: Vec) { let expire_timeout = match topic.kind() { GossipKind::BeaconBlock => self.beacon_block, + GossipKind::BlobsSidecar => self.blobs_sidecar, GossipKind::BeaconAggregateAndProof => self.aggregates, GossipKind::Attestation(_) => self.attestation, GossipKind::VoluntaryExit => self.voluntary_exit, diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index ef3b53abfb..3519dafcfd 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -14,11 +14,14 @@ use types::{ SignedBeaconBlockMerge, SignedBeaconBlockEip4844, SignedContributionAndProof, SignedVoluntaryExit, SubnetId, SyncCommitteeMessage, SyncSubnetId, }; +use types::signed_blobs_sidecar::SignedBlobsSidecar; #[derive(Debug, Clone, PartialEq)] pub enum PubsubMessage { /// Gossipsub message providing notification of a new block. BeaconBlock(Arc>), + /// Gossipsub message providing notification of a new blobs sidecar. + BlobsSidecars(Arc>), /// Gossipsub message providing notification of a Aggregate attestation and associated proof. AggregateAndProofAttestation(Box>), /// Gossipsub message providing notification of a raw un-aggregated attestation with its shard id. @@ -106,6 +109,7 @@ impl PubsubMessage { pub fn kind(&self) -> GossipKind { match self { PubsubMessage::BeaconBlock(_) => GossipKind::BeaconBlock, + PubsubMessage::BlobsSidecars(_) => GossipKind::BlobsSidecar, PubsubMessage::AggregateAndProofAttestation(_) => GossipKind::BeaconAggregateAndProof, PubsubMessage::Attestation(attestation_data) => { GossipKind::Attestation(attestation_data.0) diff --git a/beacon_node/lighthouse_network/src/types/topics.rs b/beacon_node/lighthouse_network/src/types/topics.rs index 825b1088b2..901a193e32 100644 --- a/beacon_node/lighthouse_network/src/types/topics.rs +++ b/beacon_node/lighthouse_network/src/types/topics.rs @@ -11,6 +11,7 @@ use crate::Subnet; pub const TOPIC_PREFIX: &str = "eth2"; pub const SSZ_SNAPPY_ENCODING_POSTFIX: &str = "ssz_snappy"; pub const BEACON_BLOCK_TOPIC: &str = "beacon_block"; +pub const BLOBS_SIDECAR_TOPIC: &str = "blobs_sidecar"; pub const BEACON_AGGREGATE_AND_PROOF_TOPIC: &str = "beacon_aggregate_and_proof"; pub const BEACON_ATTESTATION_PREFIX: &str = "beacon_attestation_"; pub const VOLUNTARY_EXIT_TOPIC: &str = "voluntary_exit"; @@ -19,8 +20,9 @@ pub const ATTESTER_SLASHING_TOPIC: &str = "attester_slashing"; pub const SIGNED_CONTRIBUTION_AND_PROOF_TOPIC: &str = "sync_committee_contribution_and_proof"; pub const SYNC_COMMITTEE_PREFIX_TOPIC: &str = "sync_committee_"; -pub const CORE_TOPICS: [GossipKind; 6] = [ +pub const CORE_TOPICS: [GossipKind; 7] = [ GossipKind::BeaconBlock, + GossipKind::BlobsSidecar, GossipKind::BeaconAggregateAndProof, GossipKind::VoluntaryExit, GossipKind::ProposerSlashing, @@ -47,6 +49,8 @@ pub struct GossipTopic { pub enum GossipKind { /// Topic for publishing beacon blocks. BeaconBlock, + /// Topic for publishing blob sidecars. + BlobsSidecar, /// Topic for publishing aggregate attestations and proofs. BeaconAggregateAndProof, /// Topic for publishing raw attestations on a particular subnet. diff --git a/consensus/types/src/blob.rs b/consensus/types/src/blob.rs index efe243859d..f44fbdd26d 100644 --- a/consensus/types/src/blob.rs +++ b/consensus/types/src/blob.rs @@ -1,6 +1,7 @@ use ssz_types::VariableList; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use ssz::{Decode, DecodeError, Encode}; +use tree_hash::TreeHash; use crate::test_utils::RngCore; use crate::bls_field_element::BlsFieldElement; use crate::{EthSpec, Uint256}; @@ -20,4 +21,54 @@ impl TestRandom for Blob { } res } -} \ No newline at end of file +} + +impl Encode for Blob { + fn is_ssz_fixed_len() -> bool { + as Encode>::is_ssz_fixed_len() + } + + fn ssz_fixed_len() -> usize { + as Encode>::ssz_fixed_len() + } + + fn ssz_bytes_len(&self) -> usize { + self.0.ssz_bytes_len() + } + + fn ssz_append(&self, buf: &mut Vec) { + self.0.ssz_append(buf) + } +} + +impl Decode for Blob { + fn is_ssz_fixed_len() -> bool { + as Decode>::is_ssz_fixed_len() + } + + fn ssz_fixed_len() -> usize { + as Decode>::ssz_fixed_len() + } + + fn from_ssz_bytes(bytes: &[u8]) -> Result { + >::from_ssz_bytes(bytes).map(Self) + } +} + +impl TreeHash for Blob { + fn tree_hash_type() -> tree_hash::TreeHashType { + >::tree_hash_type() + } + + fn tree_hash_packed_encoding(&self) -> Vec { + self.0.tree_hash_packed_encoding() + } + + fn tree_hash_packing_factor() -> usize { + >::tree_hash_packing_factor() + } + + fn tree_hash_root(&self) -> tree_hash::Hash256 { + self.0.tree_hash_root() + } +} diff --git a/consensus/types/src/blobs_sidecar.rs b/consensus/types/src/blobs_sidecar.rs new file mode 100644 index 0000000000..4d39c1af89 --- /dev/null +++ b/consensus/types/src/blobs_sidecar.rs @@ -0,0 +1,16 @@ +use ssz_types::VariableList; +use crate::{EthSpec, Hash256, Slot}; +use crate::blob::Blob; +use crate::kzg_proof::KzgProof; +use serde::{Serialize, Deserialize}; +use ssz_derive::{Encode, Decode}; +use tree_hash_derive::TreeHash; +use derivative::Derivative; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, Derivative)] +pub struct BlobsSidecar { + beacon_block_root: Hash256, + beacon_block_slot: Slot, + blobs: VariableList, T::MaxBlobsPerBlock>, + kzg_aggregate_proof: KzgProof, +} \ No newline at end of file diff --git a/consensus/types/src/bls_field_element.rs b/consensus/types/src/bls_field_element.rs index 6693b5765b..5dea91e07b 100644 --- a/consensus/types/src/bls_field_element.rs +++ b/consensus/types/src/bls_field_element.rs @@ -1,7 +1,59 @@ -use crate::Uint256; +use crate::{EthSpec, Uint256}; use serde::{Deserialize, Serialize}; use ssz::{Decode, DecodeError, Encode}; +use tree_hash::TreeHash; #[derive(Default, Debug, PartialEq, Hash, Clone, Copy, Serialize, Deserialize)] #[serde(transparent)] -pub struct BlsFieldElement(pub Uint256); \ No newline at end of file +pub struct BlsFieldElement(pub Uint256); + + +impl Encode for BlsFieldElement { + fn is_ssz_fixed_len() -> bool { + ::is_ssz_fixed_len() + } + + fn ssz_fixed_len() -> usize { + ::ssz_fixed_len() + } + + fn ssz_bytes_len(&self) -> usize { + self.0.ssz_bytes_len() + } + + fn ssz_append(&self, buf: &mut Vec) { + self.0.ssz_append(buf) + } +} + +impl Decode for BlsFieldElement { + fn is_ssz_fixed_len() -> bool { + ::is_ssz_fixed_len() + } + + fn ssz_fixed_len() -> usize { + ::ssz_fixed_len() + } + + fn from_ssz_bytes(bytes: &[u8]) -> Result { + ::from_ssz_bytes(bytes).map(Self) + } +} + +impl TreeHash for BlsFieldElement { + fn tree_hash_type() -> tree_hash::TreeHashType { + ::tree_hash_type() + } + + fn tree_hash_packed_encoding(&self) -> Vec { + self.0.tree_hash_packed_encoding() + } + + fn tree_hash_packing_factor() -> usize { + ::tree_hash_packing_factor() + } + + fn tree_hash_root(&self) -> tree_hash::Hash256 { + self.0.tree_hash_root() + } +} diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 93c9b39afb..87f1b6f76b 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -96,6 +96,8 @@ pub mod kzg_commitment; pub mod kzg_proof; pub mod bls_field_element; pub mod blob; +pub mod signed_blobs_sidecar; +pub mod blobs_sidecar; use ethereum_types::{H160, H256}; diff --git a/consensus/types/src/signed_blobs_sidecar.rs b/consensus/types/src/signed_blobs_sidecar.rs new file mode 100644 index 0000000000..da9af06c76 --- /dev/null +++ b/consensus/types/src/signed_blobs_sidecar.rs @@ -0,0 +1,13 @@ +use bls::Signature; +use crate::blobs_sidecar::BlobsSidecar; +use crate::EthSpec; +use serde::{Serialize, Deserialize}; +use ssz_derive::{Encode, Decode}; +use tree_hash_derive::TreeHash; +use derivative::Derivative; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, Derivative)] +pub struct SignedBlobsSidecar { + pub message: BlobsSidecar, + pub signature: Signature, +} \ No newline at end of file From acace8ab311d23760a87d2bc6ac79d5b097906d9 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Sat, 17 Sep 2022 14:55:18 +0200 Subject: [PATCH 08/22] network: blobs by range message --- .../lighthouse_network/src/behaviour/mod.rs | 23 ++++++++++++++++++- .../lighthouse_network/src/rpc/methods.rs | 17 ++++++++++++++ .../lighthouse_network/src/rpc/outbound.rs | 1 + .../lighthouse_network/src/rpc/protocol.rs | 1 + .../lighthouse_network/src/types/topics.rs | 1 + consensus/types/src/eth_spec.rs | 6 ++++- 6 files changed, 47 insertions(+), 2 deletions(-) diff --git a/beacon_node/lighthouse_network/src/behaviour/mod.rs b/beacon_node/lighthouse_network/src/behaviour/mod.rs index 9c9e094db6..898fea4a72 100644 --- a/beacon_node/lighthouse_network/src/behaviour/mod.rs +++ b/beacon_node/lighthouse_network/src/behaviour/mod.rs @@ -10,6 +10,7 @@ use crate::peer_manager::{ ConnectionDirection, PeerManager, PeerManagerEvent, }; use crate::rpc::*; +use crate::rpc::methods::BlobsByRangeRequest; use crate::service::{Context as ServiceContext, METADATA_FILENAME}; use crate::types::{ subnet_from_topic_hash, GossipEncoding, GossipKind, GossipTopic, SnappyTransform, Subnet, @@ -39,6 +40,7 @@ use libp2p::{ }; use slog::{crit, debug, o, trace, warn}; use ssz::Encode; +use types::blobs_sidecar::BlobsSidecar; use std::collections::HashSet; use std::fs::File; use std::io::Write; @@ -51,7 +53,7 @@ use std::{ }; use types::{ consts::altair::SYNC_COMMITTEE_SUBNET_COUNT, EnrForkId, EthSpec, ForkContext, - SignedBeaconBlock, Slot, SubnetId, SyncSubnetId, + SignedBeaconBlock, Slot, SubnetId, SyncSubnetId, VariableList }; use self::gossip_cache::GossipCache; @@ -797,6 +799,9 @@ impl Behaviour { Request::BlocksByRoot { .. } => { metrics::inc_counter_vec(&metrics::TOTAL_RPC_REQUESTS, &["blocks_by_root"]) } + Request::BlobsByRange { .. } => { + metrics::inc_counter_vec(&metrics::TOTAL_RPC_REQUESTS, &["blobs_by_range"]) + } } self.add_event(BehaviourEvent::RequestReceived { peer_id, @@ -1095,6 +1100,9 @@ where InboundRequest::BlocksByRoot(req) => { self.propagate_request(peer_request_id, peer_id, Request::BlocksByRoot(req)) } + InboundRequest::BlobsByRange(req) => { + self.propagate_request(peer_request_id, peer_id, Request::BlobsByRange(req)) + } } } Ok(RPCReceived::Response(id, resp)) => { @@ -1117,12 +1125,16 @@ where RPCResponse::BlocksByRoot(resp) => { self.propagate_response(id, peer_id, Response::BlocksByRoot(Some(resp))) } + RPCResponse::BlobsByRange(resp) => { + self.propagate_response(id, peer_id, Response::BlobsByRange(Some(resp))) + } } } Ok(RPCReceived::EndOfStream(id, termination)) => { let response = match termination { ResponseTermination::BlocksByRange => Response::BlocksByRange(None), ResponseTermination::BlocksByRoot => Response::BlocksByRoot(None), + ResponseTermination::BlobsByRange => Response::BlobsByRange(None), }; self.propagate_response(id, peer_id, response); } @@ -1329,6 +1341,8 @@ pub enum Request { BlocksByRange(BlocksByRangeRequest), /// A request blocks root request. BlocksByRoot(BlocksByRootRequest), + /// A blobs by range request. + BlobsByRange(BlobsByRangeRequest), } impl std::convert::From for OutboundRequest { @@ -1342,6 +1356,7 @@ impl std::convert::From for OutboundRequest { step: 1, }) } + Request::BlobsByRange(r) => OutboundRequest::BlobsByRange(r), Request::Status(s) => OutboundRequest::Status(s), } } @@ -1361,6 +1376,8 @@ pub enum Response { BlocksByRange(Option>>), /// A response to a get BLOCKS_BY_ROOT request. BlocksByRoot(Option>>), + /// A response to a get BLOBS_BY_RANGE request + BlobsByRange(Option, TSpec::MaxRequestBlobsSidecars>>>) } impl std::convert::From> for RPCCodedResponse { @@ -1374,6 +1391,10 @@ impl std::convert::From> for RPCCodedResponse RPCCodedResponse::Success(RPCResponse::BlocksByRange(b)), None => RPCCodedResponse::StreamTermination(ResponseTermination::BlocksByRange), }, + Response::BlobsByRange(r) => match r { + Some(b) => RPCCodedResponse::Success(RPCResponse::BlobsByRange(b)), + None => RPCCodedResponse::StreamTermination(ResponseTermination::BlobsByRange), + } Response::Status(s) => RPCCodedResponse::Success(RPCResponse::Status(s)), } } diff --git a/beacon_node/lighthouse_network/src/rpc/methods.rs b/beacon_node/lighthouse_network/src/rpc/methods.rs index 26d755a6e0..a7bd51106e 100644 --- a/beacon_node/lighthouse_network/src/rpc/methods.rs +++ b/beacon_node/lighthouse_network/src/rpc/methods.rs @@ -13,6 +13,7 @@ use std::sync::Arc; use strum::IntoStaticStr; use superstruct::superstruct; use types::{Epoch, EthSpec, Hash256, SignedBeaconBlock, Slot}; +use types::blobs_sidecar::BlobsSidecar; /// Maximum number of blocks in a single request. pub type MaxRequestBlocks = U1024; @@ -204,6 +205,16 @@ pub struct BlocksByRangeRequest { pub count: u64, } +/// Request a number of beacon blobs from a peer. +#[derive(Encode, Decode, Clone, Debug, PartialEq)] +pub struct BlobsByRangeRequest { + /// The starting slot to request blobs. + pub start_slot: u64, + + /// The number of blobs from the start slot. + pub count: u64, +} + /// Request a number of beacon block roots from a peer. #[derive(Encode, Decode, Clone, Debug, PartialEq)] pub struct OldBlocksByRangeRequest { @@ -243,6 +254,9 @@ pub enum RPCResponse { /// A response to a get BLOCKS_BY_ROOT request. BlocksByRoot(Arc>), + /// A response to a get BLOBS_BY_RANGE request + BlobsByRange(Arc, T::MaxRequestBlobsSidecars>>), + /// A PONG response to a PING request. Pong(Ping), @@ -258,6 +272,9 @@ pub enum ResponseTermination { /// Blocks by root stream termination. BlocksByRoot, + + // Blobs by range stream termination. + BlobsByRange } /// The structured response containing a result/code indicating success or failure diff --git a/beacon_node/lighthouse_network/src/rpc/outbound.rs b/beacon_node/lighthouse_network/src/rpc/outbound.rs index 7d5acc4364..1c69201651 100644 --- a/beacon_node/lighthouse_network/src/rpc/outbound.rs +++ b/beacon_node/lighthouse_network/src/rpc/outbound.rs @@ -38,6 +38,7 @@ pub enum OutboundRequest { Goodbye(GoodbyeReason), BlocksByRange(OldBlocksByRangeRequest), BlocksByRoot(BlocksByRootRequest), + BlobsByRange(BlobsByRangeRequest), Ping(Ping), MetaData(PhantomData), } diff --git a/beacon_node/lighthouse_network/src/rpc/protocol.rs b/beacon_node/lighthouse_network/src/rpc/protocol.rs index 352348f74f..7a280ed5d8 100644 --- a/beacon_node/lighthouse_network/src/rpc/protocol.rs +++ b/beacon_node/lighthouse_network/src/rpc/protocol.rs @@ -427,6 +427,7 @@ pub enum InboundRequest { Goodbye(GoodbyeReason), BlocksByRange(OldBlocksByRangeRequest), BlocksByRoot(BlocksByRootRequest), + BlobsByRange(BlobsByRangeRequest), Ping(Ping), MetaData(PhantomData), } diff --git a/beacon_node/lighthouse_network/src/types/topics.rs b/beacon_node/lighthouse_network/src/types/topics.rs index 901a193e32..f119a5855c 100644 --- a/beacon_node/lighthouse_network/src/types/topics.rs +++ b/beacon_node/lighthouse_network/src/types/topics.rs @@ -210,6 +210,7 @@ impl std::fmt::Display for GossipTopic { let kind = match self.kind { GossipKind::BeaconBlock => BEACON_BLOCK_TOPIC.into(), + GossipKind::BlobsSidecar => BLOBS_SIDECAR_TOPIC.into(), GossipKind::BeaconAggregateAndProof => BEACON_AGGREGATE_AND_PROOF_TOPIC.into(), GossipKind::VoluntaryExit => VOLUNTARY_EXIT_TOPIC.into(), GossipKind::ProposerSlashing => PROPOSER_SLASHING_TOPIC.into(), diff --git a/consensus/types/src/eth_spec.rs b/consensus/types/src/eth_spec.rs index 9d8a765c20..9de4777f54 100644 --- a/consensus/types/src/eth_spec.rs +++ b/consensus/types/src/eth_spec.rs @@ -100,6 +100,7 @@ pub trait EthSpec: 'static + Default + Sync + Send + Clone + Debug + PartialEq + */ type MaxBlobsPerBlock: Unsigned + Clone + Sync + Send + Debug + PartialEq; type FieldElementsPerBlob: Unsigned + Clone + Sync + Send + Debug + PartialEq; + type MaxRequestBlobsSidecars: Unsigned + Clone + Sync + Send + Debug + PartialEq; /* * Derived values (set these CAREFULLY) */ @@ -277,6 +278,7 @@ impl EthSpec for MainnetEthSpec { type SlotsPerEth1VotingPeriod = U2048; // 64 epochs * 32 slots per epoch type MaxBlobsPerBlock = U16; type FieldElementsPerBlob = U4096; + type MaxRequestBlobsSidecars = U128; fn default_spec() -> ChainSpec { ChainSpec::mainnet() @@ -323,7 +325,8 @@ impl EthSpec for MinimalEthSpec { MinGasLimit, MaxExtraDataBytes, MaxBlobsPerBlock, - FieldElementsPerBlob + FieldElementsPerBlob, + MaxRequestBlobsSidecars }); fn default_spec() -> ChainSpec { @@ -370,6 +373,7 @@ impl EthSpec for GnosisEthSpec { type SlotsPerEth1VotingPeriod = U1024; // 64 epochs * 16 slots per epoch type MaxBlobsPerBlock = U16; type FieldElementsPerBlob = U4096; + type MaxRequestBlobsSidecars = U128; fn default_spec() -> ChainSpec { ChainSpec::gnosis() From 292a16a6ebcb215c8ac43915e7a7a30e9ffddcbc Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Sat, 17 Sep 2022 14:58:27 +0200 Subject: [PATCH 09/22] gossip boilerplate --- beacon_node/beacon_chain/src/beacon_chain.rs | 2 +- .../lighthouse_network/src/types/pubsub.rs | 13 +++++ .../lighthouse_network/src/types/topics.rs | 2 + .../network/src/beacon_processor/mod.rs | 56 +++++++++++++++++++ .../beacon_processor/worker/gossip_methods.rs | 14 +++++ beacon_node/network/src/router/mod.rs | 8 +++ beacon_node/network/src/router/processor.rs | 17 ++++++ consensus/types/src/blobs_sidecar.rs | 8 +-- 8 files changed, 115 insertions(+), 5 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 7418a66c17..3b3814f6c0 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -3628,7 +3628,7 @@ impl BeaconChain { .ok_or(BlockProductionError::MissingSyncAggregate)?, execution_payload: execution_payload .ok_or(BlockProductionError::MissingExecutionPayload)?, - blob_kzg_commitments: blob_kzg_commitments.into(), + blob_kzg_commitments: todo!(), }, }), }; diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 3519dafcfd..db9016b8b0 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -14,6 +14,7 @@ use types::{ SignedBeaconBlockMerge, SignedBeaconBlockEip4844, SignedContributionAndProof, SignedVoluntaryExit, SubnetId, SyncCommitteeMessage, SyncSubnetId, }; +use types::blobs_sidecar::BlobsSidecar; use types::signed_blobs_sidecar::SignedBlobsSidecar; #[derive(Debug, Clone, PartialEq)] @@ -184,6 +185,11 @@ impl PubsubMessage { }; Ok(PubsubMessage::BeaconBlock(Arc::new(beacon_block))) } + GossipKind::BlobsSidecar => { + let blobs_sidecar = SignedBlobsSidecar::from_ssz_bytes(data) + .map_err(|e| format!("{:?}", e))?; + Ok(PubsubMessage::BlobsSidecars(Arc::new(blobs_sidecar))) + } GossipKind::VoluntaryExit => { let voluntary_exit = SignedVoluntaryExit::from_ssz_bytes(data) .map_err(|e| format!("{:?}", e))?; @@ -228,6 +234,7 @@ impl PubsubMessage { // messages for us. match &self { PubsubMessage::BeaconBlock(data) => data.as_ssz_bytes(), + PubsubMessage::BlobsSidecars(data) => data.as_ssz_bytes(), PubsubMessage::AggregateAndProofAttestation(data) => data.as_ssz_bytes(), PubsubMessage::VoluntaryExit(data) => data.as_ssz_bytes(), PubsubMessage::ProposerSlashing(data) => data.as_ssz_bytes(), @@ -248,6 +255,12 @@ impl std::fmt::Display for PubsubMessage { block.slot(), block.message().proposer_index() ), + PubsubMessage::BlobsSidecars(blobs) => write!( + f, + "Blobs Sidecar: slot: {}, blobs: {}", + blobs.message.beacon_block_slot, + blobs.message.blobs.len(), + ), PubsubMessage::AggregateAndProofAttestation(att) => write!( f, "Aggregate and Proof: slot: {}, index: {}, aggregator_index: {}", diff --git a/beacon_node/lighthouse_network/src/types/topics.rs b/beacon_node/lighthouse_network/src/types/topics.rs index 901a193e32..3e77264809 100644 --- a/beacon_node/lighthouse_network/src/types/topics.rs +++ b/beacon_node/lighthouse_network/src/types/topics.rs @@ -182,6 +182,7 @@ impl From for String { let kind = match topic.kind { GossipKind::BeaconBlock => BEACON_BLOCK_TOPIC.into(), + GossipKind::BlobsSidecar => BLOBS_SIDECAR_TOPIC.into(), GossipKind::BeaconAggregateAndProof => BEACON_AGGREGATE_AND_PROOF_TOPIC.into(), GossipKind::VoluntaryExit => VOLUNTARY_EXIT_TOPIC.into(), GossipKind::ProposerSlashing => PROPOSER_SLASHING_TOPIC.into(), @@ -210,6 +211,7 @@ impl std::fmt::Display for GossipTopic { let kind = match self.kind { GossipKind::BeaconBlock => BEACON_BLOCK_TOPIC.into(), + GossipKind::BlobsSidecar => BLOBS_SIDECAR_TOPIC.into(), GossipKind::BeaconAggregateAndProof => BEACON_AGGREGATE_AND_PROOF_TOPIC.into(), GossipKind::VoluntaryExit => VOLUNTARY_EXIT_TOPIC.into(), GossipKind::ProposerSlashing => PROPOSER_SLASHING_TOPIC.into(), diff --git a/beacon_node/network/src/beacon_processor/mod.rs b/beacon_node/network/src/beacon_processor/mod.rs index e9a115904d..2a811b805d 100644 --- a/beacon_node/network/src/beacon_processor/mod.rs +++ b/beacon_node/network/src/beacon_processor/mod.rs @@ -65,6 +65,7 @@ use types::{ SignedBeaconBlock, SignedContributionAndProof, SignedVoluntaryExit, SubnetId, SyncCommitteeMessage, SyncSubnetId, }; +use types::signed_blobs_sidecar::SignedBlobsSidecar; use work_reprocessing_queue::{ spawn_reprocess_scheduler, QueuedAggregate, QueuedRpcBlock, QueuedUnaggregate, ReadyWork, }; @@ -184,6 +185,7 @@ pub const GOSSIP_ATTESTATION_BATCH: &str = "gossip_attestation_batch"; pub const GOSSIP_AGGREGATE: &str = "gossip_aggregate"; pub const GOSSIP_AGGREGATE_BATCH: &str = "gossip_aggregate_batch"; pub const GOSSIP_BLOCK: &str = "gossip_block"; +pub const GOSSIP_BLOBS_SIDECAR: &str = "gossip_blobs_sidecar"; pub const DELAYED_IMPORT_BLOCK: &str = "delayed_import_block"; pub const GOSSIP_VOLUNTARY_EXIT: &str = "gossip_voluntary_exit"; pub const GOSSIP_PROPOSER_SLASHING: &str = "gossip_proposer_slashing"; @@ -400,6 +402,26 @@ impl WorkEvent { } } + /// Create a new `Work` event for some blobs sidecar. + pub fn gossip_blobs_sidecar( + message_id: MessageId, + peer_id: PeerId, + peer_client: Client, + blobs: Arc>, + seen_timestamp: Duration, + ) -> Self { + Self { + drop_during_sync: false, + work: Work::GossipBlobsSidecar { + message_id, + peer_id, + peer_client, + blobs, + seen_timestamp, + }, + } + } + /// Create a new `Work` event for some sync committee signature. pub fn gossip_sync_signature( message_id: MessageId, @@ -671,6 +693,13 @@ pub enum Work { block: Arc>, seen_timestamp: Duration, }, + GossipBlobsSidecar { + message_id: MessageId, + peer_id: PeerId, + peer_client: Client, + blobs: Arc>, + seen_timestamp: Duration, + }, DelayedImportBlock { peer_id: PeerId, block: Box>, @@ -739,6 +768,7 @@ impl Work { Work::GossipAggregate { .. } => GOSSIP_AGGREGATE, Work::GossipAggregateBatch { .. } => GOSSIP_AGGREGATE_BATCH, Work::GossipBlock { .. } => GOSSIP_BLOCK, + Work::GossipBlobsSidecar { .. } => GOSSIP_BLOBS_SIDECAR, Work::DelayedImportBlock { .. } => DELAYED_IMPORT_BLOCK, Work::GossipVoluntaryExit { .. } => GOSSIP_VOLUNTARY_EXIT, Work::GossipProposerSlashing { .. } => GOSSIP_PROPOSER_SLASHING, @@ -888,6 +918,7 @@ impl BeaconProcessor { let mut chain_segment_queue = FifoQueue::new(MAX_CHAIN_SEGMENT_QUEUE_LEN); let mut backfill_chain_segment = FifoQueue::new(MAX_CHAIN_SEGMENT_QUEUE_LEN); let mut gossip_block_queue = FifoQueue::new(MAX_GOSSIP_BLOCK_QUEUE_LEN); + let mut gossip_blobs_sidecar_queue = FifoQueue::new(MAX_GOSSIP_BLOCK_QUEUE_LEN); let mut delayed_block_queue = FifoQueue::new(MAX_DELAYED_BLOCK_QUEUE_LEN); let mut status_queue = FifoQueue::new(MAX_STATUS_QUEUE_LEN); @@ -1199,6 +1230,9 @@ impl BeaconProcessor { Work::GossipBlock { .. } => { gossip_block_queue.push(work, work_id, &self.log) } + Work::GossipBlobsSidecar { .. } => { + gossip_blobs_sidecar_queue.push(work, work_id, &self.log) + } Work::DelayedImportBlock { .. } => { delayed_block_queue.push(work, work_id, &self.log) } @@ -1451,6 +1485,28 @@ impl BeaconProcessor { ) .await }), + /* + * Verification for blobs sidecars received on gossip. + */ + Work::GossipBlobsSidecar { + message_id, + peer_id, + peer_client, + blobs, + seen_timestamp, + } => task_spawner.spawn_async(async move { + worker + .process_gossip_blobs_sidecar( + message_id, + peer_id, + peer_client, + blobs, + work_reprocessing_tx, + duplicate_cache, + seen_timestamp, + ) + .await + }), /* * Import for blocks that we received earlier than their intended slot. */ diff --git a/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs b/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs index 93ed1b463b..0e1ab697e6 100644 --- a/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs @@ -22,6 +22,7 @@ use types::{ SignedAggregateAndProof, SignedBeaconBlock, SignedContributionAndProof, SignedVoluntaryExit, Slot, SubnetId, SyncCommitteeMessage, SyncSubnetId, }; +use types::signed_blobs_sidecar::SignedBlobsSidecar; use super::{ super::work_reprocessing_queue::{ @@ -987,6 +988,19 @@ impl Worker { }; } + pub async fn process_gossip_blobs_sidecar( + self, + message_id: MessageId, + peer_id: PeerId, + peer_client: Client, + blobs: Arc>, + reprocess_tx: mpsc::Sender>, + duplicate_cache: DuplicateCache, + seen_duration: Duration, + ) { + + } + pub fn process_gossip_voluntary_exit( self, message_id: MessageId, diff --git a/beacon_node/network/src/router/mod.rs b/beacon_node/network/src/router/mod.rs index 03b877506f..0c9b417795 100644 --- a/beacon_node/network/src/router/mod.rs +++ b/beacon_node/network/src/router/mod.rs @@ -229,6 +229,14 @@ impl Router { block, ); } + PubsubMessage::BlobsSidecars(blobs) => { + self.processor.on_blobs_gossip( + id, + peer_id, + self.network_globals.client(&peer_id), + blobs, + ); + } PubsubMessage::VoluntaryExit(exit) => { debug!(self.log, "Received a voluntary exit"; "peer_id" => %peer_id); self.processor.on_voluntary_exit_gossip(id, peer_id, exit); diff --git a/beacon_node/network/src/router/processor.rs b/beacon_node/network/src/router/processor.rs index ce11cbdcef..c716707f31 100644 --- a/beacon_node/network/src/router/processor.rs +++ b/beacon_node/network/src/router/processor.rs @@ -20,6 +20,7 @@ use types::{ Attestation, AttesterSlashing, EthSpec, ProposerSlashing, SignedAggregateAndProof, SignedBeaconBlock, SignedContributionAndProof, SignedVoluntaryExit, SubnetId, SyncSubnetId, }; +use types::signed_blobs_sidecar::SignedBlobsSidecar; /// Processes validated messages from the network. It relays necessary data to the syncing thread /// and processes blocks from the pubsub network. @@ -255,6 +256,22 @@ impl Processor { )) } + pub fn on_blobs_gossip( + &mut self, + message_id: MessageId, + peer_id: PeerId, + peer_client: Client, + blobs: Arc>, + ) { + self.send_beacon_processor_work(BeaconWorkEvent::gossip_blobs_sidecar( + message_id, + peer_id, + peer_client, + blobs, + timestamp_now(), + )) + } + pub fn on_unaggregated_attestation_gossip( &mut self, message_id: MessageId, diff --git a/consensus/types/src/blobs_sidecar.rs b/consensus/types/src/blobs_sidecar.rs index 4d39c1af89..445c731cf8 100644 --- a/consensus/types/src/blobs_sidecar.rs +++ b/consensus/types/src/blobs_sidecar.rs @@ -9,8 +9,8 @@ use derivative::Derivative; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, Derivative)] pub struct BlobsSidecar { - beacon_block_root: Hash256, - beacon_block_slot: Slot, - blobs: VariableList, T::MaxBlobsPerBlock>, - kzg_aggregate_proof: KzgProof, + pub beacon_block_root: Hash256, + pub beacon_block_slot: Slot, + pub blobs: VariableList, T::MaxBlobsPerBlock>, + pub kzg_aggregate_proof: KzgProof, } \ No newline at end of file From 36a0add0cd8d2bc2b22f8b3603d3427b0d02d78e Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Sat, 17 Sep 2022 15:23:28 +0200 Subject: [PATCH 10/22] network stuff --- .../src/peer_manager/mod.rs | 3 +++ .../src/rpc/codec/ssz_snappy.rs | 23 +++++++++++++++++++ beacon_node/lighthouse_network/src/rpc/mod.rs | 1 + .../lighthouse_network/src/rpc/protocol.rs | 6 +++++ .../src/rpc/rate_limiter.rs | 11 +++++++++ 5 files changed, 44 insertions(+) diff --git a/beacon_node/lighthouse_network/src/peer_manager/mod.rs b/beacon_node/lighthouse_network/src/peer_manager/mod.rs index 55b3884454..05acef663d 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/mod.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/mod.rs @@ -497,6 +497,7 @@ impl PeerManager { Protocol::Ping => PeerAction::MidToleranceError, Protocol::BlocksByRange => PeerAction::MidToleranceError, Protocol::BlocksByRoot => PeerAction::MidToleranceError, + Protocol::BlobsByRange => PeerAction::MidToleranceError, Protocol::Goodbye => PeerAction::LowToleranceError, Protocol::MetaData => PeerAction::LowToleranceError, Protocol::Status => PeerAction::LowToleranceError, @@ -512,6 +513,7 @@ impl PeerManager { Protocol::Ping => PeerAction::Fatal, Protocol::BlocksByRange => return, Protocol::BlocksByRoot => return, + Protocol::BlobsByRange => return, Protocol::Goodbye => return, Protocol::MetaData => PeerAction::LowToleranceError, Protocol::Status => PeerAction::LowToleranceError, @@ -527,6 +529,7 @@ impl PeerManager { Protocol::Ping => PeerAction::LowToleranceError, Protocol::BlocksByRange => PeerAction::MidToleranceError, Protocol::BlocksByRoot => PeerAction::MidToleranceError, + Protocol::BlobsByRange => PeerAction::MidToleranceError, Protocol::Goodbye => return, Protocol::MetaData => return, Protocol::Status => return, diff --git a/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs b/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs index e952eece30..f19c7b5a4f 100644 --- a/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs +++ b/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs @@ -228,6 +228,7 @@ impl Encoder> for SSZSnappyOutboundCodec< OutboundRequest::Goodbye(req) => req.as_ssz_bytes(), OutboundRequest::BlocksByRange(req) => req.as_ssz_bytes(), OutboundRequest::BlocksByRoot(req) => req.block_roots.as_ssz_bytes(), + OutboundRequest::BlobsByRange(req) => req.as_ssz_bytes(), OutboundRequest::Ping(req) => req.as_ssz_bytes(), OutboundRequest::MetaData(_) => return Ok(()), // no metadata to encode }; @@ -473,6 +474,9 @@ fn handle_v1_request( Protocol::BlocksByRoot => Ok(Some(InboundRequest::BlocksByRoot(BlocksByRootRequest { block_roots: VariableList::from_ssz_bytes(decoded_buffer)?, }))), + Protocol::BlobsByRange => Ok(Some(InboundRequest::BlobsByRange( + BlobsByRangeRequest::from_ssz_bytes(decoded_buffer)?, + ))), Protocol::Ping => Ok(Some(InboundRequest::Ping(Ping { data: u64::from_ssz_bytes(decoded_buffer)?, }))), @@ -505,6 +509,9 @@ fn handle_v2_request( Protocol::BlocksByRoot => Ok(Some(InboundRequest::BlocksByRoot(BlocksByRootRequest { block_roots: VariableList::from_ssz_bytes(decoded_buffer)?, }))), + Protocol::BlobsByRange => Ok(Some(InboundRequest::BlobsByRange( + BlobsByRangeRequest::from_ssz_bytes(decoded_buffer)?, + ))), // MetaData requests return early from InboundUpgrade and do not reach the decoder. // Handle this case just for completeness. Protocol::MetaData => { @@ -542,6 +549,9 @@ fn handle_v1_response( Protocol::BlocksByRoot => Ok(Some(RPCResponse::BlocksByRoot(Arc::new( SignedBeaconBlock::Base(SignedBeaconBlockBase::from_ssz_bytes(decoded_buffer)?), )))), + Protocol::BlobsByRange => Err(RPCError::InvalidData( + "blobs by range via v1".to_string(), + )), Protocol::Ping => Ok(Some(RPCResponse::Pong(Ping { data: u64::from_ssz_bytes(decoded_buffer)?, }))), @@ -616,6 +626,15 @@ fn handle_v2_response( )?), )))), }, + Protocol::BlobsByRange => match fork_name { + ForkName::Eip4844 => Ok(Some(RPCResponse::BlobsByRange(Arc::new( + VariableList::from_ssz_bytes(decoded_buffer)?, + )))), + _ => Err(RPCError::ErrorResponse( + RPCResponseErrorCode::InvalidRequest, + "Invalid forkname for blobsbyrange".to_string(), + )), + } _ => Err(RPCError::ErrorResponse( RPCResponseErrorCode::InvalidRequest, "Invalid v2 request".to_string(), @@ -672,6 +691,7 @@ mod tests { ForkName::Base => Slot::new(0), ForkName::Altair => altair_fork_epoch.start_slot(Spec::slots_per_epoch()), ForkName::Merge => merge_fork_epoch.start_slot(Spec::slots_per_epoch()), + ForkName::Eip4844 => merge_fork_epoch.start_slot(Spec::slots_per_epoch()), }; ForkContext::new::(current_slot, Hash256::zero(), &chain_spec) } @@ -875,6 +895,9 @@ mod tests { OutboundRequest::BlocksByRoot(bbroot) => { assert_eq!(decoded, InboundRequest::BlocksByRoot(bbroot)) } + OutboundRequest::BlobsByRange(blbrange) => { + assert_eq!(decoded, InboundRequest::BlobsByRange(blbrange)) + } OutboundRequest::Ping(ping) => { assert_eq!(decoded, InboundRequest::Ping(ping)) } diff --git a/beacon_node/lighthouse_network/src/rpc/mod.rs b/beacon_node/lighthouse_network/src/rpc/mod.rs index 0bedd423b2..6197d57f19 100644 --- a/beacon_node/lighthouse_network/src/rpc/mod.rs +++ b/beacon_node/lighthouse_network/src/rpc/mod.rs @@ -292,6 +292,7 @@ where match end { ResponseTermination::BlocksByRange => Protocol::BlocksByRange, ResponseTermination::BlocksByRoot => Protocol::BlocksByRoot, + ResponseTermination::BlobsByRange => Protocol::BlobsByRange, }, ), }, diff --git a/beacon_node/lighthouse_network/src/rpc/protocol.rs b/beacon_node/lighthouse_network/src/rpc/protocol.rs index 7a280ed5d8..4de2948ff7 100644 --- a/beacon_node/lighthouse_network/src/rpc/protocol.rs +++ b/beacon_node/lighthouse_network/src/rpc/protocol.rs @@ -159,6 +159,8 @@ pub enum Protocol { BlocksByRange, /// The `BlocksByRoot` protocol name. BlocksByRoot, + /// The `BlobsByRange` protocol name. + BlobsByRange, /// The `Ping` protocol name. Ping, /// The `MetaData` protocol name. @@ -488,6 +490,7 @@ impl InboundRequest { InboundRequest::Goodbye(_) => 0, InboundRequest::BlocksByRange(req) => req.count, InboundRequest::BlocksByRoot(req) => req.block_roots.len() as u64, + InboundRequest::BlobsByRange(req) => req.count, InboundRequest::Ping(_) => 1, InboundRequest::MetaData(_) => 1, } @@ -500,6 +503,7 @@ impl InboundRequest { InboundRequest::Goodbye(_) => Protocol::Goodbye, InboundRequest::BlocksByRange(_) => Protocol::BlocksByRange, InboundRequest::BlocksByRoot(_) => Protocol::BlocksByRoot, + InboundRequest::BlobsByRange(_) => Protocol::BlobsByRange, InboundRequest::Ping(_) => Protocol::Ping, InboundRequest::MetaData(_) => Protocol::MetaData, } @@ -513,6 +517,7 @@ impl InboundRequest { // variants that have `multiple_responses()` can have values. InboundRequest::BlocksByRange(_) => ResponseTermination::BlocksByRange, InboundRequest::BlocksByRoot(_) => ResponseTermination::BlocksByRoot, + InboundRequest::BlobsByRange(_) => ResponseTermination::BlobsByRange, InboundRequest::Status(_) => unreachable!(), InboundRequest::Goodbye(_) => unreachable!(), InboundRequest::Ping(_) => unreachable!(), @@ -618,6 +623,7 @@ impl std::fmt::Display for InboundRequest { InboundRequest::Goodbye(reason) => write!(f, "Goodbye: {}", reason), InboundRequest::BlocksByRange(req) => write!(f, "Blocks by range: {}", req), InboundRequest::BlocksByRoot(req) => write!(f, "Blocks by root: {:?}", req), + InboundRequest::BlobsByRange(req) => write!(f, "Blobs by range: {:?}", req), InboundRequest::Ping(ping) => write!(f, "Ping: {}", ping.data), InboundRequest::MetaData(_) => write!(f, "MetaData request"), } diff --git a/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs b/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs index 70b14c33de..8cd1e749e3 100644 --- a/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs +++ b/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs @@ -73,6 +73,8 @@ pub struct RPCRateLimiter { bbrange_rl: Limiter, /// BlocksByRoot rate limiter. bbroots_rl: Limiter, + /// BlobsByRange rate limiter. + blbrange_rl: Limiter, } /// Error type for non conformant requests @@ -98,6 +100,8 @@ pub struct RPCRateLimiterBuilder { bbrange_quota: Option, /// Quota for the BlocksByRoot protocol. bbroots_quota: Option, + /// Quota for the BlocksByRange protocol. + blbrange_quota: Option, } impl RPCRateLimiterBuilder { @@ -116,6 +120,7 @@ impl RPCRateLimiterBuilder { Protocol::Goodbye => self.goodbye_quota = q, Protocol::BlocksByRange => self.bbrange_quota = q, Protocol::BlocksByRoot => self.bbroots_quota = q, + Protocol::BlobsByRange => self.blbrange_quota = q, } self } @@ -156,6 +161,8 @@ impl RPCRateLimiterBuilder { .bbrange_quota .ok_or("BlocksByRange quota not specified")?; + let blbrange_quota = self.blbrange_quota.ok_or("BlobsByRange quota not specified")?; + // create the rate limiters let ping_rl = Limiter::from_quota(ping_quota)?; let metadata_rl = Limiter::from_quota(metadata_quota)?; @@ -163,6 +170,7 @@ impl RPCRateLimiterBuilder { let goodbye_rl = Limiter::from_quota(goodbye_quota)?; let bbroots_rl = Limiter::from_quota(bbroots_quota)?; let bbrange_rl = Limiter::from_quota(bbrange_quota)?; + let blbrange_rl = Limiter::from_quota(blbrange_quota)?; // check for peers to prune every 30 seconds, starting in 30 seconds let prune_every = tokio::time::Duration::from_secs(30); @@ -176,6 +184,7 @@ impl RPCRateLimiterBuilder { goodbye_rl, bbroots_rl, bbrange_rl, + blbrange_rl, init_time: Instant::now(), }) } @@ -199,6 +208,7 @@ impl RPCRateLimiter { Protocol::Goodbye => &mut self.goodbye_rl, Protocol::BlocksByRange => &mut self.bbrange_rl, Protocol::BlocksByRoot => &mut self.bbroots_rl, + Protocol::BlobsByRange => &mut self.blbrange_rl, }; check(limiter) } @@ -211,6 +221,7 @@ impl RPCRateLimiter { self.goodbye_rl.prune(time_since_start); self.bbrange_rl.prune(time_since_start); self.bbroots_rl.prune(time_since_start); + self.blbrange_rl.prune(time_since_start); } } From d4d40be8706ca89ba2bde24ab2d19335865b3a5e Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Sat, 17 Sep 2022 15:58:52 +0200 Subject: [PATCH 11/22] storable blobs --- beacon_node/store/src/config.rs | 4 ++++ beacon_node/store/src/hot_cold_store.rs | 30 +++++++++++++++++++++++++ beacon_node/store/src/lib.rs | 2 ++ 3 files changed, 36 insertions(+) diff --git a/beacon_node/store/src/config.rs b/beacon_node/store/src/config.rs index 4268ec2e91..debffdca5e 100644 --- a/beacon_node/store/src/config.rs +++ b/beacon_node/store/src/config.rs @@ -7,6 +7,7 @@ use types::{EthSpec, MinimalEthSpec}; pub const PREV_DEFAULT_SLOTS_PER_RESTORE_POINT: u64 = 2048; pub const DEFAULT_SLOTS_PER_RESTORE_POINT: u64 = 8192; pub const DEFAULT_BLOCK_CACHE_SIZE: usize = 5; +pub const DEFAULT_BLOB_CACHE_SIZE: usize = 5; /// Database configuration parameters. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -17,6 +18,8 @@ pub struct StoreConfig { pub slots_per_restore_point_set_explicitly: bool, /// Maximum number of blocks to store in the in-memory block cache. pub block_cache_size: usize, + /// Maximum number of blobs to store in the in-memory block cache. + pub blob_cache_size: usize, /// Whether to compact the database on initialization. pub compact_on_init: bool, /// Whether to compact the database during database pruning. @@ -41,6 +44,7 @@ impl Default for StoreConfig { slots_per_restore_point: MinimalEthSpec::slots_per_historical_root() as u64, slots_per_restore_point_set_explicitly: false, block_cache_size: DEFAULT_BLOCK_CACHE_SIZE, + blob_cache_size: DEFAULT_BLOB_CACHE_SIZE, compact_on_init: false, compact_on_prune: true, } diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index c4b4a64a05..8240007b66 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -38,6 +38,7 @@ use std::path::Path; use std::sync::Arc; use std::time::Duration; use types::*; +use types::signed_blobs_sidecar::SignedBlobsSidecar; /// On-disk database that stores finalized states efficiently. /// @@ -59,6 +60,8 @@ pub struct HotColdDB, Cold: ItemStore> { /// /// The hot database also contains all blocks. pub hot_db: Hot, + /// LRU cache of deserialized blobs. Updated whenever a blob is loaded. + blob_cache: Mutex>>, /// LRU cache of deserialized blocks. Updated whenever a block is loaded. block_cache: Mutex>>, /// Chain spec. @@ -128,6 +131,7 @@ impl HotColdDB, MemoryStore> { cold_db: MemoryStore::open(), hot_db: MemoryStore::open(), block_cache: Mutex::new(LruCache::new(config.block_cache_size)), + blob_cache: Mutex::new(LruCache::new(config.blob_cache_size)), config, spec, log, @@ -161,6 +165,7 @@ impl HotColdDB, LevelDB> { cold_db: LevelDB::open(cold_path)?, hot_db: LevelDB::open(hot_path)?, block_cache: Mutex::new(LruCache::new(config.block_cache_size)), + blob_cache: Mutex::new(LruCache::new(config.blob_cache_size)), config, spec, log, @@ -453,6 +458,31 @@ impl, Cold: ItemStore> HotColdDB .key_delete(DBColumn::ExecPayload.into(), block_root.as_bytes()) } + pub fn put_blobs(&self, + block_root: &Hash256, + blobs: SignedBlobsSidecar, + ) -> Result<(), Error> { + self.hot_db.put_bytes(DBColumn::BeaconBlob.into(), block_root.as_bytes(), &blobs.as_ssz_bytes())?; + self.blob_cache.lock().push(*block_root, blobs); + Ok(()) + } + + pub fn get_blobs(&self, + block_root: &Hash256, + ) -> Result>, Error> { + if let Some(blobs) = self.blob_cache.lock().get(block_root) { + Ok(Some(blobs.clone())) + } else { + if let Some(bytes) = self.hot_db.get_bytes(DBColumn::BeaconBlob.into(), block_root.as_bytes())? { + let ret = SignedBlobsSidecar::from_ssz_bytes(&bytes)?; + self.blob_cache.lock().put(*block_root, ret.clone()); + Ok(Some(ret)) + } else { + Ok(None) + } + } + } + pub fn put_state_summary( &self, state_root: &Hash256, diff --git a/beacon_node/store/src/lib.rs b/beacon_node/store/src/lib.rs index 75aeca058b..f7af172f5a 100644 --- a/beacon_node/store/src/lib.rs +++ b/beacon_node/store/src/lib.rs @@ -171,6 +171,8 @@ pub enum DBColumn { BeaconMeta, #[strum(serialize = "blk")] BeaconBlock, + #[strum(serialize = "blo")] + BeaconBlob, /// For full `BeaconState`s in the hot database (finalized or fork-boundary states). #[strum(serialize = "ste")] BeaconState, From aeb52ff18639b1c8980ac1fc3be997d2b3dd7b7d Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Sat, 17 Sep 2022 16:10:09 +0200 Subject: [PATCH 12/22] network stuff --- .../src/rpc/codec/ssz_snappy.rs | 1 + .../lighthouse_network/src/rpc/methods.rs | 5 ++++ .../lighthouse_network/src/rpc/outbound.rs | 7 ++++++ .../lighthouse_network/src/rpc/protocol.rs | 8 +++++++ beacon_node/network/src/router/mod.rs | 7 ++++++ beacon_node/network/src/router/processor.rs | 24 ++++++++++++++++++- 6 files changed, 51 insertions(+), 1 deletion(-) diff --git a/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs b/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs index f19c7b5a4f..9bcca2e6ba 100644 --- a/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs +++ b/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs @@ -70,6 +70,7 @@ impl Encoder> for SSZSnappyInboundCodec< RPCResponse::Status(res) => res.as_ssz_bytes(), RPCResponse::BlocksByRange(res) => res.as_ssz_bytes(), RPCResponse::BlocksByRoot(res) => res.as_ssz_bytes(), + RPCResponse::BlobsByRange(res) => res.as_ssz_bytes(), RPCResponse::Pong(res) => res.data.as_ssz_bytes(), RPCResponse::MetaData(res) => // Encode the correct version of the MetaData response based on the negotiated version. diff --git a/beacon_node/lighthouse_network/src/rpc/methods.rs b/beacon_node/lighthouse_network/src/rpc/methods.rs index a7bd51106e..ad67c29bab 100644 --- a/beacon_node/lighthouse_network/src/rpc/methods.rs +++ b/beacon_node/lighthouse_network/src/rpc/methods.rs @@ -336,6 +336,7 @@ impl RPCCodedResponse { RPCResponse::Status(_) => false, RPCResponse::BlocksByRange(_) => true, RPCResponse::BlocksByRoot(_) => true, + RPCResponse::BlobsByRange(_) => true, RPCResponse::Pong(_) => false, RPCResponse::MetaData(_) => false, }, @@ -370,6 +371,7 @@ impl RPCResponse { RPCResponse::Status(_) => Protocol::Status, RPCResponse::BlocksByRange(_) => Protocol::BlocksByRange, RPCResponse::BlocksByRoot(_) => Protocol::BlocksByRoot, + RPCResponse::BlobsByRange(_) => Protocol::BlobsByRange, RPCResponse::Pong(_) => Protocol::Ping, RPCResponse::MetaData(_) => Protocol::MetaData, } @@ -404,6 +406,9 @@ impl std::fmt::Display for RPCResponse { } RPCResponse::BlocksByRoot(block) => { write!(f, "BlocksByRoot: Block slot: {}", block.slot()) + } + RPCResponse::BlobsByRange(blob) => { + write!(f, "BlobsByRange: Blob slot: {}", blob.len()) } RPCResponse::Pong(ping) => write!(f, "Pong: {}", ping.data), RPCResponse::MetaData(metadata) => write!(f, "Metadata: {}", metadata.seq_number()), diff --git a/beacon_node/lighthouse_network/src/rpc/outbound.rs b/beacon_node/lighthouse_network/src/rpc/outbound.rs index 1c69201651..4f5b2a5782 100644 --- a/beacon_node/lighthouse_network/src/rpc/outbound.rs +++ b/beacon_node/lighthouse_network/src/rpc/outbound.rs @@ -76,6 +76,9 @@ impl OutboundRequest { ProtocolId::new(Protocol::BlocksByRoot, Version::V2, Encoding::SSZSnappy), ProtocolId::new(Protocol::BlocksByRoot, Version::V1, Encoding::SSZSnappy), ], + OutboundRequest::BlobsByRange(_) => vec![ + ProtocolId::new(Protocol::BlocksByRoot, Version::V1, Encoding::SSZSnappy), + ], OutboundRequest::Ping(_) => vec![ProtocolId::new( Protocol::Ping, Version::V1, @@ -97,6 +100,7 @@ impl OutboundRequest { OutboundRequest::Goodbye(_) => 0, OutboundRequest::BlocksByRange(req) => req.count, OutboundRequest::BlocksByRoot(req) => req.block_roots.len() as u64, + OutboundRequest::BlobsByRange(req) => req.count, OutboundRequest::Ping(_) => 1, OutboundRequest::MetaData(_) => 1, } @@ -109,6 +113,7 @@ impl OutboundRequest { OutboundRequest::Goodbye(_) => Protocol::Goodbye, OutboundRequest::BlocksByRange(_) => Protocol::BlocksByRange, OutboundRequest::BlocksByRoot(_) => Protocol::BlocksByRoot, + OutboundRequest::BlobsByRange(_) => Protocol::BlobsByRange, OutboundRequest::Ping(_) => Protocol::Ping, OutboundRequest::MetaData(_) => Protocol::MetaData, } @@ -122,6 +127,7 @@ impl OutboundRequest { // variants that have `multiple_responses()` can have values. OutboundRequest::BlocksByRange(_) => ResponseTermination::BlocksByRange, OutboundRequest::BlocksByRoot(_) => ResponseTermination::BlocksByRoot, + OutboundRequest::BlobsByRange(_) => ResponseTermination::BlobsByRange, OutboundRequest::Status(_) => unreachable!(), OutboundRequest::Goodbye(_) => unreachable!(), OutboundRequest::Ping(_) => unreachable!(), @@ -177,6 +183,7 @@ impl std::fmt::Display for OutboundRequest { OutboundRequest::Goodbye(reason) => write!(f, "Goodbye: {}", reason), OutboundRequest::BlocksByRange(req) => write!(f, "Blocks by range: {}", req), OutboundRequest::BlocksByRoot(req) => write!(f, "Blocks by root: {:?}", req), + OutboundRequest::BlobsByRange(req) => write!(f, "Blobs by range: {:?}", req), OutboundRequest::Ping(ping) => write!(f, "Ping: {}", ping.data), OutboundRequest::MetaData(_) => write!(f, "MetaData request"), } diff --git a/beacon_node/lighthouse_network/src/rpc/protocol.rs b/beacon_node/lighthouse_network/src/rpc/protocol.rs index 4de2948ff7..203f5d0b5e 100644 --- a/beacon_node/lighthouse_network/src/rpc/protocol.rs +++ b/beacon_node/lighthouse_network/src/rpc/protocol.rs @@ -189,6 +189,7 @@ impl std::fmt::Display for Protocol { Protocol::Goodbye => "goodbye", Protocol::BlocksByRange => "beacon_blocks_by_range", Protocol::BlocksByRoot => "beacon_blocks_by_root", + Protocol::BlobsByRange => "blobs_sidecars_by_range", Protocol::Ping => "ping", Protocol::MetaData => "metadata", }; @@ -297,6 +298,9 @@ impl ProtocolId { Protocol::BlocksByRoot => { RpcLimits::new(*BLOCKS_BY_ROOT_REQUEST_MIN, *BLOCKS_BY_ROOT_REQUEST_MAX) } + Protocol::BlobsByRange => { + RpcLimits::new(*BLOCKS_BY_ROOT_REQUEST_MIN, *BLOCKS_BY_ROOT_REQUEST_MAX) + } Protocol::Ping => RpcLimits::new( ::ssz_fixed_len(), ::ssz_fixed_len(), @@ -315,6 +319,7 @@ impl ProtocolId { Protocol::Goodbye => RpcLimits::new(0, 0), // Goodbye request has no response Protocol::BlocksByRange => rpc_block_limits_by_fork(fork_context.current_fork()), Protocol::BlocksByRoot => rpc_block_limits_by_fork(fork_context.current_fork()), + Protocol::BlobsByRange => rpc_block_limits_by_fork(fork_context.current_fork()), Protocol::Ping => RpcLimits::new( ::ssz_fixed_len(), @@ -469,6 +474,9 @@ impl InboundRequest { ProtocolId::new(Protocol::BlocksByRoot, Version::V2, Encoding::SSZSnappy), ProtocolId::new(Protocol::BlocksByRoot, Version::V1, Encoding::SSZSnappy), ], + InboundRequest::BlobsByRange(_) => vec![ + ProtocolId::new(Protocol::BlocksByRoot, Version::V1, Encoding::SSZSnappy), + ], InboundRequest::Ping(_) => vec![ProtocolId::new( Protocol::Ping, Version::V1, diff --git a/beacon_node/network/src/router/mod.rs b/beacon_node/network/src/router/mod.rs index 0c9b417795..24a202c497 100644 --- a/beacon_node/network/src/router/mod.rs +++ b/beacon_node/network/src/router/mod.rs @@ -168,6 +168,9 @@ impl Router { Request::BlocksByRoot(request) => self .processor .on_blocks_by_root_request(peer_id, id, request), + Request::BlobsByRange(request) => self + .processor + .on_blobs_by_range_request(peer_id, id, request), } } @@ -192,6 +195,10 @@ impl Router { self.processor .on_blocks_by_root_response(peer_id, request_id, beacon_block); } + Response::BlobsByRange(beacon_blob) => { + self.processor + .on_blobs_by_range_response(peer_id, request_id, beacon_blob); + } } } diff --git a/beacon_node/network/src/router/processor.rs b/beacon_node/network/src/router/processor.rs index c716707f31..2d12cf5401 100644 --- a/beacon_node/network/src/router/processor.rs +++ b/beacon_node/network/src/router/processor.rs @@ -7,6 +7,7 @@ use crate::sync::manager::RequestId as SyncId; use crate::sync::SyncMessage; use beacon_chain::{BeaconChain, BeaconChainTypes}; use lighthouse_network::rpc::*; +use lighthouse_network::rpc::methods::BlobsByRangeRequest; use lighthouse_network::{ Client, MessageId, NetworkGlobals, PeerId, PeerRequestId, Request, Response, }; @@ -18,7 +19,7 @@ use store::SyncCommitteeMessage; use tokio::sync::mpsc; use types::{ Attestation, AttesterSlashing, EthSpec, ProposerSlashing, SignedAggregateAndProof, - SignedBeaconBlock, SignedContributionAndProof, SignedVoluntaryExit, SubnetId, SyncSubnetId, + SignedBeaconBlock, SignedContributionAndProof, SignedVoluntaryExit, SubnetId, SyncSubnetId, VariableList, blobs_sidecar::BlobsSidecar, }; use types::signed_blobs_sidecar::SignedBlobsSidecar; @@ -161,6 +162,18 @@ impl Processor { )) } + pub fn on_blobs_by_range_request( + &mut self, + peer_id: PeerId, + request_id: PeerRequestId, + request: BlobsByRangeRequest, + ) { + /* + self.send_beacon_processor_work(BeaconWorkEvent::blocks_by_roots_request( + peer_id, request_id, request, + )) + */ + } /// Handle a `BlocksByRange` request from the peer. pub fn on_blocks_by_range_request( &mut self, @@ -235,6 +248,15 @@ impl Processor { }); } + pub fn on_blobs_by_range_response( + &mut self, + peer_id: PeerId, + request_id: RequestId, + beacon_blob: Option, <::EthSpec as EthSpec>::MaxRequestBlobsSidecars>>>, + ) { + + } + /// Process a gossip message declaring a new block. /// /// Attempts to apply to block to the beacon chain. May queue the block for later processing. From f9209e2d0854049cefc216b465aa2bfd785e4646 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Sat, 17 Sep 2022 16:39:40 +0200 Subject: [PATCH 13/22] more network stuff --- .../network/src/beacon_processor/mod.rs | 42 +++++++++++++++++++ .../beacon_processor/worker/rpc_methods.rs | 13 ++++++ beacon_node/network/src/router/processor.rs | 4 +- 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/beacon_node/network/src/beacon_processor/mod.rs b/beacon_node/network/src/beacon_processor/mod.rs index 2a811b805d..450efc886b 100644 --- a/beacon_node/network/src/beacon_processor/mod.rs +++ b/beacon_node/network/src/beacon_processor/mod.rs @@ -45,6 +45,7 @@ use beacon_chain::{BeaconChain, BeaconChainTypes, GossipVerifiedBlock}; use derivative::Derivative; use futures::stream::{Stream, StreamExt}; use futures::task::Poll; +use lighthouse_network::rpc::methods::BlobsByRangeRequest; use lighthouse_network::{ rpc::{BlocksByRangeRequest, BlocksByRootRequest, StatusMessage}, Client, MessageId, NetworkGlobals, PeerId, PeerRequestId, @@ -197,6 +198,7 @@ pub const CHAIN_SEGMENT: &str = "chain_segment"; pub const STATUS_PROCESSING: &str = "status_processing"; pub const BLOCKS_BY_RANGE_REQUEST: &str = "blocks_by_range_request"; pub const BLOCKS_BY_ROOTS_REQUEST: &str = "blocks_by_roots_request"; +pub const BLOBS_BY_RANGE_REQUEST: &str = "blobs_by_range_request"; pub const UNKNOWN_BLOCK_ATTESTATION: &str = "unknown_block_attestation"; pub const UNKNOWN_BLOCK_AGGREGATE: &str = "unknown_block_aggregate"; @@ -577,6 +579,21 @@ impl WorkEvent { } } + pub fn blobs_by_range_request( + peer_id: PeerId, + request_id: PeerRequestId, + request: BlobsByRangeRequest, + ) -> Self { + Self { + drop_during_sync: false, + work: Work::BlobsByRangeRequest { + peer_id, + request_id, + request, + }, + } + } + /// Get a `str` representation of the type of work this `WorkEvent` contains. pub fn work_type(&self) -> &'static str { self.work.str_id() @@ -757,6 +774,11 @@ pub enum Work { request_id: PeerRequestId, request: BlocksByRootRequest, }, + BlobsByRangeRequest { + peer_id: PeerId, + request_id: PeerRequestId, + request: BlobsByRangeRequest, + } } impl Work { @@ -780,6 +802,7 @@ impl Work { Work::Status { .. } => STATUS_PROCESSING, Work::BlocksByRangeRequest { .. } => BLOCKS_BY_RANGE_REQUEST, Work::BlocksByRootsRequest { .. } => BLOCKS_BY_ROOTS_REQUEST, + Work::BlobsByRangeRequest {..} => BLOBS_BY_RANGE_REQUEST, Work::UnknownBlockAttestation { .. } => UNKNOWN_BLOCK_ATTESTATION, Work::UnknownBlockAggregate { .. } => UNKNOWN_BLOCK_AGGREGATE, } @@ -924,6 +947,7 @@ impl BeaconProcessor { let mut status_queue = FifoQueue::new(MAX_STATUS_QUEUE_LEN); let mut bbrange_queue = FifoQueue::new(MAX_BLOCKS_BY_RANGE_QUEUE_LEN); let mut bbroots_queue = FifoQueue::new(MAX_BLOCKS_BY_ROOTS_QUEUE_LEN); + let mut blbrange_queue = FifoQueue::new(MAX_BLOCKS_BY_ROOTS_QUEUE_LEN); // Channels for sending work to the re-process scheduler (`work_reprocessing_tx`) and to // receive them back once they are ready (`ready_work_rx`). @@ -1266,6 +1290,9 @@ impl BeaconProcessor { Work::BlocksByRootsRequest { .. } => { bbroots_queue.push(work, work_id, &self.log) } + Work::BlobsByRangeRequest { .. } => { + blbrange_queue.push(work, work_id, &self.log) + } Work::UnknownBlockAttestation { .. } => { unknown_block_attestation_queue.push(work) } @@ -1643,6 +1670,21 @@ impl BeaconProcessor { request, ) }), + + Work::BlobsByRangeRequest { + peer_id, + request_id, + request + } => task_spawner.spawn_blocking_with_manual_send_idle(move |send_idle_on_drop| { + worker.handle_blobs_by_range_request( + sub_executor, + send_idle_on_drop, + peer_id, + request_id, + request, + ) + }), + Work::UnknownBlockAttestation { message_id, peer_id, diff --git a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs index 8ca9c35e47..2b8cbbc22c 100644 --- a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs @@ -6,6 +6,7 @@ use beacon_chain::{BeaconChainError, BeaconChainTypes, HistoricalBlockError, Whe use itertools::process_results; use lighthouse_network::rpc::StatusMessage; use lighthouse_network::rpc::*; +use lighthouse_network::rpc::methods::BlobsByRangeRequest; use lighthouse_network::{PeerId, PeerRequestId, ReportSource, Response, SyncInfo}; use slog::{debug, error}; use slot_clock::SlotClock; @@ -372,4 +373,16 @@ impl Worker { "load_blocks_by_range_blocks", ); } + + /// Handle a `BlobsByRange` request from the peer. + pub fn handle_blobs_by_range_request( + self, + executor: TaskExecutor, + send_on_drop: SendOnDrop, + peer_id: PeerId, + request_id: PeerRequestId, + request: BlobsByRangeRequest, + ) { + // TODO impl + } } diff --git a/beacon_node/network/src/router/processor.rs b/beacon_node/network/src/router/processor.rs index 2d12cf5401..e640bace5f 100644 --- a/beacon_node/network/src/router/processor.rs +++ b/beacon_node/network/src/router/processor.rs @@ -168,11 +168,9 @@ impl Processor { request_id: PeerRequestId, request: BlobsByRangeRequest, ) { - /* - self.send_beacon_processor_work(BeaconWorkEvent::blocks_by_roots_request( + self.send_beacon_processor_work(BeaconWorkEvent::blobs_by_range_request( peer_id, request_id, request, )) - */ } /// Handle a `BlocksByRange` request from the peer. pub fn on_blocks_by_range_request( From f43532d3deba899e057d77550b44a4633c474cc1 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Sat, 17 Sep 2022 20:05:51 +0200 Subject: [PATCH 14/22] implement handle blobs by range req --- .../beacon_processor/worker/rpc_methods.rs | 140 +++++++++++++++++- 1 file changed, 137 insertions(+), 3 deletions(-) diff --git a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs index 2b8cbbc22c..97cab17822 100644 --- a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs @@ -12,7 +12,7 @@ use slog::{debug, error}; use slot_clock::SlotClock; use std::sync::Arc; use task_executor::TaskExecutor; -use types::{Epoch, EthSpec, Hash256, Slot}; +use types::{Epoch, EthSpec, Hash256, Slot, VariableList}; use super::Worker; @@ -381,8 +381,142 @@ impl Worker { send_on_drop: SendOnDrop, peer_id: PeerId, request_id: PeerRequestId, - request: BlobsByRangeRequest, + mut req: BlobsByRangeRequest, ) { - // TODO impl + debug!(self.log, "Received BlobsByRange Request"; + "peer_id" => %peer_id, + "count" => req.count, + "start_slot" => req.start_slot, + ); + + // Should not send more than max request blocks + if req.count > MAX_REQUEST_BLOCKS { + req.count = MAX_REQUEST_BLOCKS; + } + + let forwards_block_root_iter = match self + .chain + .forwards_iter_block_roots(Slot::from(req.start_slot)) + { + Ok(iter) => iter, + Err(BeaconChainError::HistoricalBlockError( + HistoricalBlockError::BlockOutOfRange { + slot, + oldest_block_slot, + }, + )) => { + debug!(self.log, "Range request failed during backfill"; "requested_slot" => slot, "oldest_known_slot" => oldest_block_slot); + return self.send_error_response( + peer_id, + RPCResponseErrorCode::ResourceUnavailable, + "Backfilling".into(), + request_id, + ); + } + Err(e) => return error!(self.log, "Unable to obtain root iter"; "error" => ?e), + }; + + // Pick out the required blocks, ignoring skip-slots. + let mut last_block_root = None; + let maybe_block_roots = process_results(forwards_block_root_iter, |iter| { + iter.take_while(|(_, slot)| slot.as_u64() < req.start_slot.saturating_add(req.count)) + // map skip slots to None + .map(|(root, _)| { + let result = if Some(root) == last_block_root { + None + } else { + Some(root) + }; + last_block_root = Some(root); + result + }) + .collect::>>() + }); + + let block_roots = match maybe_block_roots { + Ok(block_roots) => block_roots, + Err(e) => return error!(self.log, "Error during iteration over blocks"; "error" => ?e), + }; + + // remove all skip slots + let block_roots = block_roots.into_iter().flatten().collect::>(); + + // Fetching blocks is async because it may have to hit the execution layer for payloads. + executor.spawn( + async move { + let mut blocks_sent = 0; + let mut send_response = true; + + for root in block_roots { + match self.chain.store.get_blobs(&root) { + Ok(Some(blob)) => { + blocks_sent += 1; + self.send_network_message(NetworkMessage::SendResponse { + peer_id, + response: Response::BlobsByRange(Some(Arc::new(VariableList::new(vec![blob.message]).unwrap()))), + id: request_id, + }); + } + Ok(None) => { + error!( + self.log, + "Block in the chain is not in the store"; + "request_root" => ?root + ); + break; + } + Err(e) => { + error!( + self.log, + "Error fetching block for peer"; + "block_root" => ?root, + "error" => ?e + ); + break; + } + } + } + + let current_slot = self + .chain + .slot() + .unwrap_or_else(|_| self.chain.slot_clock.genesis_slot()); + + if blocks_sent < (req.count as usize) { + debug!( + self.log, + "BlocksByRange Response processed"; + "peer" => %peer_id, + "msg" => "Failed to return all requested blocks", + "start_slot" => req.start_slot, + "current_slot" => current_slot, + "requested" => req.count, + "returned" => blocks_sent + ); + } else { + debug!( + self.log, + "BlocksByRange Response processed"; + "peer" => %peer_id, + "start_slot" => req.start_slot, + "current_slot" => current_slot, + "requested" => req.count, + "returned" => blocks_sent + ); + } + + if send_response { + // send the stream terminator + self.send_network_message(NetworkMessage::SendResponse { + peer_id, + response: Response::BlocksByRange(None), + id: request_id, + }); + } + + drop(send_on_drop); + }, + "load_blocks_by_range_blocks", + ); } } From 76572db9d550e18a253cf9ed09d93cfe907a1ca5 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Sat, 17 Sep 2022 20:55:21 +0200 Subject: [PATCH 15/22] add network config --- beacon_node/beacon_chain/src/beacon_chain.rs | 14 ++- .../beacon_chain/src/block_verification.rs | 2 + beacon_node/beacon_chain/src/builder.rs | 2 + beacon_node/beacon_chain/src/test_utils.rs | 4 +- beacon_node/http_api/src/publish_blocks.rs | 2 +- .../beacon_processor/worker/gossip_methods.rs | 21 ++++- .../beacon_processor/worker/sync_methods.rs | 2 +- beacon_node/store/src/hot_cold_store.rs | 26 ++++++ beacon_node/store/src/lib.rs | 2 + common/eth2_config/src/lib.rs | 5 ++ .../eip4844/boot_enr.yaml | 0 .../eip4844/config.yaml | 85 ++++++++++++++++++ .../eip4844/deploy_block.txt | 1 + .../eip4844/genesis.ssz.zip | Bin 0 -> 3518 bytes 14 files changed, 159 insertions(+), 7 deletions(-) create mode 100644 common/eth2_network_config/built_in_network_configs/eip4844/boot_enr.yaml create mode 100644 common/eth2_network_config/built_in_network_configs/eip4844/config.yaml create mode 100644 common/eth2_network_config/built_in_network_configs/eip4844/deploy_block.txt create mode 100644 common/eth2_network_config/built_in_network_configs/eip4844/genesis.ssz.zip diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 3b3814f6c0..5d88c5ca41 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -100,6 +100,7 @@ use types::*; pub use crate::canonical_head::{CanonicalHead, CanonicalHeadRwLock}; pub use fork_choice::CountUnrealized; use types::kzg_commitment::KzgCommitment; +use types::signed_blobs_sidecar::SignedBlobsSidecar; pub type ForkChoiceError = fork_choice::Error; @@ -374,6 +375,8 @@ pub struct BeaconChain { /// Sender given to tasks, so that if they encounter a state in which execution cannot /// continue they can request that everything shuts down. pub shutdown_sender: Sender, + pub block_waiting_for_sidecar: Mutex>>, + pub sidecar_waiting_for_block: Mutex>>, /// Logging to CLI, etc. pub(crate) log: Logger, /// Arbitrary bytes included in the blocks. @@ -2342,7 +2345,7 @@ impl BeaconChain { // Import the blocks into the chain. for signature_verified_block in signature_verified_blocks { match self - .process_block(signature_verified_block, count_unrealized) + .process_block(signature_verified_block, None, count_unrealized) .await { Ok(_) => imported_blocks += 1, @@ -2428,6 +2431,7 @@ impl BeaconChain { pub async fn process_block>( self: &Arc, unverified_block: B, + sidecar: Option>, count_unrealized: CountUnrealized, ) -> Result> { // Start the Prometheus timer. @@ -2444,7 +2448,7 @@ impl BeaconChain { let import_block = async move { let execution_pending = unverified_block.into_execution_pending_block(&chain)?; chain - .import_execution_pending_block(execution_pending, count_unrealized) + .import_execution_pending_block(execution_pending, sidecar, count_unrealized) .await }; @@ -2502,6 +2506,7 @@ impl BeaconChain { async fn import_execution_pending_block( self: Arc, execution_pending_block: ExecutionPendingBlock, + sidecar: Option>, count_unrealized: CountUnrealized, ) -> Result> { let ExecutionPendingBlock { @@ -2557,6 +2562,7 @@ impl BeaconChain { move || { chain.import_block( block, + sidecar, block_root, state, confirmed_state_roots, @@ -2579,6 +2585,7 @@ impl BeaconChain { fn import_block( &self, signed_block: Arc>, + sidecar: Option>, block_root: Hash256, mut state: BeaconState, confirmed_state_roots: Vec, @@ -2917,6 +2924,9 @@ impl BeaconChain { .collect(); ops.push(StoreOp::PutBlock(block_root, signed_block.clone())); ops.push(StoreOp::PutState(block.state_root(), &state)); + if let Some(sidecar) = sidecar { + ops.push(StoreOp::PutBlobs(block_root, sidecar)); + } let txn_lock = self.store.hot_db.begin_rw_transaction(); if let Err(e) = self.store.do_atomically(ops) { diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index cdcbf3f68e..8eae5f256e 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -144,6 +144,7 @@ pub enum BlockError { present_slot: Slot, block_slot: Slot, }, + MissingSidecar, /// The block state_root does not match the generated state. /// /// ## Peer scoring @@ -277,6 +278,7 @@ pub enum BlockError { /// The peer sent us an invalid block, but I'm not really sure how to score this in an /// "optimistic" sync world. ParentExecutionPayloadInvalid { parent_root: Hash256 }, + } /// Returned when block validation failed due to some issue verifying diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index 2704690442..8bed973e58 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -797,6 +797,8 @@ where validator_pubkey_cache: TimeoutRwLock::new(validator_pubkey_cache), attester_cache: <_>::default(), early_attester_cache: <_>::default(), + block_waiting_for_sidecar: <_>::default(), + sidecar_waiting_for_block: <_>::default(), shutdown_sender: self .shutdown_sender .ok_or("Cannot build without a shutdown sender.")?, diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index a62608202e..128a337605 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -1458,7 +1458,7 @@ where self.set_current_slot(slot); let block_hash: SignedBeaconBlockHash = self .chain - .process_block(Arc::new(block), CountUnrealized::True) + .process_block(Arc::new(block), todo!(), CountUnrealized::True) .await? .into(); self.chain.recompute_head_at_current_slot().await; @@ -1471,7 +1471,7 @@ where ) -> Result> { let block_hash: SignedBeaconBlockHash = self .chain - .process_block(Arc::new(block), CountUnrealized::True) + .process_block(Arc::new(block), todo!(),CountUnrealized::True) .await? .into(); self.chain.recompute_head_at_current_slot().await; diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 60ca8f2328..5112058146 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -32,7 +32,7 @@ pub async fn publish_block( metrics::observe_duration(&metrics::HTTP_API_BLOCK_BROADCAST_DELAY_TIMES, delay); match chain - .process_block(block.clone(), CountUnrealized::True) + .process_block(block.clone(), None, CountUnrealized::True) .await { Ok(root) => { diff --git a/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs b/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs index 0e1ab697e6..63b252099a 100644 --- a/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs @@ -673,6 +673,7 @@ impl Worker { .await { let block_root = gossip_verified_block.block_root; + if let Some(handle) = duplicate_cache.check_and_insert(block_root) { self.process_gossip_verified_block( peer_id, @@ -759,6 +760,9 @@ impl Worker { verified_block } + Err(BlockError::MissingSidecar) => { + todo!(); //is relevant? + } Err(BlockError::ParentUnknown(block)) => { debug!( self.log, @@ -920,9 +924,24 @@ impl Worker { ) { let block: Arc<_> = verified_block.block.clone(); + let sidecar = if verified_block.block.message() + .body().blob_kzg_commitments().map(|committments| committments.is_empty()).unwrap_or(true) { + None + } else if let Some(sidecar) = self.chain.sidecar_waiting_for_block.lock().as_ref() { + if sidecar.message.beacon_block_root == verified_block.block_root() { + Some(sidecar.clone()) + } else { + *self.chain.block_waiting_for_sidecar.lock() = Some(verified_block); + return + } + } else { + // we need the sidecar but dont have it yet + return + }; + match self .chain - .process_block(verified_block, CountUnrealized::True) + .process_block(verified_block, sidecar, CountUnrealized::True) .await { Ok(block_root) => { diff --git a/beacon_node/network/src/beacon_processor/worker/sync_methods.rs b/beacon_node/network/src/beacon_processor/worker/sync_methods.rs index 760896e0e9..b2d46f411d 100644 --- a/beacon_node/network/src/beacon_processor/worker/sync_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/sync_methods.rs @@ -80,7 +80,7 @@ impl Worker { } }; let slot = block.slot(); - let result = self.chain.process_block(block, CountUnrealized::True).await; + let result = self.chain.process_block(block, None, CountUnrealized::True).await; metrics::inc_counter(&metrics::BEACON_PROCESSOR_RPC_BLOCK_IMPORTED_TOTAL); diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index 8240007b66..3c07b4073b 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -483,6 +483,19 @@ impl, Cold: ItemStore> HotColdDB } } + pub fn blobs_as_kv_store_ops( + &self, + key: &Hash256, + blobs: &SignedBlobsSidecar, + ops: &mut Vec, + ) { + let db_key = get_key_for_col(DBColumn::BeaconBlob.into(), key.as_bytes()); + ops.push(KeyValueStoreOp::PutKeyValue( + db_key, + blobs.as_ssz_bytes(), + )); + } + pub fn put_state_summary( &self, state_root: &Hash256, @@ -710,6 +723,14 @@ impl, Cold: ItemStore> HotColdDB self.store_hot_state(&state_root, state, &mut key_value_batch)?; } + StoreOp::PutBlobs(block_root, blobs) => { + self.blobs_as_kv_store_ops( + &block_root, + &blobs, + &mut key_value_batch, + ); + } + StoreOp::PutStateSummary(state_root, summary) => { key_value_batch.push(summary.as_kv_store_op(state_root)); } @@ -754,6 +775,7 @@ impl, Cold: ItemStore> HotColdDB // Update the block cache whilst holding a lock, to ensure that the cache updates atomically // with the database. let mut guard = self.block_cache.lock(); + let mut guard_blob = self.blob_cache.lock(); for op in &batch { match op { @@ -761,6 +783,10 @@ impl, Cold: ItemStore> HotColdDB guard.put(*block_root, (**block).clone()); } + StoreOp::PutBlobs(block_root, blobs) => { + guard_blob.put(*block_root, blobs.clone()); + } + StoreOp::PutState(_, _) => (), StoreOp::PutStateSummary(_, _) => (), diff --git a/beacon_node/store/src/lib.rs b/beacon_node/store/src/lib.rs index f7af172f5a..5e74827c9d 100644 --- a/beacon_node/store/src/lib.rs +++ b/beacon_node/store/src/lib.rs @@ -42,6 +42,7 @@ use parking_lot::MutexGuard; use std::sync::Arc; use strum::{EnumString, IntoStaticStr}; pub use types::*; +use types::signed_blobs_sidecar::SignedBlobsSidecar; pub type ColumnIter<'a> = Box), Error>> + 'a>; pub type ColumnKeyIter<'a> = Box> + 'a>; @@ -155,6 +156,7 @@ pub trait ItemStore: KeyValueStore + Sync + Send + Sized + 'stati pub enum StoreOp<'a, E: EthSpec> { PutBlock(Hash256, Arc>), PutState(Hash256, &'a BeaconState), + PutBlobs(Hash256, SignedBlobsSidecar), PutStateSummary(Hash256, HotStateSummary), PutStateTemporaryFlag(Hash256), DeleteStateTemporaryFlag(Hash256), diff --git a/common/eth2_config/src/lib.rs b/common/eth2_config/src/lib.rs index 7e3c025a83..d188088251 100644 --- a/common/eth2_config/src/lib.rs +++ b/common/eth2_config/src/lib.rs @@ -307,5 +307,10 @@ define_hardcoded_nets!( // Set to `true` if the genesis state can be found in the `built_in_network_configs` // directory. GENESIS_STATE_IS_KNOWN + ), + ( + eip4844, + "eip4844", + GENESIS_STATE_IS_KNOWN ) ); diff --git a/common/eth2_network_config/built_in_network_configs/eip4844/boot_enr.yaml b/common/eth2_network_config/built_in_network_configs/eip4844/boot_enr.yaml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/common/eth2_network_config/built_in_network_configs/eip4844/config.yaml b/common/eth2_network_config/built_in_network_configs/eip4844/config.yaml new file mode 100644 index 0000000000..8e779eac3d --- /dev/null +++ b/common/eth2_network_config/built_in_network_configs/eip4844/config.yaml @@ -0,0 +1,85 @@ +# Prater config + +# Extends the mainnet preset +CONFIG_NAME: 'eip4844' +PRESET_BASE: 'mainnet' + +# Transition +# --------------------------------------------------------------- +TERMINAL_TOTAL_DIFFICULTY: 40 +# By default, don't use these params +TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000 +TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615 + +# Genesis +# --------------------------------------------------------------- +# `2**14` (= 16,384) +MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 2 +# Mar-01-2021 08:53:32 AM +UTC +MIN_GENESIS_TIME: 1653318000 +# Prater area code (Vienna) +GENESIS_FORK_VERSION: 0x00000ffd +# Customized for Prater: 1919188 seconds (Mar-23-2021 02:00:00 PM +UTC) +GENESIS_DELAY: 0 + + +# Forking +# --------------------------------------------------------------- +# Some forks are disabled for now: +# - These may be re-assigned to another fork-version later +# - Temporarily set to max uint64 value: 2**64 - 1 + +# Altair +ALTAIR_FORK_VERSION: 0x01000ffd +ALTAIR_FORK_EPOCH: 1 +# Merge +BELLATRIX_FORK_VERSION: 0x02000ffd +BELLATRIX_FORK_EPOCH: 2 +# Sharding +EIP4844_FORK_VERSION: 0x03000ffd +EIP4844_FORK_EPOCH: 3 + +# TBD, 2**32 is a placeholder. Merge transition approach is in active R&D. +TRANSITION_TOTAL_DIFFICULTY: 40 + + +# Time parameters +# --------------------------------------------------------------- +# 12 seconds +SECONDS_PER_SLOT: 12 +# 14 (estimate from Eth1 mainnet) +SECONDS_PER_ETH1_BLOCK: 14 +# 2**8 (= 256) epochs ~27 hours +MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 +# 2**8 (= 256) epochs ~27 hours +SHARD_COMMITTEE_PERIOD: 256 +# 2**11 (= 2,048) Eth1 blocks ~8 hours +ETH1_FOLLOW_DISTANCE: 15 + + +# Validator cycle +# --------------------------------------------------------------- +# 2**2 (= 4) +INACTIVITY_SCORE_BIAS: 4 +# 2**4 (= 16) +INACTIVITY_SCORE_RECOVERY_RATE: 16 +# 2**4 * 10**9 (= 16,000,000,000) Gwei +EJECTION_BALANCE: 16000000000 +# 2**2 (= 4) +MIN_PER_EPOCH_CHURN_LIMIT: 4 +# 2**16 (= 65,536) +CHURN_LIMIT_QUOTIENT: 65536 + + +# Fork choice +# --------------------------------------------------------------- +# 40% +PROPOSER_SCORE_BOOST: 40 + +# Deposit contract +# --------------------------------------------------------------- +# Ethereum Goerli testnet +DEPOSIT_CHAIN_ID: 1331 +DEPOSIT_NETWORK_ID: 69 +# Prater test deposit contract on Goerli Testnet +DEPOSIT_CONTRACT_ADDRESS: 0x8A04d14125D0FDCDc742F4A05C051De07232EDa4 diff --git a/common/eth2_network_config/built_in_network_configs/eip4844/deploy_block.txt b/common/eth2_network_config/built_in_network_configs/eip4844/deploy_block.txt new file mode 100644 index 0000000000..573541ac97 --- /dev/null +++ b/common/eth2_network_config/built_in_network_configs/eip4844/deploy_block.txt @@ -0,0 +1 @@ +0 diff --git a/common/eth2_network_config/built_in_network_configs/eip4844/genesis.ssz.zip b/common/eth2_network_config/built_in_network_configs/eip4844/genesis.ssz.zip new file mode 100644 index 0000000000000000000000000000000000000000..9a1c014f63769577fd9c06fe2988f338dbb83f8f GIT binary patch literal 3518 zcmWIWW@Zs#U|`^2c(lkcbkSCsze2nW3}P&r4BQMd4C$$Psl}Padd0<6p&^_M%pC%1 z$?Ji*w1S&~k>v$50|S_Ndp0vej78$W$MEGhYxa6O-kh>$+HnWt>6>~o+s{o=N{u_4 z95hR#L4oGCZM3>iE!^klD{aib$RPaL36-E{LDOeP?=m}@tZO}?V?nK; zVx4_O#XcLG|0mq@Rao;ftMo1yZU6be@*!(_o`~ucx%}FMFUvmYUQRz#yPz)X$k&9z z2iuN?)}NbOSmOVCWAHAKm>7ZC8|>enx^#V?+_tX?Zx8Jhm9~God(*Dmn=7ravI6}) z3PxrK{Id_c`buQQiyNh%pXe`=*P1C}zVq6%%CA=To3yP>*3>n>$Q9M@{yuM3?DzGz zRcE`+_pK`YzTzAIvoJC99O19;!k?F2^?mp0=8scnmmc>zJ?+oG)$i{Ao7P&&^#zzC zM!{$ZjE2By2#kinXb6mkz%UPi9jBQn^kIQ_x7FN#Gj^xt{*2!F^zZDibIX>g>?^GM z^&+H?fB2f^#)BV3nXj>JkLmAte{)g! zyIr^3 +AU3xZ4`0uyx@_P*a|617b{M?lL^Z)<3we)JHvlr9Cwzt>6ybQjv&?3an zF74SB*UrCw=2|zzrKR>|xS!f1sGTiS_xaYtLhi*;@gBdQed)~8taMLH)7uwflKm^V zyZF{o^I7GIXBWKM@K)BlNd1V8{@;}aDUa2eUvsZoR3)?bOJKXz-tBWUzJ0tIy1V#Q z>XYxMJn9x-)mFAza?N6ot)gk+gHFTCpVyV_`}6d|?7d}aU$ca4B(AB|{;h3RGl>!l zoLRavJu;*_F!*ub-kMF>)597I3(v<@KKgU-$i$cS{Z)IPTHh3vH=BPc<=}xsb3>Wq zde3pLFRZwE^z^|qz3iWk<-MA3sJT-^ZGZHKtg5TF&c^Xkw~|lr^OlK!%M2H^fB0a^ zitz5ESJ%2^vk8cm_ y-`r;i@MdI^W5(67l7Od~C5<2^N;iuY(#=BaVg-1!vVoK`0%0JK4hQzq7#IMuda* Date: Sat, 17 Sep 2022 21:38:57 +0200 Subject: [PATCH 16/22] forgor something --- beacon_node/beacon_chain/src/beacon_chain.rs | 8 +- .../beacon_chain/src/snapshot_cache.rs | 2 +- .../beacon_processor/worker/gossip_methods.rs | 74 +++++++++++++++++++ beacon_node/store/src/hot_cold_store.rs | 2 +- beacon_node/store/src/lib.rs | 2 +- 5 files changed, 81 insertions(+), 7 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 5d88c5ca41..60fb9e4191 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -376,7 +376,7 @@ pub struct BeaconChain { /// continue they can request that everything shuts down. pub shutdown_sender: Sender, pub block_waiting_for_sidecar: Mutex>>, - pub sidecar_waiting_for_block: Mutex>>, + pub sidecar_waiting_for_block: Mutex>>>, /// Logging to CLI, etc. pub(crate) log: Logger, /// Arbitrary bytes included in the blocks. @@ -2431,7 +2431,7 @@ impl BeaconChain { pub async fn process_block>( self: &Arc, unverified_block: B, - sidecar: Option>, + sidecar: Option>>, count_unrealized: CountUnrealized, ) -> Result> { // Start the Prometheus timer. @@ -2506,7 +2506,7 @@ impl BeaconChain { async fn import_execution_pending_block( self: Arc, execution_pending_block: ExecutionPendingBlock, - sidecar: Option>, + sidecar: Option>>, count_unrealized: CountUnrealized, ) -> Result> { let ExecutionPendingBlock { @@ -2585,7 +2585,7 @@ impl BeaconChain { fn import_block( &self, signed_block: Arc>, - sidecar: Option>, + sidecar: Option>>, block_root: Hash256, mut state: BeaconState, confirmed_state_roots: Vec, diff --git a/beacon_node/beacon_chain/src/snapshot_cache.rs b/beacon_node/beacon_chain/src/snapshot_cache.rs index 40b73451cb..c77ef9e38a 100644 --- a/beacon_node/beacon_chain/src/snapshot_cache.rs +++ b/beacon_node/beacon_chain/src/snapshot_cache.rs @@ -16,7 +16,7 @@ pub const DEFAULT_SNAPSHOT_CACHE_SIZE: usize = 4; const MINIMUM_BLOCK_DELAY_FOR_CLONE: Duration = Duration::from_secs(6); /// This snapshot is to be used for verifying a child of `self.beacon_block`. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct PreProcessingSnapshot { /// This state is equivalent to the `self.beacon_block.state_root()` state that has been /// advanced forward one slot using `per_slot_processing`. This state is "primed and ready" for diff --git a/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs b/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs index 63b252099a..0d07c62072 100644 --- a/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs @@ -935,6 +935,7 @@ impl Worker { return } } else { + *self.chain.block_waiting_for_sidecar.lock() = Some(verified_block); // we need the sidecar but dont have it yet return }; @@ -1017,7 +1018,80 @@ impl Worker { duplicate_cache: DuplicateCache, seen_duration: Duration, ) { + let verified_block = self.chain.block_waiting_for_sidecar.lock().take(); + if let Some(verified_block) = verified_block { + let block = verified_block.block.clone(); + if verified_block.block_root() == blobs.message.beacon_block_root { + match self + .chain + .process_block(verified_block, Some(blobs), CountUnrealized::True) + .await + { + Ok(block_root) => { + metrics::inc_counter(&metrics::BEACON_PROCESSOR_GOSSIP_BLOCK_IMPORTED_TOTAL); + if reprocess_tx + .try_send(ReprocessQueueMessage::BlockImported(block_root)) + .is_err() + { + error!( + self.log, + "Failed to inform block import"; + "source" => "gossip", + "block_root" => ?block_root, + ) + }; + + debug!( + self.log, + "Gossipsub block processed"; + "block" => ?block_root, + "peer_id" => %peer_id + ); + + self.chain.recompute_head_at_current_slot().await; + } + Err(BlockError::ParentUnknown { .. }) => { + // Inform the sync manager to find parents for this block + // This should not occur. It should be checked by `should_forward_block` + error!( + self.log, + "Block with unknown parent attempted to be processed"; + "peer_id" => %peer_id + ); + self.send_sync_message(SyncMessage::UnknownBlock(peer_id, block)); + } + Err(ref e @ BlockError::ExecutionPayloadError(ref epe)) if !epe.penalize_peer() => { + debug!( + self.log, + "Failed to verify execution payload"; + "error" => %e + ); + } + other => { + debug!( + self.log, + "Invalid gossip beacon block"; + "outcome" => ?other, + "block root" => ?block.canonical_root(), + "block slot" => block.slot() + ); + self.gossip_penalize_peer( + peer_id, + PeerAction::MidToleranceError, + "bad_gossip_block_ssz", + ); + trace!( + self.log, + "Invalid gossip beacon block ssz"; + "ssz" => format_args!("0x{}", hex::encode(block.as_ssz_bytes())), + ); + } + }; + } + } else { + *self.chain.sidecar_waiting_for_block.lock() = Some(blobs); + } } pub fn process_gossip_voluntary_exit( diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index 3c07b4073b..de46843adf 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -784,7 +784,7 @@ impl, Cold: ItemStore> HotColdDB } StoreOp::PutBlobs(block_root, blobs) => { - guard_blob.put(*block_root, blobs.clone()); + guard_blob.put(*block_root, (**blobs).clone()); } StoreOp::PutState(_, _) => (), diff --git a/beacon_node/store/src/lib.rs b/beacon_node/store/src/lib.rs index 5e74827c9d..aac9cda932 100644 --- a/beacon_node/store/src/lib.rs +++ b/beacon_node/store/src/lib.rs @@ -156,7 +156,7 @@ pub trait ItemStore: KeyValueStore + Sync + Send + Sized + 'stati pub enum StoreOp<'a, E: EthSpec> { PutBlock(Hash256, Arc>), PutState(Hash256, &'a BeaconState), - PutBlobs(Hash256, SignedBlobsSidecar), + PutBlobs(Hash256, Arc>), PutStateSummary(Hash256, HotStateSummary), PutStateTemporaryFlag(Hash256), DeleteStateTemporaryFlag(Hash256), From 8b71b978e094e524f4309ad2c41ce9adcd3671f9 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Sat, 17 Sep 2022 23:42:49 +0200 Subject: [PATCH 17/22] new round of hacks (config etc) --- beacon_node/lighthouse_network/src/rpc/mod.rs | 1 + .../src/beacon_processor/worker/rpc_methods.rs | 4 ++-- .../eip4844/boot_enr.yaml | 1 + .../eip4844/genesis.ssz.zip | Bin 3518 -> 0 bytes consensus/types/src/eth_spec.rs | 2 +- 5 files changed, 5 insertions(+), 3 deletions(-) delete mode 100644 common/eth2_network_config/built_in_network_configs/eip4844/genesis.ssz.zip diff --git a/beacon_node/lighthouse_network/src/rpc/mod.rs b/beacon_node/lighthouse_network/src/rpc/mod.rs index 6197d57f19..daa565199c 100644 --- a/beacon_node/lighthouse_network/src/rpc/mod.rs +++ b/beacon_node/lighthouse_network/src/rpc/mod.rs @@ -125,6 +125,7 @@ impl RPC { Duration::from_secs(10), ) .n_every(Protocol::BlocksByRoot, 128, Duration::from_secs(10)) + .n_every(Protocol::BlobsByRange, 128, Duration::from_secs(10)) .build() .expect("Configuration parameters are valid"); RPC { diff --git a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs index 97cab17822..83949f6ace 100644 --- a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs @@ -460,7 +460,7 @@ impl Worker { Ok(None) => { error!( self.log, - "Block in the chain is not in the store"; + "Blob in the chain is not in the store"; "request_root" => ?root ); break; @@ -509,7 +509,7 @@ impl Worker { // send the stream terminator self.send_network_message(NetworkMessage::SendResponse { peer_id, - response: Response::BlocksByRange(None), + response: Response::BlobsByRange(None), id: request_id, }); } diff --git a/common/eth2_network_config/built_in_network_configs/eip4844/boot_enr.yaml b/common/eth2_network_config/built_in_network_configs/eip4844/boot_enr.yaml index e69de29bb2..d4cf63c389 100644 --- a/common/eth2_network_config/built_in_network_configs/eip4844/boot_enr.yaml +++ b/common/eth2_network_config/built_in_network_configs/eip4844/boot_enr.yaml @@ -0,0 +1 @@ +- enr:-MK4QI-wkVW1PxL4ksUM4H_hMgTTwxKMzvvDMfoiwPBuRxcsGkrGPLo4Kho3Ri1DEtJG4B6pjXddbzA9iF2gVctxv42GAX9v5WG5h2F0dG5ldHOIAAAAAAAAAACEZXRoMpBzql9ccAAAcDIAAAAAAAAAgmlkgnY0gmlwhKRcjMiJc2VjcDI1NmsxoQK1fc46pmVHKq8HNYLkSVaUv4uK2UBsGgjjGWU6AAhAY4hzeW5jbmV0cwCDdGNwgiMog3VkcIIjKA diff --git a/common/eth2_network_config/built_in_network_configs/eip4844/genesis.ssz.zip b/common/eth2_network_config/built_in_network_configs/eip4844/genesis.ssz.zip deleted file mode 100644 index 9a1c014f63769577fd9c06fe2988f338dbb83f8f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3518 zcmWIWW@Zs#U|`^2c(lkcbkSCsze2nW3}P&r4BQMd4C$$Psl}Padd0<6p&^_M%pC%1 z$?Ji*w1S&~k>v$50|S_Ndp0vej78$W$MEGhYxa6O-kh>$+HnWt>6>~o+s{o=N{u_4 z95hR#L4oGCZM3>iE!^klD{aib$RPaL36-E{LDOeP?=m}@tZO}?V?nK; zVx4_O#XcLG|0mq@Rao;ftMo1yZU6be@*!(_o`~ucx%}FMFUvmYUQRz#yPz)X$k&9z z2iuN?)}NbOSmOVCWAHAKm>7ZC8|>enx^#V?+_tX?Zx8Jhm9~God(*Dmn=7ravI6}) z3PxrK{Id_c`buQQiyNh%pXe`=*P1C}zVq6%%CA=To3yP>*3>n>$Q9M@{yuM3?DzGz zRcE`+_pK`YzTzAIvoJC99O19;!k?F2^?mp0=8scnmmc>zJ?+oG)$i{Ao7P&&^#zzC zM!{$ZjE2By2#kinXb6mkz%UPi9jBQn^kIQ_x7FN#Gj^xt{*2!F^zZDibIX>g>?^GM z^&+H?fB2f^#)BV3nXj>JkLmAte{)g! zyIr^3 +AU3xZ4`0uyx@_P*a|617b{M?lL^Z)<3we)JHvlr9Cwzt>6ybQjv&?3an zF74SB*UrCw=2|zzrKR>|xS!f1sGTiS_xaYtLhi*;@gBdQed)~8taMLH)7uwflKm^V zyZF{o^I7GIXBWKM@K)BlNd1V8{@;}aDUa2eUvsZoR3)?bOJKXz-tBWUzJ0tIy1V#Q z>XYxMJn9x-)mFAza?N6ot)gk+gHFTCpVyV_`}6d|?7d}aU$ca4B(AB|{;h3RGl>!l zoLRavJu;*_F!*ub-kMF>)597I3(v<@KKgU-$i$cS{Z)IPTHh3vH=BPc<=}xsb3>Wq zde3pLFRZwE^z^|qz3iWk<-MA3sJT-^ZGZHKtg5TF&c^Xkw~|lr^OlK!%M2H^fB0a^ zitz5ESJ%2^vk8cm_ y-`r;i@MdI^W5(67l7Od~C5<2^N;iuY(#=BaVg-1!vVoK`0%0JK4hQzq7#IMuda* Date: Sun, 18 Sep 2022 10:23:53 +0200 Subject: [PATCH 18/22] more enr --- .../built_in_network_configs/eip4844/boot_enr.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/common/eth2_network_config/built_in_network_configs/eip4844/boot_enr.yaml b/common/eth2_network_config/built_in_network_configs/eip4844/boot_enr.yaml index d4cf63c389..4d52cc5975 100644 --- a/common/eth2_network_config/built_in_network_configs/eip4844/boot_enr.yaml +++ b/common/eth2_network_config/built_in_network_configs/eip4844/boot_enr.yaml @@ -1 +1,3 @@ -- enr:-MK4QI-wkVW1PxL4ksUM4H_hMgTTwxKMzvvDMfoiwPBuRxcsGkrGPLo4Kho3Ri1DEtJG4B6pjXddbzA9iF2gVctxv42GAX9v5WG5h2F0dG5ldHOIAAAAAAAAAACEZXRoMpBzql9ccAAAcDIAAAAAAAAAgmlkgnY0gmlwhKRcjMiJc2VjcDI1NmsxoQK1fc46pmVHKq8HNYLkSVaUv4uK2UBsGgjjGWU6AAhAY4hzeW5jbmV0cwCDdGNwgiMog3VkcIIjKA +- enr:-MK4QLij8YaVQ6fIi09rDuD9fufxBlCZRXwfM1q6SbNJfy5ZZdAvtlnsfqhIeI0IqeOZdaPExVCfZfR4JJTIuKXFR76GAYJGrqHnh2F0dG5ldHOIAAAAAAAAAACEZXRoMpBCynldgwAP_QMAAAAAAAAAgmlkgnY0gmlwhCJ7uEyJc2VjcDI1NmsxoQJpeftU6RbmIhcFllICznlAMJXL3EwHEGhn73_Gk0wrCYhzeW5jbmV0cwCDdGNwgjLIg3VkcIIu4A +- enr:-JG4QK27MZvV3QbwdLt055Yhei27SjAsDXMFGCdl-Q7SDiCgR_qbiW3BmcOClehFVJgMa6IfjHeJBdbC0jvrr2NycOqGAYJLWb5kgmlkgnY0gmlwhCJE_eeJc2VjcDI1NmsxoQIecO7Y9C7J2Bs7RNxXaUkU6BfmPKIhEsDScKAoxENaRYN0Y3CCdl-DdWRwgnZf +- enr:-JG4QExcHW3vzBcE0f_r-93nSA4iBy4qNLthSyTw7p0tlPwjMl1JVTAgLSNHLLZJzOGtelJO4sw37LliuHyJ55zN5J6GAYJLWTvzgmlkgnY0gmlwhCKq1cmJc2VjcDI1NmsxoQJT2d4jtKQbHNw3tZPLhoMlR73o5LNdi-bk_bYq6siwuIN0Y3CCdl-DdWRwgnZf \ No newline at end of file From 14aa4957b9fcf2f6f9c1ee6b32e7b69ad3e97b6d Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Sun, 18 Sep 2022 10:46:01 +0200 Subject: [PATCH 19/22] correct fork version --- .../built_in_network_configs/eip4844/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/eth2_network_config/built_in_network_configs/eip4844/config.yaml b/common/eth2_network_config/built_in_network_configs/eip4844/config.yaml index 8e779eac3d..d6e6aef57a 100644 --- a/common/eth2_network_config/built_in_network_configs/eip4844/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/eip4844/config.yaml @@ -36,7 +36,7 @@ ALTAIR_FORK_EPOCH: 1 BELLATRIX_FORK_VERSION: 0x02000ffd BELLATRIX_FORK_EPOCH: 2 # Sharding -EIP4844_FORK_VERSION: 0x03000ffd +EIP4844_FORK_VERSION: 0x83000ffd EIP4844_FORK_EPOCH: 3 # TBD, 2**32 is a placeholder. Merge transition approach is in active R&D. From 285dbf43edd6c7022a87b2f60ea7240c3920b7ad Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Sun, 18 Sep 2022 11:34:46 +0200 Subject: [PATCH 20/22] hacky hacks --- beacon_node/network/src/status.rs | 2 +- consensus/types/src/chain_spec.rs | 6 +++--- consensus/types/src/payload.rs | 15 ++------------- 3 files changed, 6 insertions(+), 17 deletions(-) diff --git a/beacon_node/network/src/status.rs b/beacon_node/network/src/status.rs index 865f8ee933..27689a7416 100644 --- a/beacon_node/network/src/status.rs +++ b/beacon_node/network/src/status.rs @@ -18,7 +18,7 @@ impl ToStatusMessage for BeaconChain { /// Build a `StatusMessage` representing the state of the given `beacon_chain`. pub(crate) fn status_message(beacon_chain: &BeaconChain) -> StatusMessage { - let fork_digest = beacon_chain.enr_fork_id().fork_digest; + let fork_digest = [0x9c, 0x67, 0x11, 0x28]; let cached_head = beacon_chain.canonical_head.cached_head(); let mut finalized_checkpoint = cached_head.finalized_checkpoint(); diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index b894bef12e..9e3d40c997 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -560,7 +560,7 @@ impl ChainSpec { domain_sync_committee: 7, domain_sync_committee_selection_proof: 8, domain_contribution_and_proof: 9, - altair_fork_version: [0x01, 0x00, 0x00, 0x00], + altair_fork_version: [0x01, 0x00, 0x0f, 0xfd], altair_fork_epoch: Some(Epoch::new(74240)), /* @@ -571,7 +571,7 @@ impl ChainSpec { min_slashing_penalty_quotient_bellatrix: u64::checked_pow(2, 5) .expect("pow does not overflow"), proportional_slashing_multiplier_bellatrix: 3, - bellatrix_fork_version: [0x02, 0x00, 0x00, 0x00], + bellatrix_fork_version: [0x02, 0x00, 0x0f, 0xfd], bellatrix_fork_epoch: Some(Epoch::new(144896)), terminal_total_difficulty: Uint256::from_dec_str("58750000000000000000000") .expect("terminal_total_difficulty is a valid integer"), @@ -583,7 +583,7 @@ impl ChainSpec { * Eip4844 hard fork params */ eip4844_fork_epoch: None, - eip4844_fork_version: [0x03, 0x00, 0x00, 0x00], + eip4844_fork_version: [0x83, 0x00, 0x0f, 0xfd], /* * Network specific diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index 23db2d961f..6f4e3a39e6 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -227,7 +227,8 @@ impl Encode for BlindedPayload { } } -#[derive(Default, Debug, Clone, Serialize, Deserialize)] +#[derive(Default, Debug, Clone, Serialize, Deserialize, Derivative)] +#[derivative(PartialEq, Hash(bound = "T: EthSpec"))] #[serde(bound = "T: EthSpec")] pub struct FullPayload { pub execution_payload: ExecutionPayload @@ -239,18 +240,6 @@ impl TestRandom for FullPayload { } } -impl PartialEq for FullPayload { - fn eq(&self, other: &FullPayload) -> bool { - todo!() - } -} - -impl Hash for FullPayload { - fn hash(&self, into: &mut H) { - todo!() - } -} - impl From> for FullPayload { fn from(execution_payload: ExecutionPayload) -> Self { Self { From 257087b01085afdadc3c2e8d2a4f5116f9a3a3f0 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Sun, 18 Sep 2022 11:43:53 +0200 Subject: [PATCH 21/22] correct fork version --- consensus/types/src/fork_context.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/consensus/types/src/fork_context.rs b/consensus/types/src/fork_context.rs index c5316a6567..c9e9bed0bf 100644 --- a/consensus/types/src/fork_context.rs +++ b/consensus/types/src/fork_context.rs @@ -47,11 +47,11 @@ impl ForkContext { )); } - if spec.bellatrix_fork_epoch.is_some() { + if spec.eip4844_fork_epoch.is_some() { fork_to_digest.push(( - ForkName::Merge, + ForkName::Eip4844, ChainSpec::compute_fork_digest( - spec.bellatrix_fork_version, + spec.eip4844_fork_version, genesis_validators_root, ), )); From 6f7d21c542defc103652e610f820d438e9414cd1 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Sun, 18 Sep 2022 12:13:03 +0200 Subject: [PATCH 22/22] enable 4844 at epoch 3 --- beacon_node/network/src/status.rs | 2 +- consensus/types/src/chain_spec.rs | 18 +++++++++++------- consensus/types/src/fork_name.rs | 3 ++- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/beacon_node/network/src/status.rs b/beacon_node/network/src/status.rs index 27689a7416..865f8ee933 100644 --- a/beacon_node/network/src/status.rs +++ b/beacon_node/network/src/status.rs @@ -18,7 +18,7 @@ impl ToStatusMessage for BeaconChain { /// Build a `StatusMessage` representing the state of the given `beacon_chain`. pub(crate) fn status_message(beacon_chain: &BeaconChain) -> StatusMessage { - let fork_digest = [0x9c, 0x67, 0x11, 0x28]; + let fork_digest = beacon_chain.enr_fork_id().fork_digest; let cached_head = beacon_chain.canonical_head.cached_head(); let mut finalized_checkpoint = cached_head.finalized_checkpoint(); diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index 9e3d40c997..e07d53bf8b 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -236,13 +236,17 @@ impl ChainSpec { /// Returns the name of the fork which is active at `epoch`. pub fn fork_name_at_epoch(&self, epoch: Epoch) -> ForkName { - match self.bellatrix_fork_epoch { - Some(fork_epoch) if epoch >= fork_epoch => ForkName::Merge, - _ => match self.altair_fork_epoch { - Some(fork_epoch) if epoch >= fork_epoch => ForkName::Altair, - _ => ForkName::Base, - }, + match self.eip4844_fork_epoch { + Some(fork_epoch) if epoch >= fork_epoch => ForkName::Eip4844, + _ => match self.bellatrix_fork_epoch { + Some(fork_epoch) if epoch >= fork_epoch => ForkName::Merge, + _ => match self.altair_fork_epoch { + Some(fork_epoch) if epoch >= fork_epoch => ForkName::Altair, + _ => ForkName::Base, + }, + } } + } /// Returns the fork version for a named fork. @@ -582,7 +586,7 @@ impl ChainSpec { /* * Eip4844 hard fork params */ - eip4844_fork_epoch: None, + eip4844_fork_epoch: Some(Epoch::new(3)), eip4844_fork_version: [0x83, 0x00, 0x0f, 0xfd], /* diff --git a/consensus/types/src/fork_name.rs b/consensus/types/src/fork_name.rs index 7afe0d7530..5e6ee3e5e9 100644 --- a/consensus/types/src/fork_name.rs +++ b/consensus/types/src/fork_name.rs @@ -42,7 +42,7 @@ impl ForkName { ForkName::Eip4844 => { spec.altair_fork_epoch = Some(Epoch::new(0)); spec.bellatrix_fork_epoch = Some(Epoch::new(0)); - spec.eip4844_fork_epoch = Some(Epoch::new(0)); + spec.eip4844_fork_epoch = Some(Epoch::new(3)); spec } } @@ -126,6 +126,7 @@ impl FromStr for ForkName { "phase0" | "base" => ForkName::Base, "altair" => ForkName::Altair, "bellatrix" | "merge" => ForkName::Merge, + "eip4844" => ForkName::Eip4844, _ => return Err(format!("unknown fork name: {}", fork_name)), }) }