gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-05-07 16:55:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'deneb-free-blobs' of https://github.com/sigp/lighthouse into some-blob-reprocessing-work

Browse Source
This commit is contained in:
realbigsean
2023-05-18 15:27:59 -04:00
parent 542e3c16db 9b55d74c1c
commit 4dd1184968
39 changed files with 3016 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
consensus/fork_choice/src/fork_choice.rs
View File

@@ -192,7 +192,8 @@ impl CountUnrealized {
/// Indicates if a block has been verified by an execution payload.
///
/// There is no variant for "invalid", since such a block should never be added to fork choice.
#[derive(Clone, Copy, Debug, PartialEq)]
#[derive(Clone, Copy, Debug, PartialEq, Encode, Decode)]
#[ssz(enum_behaviour = "tag")]
pub enum PayloadVerificationStatus {
/// An EL has declared the execution payload to be valid.
Verified,

145
consensus/ssz_derive/src/lib.rs
View File

@@ -4,6 +4,7 @@
//!
//! The following struct/enum attributes are available:
//!
//! - `#[ssz(enum_behaviour = "tag")]`: encodes and decodes an `enum` with 0 fields per variant
//! - `#[ssz(enum_behaviour = "union")]`: encodes and decodes an `enum` with a one-byte variant selector.
//! - `#[ssz(enum_behaviour = "transparent")]`: allows encoding an `enum` by serializing only the
//! value whilst ignoring outermost the `enum`.
@@ -140,6 +141,22 @@
//! TransparentEnum::Bar(vec![42, 42]).as_ssz_bytes(),
//! vec![42, 42]
//! );
//!
//! /// Representated as an SSZ "uint8"
//! #[derive(Debug, PartialEq, Encode, Decode)]
//! #[ssz(enum_behaviour = "tag")]
//! enum TagEnum {
//! Foo,
//! Bar,
//! }
//! assert_eq!(
//! TagEnum::Foo.as_ssz_bytes(),
//! vec![0]
//! );
//! assert_eq!(
//! TagEnum::from_ssz_bytes(&[1]).unwrap(),
//! TagEnum::Bar,
//! );
//! ```
use darling::{FromDeriveInput, FromMeta};
@@ -154,8 +171,9 @@ const MAX_UNION_SELECTOR: u8 = 127;
const ENUM_TRANSPARENT: &str = "transparent";
const ENUM_UNION: &str = "union";
const ENUM_TAG: &str = "tag";
const NO_ENUM_BEHAVIOUR_ERROR: &str = "enums require an \"enum_behaviour\" attribute with \
a \"transparent\" or \"union\" value, e.g., #[ssz(enum_behaviour = \"transparent\")]";
a \"transparent\", \"union\", or \"tag\" value, e.g., #[ssz(enum_behaviour = \"transparent\")]";
#[derive(Debug, FromDeriveInput)]
#[darling(attributes(ssz))]
@@ -196,6 +214,7 @@ enum StructBehaviour {
enum EnumBehaviour {
Union,
Transparent,
Tag,
}
impl<'a> Procedure<'a> {
@@ -237,6 +256,10 @@ impl<'a> Procedure<'a> {
data,
behaviour: EnumBehaviour::Transparent,
},
Some("tag") => Procedure::Enum {
data,
behaviour: EnumBehaviour::Tag,
},
Some(other) => panic!(
"{} is not a valid enum behaviour, use \"container\" or \"transparent\"",
other
@@ -296,6 +319,7 @@ pub fn ssz_encode_derive(input: TokenStream) -> TokenStream {
Procedure::Enum { data, behaviour } => match behaviour {
EnumBehaviour::Transparent => ssz_encode_derive_enum_transparent(&item, data),
EnumBehaviour::Union => ssz_encode_derive_enum_union(&item, data),
EnumBehaviour::Tag => ssz_encode_derive_enum_tag(&item, data),
},
}
}
@@ -573,6 +597,67 @@ fn ssz_encode_derive_enum_transparent(
output.into()
}
/// Derive `ssz::Encode` for an `enum` following the "tag" method.
///
/// The union selector will be determined based upon the order in which the enum variants are
/// defined. E.g., the top-most variant in the enum will have a selector of `0`, the variant
/// beneath it will have a selector of `1` and so on.
///
/// # Limitations
///
/// Only supports enums where each variant has no fields
fn ssz_encode_derive_enum_tag(derive_input: &DeriveInput, enum_data: &DataEnum) -> TokenStream {
let name = &derive_input.ident;
let (impl_generics, ty_generics, where_clause) = &derive_input.generics.split_for_impl();
let patterns: Vec<_> = enum_data
.variants
.iter()
.map(|variant| {
let variant_name = &variant.ident;
if !variant.fields.is_empty() {
panic!("ssz::Encode tag behaviour can only be derived for enums with no fields");
}
quote! {
#name::#variant_name
}
})
.collect();
let union_selectors = compute_union_selectors(patterns.len());
let output = quote! {
impl #impl_generics ssz::Encode for #name #ty_generics #where_clause {
fn is_ssz_fixed_len() -> bool {
true
}
fn ssz_fixed_len() -> usize {
1
}
fn ssz_bytes_len(&self) -> usize {
1
}
fn ssz_append(&self, buf: &mut Vec<u8>) {
match self {
#(
#patterns => {
let union_selector: u8 = #union_selectors;
debug_assert!(union_selector <= ssz::MAX_UNION_SELECTOR);
buf.push(union_selector);
},
)*
}
}
}
};
output.into()
}
/// Derive `ssz::Encode` for an `enum` following the "union" SSZ spec.
///
/// The union selector will be determined based upon the order in which the enum variants are
@@ -652,9 +737,10 @@ pub fn ssz_decode_derive(input: TokenStream) -> TokenStream {
},
Procedure::Enum { data, behaviour } => match behaviour {
EnumBehaviour::Union => ssz_decode_derive_enum_union(&item, data),
EnumBehaviour::Tag => ssz_decode_derive_enum_tag(&item, data),
EnumBehaviour::Transparent => panic!(
"Decode cannot be derived for enum_behaviour \"{}\", only \"{}\" is valid.",
ENUM_TRANSPARENT, ENUM_UNION
"Decode cannot be derived for enum_behaviour \"{}\", only \"{}\" and \"{}\" is valid.",
ENUM_TRANSPARENT, ENUM_UNION, ENUM_TAG,
),
},
}
@@ -908,6 +994,59 @@ fn ssz_decode_derive_struct_transparent(
output.into()
}
/// Derive `ssz::Decode` for an `enum` following the "tag" SSZ spec.
fn ssz_decode_derive_enum_tag(derive_input: &DeriveInput, enum_data: &DataEnum) -> TokenStream {
let name = &derive_input.ident;
let (impl_generics, ty_generics, where_clause) = &derive_input.generics.split_for_impl();
let patterns: Vec<_> = enum_data
.variants
.iter()
.map(|variant| {
let variant_name = &variant.ident;
if !variant.fields.is_empty() {
panic!("ssz::Decode tag behaviour can only be derived for enums with no fields");
}
quote! {
#name::#variant_name
}
})
.collect();
let union_selectors = compute_union_selectors(patterns.len());
let output = quote! {
impl #impl_generics ssz::Decode for #name #ty_generics #where_clause {
fn is_ssz_fixed_len() -> bool {
true
}
fn ssz_fixed_len() -> usize {
1
}
fn from_ssz_bytes(bytes: &[u8]) -> std::result::Result<Self, ssz::DecodeError> {
let byte = bytes
.first()
.copied()
.ok_or(ssz::DecodeError::OutOfBoundsByte { i: 0 })?;
match byte {
#(
#union_selectors => {
Ok(#patterns)
},
)*
other => Err(ssz::DecodeError::UnionSelectorInvalid(other)),
}
}
}
};
output.into()
}
/// Derive `ssz::Decode` for an `enum` following the "union" SSZ spec.
fn ssz_decode_derive_enum_union(derive_input: &DeriveInput, enum_data: &DataEnum) -> TokenStream {
let name = &derive_input.ident;

15
consensus/ssz_derive/tests/tests.rs
View File

@@ -12,6 +12,14 @@ fn assert_encode_decode<T: Encode + Decode + PartialEq + Debug>(item: &T, bytes:
assert_eq!(T::from_ssz_bytes(bytes).unwrap(), *item);
}
#[derive(PartialEq, Debug, Encode, Decode)]
#[ssz(enum_behaviour = "tag")]
enum TagEnum {
A,
B,
C,
}
#[derive(PartialEq, Debug, Encode, Decode)]
#[ssz(enum_behaviour = "union")]
enum TwoFixedUnion {
@@ -120,6 +128,13 @@ fn two_variable_union() {
);
}
#[test]
fn tag_enum() {
assert_encode_decode(&TagEnum::A, &[0]);
assert_encode_decode(&TagEnum::B, &[1]);
assert_encode_decode(&TagEnum::C, &[2]);
}
#[derive(PartialEq, Debug, Encode, Decode)]
#[ssz(enum_behaviour = "union")]
enum TwoVecUnion {

5
consensus/state_processing/src/consensus_context.rs
View File

@@ -1,5 +1,6 @@
use crate::common::get_indexed_attestation;
use crate::per_block_processing::errors::{AttestationInvalid, BlockOperationError};
use ssz_derive::{Decode, Encode};
use std::collections::{hash_map::Entry, HashMap};
use tree_hash::TreeHash;
use types::{
@@ -7,7 +8,7 @@ use types::{
ChainSpec, Epoch, EthSpec, Hash256, IndexedAttestation, SignedBeaconBlock, Slot,
};
#[derive(Debug, Clone)]
#[derive(Debug, PartialEq, Clone, Encode, Decode)]
pub struct ConsensusContext<T: EthSpec> {
/// Slot to act as an identifier/safeguard
slot: Slot,
@@ -16,6 +17,8 @@ pub struct ConsensusContext<T: EthSpec> {
/// Block root of the block at `slot`.
current_block_root: Option<Hash256>,
/// Cache of indexed attestations constructed during block processing.
/// We can skip serializing / deserializing this as the cache will just be rebuilt
#[ssz(skip_serializing, skip_deserializing)]
indexed_attestations:
HashMap<(AttestationData, BitList<T::MaxValidatorsPerCommittee>), IndexedAttestation<T>>,
/// Whether `verify_kzg_commitments_against_transactions` has successfully passed.

21
consensus/types/src/beacon_block.rs
View File

@@ -201,13 +201,7 @@ impl<'a, T: EthSpec, Payload: AbstractExecPayload<T>> BeaconBlockRef<'a, T, Payl
/// dictated by `self.slot()`.
pub fn fork_name(&self, spec: &ChainSpec) -> Result<ForkName, InconsistentFork> {
let fork_at_slot = spec.fork_name_at_slot::<T>(self.slot());
let object_fork = match self {
BeaconBlockRef::Base { .. } => ForkName::Base,
BeaconBlockRef::Altair { .. } => ForkName::Altair,
BeaconBlockRef::Merge { .. } => ForkName::Merge,
BeaconBlockRef::Capella { .. } => ForkName::Capella,
BeaconBlockRef::Deneb { .. } => ForkName::Deneb,
};
let object_fork = self.fork_name_unchecked();
if fork_at_slot == object_fork {
Ok(object_fork)
@@ -219,6 +213,19 @@ impl<'a, T: EthSpec, Payload: AbstractExecPayload<T>> BeaconBlockRef<'a, T, Payl
}
}
/// Returns the name of the fork pertaining to `self`.
///
/// Does not check that the fork is consistent with the slot.
pub fn fork_name_unchecked(&self) -> ForkName {
match self {
BeaconBlockRef::Base { .. } => ForkName::Base,
BeaconBlockRef::Altair { .. } => ForkName::Altair,
BeaconBlockRef::Merge { .. } => ForkName::Merge,
BeaconBlockRef::Capella { .. } => ForkName::Capella,
BeaconBlockRef::Deneb { .. } => ForkName::Deneb,
}
}
/// Convenience accessor for the `body` as a `BeaconBlockBodyRef`.
pub fn body(&self) -> BeaconBlockBodyRef<'a, T, Payload> {
map_beacon_block_ref_into_beacon_block_body_ref!(&'a _, *self, |block, cons| cons(

98
consensus/types/src/beacon_state.rs
View File

@@ -415,13 +415,7 @@ impl<T: EthSpec> BeaconState<T> {
/// dictated by `self.slot()`.
pub fn fork_name(&self, spec: &ChainSpec) -> Result<ForkName, InconsistentFork> {
let fork_at_slot = spec.fork_name_at_epoch(self.current_epoch());
let object_fork = match self {
BeaconState::Base { .. } => ForkName::Base,
BeaconState::Altair { .. } => ForkName::Altair,
BeaconState::Merge { .. } => ForkName::Merge,
BeaconState::Capella { .. } => ForkName::Capella,
BeaconState::Deneb { .. } => ForkName::Deneb,
};
let object_fork = self.fork_name_unchecked();
if fork_at_slot == object_fork {
Ok(object_fork)
@@ -433,6 +427,19 @@ impl<T: EthSpec> BeaconState<T> {
}
}
/// Returns the name of the fork pertaining to `self`.
///
/// Does not check if `self` is consistent with the fork dictated by `self.slot()`.
pub fn fork_name_unchecked(&self) -> ForkName {
match self {
BeaconState::Base { .. } => ForkName::Base,
BeaconState::Altair { .. } => ForkName::Altair,
BeaconState::Merge { .. } => ForkName::Merge,
BeaconState::Capella { .. } => ForkName::Capella,
BeaconState::Deneb { .. } => ForkName::Deneb,
}
}
/// Specialised deserialisation method that uses the `ChainSpec` as context.
#[allow(clippy::integer_arithmetic)]
pub fn from_ssz_bytes(bytes: &[u8], spec: &ChainSpec) -> Result<Self, ssz::DecodeError> {
@@ -1870,3 +1877,80 @@ impl<T: EthSpec> ForkVersionDeserialize for BeaconState<T> {
))
}
}
/// This module can be used to encode and decode a `BeaconState` the same way it
/// would be done if we had tagged the superstruct enum with
/// `#[ssz(enum_behaviour = "union")]`
/// This should _only_ be used for *some* cases to store these objects in the
/// database and _NEVER_ for encoding / decoding states sent over the network!
pub mod ssz_tagged_beacon_state {
use super::*;
pub mod encode {
use super::*;
#[allow(unused_imports)]
use ssz::*;
pub fn is_ssz_fixed_len() -> bool {
false
}
pub fn ssz_fixed_len() -> usize {
BYTES_PER_LENGTH_OFFSET
}
pub fn ssz_bytes_len<E: EthSpec>(state: &BeaconState<E>) -> usize {
state
.ssz_bytes_len()
.checked_add(1)
.expect("encoded length must be less than usize::max")
}
pub fn ssz_append<E: EthSpec>(state: &BeaconState<E>, buf: &mut Vec<u8>) {
let fork_name = state.fork_name_unchecked();
fork_name.ssz_append(buf);
state.ssz_append(buf);
}
pub fn as_ssz_bytes<E: EthSpec>(state: &BeaconState<E>) -> Vec<u8> {
let mut buf = vec![];
ssz_append(state, &mut buf);
buf
}
}
pub mod decode {
use super::*;
#[allow(unused_imports)]
use ssz::*;
pub fn is_ssz_fixed_len() -> bool {
false
}
pub fn ssz_fixed_len() -> usize {
BYTES_PER_LENGTH_OFFSET
}
pub fn from_ssz_bytes<E: EthSpec>(bytes: &[u8]) -> Result<BeaconState<E>, DecodeError> {
let fork_byte = bytes
.first()
.copied()
.ok_or(DecodeError::OutOfBoundsByte { i: 0 })?;
let body = bytes
.get(1..)
.ok_or(DecodeError::OutOfBoundsByte { i: 1 })?;
match ForkName::from_ssz_bytes(&[fork_byte])? {
ForkName::Base => Ok(BeaconState::Base(BeaconStateBase::from_ssz_bytes(body)?)),
ForkName::Altair => Ok(BeaconState::Altair(BeaconStateAltair::from_ssz_bytes(
body,
)?)),
ForkName::Merge => Ok(BeaconState::Merge(BeaconStateMerge::from_ssz_bytes(body)?)),
ForkName::Capella => Ok(BeaconState::Capella(BeaconStateCapella::from_ssz_bytes(
body,
)?)),
ForkName::Deneb => Ok(BeaconState::Deneb(BeaconStateDeneb::from_ssz_bytes(body)?)),
}
}
}
}

4
consensus/types/src/fork_name.rs
View File

@@ -1,12 +1,14 @@
use crate::{ChainSpec, Epoch};
use serde_derive::{Deserialize, Serialize};
use ssz_derive::{Decode, Encode};
use std::convert::TryFrom;
use std::fmt::{self, Display, Formatter};
use std::str::FromStr;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[derive(Debug, Clone, Copy, Decode, Encode, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(try_from = "String")]
#[serde(into = "String")]
#[ssz(enum_behaviour = "tag")]
pub enum ForkName {
Base,
Altair,

6
consensus/types/src/lib.rs
View File

@@ -170,9 +170,9 @@ pub use crate::selection_proof::SelectionProof;
pub use crate::shuffling_id::AttestationShufflingId;
pub use crate::signed_aggregate_and_proof::SignedAggregateAndProof;
pub use crate::signed_beacon_block::{
SignedBeaconBlock, SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockCapella,
SignedBeaconBlockDeneb, SignedBeaconBlockHash, SignedBeaconBlockMerge,
SignedBlindedBeaconBlock,
ssz_tagged_signed_beacon_block, SignedBeaconBlock, SignedBeaconBlockAltair,
SignedBeaconBlockBase, SignedBeaconBlockCapella, SignedBeaconBlockDeneb, SignedBeaconBlockHash,
SignedBeaconBlockMerge, SignedBlindedBeaconBlock,
};
pub use crate::signed_beacon_block_header::SignedBeaconBlockHeader;
pub use crate::signed_blob::*;

133
consensus/types/src/signed_beacon_block.rs
View File

@@ -93,6 +93,12 @@ impl<E: EthSpec, Payload: AbstractExecPayload<E>> SignedBeaconBlock<E, Payload>
self.message().fork_name(spec)
}
/// Returns the name of the fork pertaining to `self`
/// Does not check that the fork is consistent with the slot.
pub fn fork_name_unchecked(&self) -> ForkName {
self.message().fork_name_unchecked()
}
/// SSZ decode with fork variant determined by slot.
pub fn from_ssz_bytes(bytes: &[u8], spec: &ChainSpec) -> Result<Self, ssz::DecodeError> {
Self::from_ssz_bytes_with(bytes, |bytes| BeaconBlock::from_ssz_bytes(bytes, spec))
@@ -543,6 +549,99 @@ impl<E: EthSpec, Payload: AbstractExecPayload<E>> ForkVersionDeserialize
}
}
/// This module can be used to encode and decode a `SignedBeaconBlock` the same way it
/// would be done if we had tagged the superstruct enum with
/// `#[ssz(enum_behaviour = "union")]`
/// This should _only_ be used *some* cases when storing these objects in the database
/// and _NEVER_ for encoding / decoding blocks sent over the network!
pub mod ssz_tagged_signed_beacon_block {
use super::*;
pub mod encode {
use super::*;
#[allow(unused_imports)]
use ssz::*;
pub fn is_ssz_fixed_len() -> bool {
false
}
pub fn ssz_fixed_len() -> usize {
BYTES_PER_LENGTH_OFFSET
}
pub fn ssz_bytes_len<E: EthSpec, Payload: AbstractExecPayload<E>>(
block: &SignedBeaconBlock<E, Payload>,
) -> usize {
block
.ssz_bytes_len()
.checked_add(1)
.expect("encoded length must be less than usize::max")
}
pub fn ssz_append<E: EthSpec, Payload: AbstractExecPayload<E>>(
block: &SignedBeaconBlock<E, Payload>,
buf: &mut Vec<u8>,
) {
let fork_name = block.fork_name_unchecked();
fork_name.ssz_append(buf);
block.ssz_append(buf);
}
pub fn as_ssz_bytes<E: EthSpec, Payload: AbstractExecPayload<E>>(
block: &SignedBeaconBlock<E, Payload>,
) -> Vec<u8> {
let mut buf = vec![];
ssz_append(block, &mut buf);
buf
}
}
pub mod decode {
use super::*;
#[allow(unused_imports)]
use ssz::*;
pub fn is_ssz_fixed_len() -> bool {
false
}
pub fn ssz_fixed_len() -> usize {
BYTES_PER_LENGTH_OFFSET
}
pub fn from_ssz_bytes<E: EthSpec, Payload: AbstractExecPayload<E>>(
bytes: &[u8],
) -> Result<SignedBeaconBlock<E, Payload>, DecodeError> {
let fork_byte = bytes
.first()
.copied()
.ok_or(DecodeError::OutOfBoundsByte { i: 0 })?;
let body = bytes
.get(1..)
.ok_or(DecodeError::OutOfBoundsByte { i: 1 })?;
match ForkName::from_ssz_bytes(&[fork_byte])? {
ForkName::Base => Ok(SignedBeaconBlock::Base(
SignedBeaconBlockBase::from_ssz_bytes(body)?,
)),
ForkName::Altair => Ok(SignedBeaconBlock::Altair(
SignedBeaconBlockAltair::from_ssz_bytes(body)?,
)),
ForkName::Merge => Ok(SignedBeaconBlock::Merge(
SignedBeaconBlockMerge::from_ssz_bytes(body)?,
)),
ForkName::Capella => Ok(SignedBeaconBlock::Capella(
SignedBeaconBlockCapella::from_ssz_bytes(body)?,
)),
ForkName::Deneb => Ok(SignedBeaconBlock::Deneb(
SignedBeaconBlockDeneb::from_ssz_bytes(body)?,
)),
}
}
}
}
#[cfg(test)]
mod test {
use super::*;
@@ -584,4 +683,38 @@ mod test {
assert_eq!(reconstructed, block);
}
}
#[test]
fn test_ssz_tagged_signed_beacon_block() {
type E = MainnetEthSpec;
let spec = &E::default_spec();
let sig = Signature::empty();
let blocks = vec![
SignedBeaconBlock::<E>::from_block(
BeaconBlock::Base(BeaconBlockBase::empty(spec)),
sig.clone(),
),
SignedBeaconBlock::from_block(
BeaconBlock::Altair(BeaconBlockAltair::empty(spec)),
sig.clone(),
),
SignedBeaconBlock::from_block(
BeaconBlock::Merge(BeaconBlockMerge::empty(spec)),
sig.clone(),
),
SignedBeaconBlock::from_block(
BeaconBlock::Capella(BeaconBlockCapella::empty(spec)),
sig.clone(),
),
SignedBeaconBlock::from_block(BeaconBlock::Deneb(BeaconBlockDeneb::empty(spec)), sig),
];
for block in blocks {
let encoded = ssz_tagged_signed_beacon_block::encode::as_ssz_bytes(&block);
let decoded = ssz_tagged_signed_beacon_block::decode::from_ssz_bytes::<E, _>(&encoded)
.expect("should decode");
assert_eq!(decoded, block);
}
}
}