gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-07-05 05:44:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'deneb-free-blobs' of https://github.com/sigp/lighthouse into some-blob-reprocessing-work

Browse Source
This commit is contained in:
realbigsean
2023-05-18 15:27:59 -04:00
parent 542e3c16db 9b55d74c1c
commit 4dd1184968
39 changed files with 3016 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Cargo.lock generated
View File

@@ -2004,6 +2004,8 @@ dependencies = [
"compare_fields",
"compare_fields_derive",
"derivative",
"eth2_network_config",
"eth2_serde_utils",
"eth2_ssz",
"eth2_ssz_derive",
"ethereum-types 0.14.1",
@@ -2011,9 +2013,11 @@ dependencies = [
"fork_choice",
"fs2",
"hex",
"kzg",
"rayon",
"serde",
"serde_derive",
"serde_json",
"serde_repr",
"serde_yaml",
"snap",

10
beacon_node/beacon_chain/src/beacon_chain.rs
View File

@@ -464,7 +464,7 @@ pub struct BeaconChain<T: BeaconChainTypes> {
/// Provides monitoring of a set of explicitly defined validators.
pub validator_monitor: RwLock<ValidatorMonitor<T::EthSpec>>,
pub proposal_blob_cache: BlobCache<T::EthSpec>,
pub data_availability_checker: Arc<DataAvailabilityChecker<T::EthSpec, T::SlotClock>>,
pub data_availability_checker: Arc<DataAvailabilityChecker<T>>,
pub kzg: Option<Arc<Kzg>>,
}
@@ -608,6 +608,13 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
Ok(())
}
pub fn persist_data_availabilty_checker(&self) -> Result<(), Error> {
let _timer = metrics::start_timer(&metrics::PERSIST_DATA_AVAILABILITY_CHECKER);
self.data_availability_checker.persist_all()?;
Ok(())
}
/// Returns the slot _right now_ according to `self.slot_clock`. Returns `Err` if the slot is
/// unavailable.
///
@@ -6279,6 +6286,7 @@ impl<T: BeaconChainTypes> Drop for BeaconChain<T> {
let drop = || -> Result<(), Error> {
self.persist_head_and_fork_choice()?;
self.persist_op_pool()?;
self.persist_data_availabilty_checker()?;
self.persist_eth1_cache()
};

4
beacon_node/beacon_chain/src/blob_verification.rs
View File

@@ -15,6 +15,7 @@ use crate::BeaconChainError;
use eth2::types::BlockContentsTuple;
use kzg::Kzg;
use slog::{debug, warn};
use ssz_derive::{Decode, Encode};
use ssz_types::FixedVector;
use std::borrow::Cow;
use types::blob_sidecar::{BlobIdentifier, FixedBlobSidecarList};
@@ -403,8 +404,9 @@ fn cheap_state_advance_to_obtain_committees<'a, E: EthSpec>(
/// Wrapper over a `BlobSidecar` for which we have completed kzg verification.
/// i.e. `verify_blob_kzg_proof(blob, commitment, proof) == true`.
#[derive(Debug, Derivative, Clone)]
#[derive(Debug, Derivative, Clone, Encode, Decode)]
#[derivative(PartialEq, Eq)]
#[ssz(struct_behaviour = "transparent")]
pub struct KzgVerifiedBlob<T: EthSpec> {
blob: Arc<BlobSidecar<T>>,
}

11
beacon_node/beacon_chain/src/block_verification.rs
View File

@@ -77,6 +77,7 @@ use safe_arith::ArithError;
use slog::{debug, error, warn, Logger};
use slot_clock::SlotClock;
use ssz::Encode;
use ssz_derive::{Decode, Encode};
use state_processing::per_block_processing::{errors::IntoWithIndex, is_merge_transition_block};
use state_processing::{
block_signature_verifier::{BlockSignatureVerifier, Error as BlockSignatureVerifierError},
@@ -95,6 +96,7 @@ use task_executor::JoinHandle;
use tree_hash::TreeHash;
use types::blob_sidecar::BlobIdentifier;
use types::ExecPayload;
use types::{ssz_tagged_beacon_state, ssz_tagged_signed_beacon_block};
use types::{
BeaconBlockRef, BeaconState, BeaconStateError, BlindedPayload, ChainSpec, CloneConfig, Epoch,
EthSpec, ExecutionBlockHash, Hash256, InconsistentFork, PublicKey, PublicKeyBytes,
@@ -500,7 +502,7 @@ impl<T: EthSpec> From<ArithError> for BlockError<T> {
}
/// Stores information about verifying a payload against an execution engine.
#[derive(Clone)]
#[derive(Debug, PartialEq, Clone, Encode, Decode)]
pub struct PayloadVerificationOutcome {
pub payload_verification_status: PayloadVerificationStatus,
pub is_valid_merge_transition_block: bool,
@@ -719,6 +721,7 @@ impl<E: EthSpec> ExecutedBlock<E> {
}
}
#[derive(Debug, PartialEq)]
pub struct AvailableExecutedBlock<E: EthSpec> {
pub block: AvailableBlock<E>,
pub import_data: BlockImportData<E>,
@@ -756,6 +759,7 @@ impl<E: EthSpec> AvailableExecutedBlock<E> {
}
}
#[derive(Encode, Decode, Clone)]
pub struct AvailabilityPendingExecutedBlock<E: EthSpec> {
pub block: AvailabilityPendingBlock<E>,
pub import_data: BlockImportData<E>,
@@ -796,9 +800,14 @@ impl<E: EthSpec> AvailabilityPendingExecutedBlock<E> {
}
}
#[derive(Debug, PartialEq, Encode, Decode, Clone)]
// TODO (mark): investigate using an Arc<state> / Arc<parent_block>
// here to make this cheaper to clone
pub struct BlockImportData<E: EthSpec> {
pub block_root: Hash256,
#[ssz(with = "ssz_tagged_beacon_state")]
pub state: BeaconState<E>,
#[ssz(with = "ssz_tagged_signed_beacon_block")]
pub parent_block: SignedBeaconBlock<E, BlindedPayload<E>>,
pub parent_eth1_finalization_data: Eth1FinalizationData,
pub confirmed_state_roots: Vec<Hash256>,

5
beacon_node/beacon_chain/src/builder.rs
View File

@@ -794,7 +794,7 @@ where
let beacon_chain = BeaconChain {
spec: self.spec.clone(),
config: self.chain_config,
store,
store: store.clone(),
task_executor: self
.task_executor
.ok_or("Cannot build without task executor")?,
@@ -864,8 +864,9 @@ where
data_availability_checker: Arc::new(DataAvailabilityChecker::new(
slot_clock,
kzg.clone(),
store,
self.spec,
)),
).map_err(|e| format!("Error initializing DataAvailabiltyChecker: {:?}", e))?),
proposal_blob_cache: BlobCache::default(),
kzg,
};

340
beacon_node/beacon_chain/src/data_availability_checker.rs
View File

@@ -4,24 +4,32 @@ use crate::blob_verification::{
};
use crate::block_verification::{AvailabilityPendingExecutedBlock, AvailableExecutedBlock};
use crate::data_availability_checker::overflow_lru_cache::OverflowLRUCache;
use crate::{BeaconChain, BeaconChainTypes, BeaconStore};
use kzg::Error as KzgError;
use kzg::Kzg;
use parking_lot::RwLock;
use slog::{debug, error};
use slot_clock::SlotClock;
use ssz_types::{Error, FixedVector, VariableList};
use ssz_types::{Error, VariableList};
use state_processing::per_block_processing::deneb::deneb::verify_kzg_commitments_against_transactions;
use std::collections::hash_map::{Entry, OccupiedEntry};
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use task_executor::TaskExecutor;
use strum::IntoStaticStr;
use types::beacon_block_body::KzgCommitments;
use types::blob_sidecar::{BlobIdentifier, BlobSidecar, FixedBlobSidecarList};
use types::consts::deneb::MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS;
use types::ssz_tagged_signed_beacon_block;
use types::{
BeaconBlockRef, BlobSidecarList, ChainSpec, Epoch, EthSpec, ExecPayload, FullPayload, Hash256,
SignedBeaconBlock, SignedBeaconBlockHeader, Slot,
};
mod overflow_lru_cache;
pub const OVERFLOW_LRU_CAPACITY: usize = 1024;
#[derive(Debug, IntoStaticStr)]
pub enum AvailabilityCheckError {
Kzg(KzgError),
@@ -38,6 +46,9 @@ pub enum AvailabilityCheckError {
blob_index: u64,
},
IncorrectFork,
BlobIndexInvalid(u64),
StoreError(store::Error),
DecodeError(ssz::DecodeError),
BlockBlobRootMismatch {
block_root: Hash256,
blob_block_root: Hash256,
@@ -54,14 +65,26 @@ impl From<ssz_types::Error> for AvailabilityCheckError {
}
}
impl From<store::Error> for AvailabilityCheckError {
fn from(value: store::Error) -> Self {
Self::StoreError(value)
}
}
impl From<ssz::DecodeError> for AvailabilityCheckError {
fn from(value: ssz::DecodeError) -> Self {
Self::DecodeError(value)
}
}
/// This cache contains
/// - blobs that have been gossip verified
/// - commitments for blocks that have been gossip verified, but the commitments themselves
/// have not been verified against blobs
/// - blocks that have been fully verified and only require a data availability check
pub struct DataAvailabilityChecker<T: EthSpec, S: SlotClock> {
availability_cache: RwLock<HashMap<Hash256, ReceivedComponents<T>>>,
slot_clock: S,
pub struct DataAvailabilityChecker<T: BeaconChainTypes> {
availability_cache: Arc<OverflowLRUCache<T>>,
slot_clock: T::SlotClock,
kzg: Option<Arc<Kzg>>,
spec: ChainSpec,
}
@@ -120,19 +143,38 @@ impl<T: EthSpec> ReceivedComponents<T> {
///
/// Indicates if the block is fully `Available` or if we need blobs or blocks
/// to "complete" the requirements for an `AvailableBlock`.
#[derive(Debug, PartialEq)]
pub enum Availability<T: EthSpec> {
MissingComponents(Hash256),
Available(Box<AvailableExecutedBlock<T>>),
}
impl<T: EthSpec, S: SlotClock> DataAvailabilityChecker<T, S> {
pub fn new(slot_clock: S, kzg: Option<Arc<Kzg>>, spec: ChainSpec) -> Self {
Self {
availability_cache: <_>::default(),
impl<T: EthSpec> Availability<T> {
/// Returns all the blob identifiers associated with an `AvailableBlock`.
/// Returns `None` if avaiability hasn't been fully satisfied yet.
pub fn get_available_blob_ids(&self) -> Option<Vec<BlobIdentifier>> {
if let Self::Available(block) = self {
Some(block.get_all_blob_ids())
} else {
None
}
}
}
impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
pub fn new(
slot_clock: T::SlotClock,
kzg: Option<Arc<Kzg>>,
store: BeaconStore<T>,
spec: ChainSpec,
) -> Result<Self, AvailabilityCheckError> {
let overflow_cache = OverflowLRUCache::new(OVERFLOW_LRU_CAPACITY, store)?;
Ok(Self {
availability_cache: Arc::new(overflow_cache),
slot_clock,
kzg,
spec,
}
})
}
pub fn has_block(&self, block_root: &Hash256) -> bool {
@@ -201,14 +243,11 @@ impl<T: EthSpec, S: SlotClock> DataAvailabilityChecker<T, S> {
}
/// Get a blob from the availability cache.
pub fn get_blob(&self, blob_id: &BlobIdentifier) -> Option<Arc<BlobSidecar<T>>> {
self.availability_cache
.read()
.get(&blob_id.block_root)?
.verified_blobs
.get(blob_id.index as usize)?
.as_ref()
.map(|kzg_verified_blob| kzg_verified_blob.clone_blob())
pub fn get_blob(
&self,
blob_id: &BlobIdentifier,
) -> Result<Option<Arc<BlobSidecar<T::EthSpec>>>, AvailabilityCheckError> {
self.availability_cache.peek_blob(blob_id)
}
pub fn put_rpc_blobs(
@@ -238,8 +277,8 @@ impl<T: EthSpec, S: SlotClock> DataAvailabilityChecker<T, S> {
/// This should only accept gossip verified blobs, so we should not have to worry about dupes.
pub fn put_gossip_blob(
&self,
gossip_blob: GossipVerifiedBlob<T>,
) -> Result<Availability<T>, AvailabilityCheckError> {
gossip_blob: GossipVerifiedBlob<T::EthSpec>,
) -> Result<Availability<T::EthSpec>, AvailabilityCheckError> {
// Verify the KZG commitments.
let kzg_verified_blob = if let Some(kzg) = self.kzg.as_ref() {
verify_kzg_for_blob(gossip_blob.to_blob(), kzg)?
@@ -247,6 +286,8 @@ impl<T: EthSpec, S: SlotClock> DataAvailabilityChecker<T, S> {
return Err(AvailabilityCheckError::KzgNotInitialized);
};
self.availability_cache
.put_kzg_verified_blob(kzg_verified_blob)
self.put_kzg_verified_blobs(kzg_verified_blob.block_root(), &[kzg_verified_blob])
}
@@ -299,24 +340,10 @@ impl<T: EthSpec, S: SlotClock> DataAvailabilityChecker<T, S> {
/// triggers import of the block.
pub fn put_pending_executed_block(
&self,
executed_block: AvailabilityPendingExecutedBlock<T>,
) -> Result<Availability<T>, AvailabilityCheckError> {
let availability = match self
.availability_cache
.write()
.entry(executed_block.import_data.block_root)
{
Entry::Occupied(occupied_entry) => {
self.check_block_availability_maybe_cache(occupied_entry, executed_block)?
}
Entry::Vacant(vacant_entry) => {
let block_root = executed_block.import_data.block_root;
vacant_entry.insert(ReceivedComponents::new_from_block(executed_block));
Availability::MissingComponents(block_root)
}
};
Ok(availability)
executed_block: AvailabilityPendingExecutedBlock<T::EthSpec>,
) -> Result<Availability<T::EthSpec>, AvailabilityCheckError> {
self.availability_cache
.put_pending_executed_block(executed_block)
}
/// Checks if the provided `executed_block` contains all required blobs to be considered an
@@ -376,8 +403,8 @@ impl<T: EthSpec, S: SlotClock> DataAvailabilityChecker<T, S> {
/// available block.
pub fn check_availability(
&self,
block: BlockWrapper<T>,
) -> Result<MaybeAvailableBlock<T>, AvailabilityCheckError> {
block: BlockWrapper<T::EthSpec>,
) -> Result<MaybeAvailableBlock<T::EthSpec>, AvailabilityCheckError> {
match block {
BlockWrapper::Block(block) => self.check_availability_without_blobs(block),
BlockWrapper::BlockAndBlobs(block, blob_list) => {
@@ -395,17 +422,38 @@ impl<T: EthSpec, S: SlotClock> DataAvailabilityChecker<T, S> {
}
}
/// Checks if a block is available, returning an error if the block is not immediately available.
/// Does not access the gossip cache.
pub fn try_check_availability(
&self,
block: BlockWrapper<T::EthSpec>,
) -> Result<AvailableBlock<T::EthSpec>, AvailabilityCheckError> {
match block {
BlockWrapper::Block(block) => {
let blob_requirements = self.get_blob_requirements(&block)?;
let blobs = match blob_requirements {
BlobRequirements::EmptyBlobs => VerifiedBlobs::EmptyBlobs,
BlobRequirements::NotRequired => VerifiedBlobs::NotRequired,
BlobRequirements::PreDeneb => VerifiedBlobs::PreDeneb,
BlobRequirements::Required => return Err(AvailabilityCheckError::MissingBlobs),
};
Ok(AvailableBlock { block, blobs })
}
BlockWrapper::BlockAndBlobs(_, _) => Err(AvailabilityCheckError::Pending),
}
}
/// Verifies a block against a set of KZG verified blobs. Returns an AvailableBlock if block's
/// commitments are consistent with the provided verified blob commitments.
pub fn check_availability_with_blobs(
&self,
block: Arc<SignedBeaconBlock<T>>,
blobs: KzgVerifiedBlobList<T>,
) -> Result<AvailableBlock<T>, AvailabilityCheckError> {
block: Arc<SignedBeaconBlock<T::EthSpec>>,
blobs: KzgVerifiedBlobList<T::EthSpec>,
) -> Result<AvailableBlock<T::EthSpec>, AvailabilityCheckError> {
match self.check_availability_without_blobs(block)? {
MaybeAvailableBlock::Available(block) => Ok(block),
MaybeAvailableBlock::AvailabilityPending(pending_block) => {
self.make_available(pending_block, blobs)
pending_block.make_available(blobs)
}
}
}
@@ -414,8 +462,8 @@ impl<T: EthSpec, S: SlotClock> DataAvailabilityChecker<T, S> {
/// an AvailableBlock if no blobs are required. Otherwise this will return an AvailabilityPendingBlock.
pub fn check_availability_without_blobs(
&self,
block: Arc<SignedBeaconBlock<T>>,
) -> Result<MaybeAvailableBlock<T>, AvailabilityCheckError> {
block: Arc<SignedBeaconBlock<T::EthSpec>>,
) -> Result<MaybeAvailableBlock<T::EthSpec>, AvailabilityCheckError> {
let blob_requirements = self.get_blob_requirements(&block)?;
let blobs = match blob_requirements {
BlobRequirements::EmptyBlobs => VerifiedBlobs::EmptyBlobs,
@@ -433,50 +481,18 @@ impl<T: EthSpec, S: SlotClock> DataAvailabilityChecker<T, S> {
}))
}
/// Verifies an AvailabilityPendingBlock against a set of KZG verified blobs.
/// This does not check whether a block *should* have blobs, these checks should must have been
/// completed when producing the `AvailabilityPendingBlock`.
pub fn make_available(
&self,
block: AvailabilityPendingBlock<T>,
blobs: Vec<KzgVerifiedBlob<T>>,
) -> Result<AvailableBlock<T>, AvailabilityCheckError> {
let block_kzg_commitments = block.kzg_commitments()?;
if blobs.len() != block_kzg_commitments.len() {
return Err(AvailabilityCheckError::NumBlobsMismatch {
num_kzg_commitments: block_kzg_commitments.len(),
num_blobs: blobs.len(),
});
}
for (block_commitment, blob) in block_kzg_commitments.iter().zip(blobs.iter()) {
if *block_commitment != blob.kzg_commitment() {
return Err(AvailabilityCheckError::KzgCommitmentMismatch {
blob_index: blob.as_blob().index,
});
}
}
let blobs = VariableList::new(blobs.into_iter().map(|blob| blob.to_blob()).collect())?;
Ok(AvailableBlock {
block: block.block,
blobs: VerifiedBlobs::Available(blobs),
})
}
/// Determines the blob requirements for a block. Answers the question: "Does this block require
/// blobs?".
fn get_blob_requirements(
&self,
block: &Arc<SignedBeaconBlock<T, FullPayload<T>>>,
block: &Arc<SignedBeaconBlock<T::EthSpec, FullPayload<T::EthSpec>>>,
) -> Result<BlobRequirements, AvailabilityCheckError> {
let verified_blobs = if let (Ok(block_kzg_commitments), Ok(payload)) = (
block.message().body().blob_kzg_commitments(),
block.message().body().execution_payload(),
) {
if let Some(transactions) = payload.transactions() {
let verified = verify_kzg_commitments_against_transactions::<T>(
let verified = verify_kzg_commitments_against_transactions::<T::EthSpec>(
transactions,
block_kzg_commitments,
)
@@ -509,7 +525,7 @@ impl<T: EthSpec, S: SlotClock> DataAvailabilityChecker<T, S> {
self.spec.deneb_fork_epoch.and_then(|fork_epoch| {
self.slot_clock
.now()
.map(|slot| slot.epoch(T::slots_per_epoch()))
.map(|slot| slot.epoch(T::EthSpec::slots_per_epoch()))
.map(|current_epoch| {
std::cmp::max(
fork_epoch,
@@ -524,6 +540,96 @@ impl<T: EthSpec, S: SlotClock> DataAvailabilityChecker<T, S> {
self.data_availability_boundary()
.map_or(false, |da_epoch| block_epoch >= da_epoch)
}
/// Persist all in memory components to disk
pub fn persist_all(&self) -> Result<(), AvailabilityCheckError> {
self.availability_cache.write_all_to_disk()
}
}
pub fn start_availability_cache_maintenance_service<T: BeaconChainTypes>(
executor: TaskExecutor,
chain: Arc<BeaconChain<T>>,
) {
// this cache only needs to be maintained if deneb is configured
if chain.spec.deneb_fork_epoch.is_some() {
let overflow_cache = chain.data_availability_checker.availability_cache.clone();
executor.spawn(
async move { availability_cache_maintenance_service(chain, overflow_cache).await },
"availability_cache_service",
);
} else {
debug!(
chain.log,
"Deneb fork not configured, not starting availability cache maintenance service"
);
}
}
async fn availability_cache_maintenance_service<T: BeaconChainTypes>(
chain: Arc<BeaconChain<T>>,
overflow_cache: Arc<OverflowLRUCache<T>>,
) {
let epoch_duration = chain.slot_clock.slot_duration() * T::EthSpec::slots_per_epoch() as u32;
loop {
match chain
.slot_clock
.duration_to_next_epoch(T::EthSpec::slots_per_epoch())
{
Some(duration) => {
// this service should run 3/4 of the way through the epoch
let additional_delay = (epoch_duration * 3) / 4;
tokio::time::sleep(duration + additional_delay).await;
let deneb_fork_epoch = match chain.spec.deneb_fork_epoch {
Some(epoch) => epoch,
None => break, // shutdown service if deneb fork epoch not set
};
debug!(
chain.log,
"Availability cache maintenance service firing";
);
let current_epoch = match chain
.slot_clock
.now()
.map(|slot| slot.epoch(T::EthSpec::slots_per_epoch()))
{
Some(epoch) => epoch,
None => continue, // we'll have to try again next time I suppose..
};
if current_epoch < deneb_fork_epoch {
// we are not in deneb yet
continue;
}
let finalized_epoch = chain
.canonical_head
.fork_choice_read_lock()
.finalized_checkpoint()
.epoch;
// any data belonging to an epoch before this should be pruned
let cutoff_epoch = std::cmp::max(
finalized_epoch + 1,
std::cmp::max(
current_epoch.saturating_sub(*MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS),
deneb_fork_epoch,
),
);
if let Err(e) = overflow_cache.do_maintenance(cutoff_epoch) {
error!(chain.log, "Failed to maintain availability cache"; "error" => ?e);
}
}
None => {
error!(chain.log, "Failed to read slot clock");
// If we can't read the slot clock, just wait another slot.
tokio::time::sleep(chain.slot_clock.slot_duration()).await;
}
};
}
}
pub enum BlobRequirements {
@@ -586,6 +692,37 @@ impl<E: EthSpec> AvailabilityPendingBlock<E> {
.blob_kzg_commitments()
.map_err(|_| AvailabilityCheckError::IncorrectFork)
}
/// Verifies an AvailabilityPendingBlock against a set of KZG verified blobs.
/// This does not check whether a block *should* have blobs, these checks should must have been
/// completed when producing the `AvailabilityPendingBlock`.
pub fn make_available(
self,
blobs: Vec<KzgVerifiedBlob<E>>,
) -> Result<AvailableBlock<E>, AvailabilityCheckError> {
let block_kzg_commitments = self.kzg_commitments()?;
if blobs.len() != block_kzg_commitments.len() {
return Err(AvailabilityCheckError::NumBlobsMismatch {
num_kzg_commitments: block_kzg_commitments.len(),
num_blobs: blobs.len(),
});
}
for (block_commitment, blob) in block_kzg_commitments.iter().zip(blobs.iter()) {
if *block_commitment != blob.kzg_commitment() {
return Err(AvailabilityCheckError::KzgCommitmentMismatch {
blob_index: blob.as_blob().index,
});
}
}
let blobs = VariableList::new(blobs.into_iter().map(|blob| blob.to_blob()).collect())?;
Ok(AvailableBlock {
block: self.block,
blobs: VerifiedBlobs::Available(blobs),
})
}
}
#[derive(Clone, Debug, PartialEq)]
@@ -677,3 +814,44 @@ impl<E: EthSpec> AsBlock<E> for AvailableBlock<E> {
}
}
}
// The standard implementation of Encode for SignedBeaconBlock
// requires us to use ssz(enum_behaviour = "transparent"). This
// prevents us from implementing Decode. We need to use a
// custom Encode and Decode in this wrapper object that essentially
// encodes it as if it were ssz(enum_behaviour = "union")
impl<E: EthSpec> ssz::Encode for AvailabilityPendingBlock<E> {
fn is_ssz_fixed_len() -> bool {
ssz_tagged_signed_beacon_block::encode::is_ssz_fixed_len()
}
fn ssz_append(&self, buf: &mut Vec<u8>) {
ssz_tagged_signed_beacon_block::encode::ssz_append(self.block.as_ref(), buf);
}
fn ssz_bytes_len(&self) -> usize {
ssz_tagged_signed_beacon_block::encode::ssz_bytes_len(self.block.as_ref())
}
}
impl<E: EthSpec> ssz::Decode for AvailabilityPendingBlock<E> {
fn is_ssz_fixed_len() -> bool {
ssz_tagged_signed_beacon_block::decode::is_ssz_fixed_len()
}
fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, ssz::DecodeError> {
Ok(Self {
block: Arc::new(ssz_tagged_signed_beacon_block::decode::from_ssz_bytes(
bytes,
)?),
})
}
}
#[cfg(test)]
mod test {
#[test]
fn check_encode_decode_availability_pending_block() {
// todo.. (difficult to create default beacon blocks to test)
}
}

1645
beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs Normal file
View File

File diff suppressed because it is too large Load Diff

3
beacon_node/beacon_chain/src/errors.rs
View File

@@ -2,6 +2,7 @@ use crate::attester_cache::Error as AttesterCacheError;
use crate::beacon_block_streamer::Error as BlockStreamerError;
use crate::beacon_chain::ForkChoiceError;
use crate::beacon_fork_choice_store::Error as ForkChoiceStoreError;
use crate::data_availability_checker::AvailabilityCheckError;
use crate::eth1_chain::Error as Eth1ChainError;
use crate::historical_blocks::HistoricalBlockError;
use crate::migrate::PruningError;
@@ -215,6 +216,7 @@ pub enum BeaconChainError {
BlsToExecutionConflictsWithPool,
InconsistentFork(InconsistentFork),
ProposerHeadForkChoiceError(fork_choice::Error<proto_array::Error>),
AvailabilityCheckError(AvailabilityCheckError),
}
easy_from_to!(SlotProcessingError, BeaconChainError);
@@ -240,6 +242,7 @@ easy_from_to!(HistoricalBlockError, BeaconChainError);
easy_from_to!(StateAdvanceError, BeaconChainError);
easy_from_to!(BlockReplayError, BeaconChainError);
easy_from_to!(InconsistentFork, BeaconChainError);
easy_from_to!(AvailabilityCheckError, BeaconChainError);
#[derive(Debug)]
pub enum BlockProductionError {

3
beacon_node/beacon_chain/src/eth1_finalization_cache.rs
View File

@@ -1,4 +1,5 @@
use slog::{debug, Logger};
use ssz_derive::{Decode, Encode};
use std::cmp;
use std::collections::BTreeMap;
use types::{Checkpoint, Epoch, Eth1Data, Hash256 as Root};
@@ -10,7 +11,7 @@ pub const DEFAULT_ETH1_CACHE_SIZE: usize = 5;
/// These fields are named the same as the corresponding fields in the `BeaconState`
/// as this structure stores these values from the `BeaconState` at a `Checkpoint`
#[derive(Clone)]
#[derive(Clone, Debug, PartialEq, Encode, Decode)]
pub struct Eth1FinalizationData {
pub eth1_data: Eth1Data,
pub eth1_deposit_index: u64,

24
beacon_node/beacon_chain/src/kzg_utils.rs
View File

@@ -1,5 +1,5 @@
use kzg::{Error as KzgError, Kzg, BYTES_PER_BLOB};
use types::{Blob, EthSpec, KzgCommitment, KzgProof};
use types::{Blob, EthSpec, Hash256, KzgCommitment, KzgProof};
/// Converts a blob ssz List object to an array to be used with the kzg
/// crypto library.
@@ -56,3 +56,25 @@ pub fn blob_to_kzg_commitment<T: EthSpec>(
) -> Result<KzgCommitment, KzgError> {
kzg.blob_to_kzg_commitment(ssz_blob_to_crypto_blob::<T>(blob))
}
/// Compute the kzg proof for a given blob and an evaluation point z.
pub fn compute_kzg_proof<T: EthSpec>(
kzg: &Kzg,
blob: Blob<T>,
z: Hash256,
) -> Result<(KzgProof, Hash256), KzgError> {
let z = z.0.into();
kzg.compute_kzg_proof(ssz_blob_to_crypto_blob::<T>(blob), z)
.map(|(proof, z)| (proof, Hash256::from_slice(&z.to_vec())))
}
/// Verify a `kzg_proof` for a `kzg_commitment` that evaluating a polynomial at `z` results in `y`
pub fn verify_kzg_proof<T: EthSpec>(
kzg: &Kzg,
kzg_commitment: KzgCommitment,
kzg_proof: KzgProof,
z: Hash256,
y: Hash256,
) -> Result<bool, KzgError> {
kzg.verify_kzg_proof(kzg_commitment, z.0.into(), y.0.into(), kzg_proof)
}

2
beacon_node/beacon_chain/src/metrics.rs
View File

@@ -380,6 +380,8 @@ lazy_static! {
try_create_histogram("beacon_persist_eth1_cache", "Time taken to persist the eth1 caches");
pub static ref PERSIST_FORK_CHOICE: Result<Histogram> =
try_create_histogram("beacon_persist_fork_choice", "Time taken to persist the fork choice struct");
pub static ref PERSIST_DATA_AVAILABILITY_CHECKER: Result<Histogram> =
try_create_histogram("beacon_persist_data_availability_checker", "Time taken to persist the data availability checker");
/*
* Eth1

5
beacon_node/client/src/builder.rs
View File

@@ -2,6 +2,7 @@ use crate::address_change_broadcast::broadcast_address_changes_at_capella;
use crate::config::{ClientGenesis, Config as ClientConfig};
use crate::notifier::spawn_notifier;
use crate::Client;
use beacon_chain::data_availability_checker::start_availability_cache_maintenance_service;
use beacon_chain::otb_verification_service::start_otb_verification_service;
use beacon_chain::proposer_prep_service::start_proposer_prep_service;
use beacon_chain::schema_change::migrate_schema;
@@ -828,6 +829,10 @@ where
start_proposer_prep_service(runtime_context.executor.clone(), beacon_chain.clone());
start_otb_verification_service(runtime_context.executor.clone(), beacon_chain.clone());
start_availability_cache_maintenance_service(
runtime_context.executor.clone(),
beacon_chain.clone(),
);
}
Ok(Client {

56
beacon_node/execution_layer/src/test_utils/execution_block_generator.rs
View File

@@ -658,34 +658,34 @@ pub fn generate_random_blobs<T: EthSpec>(
let versioned_hash = commitment.calculate_versioned_hash();
let blob_transaction = BlobTransaction {
chain_id: Default::default(),
nonce: 0,
max_priority_fee_per_gas: Default::default(),
max_fee_per_gas: Default::default(),
gas: 100000,
to: None,
value: Default::default(),
data: Default::default(),
access_list: Default::default(),
max_fee_per_data_gas: Default::default(),
versioned_hashes: vec![versioned_hash].into(),
};
let bad_signature = EcdsaSignature {
y_parity: false,
r: Uint256::from(0),
s: Uint256::from(0),
};
let signed_blob_transaction = SignedBlobTransaction {
message: blob_transaction,
signature: bad_signature,
};
// calculate transaction bytes
let tx_bytes = [BLOB_TX_TYPE]
.into_iter()
.chain(signed_blob_transaction.as_ssz_bytes().into_iter())
.collect::<Vec<_>>();
let tx = Transaction::<T::MaxBytesPerTransaction>::from(tx_bytes);
let blob_transaction = BlobTransaction {
chain_id: Default::default(),
nonce: 0,
max_priority_fee_per_gas: Default::default(),
max_fee_per_gas: Default::default(),
gas: 100000,
to: None,
value: Default::default(),
data: Default::default(),
access_list: Default::default(),
max_fee_per_data_gas: Default::default(),
versioned_hashes: vec![versioned_hash].into(),
};
let bad_signature = EcdsaSignature {
y_parity: false,
r: Uint256::from(0),
s: Uint256::from(0),
};
let signed_blob_transaction = SignedBlobTransaction {
message: blob_transaction,
signature: bad_signature,
};
// calculate transaction bytes
let tx_bytes = [BLOB_TX_TYPE]
.into_iter()
.chain(signed_blob_transaction.as_ssz_bytes().into_iter())
.collect::<Vec<_>>();
let tx = Transaction::<T::MaxBytesPerTransaction>::from(tx_bytes);
transactions.push(tx);
bundle

2
beacon_node/lighthouse_network/src/rpc/outbound.rs
View File

@@ -112,7 +112,7 @@ impl<TSpec: EthSpec> OutboundRequest<TSpec> {
OutboundRequest::Goodbye(_) => 0,
OutboundRequest::BlocksByRange(req) => req.count,
OutboundRequest::BlocksByRoot(req) => req.block_roots.len() as u64,
OutboundRequest::BlobsByRange(req) => req.count,
OutboundRequest::BlobsByRange(req) => req.count * TSpec::max_blobs_per_block() as u64,
OutboundRequest::BlobsByRoot(req) => req.blob_ids.len() as u64,
OutboundRequest::Ping(_) => 1,
OutboundRequest::MetaData(_) => 1,

2
beacon_node/lighthouse_network/src/rpc/protocol.rs
View File

@@ -528,7 +528,7 @@ impl<TSpec: EthSpec> InboundRequest<TSpec> {
InboundRequest::Goodbye(_) => 0,
InboundRequest::BlocksByRange(req) => req.count,
InboundRequest::BlocksByRoot(req) => req.block_roots.len() as u64,
InboundRequest::BlobsByRange(req) => req.count,
InboundRequest::BlobsByRange(req) => req.count * TSpec::max_blobs_per_block() as u64,
InboundRequest::BlobsByRoot(req) => req.blob_ids.len() as u64,
InboundRequest::Ping(_) => 1,
InboundRequest::MetaData(_) => 1,

35
beacon_node/network/src/beacon_processor/worker/rpc_methods.rs
View File

@@ -230,7 +230,7 @@ impl<T: BeaconChainTypes> Worker<T> {
let mut blob_list_results = HashMap::new();
for id in request.blob_ids.into_iter() {
// First attempt to get the blobs from the RPC cache.
if let Some(blob) = self.chain.data_availability_checker.get_blob(&id) {
if let Ok(Some(blob)) = self.chain.data_availability_checker.get_blob(&id) {
self.send_response(peer_id, Response::BlobsByRoot(Some(blob)), request_id);
send_blob_count += 1;
} else {
@@ -625,7 +625,7 @@ impl<T: BeaconChainTypes> Worker<T> {
);
// Should not send more than max request blocks
if req.count > MAX_REQUEST_BLOB_SIDECARS {
if req.count * T::EthSpec::max_blobs_per_block() as u64 > MAX_REQUEST_BLOB_SIDECARS {
return self.send_error_response(
peer_id,
RPCResponseErrorCode::InvalidRequest,
@@ -808,28 +808,15 @@ impl<T: BeaconChainTypes> Worker<T> {
.slot()
.unwrap_or_else(|_| self.chain.slot_clock.genesis_slot());
if blobs_sent < (req.count as usize) {
debug!(
self.log,
"BlobsByRange Response processed";
"peer" => %peer_id,
"msg" => "Failed to return all requested blobs",
"start_slot" => req.start_slot,
"current_slot" => current_slot,
"requested" => req.count,
"returned" => blobs_sent
);
} else {
debug!(
self.log,
"BlobsByRange Response processed";
"peer" => %peer_id,
"start_slot" => req.start_slot,
"current_slot" => current_slot,
"requested" => req.count,
"returned" => blobs_sent
);
}
debug!(
self.log,
"BlobsByRange Response processed";
"peer" => %peer_id,
"start_slot" => req.start_slot,
"current_slot" => current_slot,
"requested" => req.count,
"returned" => blobs_sent
);
if send_response {
// send the stream terminator

3
beacon_node/store/src/hot_cold_store.rs
View File

@@ -260,8 +260,7 @@ impl<E: EthSpec> HotColdDB<E, LevelDB<E>, LevelDB<E>> {
db.blobs_db = Some(LevelDB::open(path.as_path())?);
}
}
let blob_info = blob_info.unwrap_or_else(|| db.get_blob_info());
db.compare_and_set_blob_info_with_write(blob_info, new_blob_info)?;
db.compare_and_set_blob_info_with_write(<_>::default(), new_blob_info)?;
info!(
db.log,
"Blobs DB initialized";

30
beacon_node/store/src/leveldb_store.rs
View File

@@ -198,6 +198,36 @@ impl<E: EthSpec> KeyValueStore<E> for LevelDB<E> {
)
}
fn iter_raw_entries(&self, column: DBColumn, prefix: &[u8]) -> RawEntryIter {
let start_key = BytesKey::from_vec(get_key_for_col(column.into(), prefix));
let iter = self.db.iter(self.read_options());
iter.seek(&start_key);
Box::new(
iter.take_while(move |(key, _)| key.key.starts_with(start_key.key.as_slice()))
.map(move |(bytes_key, value)| {
let subkey = &bytes_key.key[column.as_bytes().len()..];
Ok((Vec::from(subkey), value))
}),
)
}
fn iter_raw_keys(&self, column: DBColumn, prefix: &[u8]) -> RawKeyIter {
let start_key = BytesKey::from_vec(get_key_for_col(column.into(), prefix));
let iter = self.db.keys_iter(self.read_options());
iter.seek(&start_key);
Box::new(
iter.take_while(move |key| key.key.starts_with(start_key.key.as_slice()))
.map(move |bytes_key| {
let subkey = &bytes_key.key[column.as_bytes().len()..];
Ok(Vec::from(subkey))
}),
)
}
/// Iterate through all keys and values in a particular column.
fn iter_column_keys(&self, column: DBColumn) -> ColumnKeyIter {
let start_key =

13
beacon_node/store/src/lib.rs
View File

@@ -49,6 +49,9 @@ pub use types::*;
pub type ColumnIter<'a> = Box<dyn Iterator<Item = Result<(Hash256, Vec<u8>), Error>> + 'a>;
pub type ColumnKeyIter<'a> = Box<dyn Iterator<Item = Result<Hash256, Error>> + 'a>;
pub type RawEntryIter<'a> = Box<dyn Iterator<Item = Result<(Vec<u8>, Vec<u8>), Error>> + 'a>;
pub type RawKeyIter<'a> = Box<dyn Iterator<Item = Result<Vec<u8>, Error>> + 'a>;
pub trait KeyValueStore<E: EthSpec>: Sync + Send + Sized + 'static {
/// Retrieve some bytes in `column` with `key`.
fn get_bytes(&self, column: &str, key: &[u8]) -> Result<Option<Vec<u8>>, Error>;
@@ -88,6 +91,14 @@ pub trait KeyValueStore<E: EthSpec>: Sync + Send + Sized + 'static {
Box::new(std::iter::empty())
}
fn iter_raw_entries(&self, _column: DBColumn, _prefix: &[u8]) -> RawEntryIter {
Box::new(std::iter::empty())
}
fn iter_raw_keys(&self, _column: DBColumn, _prefix: &[u8]) -> RawKeyIter {
Box::new(std::iter::empty())
}
/// Iterate through all keys in a particular column.
fn iter_column_keys(&self, _column: DBColumn) -> ColumnKeyIter {
// Default impl for non LevelDB databases
@@ -227,6 +238,8 @@ pub enum DBColumn {
OptimisticTransitionBlock,
#[strum(serialize = "bhs")]
BeaconHistoricalSummaries,
#[strum(serialize = "olc")]
OverflowLRUCache,
}
/// A block from the database, which might have an execution payload or not.

3
consensus/fork_choice/src/fork_choice.rs
View File

@@ -192,7 +192,8 @@ impl CountUnrealized {
/// Indicates if a block has been verified by an execution payload.
///
/// There is no variant for "invalid", since such a block should never be added to fork choice.
#[derive(Clone, Copy, Debug, PartialEq)]
#[derive(Clone, Copy, Debug, PartialEq, Encode, Decode)]
#[ssz(enum_behaviour = "tag")]
pub enum PayloadVerificationStatus {
/// An EL has declared the execution payload to be valid.
Verified,

145
consensus/ssz_derive/src/lib.rs
View File

@@ -4,6 +4,7 @@
//!
//! The following struct/enum attributes are available:
//!
//! - `#[ssz(enum_behaviour = "tag")]`: encodes and decodes an `enum` with 0 fields per variant
//! - `#[ssz(enum_behaviour = "union")]`: encodes and decodes an `enum` with a one-byte variant selector.
//! - `#[ssz(enum_behaviour = "transparent")]`: allows encoding an `enum` by serializing only the
//! value whilst ignoring outermost the `enum`.
@@ -140,6 +141,22 @@
//! TransparentEnum::Bar(vec![42, 42]).as_ssz_bytes(),
//! vec![42, 42]
//! );
//!
//! /// Representated as an SSZ "uint8"
//! #[derive(Debug, PartialEq, Encode, Decode)]
//! #[ssz(enum_behaviour = "tag")]
//! enum TagEnum {
//! Foo,
//! Bar,
//! }
//! assert_eq!(
//! TagEnum::Foo.as_ssz_bytes(),
//! vec![0]
//! );
//! assert_eq!(
//! TagEnum::from_ssz_bytes(&[1]).unwrap(),
//! TagEnum::Bar,
//! );
//! ```
use darling::{FromDeriveInput, FromMeta};
@@ -154,8 +171,9 @@ const MAX_UNION_SELECTOR: u8 = 127;
const ENUM_TRANSPARENT: &str = "transparent";
const ENUM_UNION: &str = "union";
const ENUM_TAG: &str = "tag";
const NO_ENUM_BEHAVIOUR_ERROR: &str = "enums require an \"enum_behaviour\" attribute with \
a \"transparent\" or \"union\" value, e.g., #[ssz(enum_behaviour = \"transparent\")]";
a \"transparent\", \"union\", or \"tag\" value, e.g., #[ssz(enum_behaviour = \"transparent\")]";
#[derive(Debug, FromDeriveInput)]
#[darling(attributes(ssz))]
@@ -196,6 +214,7 @@ enum StructBehaviour {
enum EnumBehaviour {
Union,
Transparent,
Tag,
}
impl<'a> Procedure<'a> {
@@ -237,6 +256,10 @@ impl<'a> Procedure<'a> {
data,
behaviour: EnumBehaviour::Transparent,
},
Some("tag") => Procedure::Enum {
data,
behaviour: EnumBehaviour::Tag,
},
Some(other) => panic!(
"{} is not a valid enum behaviour, use \"container\" or \"transparent\"",
other
@@ -296,6 +319,7 @@ pub fn ssz_encode_derive(input: TokenStream) -> TokenStream {
Procedure::Enum { data, behaviour } => match behaviour {
EnumBehaviour::Transparent => ssz_encode_derive_enum_transparent(&item, data),
EnumBehaviour::Union => ssz_encode_derive_enum_union(&item, data),
EnumBehaviour::Tag => ssz_encode_derive_enum_tag(&item, data),
},
}
}
@@ -573,6 +597,67 @@ fn ssz_encode_derive_enum_transparent(
output.into()
}
/// Derive `ssz::Encode` for an `enum` following the "tag" method.
///
/// The union selector will be determined based upon the order in which the enum variants are
/// defined. E.g., the top-most variant in the enum will have a selector of `0`, the variant
/// beneath it will have a selector of `1` and so on.
///
/// # Limitations
///
/// Only supports enums where each variant has no fields
fn ssz_encode_derive_enum_tag(derive_input: &DeriveInput, enum_data: &DataEnum) -> TokenStream {
let name = &derive_input.ident;
let (impl_generics, ty_generics, where_clause) = &derive_input.generics.split_for_impl();
let patterns: Vec<_> = enum_data
.variants
.iter()
.map(|variant| {
let variant_name = &variant.ident;
if !variant.fields.is_empty() {
panic!("ssz::Encode tag behaviour can only be derived for enums with no fields");
}
quote! {
#name::#variant_name
}
})
.collect();
let union_selectors = compute_union_selectors(patterns.len());
let output = quote! {
impl #impl_generics ssz::Encode for #name #ty_generics #where_clause {
fn is_ssz_fixed_len() -> bool {
true
}
fn ssz_fixed_len() -> usize {
1
}
fn ssz_bytes_len(&self) -> usize {
1
}
fn ssz_append(&self, buf: &mut Vec<u8>) {
match self {
#(
#patterns => {
let union_selector: u8 = #union_selectors;
debug_assert!(union_selector <= ssz::MAX_UNION_SELECTOR);
buf.push(union_selector);
},
)*
}
}
}
};
output.into()
}
/// Derive `ssz::Encode` for an `enum` following the "union" SSZ spec.
///
/// The union selector will be determined based upon the order in which the enum variants are
@@ -652,9 +737,10 @@ pub fn ssz_decode_derive(input: TokenStream) -> TokenStream {
},
Procedure::Enum { data, behaviour } => match behaviour {
EnumBehaviour::Union => ssz_decode_derive_enum_union(&item, data),
EnumBehaviour::Tag => ssz_decode_derive_enum_tag(&item, data),
EnumBehaviour::Transparent => panic!(
"Decode cannot be derived for enum_behaviour \"{}\", only \"{}\" is valid.",
ENUM_TRANSPARENT, ENUM_UNION
"Decode cannot be derived for enum_behaviour \"{}\", only \"{}\" and \"{}\" is valid.",
ENUM_TRANSPARENT, ENUM_UNION, ENUM_TAG,
),
},
}
@@ -908,6 +994,59 @@ fn ssz_decode_derive_struct_transparent(
output.into()
}
/// Derive `ssz::Decode` for an `enum` following the "tag" SSZ spec.
fn ssz_decode_derive_enum_tag(derive_input: &DeriveInput, enum_data: &DataEnum) -> TokenStream {
let name = &derive_input.ident;
let (impl_generics, ty_generics, where_clause) = &derive_input.generics.split_for_impl();
let patterns: Vec<_> = enum_data
.variants
.iter()
.map(|variant| {
let variant_name = &variant.ident;
if !variant.fields.is_empty() {
panic!("ssz::Decode tag behaviour can only be derived for enums with no fields");
}
quote! {
#name::#variant_name
}
})
.collect();
let union_selectors = compute_union_selectors(patterns.len());
let output = quote! {
impl #impl_generics ssz::Decode for #name #ty_generics #where_clause {
fn is_ssz_fixed_len() -> bool {
true
}
fn ssz_fixed_len() -> usize {
1
}
fn from_ssz_bytes(bytes: &[u8]) -> std::result::Result<Self, ssz::DecodeError> {
let byte = bytes
.first()
.copied()
.ok_or(ssz::DecodeError::OutOfBoundsByte { i: 0 })?;
match byte {
#(
#union_selectors => {
Ok(#patterns)
},
)*
other => Err(ssz::DecodeError::UnionSelectorInvalid(other)),
}
}
}
};
output.into()
}
/// Derive `ssz::Decode` for an `enum` following the "union" SSZ spec.
fn ssz_decode_derive_enum_union(derive_input: &DeriveInput, enum_data: &DataEnum) -> TokenStream {
let name = &derive_input.ident;

15
consensus/ssz_derive/tests/tests.rs
View File

@@ -12,6 +12,14 @@ fn assert_encode_decode<T: Encode + Decode + PartialEq + Debug>(item: &T, bytes:
assert_eq!(T::from_ssz_bytes(bytes).unwrap(), *item);
}
#[derive(PartialEq, Debug, Encode, Decode)]
#[ssz(enum_behaviour = "tag")]
enum TagEnum {
A,
B,
C,
}
#[derive(PartialEq, Debug, Encode, Decode)]
#[ssz(enum_behaviour = "union")]
enum TwoFixedUnion {
@@ -120,6 +128,13 @@ fn two_variable_union() {
);
}
#[test]
fn tag_enum() {
assert_encode_decode(&TagEnum::A, &[0]);
assert_encode_decode(&TagEnum::B, &[1]);
assert_encode_decode(&TagEnum::C, &[2]);
}
#[derive(PartialEq, Debug, Encode, Decode)]
#[ssz(enum_behaviour = "union")]
enum TwoVecUnion {

5
consensus/state_processing/src/consensus_context.rs
View File

@@ -1,5 +1,6 @@
use crate::common::get_indexed_attestation;
use crate::per_block_processing::errors::{AttestationInvalid, BlockOperationError};
use ssz_derive::{Decode, Encode};
use std::collections::{hash_map::Entry, HashMap};
use tree_hash::TreeHash;
use types::{
@@ -7,7 +8,7 @@ use types::{
ChainSpec, Epoch, EthSpec, Hash256, IndexedAttestation, SignedBeaconBlock, Slot,
};
#[derive(Debug, Clone)]
#[derive(Debug, PartialEq, Clone, Encode, Decode)]
pub struct ConsensusContext<T: EthSpec> {
/// Slot to act as an identifier/safeguard
slot: Slot,
@@ -16,6 +17,8 @@ pub struct ConsensusContext<T: EthSpec> {
/// Block root of the block at `slot`.
current_block_root: Option<Hash256>,
/// Cache of indexed attestations constructed during block processing.
/// We can skip serializing / deserializing this as the cache will just be rebuilt
#[ssz(skip_serializing, skip_deserializing)]
indexed_attestations:
HashMap<(AttestationData, BitList<T::MaxValidatorsPerCommittee>), IndexedAttestation<T>>,
/// Whether `verify_kzg_commitments_against_transactions` has successfully passed.

21
consensus/types/src/beacon_block.rs
View File

@@ -201,13 +201,7 @@ impl<'a, T: EthSpec, Payload: AbstractExecPayload<T>> BeaconBlockRef<'a, T, Payl
/// dictated by `self.slot()`.
pub fn fork_name(&self, spec: &ChainSpec) -> Result<ForkName, InconsistentFork> {
let fork_at_slot = spec.fork_name_at_slot::<T>(self.slot());
let object_fork = match self {
BeaconBlockRef::Base { .. } => ForkName::Base,
BeaconBlockRef::Altair { .. } => ForkName::Altair,
BeaconBlockRef::Merge { .. } => ForkName::Merge,
BeaconBlockRef::Capella { .. } => ForkName::Capella,
BeaconBlockRef::Deneb { .. } => ForkName::Deneb,
};
let object_fork = self.fork_name_unchecked();
if fork_at_slot == object_fork {
Ok(object_fork)
@@ -219,6 +213,19 @@ impl<'a, T: EthSpec, Payload: AbstractExecPayload<T>> BeaconBlockRef<'a, T, Payl
}
}
/// Returns the name of the fork pertaining to `self`.
///
/// Does not check that the fork is consistent with the slot.
pub fn fork_name_unchecked(&self) -> ForkName {
match self {
BeaconBlockRef::Base { .. } => ForkName::Base,
BeaconBlockRef::Altair { .. } => ForkName::Altair,
BeaconBlockRef::Merge { .. } => ForkName::Merge,
BeaconBlockRef::Capella { .. } => ForkName::Capella,
BeaconBlockRef::Deneb { .. } => ForkName::Deneb,
}
}
/// Convenience accessor for the `body` as a `BeaconBlockBodyRef`.
pub fn body(&self) -> BeaconBlockBodyRef<'a, T, Payload> {
map_beacon_block_ref_into_beacon_block_body_ref!(&'a _, *self, |block, cons| cons(

98
consensus/types/src/beacon_state.rs
View File

@@ -415,13 +415,7 @@ impl<T: EthSpec> BeaconState<T> {
/// dictated by `self.slot()`.
pub fn fork_name(&self, spec: &ChainSpec) -> Result<ForkName, InconsistentFork> {
let fork_at_slot = spec.fork_name_at_epoch(self.current_epoch());
let object_fork = match self {
BeaconState::Base { .. } => ForkName::Base,
BeaconState::Altair { .. } => ForkName::Altair,
BeaconState::Merge { .. } => ForkName::Merge,
BeaconState::Capella { .. } => ForkName::Capella,
BeaconState::Deneb { .. } => ForkName::Deneb,
};
let object_fork = self.fork_name_unchecked();
if fork_at_slot == object_fork {
Ok(object_fork)
@@ -433,6 +427,19 @@ impl<T: EthSpec> BeaconState<T> {
}
}
/// Returns the name of the fork pertaining to `self`.
///
/// Does not check if `self` is consistent with the fork dictated by `self.slot()`.
pub fn fork_name_unchecked(&self) -> ForkName {
match self {
BeaconState::Base { .. } => ForkName::Base,
BeaconState::Altair { .. } => ForkName::Altair,
BeaconState::Merge { .. } => ForkName::Merge,
BeaconState::Capella { .. } => ForkName::Capella,
BeaconState::Deneb { .. } => ForkName::Deneb,
}
}
/// Specialised deserialisation method that uses the `ChainSpec` as context.
#[allow(clippy::integer_arithmetic)]
pub fn from_ssz_bytes(bytes: &[u8], spec: &ChainSpec) -> Result<Self, ssz::DecodeError> {
@@ -1870,3 +1877,80 @@ impl<T: EthSpec> ForkVersionDeserialize for BeaconState<T> {
))
}
}
/// This module can be used to encode and decode a `BeaconState` the same way it
/// would be done if we had tagged the superstruct enum with
/// `#[ssz(enum_behaviour = "union")]`
/// This should _only_ be used for *some* cases to store these objects in the
/// database and _NEVER_ for encoding / decoding states sent over the network!
pub mod ssz_tagged_beacon_state {
use super::*;
pub mod encode {
use super::*;
#[allow(unused_imports)]
use ssz::*;
pub fn is_ssz_fixed_len() -> bool {
false
}
pub fn ssz_fixed_len() -> usize {
BYTES_PER_LENGTH_OFFSET
}
pub fn ssz_bytes_len<E: EthSpec>(state: &BeaconState<E>) -> usize {
state
.ssz_bytes_len()
.checked_add(1)
.expect("encoded length must be less than usize::max")
}
pub fn ssz_append<E: EthSpec>(state: &BeaconState<E>, buf: &mut Vec<u8>) {
let fork_name = state.fork_name_unchecked();
fork_name.ssz_append(buf);
state.ssz_append(buf);
}
pub fn as_ssz_bytes<E: EthSpec>(state: &BeaconState<E>) -> Vec<u8> {
let mut buf = vec![];
ssz_append(state, &mut buf);
buf
}
}
pub mod decode {
use super::*;
#[allow(unused_imports)]
use ssz::*;
pub fn is_ssz_fixed_len() -> bool {
false
}
pub fn ssz_fixed_len() -> usize {
BYTES_PER_LENGTH_OFFSET
}
pub fn from_ssz_bytes<E: EthSpec>(bytes: &[u8]) -> Result<BeaconState<E>, DecodeError> {
let fork_byte = bytes
.first()
.copied()
.ok_or(DecodeError::OutOfBoundsByte { i: 0 })?;
let body = bytes
.get(1..)
.ok_or(DecodeError::OutOfBoundsByte { i: 1 })?;
match ForkName::from_ssz_bytes(&[fork_byte])? {
ForkName::Base => Ok(BeaconState::Base(BeaconStateBase::from_ssz_bytes(body)?)),
ForkName::Altair => Ok(BeaconState::Altair(BeaconStateAltair::from_ssz_bytes(
body,
)?)),
ForkName::Merge => Ok(BeaconState::Merge(BeaconStateMerge::from_ssz_bytes(body)?)),
ForkName::Capella => Ok(BeaconState::Capella(BeaconStateCapella::from_ssz_bytes(
body,
)?)),
ForkName::Deneb => Ok(BeaconState::Deneb(BeaconStateDeneb::from_ssz_bytes(body)?)),
}
}
}
}

4
consensus/types/src/fork_name.rs
View File

@@ -1,12 +1,14 @@
use crate::{ChainSpec, Epoch};
use serde_derive::{Deserialize, Serialize};
use ssz_derive::{Decode, Encode};
use std::convert::TryFrom;
use std::fmt::{self, Display, Formatter};
use std::str::FromStr;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[derive(Debug, Clone, Copy, Decode, Encode, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(try_from = "String")]
#[serde(into = "String")]
#[ssz(enum_behaviour = "tag")]
pub enum ForkName {
Base,
Altair,

6
consensus/types/src/lib.rs
View File

@@ -170,9 +170,9 @@ pub use crate::selection_proof::SelectionProof;
pub use crate::shuffling_id::AttestationShufflingId;
pub use crate::signed_aggregate_and_proof::SignedAggregateAndProof;
pub use crate::signed_beacon_block::{
SignedBeaconBlock, SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockCapella,
SignedBeaconBlockDeneb, SignedBeaconBlockHash, SignedBeaconBlockMerge,
SignedBlindedBeaconBlock,
ssz_tagged_signed_beacon_block, SignedBeaconBlock, SignedBeaconBlockAltair,
SignedBeaconBlockBase, SignedBeaconBlockCapella, SignedBeaconBlockDeneb, SignedBeaconBlockHash,
SignedBeaconBlockMerge, SignedBlindedBeaconBlock,
};
pub use crate::signed_beacon_block_header::SignedBeaconBlockHeader;
pub use crate::signed_blob::*;

133
consensus/types/src/signed_beacon_block.rs
View File

@@ -93,6 +93,12 @@ impl<E: EthSpec, Payload: AbstractExecPayload<E>> SignedBeaconBlock<E, Payload>
self.message().fork_name(spec)
}
/// Returns the name of the fork pertaining to `self`
/// Does not check that the fork is consistent with the slot.
pub fn fork_name_unchecked(&self) -> ForkName {
self.message().fork_name_unchecked()
}
/// SSZ decode with fork variant determined by slot.
pub fn from_ssz_bytes(bytes: &[u8], spec: &ChainSpec) -> Result<Self, ssz::DecodeError> {
Self::from_ssz_bytes_with(bytes, |bytes| BeaconBlock::from_ssz_bytes(bytes, spec))
@@ -543,6 +549,99 @@ impl<E: EthSpec, Payload: AbstractExecPayload<E>> ForkVersionDeserialize
}
}
/// This module can be used to encode and decode a `SignedBeaconBlock` the same way it
/// would be done if we had tagged the superstruct enum with
/// `#[ssz(enum_behaviour = "union")]`
/// This should _only_ be used *some* cases when storing these objects in the database
/// and _NEVER_ for encoding / decoding blocks sent over the network!
pub mod ssz_tagged_signed_beacon_block {
use super::*;
pub mod encode {
use super::*;
#[allow(unused_imports)]
use ssz::*;
pub fn is_ssz_fixed_len() -> bool {
false
}
pub fn ssz_fixed_len() -> usize {
BYTES_PER_LENGTH_OFFSET
}
pub fn ssz_bytes_len<E: EthSpec, Payload: AbstractExecPayload<E>>(
block: &SignedBeaconBlock<E, Payload>,
) -> usize {
block
.ssz_bytes_len()
.checked_add(1)
.expect("encoded length must be less than usize::max")
}
pub fn ssz_append<E: EthSpec, Payload: AbstractExecPayload<E>>(
block: &SignedBeaconBlock<E, Payload>,
buf: &mut Vec<u8>,
) {
let fork_name = block.fork_name_unchecked();
fork_name.ssz_append(buf);
block.ssz_append(buf);
}
pub fn as_ssz_bytes<E: EthSpec, Payload: AbstractExecPayload<E>>(
block: &SignedBeaconBlock<E, Payload>,
) -> Vec<u8> {
let mut buf = vec![];
ssz_append(block, &mut buf);
buf
}
}
pub mod decode {
use super::*;
#[allow(unused_imports)]
use ssz::*;
pub fn is_ssz_fixed_len() -> bool {
false
}
pub fn ssz_fixed_len() -> usize {
BYTES_PER_LENGTH_OFFSET
}
pub fn from_ssz_bytes<E: EthSpec, Payload: AbstractExecPayload<E>>(
bytes: &[u8],
) -> Result<SignedBeaconBlock<E, Payload>, DecodeError> {
let fork_byte = bytes
.first()
.copied()
.ok_or(DecodeError::OutOfBoundsByte { i: 0 })?;
let body = bytes
.get(1..)
.ok_or(DecodeError::OutOfBoundsByte { i: 1 })?;
match ForkName::from_ssz_bytes(&[fork_byte])? {
ForkName::Base => Ok(SignedBeaconBlock::Base(
SignedBeaconBlockBase::from_ssz_bytes(body)?,
)),
ForkName::Altair => Ok(SignedBeaconBlock::Altair(
SignedBeaconBlockAltair::from_ssz_bytes(body)?,
)),
ForkName::Merge => Ok(SignedBeaconBlock::Merge(
SignedBeaconBlockMerge::from_ssz_bytes(body)?,
)),
ForkName::Capella => Ok(SignedBeaconBlock::Capella(
SignedBeaconBlockCapella::from_ssz_bytes(body)?,
)),
ForkName::Deneb => Ok(SignedBeaconBlock::Deneb(
SignedBeaconBlockDeneb::from_ssz_bytes(body)?,
)),
}
}
}
}
#[cfg(test)]
mod test {
use super::*;
@@ -584,4 +683,38 @@ mod test {
assert_eq!(reconstructed, block);
}
}
#[test]
fn test_ssz_tagged_signed_beacon_block() {
type E = MainnetEthSpec;
let spec = &E::default_spec();
let sig = Signature::empty();
let blocks = vec![
SignedBeaconBlock::<E>::from_block(
BeaconBlock::Base(BeaconBlockBase::empty(spec)),
sig.clone(),
),
SignedBeaconBlock::from_block(
BeaconBlock::Altair(BeaconBlockAltair::empty(spec)),
sig.clone(),
),
SignedBeaconBlock::from_block(
BeaconBlock::Merge(BeaconBlockMerge::empty(spec)),
sig.clone(),
),
SignedBeaconBlock::from_block(
BeaconBlock::Capella(BeaconBlockCapella::empty(spec)),
sig.clone(),
),
SignedBeaconBlock::from_block(BeaconBlock::Deneb(BeaconBlockDeneb::empty(spec)), sig),
];
for block in blocks {
let encoded = ssz_tagged_signed_beacon_block::encode::as_ssz_bytes(&block);
let decoded = ssz_tagged_signed_beacon_block::decode::from_ssz_bytes::<E, _>(&encoded)
.expect("should decode");
assert_eq!(decoded, block);
}
}
}

27
crypto/kzg/src/lib.rs
View File

@@ -3,11 +3,11 @@ mod kzg_proof;
mod trusted_setup;
pub use crate::{kzg_commitment::KzgCommitment, kzg_proof::KzgProof, trusted_setup::TrustedSetup};
use c_kzg::Bytes48;
pub use c_kzg::{
Blob, Error as CKzgError, KzgSettings, BYTES_PER_BLOB, BYTES_PER_FIELD_ELEMENT,
FIELD_ELEMENTS_PER_BLOB,
};
use c_kzg::{Bytes32, Bytes48};
use std::path::PathBuf;
#[derive(Debug)]
@@ -116,4 +116,29 @@ impl Kzg {
.map_err(Error::InvalidBlob)
.map(|com| KzgCommitment(com.to_bytes().into_inner()))
}
/// Computes the kzg proof for a given `blob` and an evaluation point `z`
pub fn compute_kzg_proof(&self, blob: Blob, z: Bytes32) -> Result<(KzgProof, Bytes32), Error> {
c_kzg::KzgProof::compute_kzg_proof(blob, z, &self.trusted_setup)
.map_err(Error::KzgProofComputationFailed)
.map(|(proof, y)| (KzgProof(proof.to_bytes().into_inner()), y))
}
/// Verifies a `kzg_proof` for a `kzg_commitment` that evaluating a polynomial at `z` results in `y`
pub fn verify_kzg_proof(
&self,
kzg_commitment: KzgCommitment,
z: Bytes32,
y: Bytes32,
kzg_proof: KzgProof,
) -> Result<bool, Error> {
c_kzg::KzgProof::verify_kzg_proof(
kzg_commitment.into(),
z,
y,
kzg_proof.into(),
&self.trusted_setup,
)
.map_err(Error::InvalidKzgProof)
}
}

4
testing/ef_tests/Cargo.toml
View File

@@ -17,11 +17,15 @@ compare_fields_derive = { path = "../../common/compare_fields_derive" }
derivative = "2.1.1"
ethereum-types = "0.14.1"
hex = "0.4.2"
kzg = { path = "../../crypto/kzg" }
rayon = "1.4.1"
serde = "1.0.116"
serde_derive = "1.0.116"
serde_json = "1.0.58"
serde_repr = "0.1.6"
serde_yaml = "0.8.13"
eth2_network_config = { path = "../../common/eth2_network_config" }
eth2_serde_utils = { path = "../../consensus/serde_utils" }
eth2_ssz = "0.4.1"
eth2_ssz_derive = "0.3.1"
tree_hash = "0.4.1"

12
testing/ef_tests/src/cases.rs
View File

@@ -18,6 +18,12 @@ mod fork;
mod fork_choice;
mod genesis_initialization;
mod genesis_validity;
mod kzg_blob_to_kzg_commitment;
mod kzg_compute_blob_kzg_proof;
mod kzg_compute_kzg_proof;
mod kzg_verify_blob_kzg_proof;
mod kzg_verify_blob_kzg_proof_batch;
mod kzg_verify_kzg_proof;
mod merkle_proof_validity;
mod operations;
mod rewards;
@@ -42,6 +48,12 @@ pub use epoch_processing::*;
pub use fork::ForkTest;
pub use genesis_initialization::*;
pub use genesis_validity::*;
pub use kzg_blob_to_kzg_commitment::*;
pub use kzg_compute_blob_kzg_proof::*;
pub use kzg_compute_kzg_proof::*;
pub use kzg_verify_blob_kzg_proof::*;
pub use kzg_verify_blob_kzg_proof_batch::*;
pub use kzg_verify_kzg_proof::*;
pub use merkle_proof_validity::*;
pub use operations::*;
pub use rewards::RewardsTest;

46
testing/ef_tests/src/cases/kzg_blob_to_kzg_commitment.rs
View File

@@ -0,0 +1,46 @@
use super::*;
use crate::case_result::compare_result;
use beacon_chain::kzg_utils::blob_to_kzg_commitment;
use kzg::KzgCommitment;
use serde_derive::Deserialize;
use std::marker::PhantomData;
#[derive(Debug, Clone, Deserialize)]
pub struct KZGBlobToKZGCommitmentInput {
pub blob: String,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(bound = "E: EthSpec")]
pub struct KZGBlobToKZGCommitment<E: EthSpec> {
pub input: KZGBlobToKZGCommitmentInput,
pub output: Option<String>,
#[serde(skip)]
_phantom: PhantomData<E>,
}
impl<E: EthSpec> LoadCase for KZGBlobToKZGCommitment<E> {
fn load_from_dir(path: &Path, _fork_name: ForkName) -> Result<Self, Error> {
decode::yaml_decode_file(path.join("data.yaml").as_path())
}
}
impl<E: EthSpec> Case for KZGBlobToKZGCommitment<E> {
fn is_enabled_for_fork(fork_name: ForkName) -> bool {
fork_name == ForkName::Deneb
}
fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
let kzg = get_kzg()?;
let commitment = parse_blob::<E>(&self.input.blob).and_then(|blob| {
blob_to_kzg_commitment::<E>(&kzg, blob).map_err(|e| {
Error::InternalError(format!("Failed to compute kzg commitment: {:?}", e))
})
});
let expected = self.output.as_ref().and_then(|s| parse_commitment(s).ok());
compare_result::<KzgCommitment, _>(&commitment, &expected)
}
}

51
testing/ef_tests/src/cases/kzg_compute_blob_kzg_proof.rs
View File

@@ -0,0 +1,51 @@
use super::*;
use crate::case_result::compare_result;
use beacon_chain::kzg_utils::compute_blob_kzg_proof;
use kzg::KzgProof;
use serde_derive::Deserialize;
use std::marker::PhantomData;
#[derive(Debug, Clone, Deserialize)]
pub struct KZGComputeBlobKZGProofInput {
pub blob: String,
pub commitment: String,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(bound = "E: EthSpec")]
pub struct KZGComputeBlobKZGProof<E: EthSpec> {
pub input: KZGComputeBlobKZGProofInput,
pub output: Option<String>,
#[serde(skip)]
_phantom: PhantomData<E>,
}
impl<E: EthSpec> LoadCase for KZGComputeBlobKZGProof<E> {
fn load_from_dir(path: &Path, _fork_name: ForkName) -> Result<Self, Error> {
decode::yaml_decode_file(path.join("data.yaml").as_path())
}
}
impl<E: EthSpec> Case for KZGComputeBlobKZGProof<E> {
fn is_enabled_for_fork(fork_name: ForkName) -> bool {
fork_name == ForkName::Deneb
}
fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
let parse_input = |input: &KZGComputeBlobKZGProofInput| -> Result<_, Error> {
let blob = parse_blob::<E>(&input.blob)?;
let commitment = parse_commitment(&input.commitment)?;
Ok((blob, commitment))
};
let kzg = get_kzg()?;
let proof = parse_input(&self.input).and_then(|(blob, commitment)| {
compute_blob_kzg_proof::<E>(&kzg, &blob, commitment)
.map_err(|e| Error::InternalError(format!("Failed to compute kzg proof: {:?}", e)))
});
let expected = self.output.as_ref().and_then(|s| parse_proof(s).ok());
compare_result::<KzgProof, _>(&proof, &expected)
}
}

61
testing/ef_tests/src/cases/kzg_compute_kzg_proof.rs
View File

@@ -0,0 +1,61 @@
use super::*;
use crate::case_result::compare_result;
use beacon_chain::kzg_utils::compute_kzg_proof;
use kzg::KzgProof;
use serde_derive::Deserialize;
use std::marker::PhantomData;
use std::str::FromStr;
use types::Hash256;
pub fn parse_point(point: &str) -> Result<Hash256, Error> {
Hash256::from_str(&point[2..])
.map_err(|e| Error::FailedToParseTest(format!("Failed to parse point: {:?}", e)))
}
#[derive(Debug, Clone, Deserialize)]
pub struct KZGComputeKZGProofInput {
pub blob: String,
pub z: String,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(bound = "E: EthSpec")]
pub struct KZGComputeKZGProof<E: EthSpec> {
pub input: KZGComputeKZGProofInput,
pub output: Option<(String, Hash256)>,
#[serde(skip)]
_phantom: PhantomData<E>,
}
impl<E: EthSpec> LoadCase for KZGComputeKZGProof<E> {
fn load_from_dir(path: &Path, _fork_name: ForkName) -> Result<Self, Error> {
decode::yaml_decode_file(path.join("data.yaml").as_path())
}
}
impl<E: EthSpec> Case for KZGComputeKZGProof<E> {
fn is_enabled_for_fork(fork_name: ForkName) -> bool {
fork_name == ForkName::Deneb
}
fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
let parse_input = |input: &KZGComputeKZGProofInput| -> Result<_, Error> {
let blob = parse_blob::<E>(&input.blob)?;
let z = parse_point(&input.z)?;
Ok((blob, z))
};
let kzg = get_kzg()?;
let proof = parse_input(&self.input).and_then(|(blob, z)| {
compute_kzg_proof::<E>(&kzg, blob, z)
.map_err(|e| Error::InternalError(format!("Failed to compute kzg proof: {:?}", e)))
});
let expected = self
.output
.as_ref()
.and_then(|(s, z)| parse_proof(s).ok().map(|proof| (proof, *z)));
compare_result::<(KzgProof, Hash256), _>(&proof, &expected)
}
}

92
testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof.rs
View File

@@ -0,0 +1,92 @@
use super::*;
use crate::case_result::compare_result;
use beacon_chain::kzg_utils::validate_blob;
use eth2_network_config::TRUSTED_SETUP;
use kzg::{Kzg, KzgCommitment, KzgProof, TrustedSetup};
use serde_derive::Deserialize;
use std::convert::TryInto;
use std::marker::PhantomData;
use types::Blob;
pub fn get_kzg() -> Result<Kzg, Error> {
let trusted_setup: TrustedSetup = serde_json::from_reader(TRUSTED_SETUP)
.map_err(|e| Error::InternalError(format!("Failed to initialize kzg: {:?}", e)))?;
Kzg::new_from_trusted_setup(trusted_setup)
.map_err(|e| Error::InternalError(format!("Failed to initialize kzg: {:?}", e)))
}
pub fn parse_proof(proof: &str) -> Result<KzgProof, Error> {
hex::decode(&proof[2..])
.map_err(|e| Error::FailedToParseTest(format!("Failed to parse proof: {:?}", e)))
.and_then(|bytes| {
bytes
.try_into()
.map_err(|e| Error::FailedToParseTest(format!("Failed to parse proof: {:?}", e)))
})
.map(KzgProof)
}
pub fn parse_commitment(commitment: &str) -> Result<KzgCommitment, Error> {
hex::decode(&commitment[2..])
.map_err(|e| Error::FailedToParseTest(format!("Failed to parse commitment: {:?}", e)))
.and_then(|bytes| {
bytes.try_into().map_err(|e| {
Error::FailedToParseTest(format!("Failed to parse commitment: {:?}", e))
})
})
.map(KzgCommitment)
}
pub fn parse_blob<E: EthSpec>(blob: &str) -> Result<Blob<E>, Error> {
hex::decode(&blob[2..])
.map_err(|e| Error::FailedToParseTest(format!("Failed to parse blob: {:?}", e)))
.and_then(|bytes| {
Blob::<E>::new(bytes)
.map_err(|e| Error::FailedToParseTest(format!("Failed to parse blob: {:?}", e)))
})
}
#[derive(Debug, Clone, Deserialize)]
pub struct KZGVerifyBlobKZGProofInput {
pub blob: String,
pub commitment: String,
pub proof: String,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(bound = "E: EthSpec")]
pub struct KZGVerifyBlobKZGProof<E: EthSpec> {
pub input: KZGVerifyBlobKZGProofInput,
pub output: Option<bool>,
#[serde(skip)]
_phantom: PhantomData<E>,
}
impl<E: EthSpec> LoadCase for KZGVerifyBlobKZGProof<E> {
fn load_from_dir(path: &Path, _fork_name: ForkName) -> Result<Self, Error> {
decode::yaml_decode_file(path.join("data.yaml").as_path())
}
}
impl<E: EthSpec> Case for KZGVerifyBlobKZGProof<E> {
fn is_enabled_for_fork(fork_name: ForkName) -> bool {
fork_name == ForkName::Deneb
}
fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
let parse_input = |input: &KZGVerifyBlobKZGProofInput| -> Result<(Blob<E>, KzgCommitment, KzgProof), Error> {
let blob = parse_blob::<E>(&input.blob)?;
let commitment = parse_commitment(&input.commitment)?;
let proof = parse_proof(&input.proof)?;
Ok((blob, commitment, proof))
};
let kzg = get_kzg()?;
let result = parse_input(&self.input).and_then(|(blob, commitment, proof)| {
validate_blob::<E>(&kzg, blob, commitment, proof)
.map_err(|e| Error::InternalError(format!("Failed to validate blob: {:?}", e)))
});
compare_result::<bool, _>(&result, &self.output)
}
}

62
testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof_batch.rs
View File

@@ -0,0 +1,62 @@
use super::*;
use crate::case_result::compare_result;
use beacon_chain::kzg_utils::validate_blobs;
use serde_derive::Deserialize;
use std::marker::PhantomData;
#[derive(Debug, Clone, Deserialize)]
pub struct KZGVerifyBlobKZGProofBatchInput {
pub blobs: Vec<String>,
pub commitments: Vec<String>,
pub proofs: Vec<String>,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(bound = "E: EthSpec")]
pub struct KZGVerifyBlobKZGProofBatch<E: EthSpec> {
pub input: KZGVerifyBlobKZGProofBatchInput,
pub output: Option<bool>,
#[serde(skip)]
_phantom: PhantomData<E>,
}
impl<E: EthSpec> LoadCase for KZGVerifyBlobKZGProofBatch<E> {
fn load_from_dir(path: &Path, _fork_name: ForkName) -> Result<Self, Error> {
decode::yaml_decode_file(path.join("data.yaml").as_path())
}
}
impl<E: EthSpec> Case for KZGVerifyBlobKZGProofBatch<E> {
fn is_enabled_for_fork(fork_name: ForkName) -> bool {
fork_name == ForkName::Deneb
}
fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
let parse_input = |input: &KZGVerifyBlobKZGProofBatchInput| -> Result<_, Error> {
let blobs = input
.blobs
.iter()
.map(|s| parse_blob::<E>(s))
.collect::<Result<Vec<_>, _>>()?;
let commitments = input
.commitments
.iter()
.map(|s| parse_commitment(s))
.collect::<Result<Vec<_>, _>>()?;
let proofs = input
.proofs
.iter()
.map(|s| parse_proof(s))
.collect::<Result<Vec<_>, _>>()?;
Ok((commitments, blobs, proofs))
};
let kzg = get_kzg()?;
let result = parse_input(&self.input).and_then(|(commitments, blobs, proofs)| {
validate_blobs::<E>(&kzg, &commitments, &blobs, &proofs)
.map_err(|e| Error::InternalError(format!("Failed to validate blobs: {:?}", e)))
});
compare_result::<bool, _>(&result, &self.output)
}
}

52
testing/ef_tests/src/cases/kzg_verify_kzg_proof.rs
View File

@@ -0,0 +1,52 @@
use super::*;
use crate::case_result::compare_result;
use beacon_chain::kzg_utils::verify_kzg_proof;
use serde_derive::Deserialize;
use std::marker::PhantomData;
#[derive(Debug, Clone, Deserialize)]
pub struct KZGVerifyKZGProofInput {
pub commitment: String,
pub z: String,
pub y: String,
pub proof: String,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(bound = "E: EthSpec")]
pub struct KZGVerifyKZGProof<E: EthSpec> {
pub input: KZGVerifyKZGProofInput,
pub output: Option<bool>,
#[serde(skip)]
_phantom: PhantomData<E>,
}
impl<E: EthSpec> LoadCase for KZGVerifyKZGProof<E> {
fn load_from_dir(path: &Path, _fork_name: ForkName) -> Result<Self, Error> {
decode::yaml_decode_file(path.join("data.yaml").as_path())
}
}
impl<E: EthSpec> Case for KZGVerifyKZGProof<E> {
fn is_enabled_for_fork(fork_name: ForkName) -> bool {
fork_name == ForkName::Deneb
}
fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
let parse_input = |input: &KZGVerifyKZGProofInput| -> Result<_, Error> {
let commitment = parse_commitment(&input.commitment)?;
let z = parse_point(&input.z)?;
let y = parse_point(&input.y)?;
let proof = parse_proof(&input.proof)?;
Ok((commitment, z, y, proof))
};
let kzg = get_kzg()?;
let result = parse_input(&self.input).and_then(|(commitment, z, y, proof)| {
verify_kzg_proof::<E>(&kzg, commitment, proof, z, y)
.map_err(|e| Error::InternalError(format!("Failed to validate proof: {:?}", e)))
});
compare_result::<bool, _>(&result, &self.output)
}
}

120
testing/ef_tests/src/handler.rs
View File

@@ -637,6 +637,126 @@ impl<E: EthSpec + TypeName> Handler for GenesisInitializationHandler<E> {
}
}
#[derive(Derivative)]
#[derivative(Default(bound = ""))]
pub struct KZGBlobToKZGCommitmentHandler<E>(PhantomData<E>);
impl<E: EthSpec> Handler for KZGBlobToKZGCommitmentHandler<E> {
type Case = cases::KZGBlobToKZGCommitment<E>;
fn config_name() -> &'static str {
"general"
}
fn runner_name() -> &'static str {
"kzg"
}
fn handler_name(&self) -> String {
"blob_to_kzg_commitment".into()
}
}
#[derive(Derivative)]
#[derivative(Default(bound = ""))]
pub struct KZGComputeBlobKZGProofHandler<E>(PhantomData<E>);
impl<E: EthSpec> Handler for KZGComputeBlobKZGProofHandler<E> {
type Case = cases::KZGComputeBlobKZGProof<E>;
fn config_name() -> &'static str {
"general"
}
fn runner_name() -> &'static str {
"kzg"
}
fn handler_name(&self) -> String {
"compute_blob_kzg_proof".into()
}
}
#[derive(Derivative)]
#[derivative(Default(bound = ""))]
pub struct KZGComputeKZGProofHandler<E>(PhantomData<E>);
impl<E: EthSpec> Handler for KZGComputeKZGProofHandler<E> {
type Case = cases::KZGComputeKZGProof<E>;
fn config_name() -> &'static str {
"general"
}
fn runner_name() -> &'static str {
"kzg"
}
fn handler_name(&self) -> String {
"compute_kzg_proof".into()
}
}
#[derive(Derivative)]
#[derivative(Default(bound = ""))]
pub struct KZGVerifyBlobKZGProofHandler<E>(PhantomData<E>);
impl<E: EthSpec> Handler for KZGVerifyBlobKZGProofHandler<E> {
type Case = cases::KZGVerifyBlobKZGProof<E>;
fn config_name() -> &'static str {
"general"
}
fn runner_name() -> &'static str {
"kzg"
}
fn handler_name(&self) -> String {
"verify_blob_kzg_proof".into()
}
}
#[derive(Derivative)]
#[derivative(Default(bound = ""))]
pub struct KZGVerifyBlobKZGProofBatchHandler<E>(PhantomData<E>);
impl<E: EthSpec> Handler for KZGVerifyBlobKZGProofBatchHandler<E> {
type Case = cases::KZGVerifyBlobKZGProofBatch<E>;
fn config_name() -> &'static str {
"general"
}
fn runner_name() -> &'static str {
"kzg"
}
fn handler_name(&self) -> String {
"verify_blob_kzg_proof_batch".into()
}
}
#[derive(Derivative)]
#[derivative(Default(bound = ""))]
pub struct KZGVerifyKZGProofHandler<E>(PhantomData<E>);
impl<E: EthSpec> Handler for KZGVerifyKZGProofHandler<E> {
type Case = cases::KZGVerifyKZGProof<E>;
fn config_name() -> &'static str {
"general"
}
fn runner_name() -> &'static str {
"kzg"
}
fn handler_name(&self) -> String {
"verify_kzg_proof".into()
}
}
#[derive(Derivative)]
#[derivative(Default(bound = ""))]
pub struct MerkleProofValidityHandler<E>(PhantomData<E>);

30
testing/ef_tests/tests/tests.rs
View File

@@ -563,6 +563,36 @@ fn genesis_validity() {
// Note: there are no genesis validity tests for mainnet
}
#[test]
fn kzg_blob_to_kzg_commitment() {
KZGBlobToKZGCommitmentHandler::<MainnetEthSpec>::default().run();
}
#[test]
fn kzg_compute_blob_kzg_proof() {
KZGComputeBlobKZGProofHandler::<MainnetEthSpec>::default().run();
}
#[test]
fn kzg_compute_kzg_proof() {
KZGComputeKZGProofHandler::<MainnetEthSpec>::default().run();
}
#[test]
fn kzg_verify_blob_kzg_proof() {
KZGVerifyBlobKZGProofHandler::<MainnetEthSpec>::default().run();
}
#[test]
fn kzg_verify_blob_kzg_proof_batch() {
KZGVerifyBlobKZGProofBatchHandler::<MainnetEthSpec>::default().run();
}
#[test]
fn kzg_verify_kzg_proof() {
KZGVerifyKZGProofHandler::<MainnetEthSpec>::default().run();
}
#[test]
fn merkle_proof_validity() {
MerkleProofValidityHandler::<MainnetEthSpec>::default().run();