gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-04-17 12:58:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

compile after merge

Browse Source
This commit is contained in:
realbigsean
2023-05-18 16:42:13 -04:00
parent 4dd1184968
commit bef63e42f7
8 changed files with 150 additions and 327 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
beacon_node/beacon_chain/src/blob_verification.rs
View File

@@ -511,9 +511,7 @@ impl<T: EthSpec> TryInto<AvailableBlock<T>> for MaybeAvailableBlock<T> {
fn try_into(self) -> Result<AvailableBlock<T>, Self::Error> {
match self {
Self::Available(block) => Ok(block),
Self::AvailabilityPending(block) => Err(AvailabilityCheckError::MissingBlobs(
block.as_block().canonical_root(),
)),
Self::AvailabilityPending(block) => Err(AvailabilityCheckError::MissingBlobs),
}
}
}

10
beacon_node/beacon_chain/src/builder.rs
View File

@@ -861,12 +861,10 @@ where
slasher: self.slasher.clone(),
validator_monitor: RwLock::new(validator_monitor),
//TODO(sean) should we move kzg solely to the da checker?
data_availability_checker: Arc::new(DataAvailabilityChecker::new(
slot_clock,
kzg.clone(),
store,
self.spec,
).map_err(|e| format!("Error initializing DataAvailabiltyChecker: {:?}", e))?),
data_availability_checker: Arc::new(
DataAvailabilityChecker::new(slot_clock, kzg.clone(), store, self.spec)
.map_err(|e| format!("Error initializing DataAvailabiltyChecker: {:?}", e))?,
),
proposal_blob_cache: BlobCache::default(),
kzg,
};

224
beacon_node/beacon_chain/src/data_availability_checker.rs
View File

@@ -10,13 +10,12 @@ use kzg::Error as KzgError;
use kzg::Kzg;
use slog::{debug, error};
use slot_clock::SlotClock;
use ssz_types::{Error, VariableList};
use ssz_types::{Error, FixedVector, VariableList};
use state_processing::per_block_processing::deneb::deneb::verify_kzg_commitments_against_transactions;
use std::collections::hash_map::{Entry, OccupiedEntry};
use std::collections::{HashMap, HashSet};
use std::collections::HashSet;
use std::sync::Arc;
use task_executor::TaskExecutor;
use strum::IntoStaticStr;
use task_executor::TaskExecutor;
use types::beacon_block_body::KzgCommitments;
use types::blob_sidecar::{BlobIdentifier, BlobSidecar, FixedBlobSidecarList};
use types::consts::deneb::MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS;
@@ -36,7 +35,7 @@ pub enum AvailabilityCheckError {
KzgVerificationFailed,
KzgNotInitialized,
SszTypes(ssz_types::Error),
MissingBlobs(Hash256),
MissingBlobs,
NumBlobsMismatch {
num_kzg_commitments: usize,
num_blobs: usize,
@@ -89,56 +88,6 @@ pub struct DataAvailabilityChecker<T: BeaconChainTypes> {
spec: ChainSpec,
}
/// Caches partially available blobs and execution verified blocks corresponding
/// to a given `block_hash` that are received over gossip.
///
/// The blobs are all gossip and kzg verified.
/// The block has completed all verifications except the availability check.
struct ReceivedComponents<T: EthSpec> {
verified_blobs: FixedVector<Option<KzgVerifiedBlob<T>>, T::MaxBlobsPerBlock>,
executed_block: Option<AvailabilityPendingExecutedBlock<T>>,
}
impl<T: EthSpec> ReceivedComponents<T> {
fn new_from_blobs(blobs: &[KzgVerifiedBlob<T>]) -> Self {
let mut verified_blobs = FixedVector::<_, _>::default();
for blob in blobs {
// TODO: verify that we've already ensured the blob index < T::MaxBlobsPerBlock
if let Some(mut_maybe_blob) = verified_blobs.get_mut(blob.blob_index() as usize) {
*mut_maybe_blob = Some(blob.clone());
}
}
Self {
verified_blobs,
executed_block: None,
}
}
fn new_from_block(block: AvailabilityPendingExecutedBlock<T>) -> Self {
Self {
verified_blobs: <_>::default(),
executed_block: Some(block),
}
}
/// Returns `true` if the cache has all blobs corresponding to the
/// kzg commitments in the block.
fn has_all_blobs(&self, block: &AvailabilityPendingExecutedBlock<T>) -> bool {
for i in 0..block.num_blobs_expected() {
if self
.verified_blobs
.get(i)
.map(|maybe_blob| maybe_blob.is_none())
.unwrap_or(true)
{
return false;
}
}
true
}
}
/// This type is returned after adding a block / blob to the `DataAvailabilityChecker`.
///
/// Indicates if the block is fully `Available` or if we need blobs or blocks
@@ -178,34 +127,17 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
}
pub fn has_block(&self, block_root: &Hash256) -> bool {
self.availability_cache
.read()
.get(block_root)
.map_or(false, |cache| cache.executed_block.is_some())
self.availability_cache.has_block(block_root)
}
pub fn get_missing_blob_ids_checking_cache(
&self,
block_root: Hash256,
) -> Option<Vec<BlobIdentifier>> {
let guard = self.availability_cache.read();
let (block, blob_indices) = guard
.get(&block_root)
.map(|cache| {
let block_opt = cache
.executed_block
.as_ref()
.map(|block| &block.block.block);
let blobs = cache
.verified_blobs
.iter()
.enumerate()
.filter_map(|(i, maybe_blob)| maybe_blob.as_ref().map(|_| i))
.collect::<HashSet<_>>();
(block_opt, blobs)
})
.unwrap_or_default();
self.get_missing_blob_ids(block_root, block, Some(blob_indices))
let (block, blob_indices) = self
.availability_cache
.get_missing_blob_ids_checking_cache(block_root);
self.get_missing_blob_ids(block_root, block.as_ref(), Some(blob_indices))
}
/// A `None` indicates blobs are not required.
@@ -215,10 +147,10 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
pub fn get_missing_blob_ids(
&self,
block_root: Hash256,
block_opt: Option<&Arc<SignedBeaconBlock<T>>>,
block_opt: Option<&Arc<SignedBeaconBlock<T::EthSpec>>>,
blobs_opt: Option<HashSet<usize>>,
) -> Option<Vec<BlobIdentifier>> {
let epoch = self.slot_clock.now()?.epoch(T::slots_per_epoch());
let epoch = self.slot_clock.now()?.epoch(T::EthSpec::slots_per_epoch());
self.da_check_required(epoch).then(|| {
block_opt
@@ -228,8 +160,8 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
})
})
.unwrap_or_else(|| {
let mut blob_ids = Vec::with_capacity(T::max_blobs_per_block());
for i in 0..T::max_blobs_per_block() {
let mut blob_ids = Vec::with_capacity(T::EthSpec::max_blobs_per_block());
for i in 0..T::EthSpec::max_blobs_per_block() {
if blobs_opt.as_ref().map_or(true, |blobs| !blobs.contains(&i)) {
blob_ids.push(BlobIdentifier {
block_root,
@@ -253,8 +185,8 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
pub fn put_rpc_blobs(
&self,
block_root: Hash256,
blobs: FixedBlobSidecarList<T>,
) -> Result<Availability<T>, AvailabilityCheckError> {
blobs: FixedBlobSidecarList<T::EthSpec>,
) -> Result<Availability<T::EthSpec>, AvailabilityCheckError> {
// TODO(sean) we may duplicated kzg verification on some blobs we already have cached so we could optimize this
let mut verified_blobs = vec![];
@@ -265,8 +197,8 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
} else {
return Err(AvailabilityCheckError::KzgNotInitialized);
};
self.put_kzg_verified_blobs(block_root, &verified_blobs)
self.availability_cache
.put_kzg_verified_blobs(block_root, &verified_blobs)
}
/// This first validates the KZG commitments included in the blob sidecar.
@@ -287,53 +219,7 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
};
self.availability_cache
.put_kzg_verified_blob(kzg_verified_blob)
self.put_kzg_verified_blobs(kzg_verified_blob.block_root(), &[kzg_verified_blob])
}
fn put_kzg_verified_blobs(
&self,
block_root: Hash256,
kzg_verified_blobs: &[KzgVerifiedBlob<T>],
) -> Result<Availability<T>, AvailabilityCheckError> {
for blob in kzg_verified_blobs {
let blob_block_root = blob.block_root();
if blob_block_root != block_root {
return Err(AvailabilityCheckError::BlockBlobRootMismatch {
block_root,
blob_block_root,
});
}
}
let availability = match self.availability_cache.write().entry(block_root) {
Entry::Occupied(mut occupied_entry) => {
// All blobs reaching this cache should be gossip verified and gossip verification
// should filter duplicates, as well as validate indices.
let received_components = occupied_entry.get_mut();
for kzg_verified_blob in kzg_verified_blobs {
if let Some(maybe_verified_blob) = received_components
.verified_blobs
.get_mut(kzg_verified_blob.blob_index() as usize)
{
*maybe_verified_blob = Some(kzg_verified_blob.clone())
}
}
if let Some(executed_block) = received_components.executed_block.take() {
self.check_block_availability_maybe_cache(occupied_entry, executed_block)?
} else {
Availability::MissingComponents(block_root)
}
}
Entry::Vacant(vacant_entry) => {
vacant_entry.insert(ReceivedComponents::new_from_blobs(kzg_verified_blobs));
Availability::MissingComponents(block_root)
}
};
Ok(availability)
.put_kzg_verified_blobs(kzg_verified_blob.block_root(), &[kzg_verified_blob])
}
/// Check if we have all the blobs for a block. If we do, return the Availability variant that
@@ -346,59 +232,6 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
.put_pending_executed_block(executed_block)
}
/// Checks if the provided `executed_block` contains all required blobs to be considered an
/// `AvailableBlock` based on blobs that are cached.
///
/// Returns an error if there was an error when matching the block commitments against blob commitments.
///
/// Returns `Ok(Availability::Available(_))` if all blobs for the block are present in cache.
/// Returns `Ok(Availability::PendingBlobs(_))` if all corresponding blobs have not been received in the cache.
fn check_block_availability_maybe_cache(
&self,
mut occupied_entry: OccupiedEntry<Hash256, ReceivedComponents<T>>,
executed_block: AvailabilityPendingExecutedBlock<T>,
) -> Result<Availability<T>, AvailabilityCheckError> {
if occupied_entry.get().has_all_blobs(&executed_block) {
let num_blobs_expected = executed_block.num_blobs_expected();
let block_root = executed_block.import_data.block_root;
let AvailabilityPendingExecutedBlock {
block,
import_data,
payload_verification_outcome,
} = executed_block;
let ReceivedComponents {
verified_blobs,
executed_block: _,
} = occupied_entry.remove();
let verified_blobs = Vec::from(verified_blobs)
.into_iter()
.take(num_blobs_expected)
.map(|maybe_blob| {
maybe_blob.ok_or(AvailabilityCheckError::MissingBlobs(block_root))
})
.collect::<Result<Vec<_>, _>>()?;
let available_block = self.make_available(block, verified_blobs)?;
Ok(Availability::Available(Box::new(
AvailableExecutedBlock::new(
available_block,
import_data,
payload_verification_outcome,
),
)))
} else {
let received_components = occupied_entry.get_mut();
let block_root = executed_block.import_data.block_root;
let _ = received_components.executed_block.insert(executed_block);
Ok(Availability::MissingComponents(block_root))
}
}
/// Checks if a block is available, returns a `MaybeAvailableBlock` that may include the fully
/// available block.
pub fn check_availability(
@@ -422,27 +255,6 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
}
}
/// Checks if a block is available, returning an error if the block is not immediately available.
/// Does not access the gossip cache.
pub fn try_check_availability(
&self,
block: BlockWrapper<T::EthSpec>,
) -> Result<AvailableBlock<T::EthSpec>, AvailabilityCheckError> {
match block {
BlockWrapper::Block(block) => {
let blob_requirements = self.get_blob_requirements(&block)?;
let blobs = match blob_requirements {
BlobRequirements::EmptyBlobs => VerifiedBlobs::EmptyBlobs,
BlobRequirements::NotRequired => VerifiedBlobs::NotRequired,
BlobRequirements::PreDeneb => VerifiedBlobs::PreDeneb,
BlobRequirements::Required => return Err(AvailabilityCheckError::MissingBlobs),
};
Ok(AvailableBlock { block, blobs })
}
BlockWrapper::BlockAndBlobs(_, _) => Err(AvailabilityCheckError::Pending),
}
}
/// Verifies a block against a set of KZG verified blobs. Returns an AvailableBlock if block's
/// commitments are consistent with the provided verified blob commitments.
pub fn check_availability_with_blobs(

129
beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs
View File

@@ -11,7 +11,7 @@ use ssz_derive::{Decode, Encode};
use ssz_types::FixedVector;
use std::{collections::HashSet, sync::Arc};
use types::blob_sidecar::BlobIdentifier;
use types::{BlobSidecar, Epoch, EthSpec, Hash256};
use types::{BlobSidecar, Epoch, EthSpec, Hash256, SignedBeaconBlock};
/// Caches partially available blobs and execution verified blocks corresponding
/// to a given `block_hash` that are received over gossip.
@@ -25,10 +25,12 @@ pub struct PendingComponents<T: EthSpec> {
}
impl<T: EthSpec> PendingComponents<T> {
pub fn new_from_blob(blob: KzgVerifiedBlob<T>) -> Self {
pub fn new_from_blobs(blobs: &[KzgVerifiedBlob<T>]) -> Self {
let mut verified_blobs = FixedVector::<_, _>::default();
if let Some(mut_maybe_blob) = verified_blobs.get_mut(blob.blob_index() as usize) {
*mut_maybe_blob = Some(blob);
for blob in blobs {
if let Some(mut_maybe_blob) = verified_blobs.get_mut(blob.blob_index() as usize) {
*mut_maybe_blob = Some(blob.clone());
}
}
Self {
@@ -240,6 +242,12 @@ impl<T: BeaconChainTypes> Critical<T> {
Ok(())
}
pub fn has_block(&self, block_root: &Hash256) -> bool {
self.in_memory
.peek(block_root)
.map_or(false, |cache| cache.executed_block.is_some())
}
/// This only checks for the blobs in memory
pub fn peek_blob(
&self,
@@ -320,6 +328,34 @@ impl<T: BeaconChainTypes> OverflowLRUCache<T> {
})
}
pub fn has_block(&self, block_root: &Hash256) -> bool {
self.critical.read().has_block(block_root)
}
pub fn get_missing_blob_ids_checking_cache(
&self,
block_root: Hash256,
) -> (Option<Arc<SignedBeaconBlock<T::EthSpec>>>, HashSet<usize>) {
self.critical
.read()
.in_memory
.peek(&block_root)
.map(|cache| {
let block_opt = cache
.executed_block
.as_ref()
.map(|block| block.block.block.clone());
let blobs = cache
.verified_blobs
.iter()
.enumerate()
.filter_map(|(i, maybe_blob)| maybe_blob.as_ref().map(|_| i))
.collect::<HashSet<_>>();
(block_opt, blobs)
})
.unwrap_or_default()
}
pub fn peek_blob(
&self,
blob_id: &BlobIdentifier,
@@ -335,27 +371,39 @@ impl<T: BeaconChainTypes> OverflowLRUCache<T> {
}
}
pub fn put_kzg_verified_blob(
pub fn put_kzg_verified_blobs(
&self,
kzg_verified_blob: KzgVerifiedBlob<T::EthSpec>,
block_root: Hash256,
kzg_verified_blobs: &[KzgVerifiedBlob<T::EthSpec>],
) -> Result<Availability<T::EthSpec>, AvailabilityCheckError> {
for blob in kzg_verified_blobs {
let blob_block_root = blob.block_root();
if blob_block_root != block_root {
return Err(AvailabilityCheckError::BlockBlobRootMismatch {
block_root,
blob_block_root,
});
}
}
let mut write_lock = self.critical.write();
let block_root = kzg_verified_blob.block_root();
let availability = if let Some(mut pending_components) =
write_lock.pop_pending_components(block_root, &self.overflow_store)?
{
let blob_index = kzg_verified_blob.blob_index();
*pending_components
.verified_blobs
.get_mut(blob_index as usize)
.ok_or(AvailabilityCheckError::BlobIndexInvalid(blob_index))? =
Some(kzg_verified_blob);
for kzg_verified_blob in kzg_verified_blobs {
let blob_index = kzg_verified_blob.blob_index() as usize;
if let Some(maybe_verified_blob) =
pending_components.verified_blobs.get_mut(blob_index)
{
*maybe_verified_blob = Some(kzg_verified_blob.clone())
} else {
return Err(AvailabilityCheckError::BlobIndexInvalid(blob_index as u64));
}
}
if let Some(executed_block) = pending_components.executed_block.take() {
self.check_block_availability_maybe_cache(
write_lock,
block_root,
pending_components,
executed_block,
)?
@@ -365,17 +413,17 @@ impl<T: BeaconChainTypes> OverflowLRUCache<T> {
pending_components,
&self.overflow_store,
)?;
Availability::PendingBlock(block_root)
Availability::MissingComponents(block_root)
}
} else {
// not in memory or store -> put new in memory
let new_pending_components = PendingComponents::new_from_blob(kzg_verified_blob);
let new_pending_components = PendingComponents::new_from_blobs(kzg_verified_blobs);
write_lock.put_pending_components(
block_root,
new_pending_components,
&self.overflow_store,
)?;
Availability::PendingBlock(block_root)
Availability::MissingComponents(block_root)
};
Ok(availability)
@@ -394,7 +442,6 @@ impl<T: BeaconChainTypes> OverflowLRUCache<T> {
match write_lock.pop_pending_components(block_root, &self.overflow_store)? {
Some(pending_components) => self.check_block_availability_maybe_cache(
write_lock,
block_root,
pending_components,
executed_block,
)?,
@@ -422,7 +469,7 @@ impl<T: BeaconChainTypes> OverflowLRUCache<T> {
new_pending_components,
&self.overflow_store,
)?;
Availability::PendingBlobs(all_blob_ids)
Availability::MissingComponents(block_root)
}
};
@@ -435,11 +482,10 @@ impl<T: BeaconChainTypes> OverflowLRUCache<T> {
/// Returns an error if there was an error when matching the block commitments against blob commitments.
///
/// Returns `Ok(Availability::Available(_))` if all blobs for the block are present in cache.
/// Returns `Ok(Availability::PendingBlobs(_))` if all corresponding blobs have not been received in the cache.
/// Returns `Ok(Availability::MissingComponents(_))` if all corresponding blobs have not been received in the cache.
fn check_block_availability_maybe_cache(
&self,
mut write_lock: RwLockWriteGuard<Critical<T>>,
block_root: Hash256,
mut pending_components: PendingComponents<T::EthSpec>,
executed_block: AvailabilityPendingExecutedBlock<T::EthSpec>,
) -> Result<Availability<T::EthSpec>, AvailabilityCheckError> {
@@ -466,14 +512,7 @@ impl<T: BeaconChainTypes> OverflowLRUCache<T> {
),
)))
} else {
let missing_blob_ids = executed_block.get_filtered_blob_ids(|index| {
pending_components
.verified_blobs
.get(index as usize)
.map(|maybe_blob| maybe_blob.is_none())
.unwrap_or(true)
});
let block_root = executed_block.import_data.block_root;
let _ = pending_components.executed_block.insert(executed_block);
write_lock.put_pending_components(
block_root,
@@ -481,7 +520,7 @@ impl<T: BeaconChainTypes> OverflowLRUCache<T> {
&self.overflow_store,
)?;
Ok(Availability::PendingBlobs(missing_blob_ids))
Ok(Availability::MissingComponents(block_root))
}
}
@@ -1080,7 +1119,7 @@ mod test {
);
} else {
assert!(
matches!(availability, Availability::PendingBlobs(_)),
matches!(availability, Availability::MissingComponents(_)),
"should be pending blobs"
);
assert_eq!(
@@ -1109,7 +1148,7 @@ mod test {
if blob_index == blobs_expected - 1 {
assert!(matches!(availability, Availability::Available(_)));
} else {
assert!(matches!(availability, Availability::PendingBlobs(_)));
assert!(matches!(availability, Availability::MissingComponents(_)));
assert_eq!(cache.critical.read().in_memory.len(), 1);
}
}
@@ -1134,7 +1173,7 @@ mod test {
.expect("should put blob");
assert_eq!(
availability,
Availability::PendingBlock(root),
Availability::MissingComponents(root),
"should be pending block"
);
assert_eq!(cache.critical.read().in_memory.len(), 1);
@@ -1284,7 +1323,7 @@ mod test {
cache.critical.read().in_memory.peek(&roots[0]).is_some(),
"first block should be in memory"
);
assert!(matches!(availability, Availability::PendingBlobs(_)));
assert!(matches!(availability, Availability::MissingComponents(_)));
}
}
assert_eq!(
@@ -1374,14 +1413,14 @@ mod test {
.put_pending_executed_block(pending_block)
.expect("should put block");
assert!(
matches!(availability, Availability::PendingBlobs(_)),
matches!(availability, Availability::MissingComponents(_)),
"should have pending blobs"
);
let availability = cache
.put_kzg_verified_blob(kzg_verified_blob)
.expect("should put blob");
assert!(
matches!(availability, Availability::PendingBlobs(_)),
matches!(availability, Availability::MissingComponents(_)),
"availabilty should be pending blobs: {:?}",
availability
);
@@ -1392,14 +1431,14 @@ mod test {
let root = pending_block.block.as_block().canonical_root();
assert_eq!(
availability,
Availability::PendingBlock(root),
Availability::MissingComponents(root),
"should be pending block"
);
let availability = cache
.put_pending_executed_block(pending_block)
.expect("should put block");
assert!(
matches!(availability, Availability::PendingBlobs(_)),
matches!(availability, Availability::MissingComponents(_)),
"should have pending blobs"
);
}
@@ -1410,7 +1449,7 @@ mod test {
.put_pending_executed_block(pending_block)
.expect("should put block");
assert!(
matches!(availability, Availability::PendingBlobs(_)),
matches!(availability, Availability::MissingComponents(_)),
"should be pending blobs"
);
}
@@ -1527,14 +1566,14 @@ mod test {
.put_pending_executed_block(pending_block)
.expect("should put block");
assert!(
matches!(availability, Availability::PendingBlobs(_)),
matches!(availability, Availability::MissingComponents(_)),
"should have pending blobs"
);
let availability = cache
.put_kzg_verified_blob(kzg_verified_blob)
.expect("should put blob");
assert!(
matches!(availability, Availability::PendingBlobs(_)),
matches!(availability, Availability::MissingComponents(_)),
"availabilty should be pending blobs: {:?}",
availability
);
@@ -1545,14 +1584,14 @@ mod test {
let root = pending_block.block.as_block().canonical_root();
assert_eq!(
availability,
Availability::PendingBlock(root),
Availability::MissingComponents(root),
"should be pending block"
);
let availability = cache
.put_pending_executed_block(pending_block)
.expect("should put block");
assert!(
matches!(availability, Availability::PendingBlobs(_)),
matches!(availability, Availability::MissingComponents(_)),
"should have pending blobs"
);
}
@@ -1564,7 +1603,7 @@ mod test {
.put_pending_executed_block(pending_block)
.expect("should put block");
assert!(
matches!(availability, Availability::PendingBlobs(_)),
matches!(availability, Availability::MissingComponents(_)),
"should be pending blobs"
);
}
@@ -1637,7 +1676,7 @@ mod test {
if i == additional_blobs - 1 {
assert!(matches!(availability, Availability::Available(_)))
} else {
assert!(matches!(availability, Availability::PendingBlobs(_)));
assert!(matches!(availability, Availability::MissingComponents(_)));
}
}
}