Rework block processing (#4092)

* introduce availability pending block

* add intoavailableblock trait

* small fixes

* add 'gossip blob cache' and start to clean up processing and transition types

* shard memory blob cache

* Initial commit

* Fix after rebase

* Add gossip verification conditions

* cache cleanup

* general chaos

* extended chaos

* cargo fmt

* more progress

* more progress

* tons of changes, just tryna compile

* everything, everywhere, all at once

* Reprocess an ExecutedBlock on unavailable blobs

* Add sus gossip verification for blobs

* Merge stuff

* Remove reprocessing cache stuff

* lint

* Add a wrapper to allow construction of only valid `AvailableBlock`s

* rename blob arc list to blob list

* merge cleanuo

* Revert "merge cleanuo"

This reverts commit 5e98326878.

* Revert "Revert "merge cleanuo""

This reverts commit 3a4009443a.

* fix rpc methods

* move beacon block and blob to eth2/types

* rename gossip blob cache to data availability checker

* lots of changes

* fix some compilation issues

* fix compilation issues

* fix compilation issues

* fix compilation issues

* fix compilation issues

* fix compilation issues

* cargo fmt

* use a common data structure for block import types

* fix availability check on proposal import

* refactor the blob cache and split the block wrapper into two types

* add type conversion for signed block and block wrapper

* fix beacon chain tests and do some renaming, add some comments

* Partial processing (#4)

* move beacon block and blob to eth2/types

* rename gossip blob cache to data availability checker

* lots of changes

* fix some compilation issues

* fix compilation issues

* fix compilation issues

* fix compilation issues

* fix compilation issues

* fix compilation issues

* cargo fmt

* use a common data structure for block import types

* fix availability check on proposal import

* refactor the blob cache and split the block wrapper into two types

* add type conversion for signed block and block wrapper

* fix beacon chain tests and do some renaming, add some comments

* cargo update (#6)

---------

Co-authored-by: realbigsean <sean@sigmaprime.io>
Co-authored-by: realbigsean <seananderson33@gmail.com>

consensus/types/src/beacon_block_and_blob_sidecars.rs

@@ -1,37 +0,0 @@
-use crate::{
-    AbstractExecPayload, BeaconBlock, BlobSidecarList, EthSpec, ForkName, ForkVersionDeserialize,
-};
-use derivative::Derivative;
-use serde_derive::{Deserialize, Serialize};
-use ssz_derive::Encode;
-use tree_hash_derive::TreeHash;
-
-#[derive(Debug, Clone, Serialize, Deserialize, Encode, TreeHash, Derivative)]
-#[derivative(PartialEq, Hash(bound = "T: EthSpec"))]
-#[serde(bound = "T: EthSpec, Payload: AbstractExecPayload<T>")]
-pub struct BeaconBlockAndBlobSidecars<T: EthSpec, Payload: AbstractExecPayload<T>> {
-    pub block: BeaconBlock<T, Payload>,
-    pub blob_sidecars: BlobSidecarList<T>,
-}
-
-impl<T: EthSpec, Payload: AbstractExecPayload<T>> ForkVersionDeserialize
-    for BeaconBlockAndBlobSidecars<T, Payload>
-{
-    fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>(
-        value: serde_json::value::Value,
-        fork_name: ForkName,
-    ) -> Result<Self, D::Error> {
-        #[derive(Deserialize)]
-        #[serde(bound = "T: EthSpec")]
-        struct Helper<T: EthSpec> {
-            block: serde_json::Value,
-            blob_sidecars: BlobSidecarList<T>,
-        }
-        let helper: Helper<T> = serde_json::from_value(value).map_err(serde::de::Error::custom)?;
-
-        Ok(Self {
-            block: BeaconBlock::deserialize_by_fork::<'de, D>(helper.block, fork_name)?,
-            blob_sidecars: helper.blob_sidecars,
-        })
-    }
-}

consensus/types/src/beacon_block_body.rs

@@ -1,5 +1,5 @@
+use crate::test_utils::TestRandom;
 use crate::*;
-use crate::{blobs_sidecar::KzgCommitments, test_utils::TestRandom};
 use derivative::Derivative;
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
@@ -9,6 +9,8 @@ use superstruct::superstruct;
 use test_random_derive::TestRandom;
 use tree_hash_derive::TreeHash;
 
+pub type KzgCommitments<T> = VariableList<KzgCommitment, <T as EthSpec>::MaxBlobsPerBlock>;
+
 /// The body of a `BeaconChain` block, containing operations.
 ///
 /// This *superstruct* abstracts over the hard-fork.

consensus/types/src/blob_sidecar.rs

@@ -11,7 +11,7 @@ use test_random_derive::TestRandom;
 use tree_hash_derive::TreeHash;
 
 /// Container of the data that identifies an individual blob.
-#[derive(Encode, Decode, Clone, Debug, PartialEq)]
+#[derive(Encode, Decode, Clone, Debug, PartialEq, Eq, Hash)]
 pub struct BlobIdentifier {
     pub block_root: Hash256,
     pub index: u64,
@@ -35,7 +35,6 @@ pub struct BlobIdentifier {
 #[derivative(PartialEq, Hash(bound = "T: EthSpec"))]
 pub struct BlobSidecar<T: EthSpec> {
     pub block_root: Hash256,
-    // TODO: fix the type, should fit in u8 as well
     #[serde(with = "eth2_serde_utils::quoted_u64")]
     pub index: u64,
     pub slot: Slot,
@@ -49,10 +48,18 @@ pub struct BlobSidecar<T: EthSpec> {
 }
 
 pub type BlobSidecarList<T> = VariableList<Arc<BlobSidecar<T>>, <T as EthSpec>::MaxBlobsPerBlock>;
+pub type Blobs<T> = VariableList<Blob<T>, <T as EthSpec>::MaxExtraDataBytes>;
 
 impl<T: EthSpec> SignedRoot for BlobSidecar<T> {}
 
 impl<T: EthSpec> BlobSidecar<T> {
+    pub fn id(&self) -> BlobIdentifier {
+        BlobIdentifier {
+            block_root: self.block_root,
+            index: self.index,
+        }
+    }
+
     pub fn empty() -> Self {
         Self::default()
     }

consensus/types/src/blobs_sidecar.rs

@@ -1,62 +0,0 @@
-use crate::test_utils::TestRandom;
-use crate::{Blob, EthSpec, Hash256, KzgCommitment, SignedRoot, Slot};
-use derivative::Derivative;
-use kzg::KzgProof;
-use serde_derive::{Deserialize, Serialize};
-use ssz::Encode;
-use ssz_derive::{Decode, Encode};
-use ssz_types::VariableList;
-use test_random_derive::TestRandom;
-use tree_hash_derive::TreeHash;
-
-pub type KzgCommitments<T> = VariableList<KzgCommitment, <T as EthSpec>::MaxBlobsPerBlock>;
-pub type Blobs<T> = VariableList<Blob<T>, <T as EthSpec>::MaxBlobsPerBlock>;
-
-#[derive(
-    Debug,
-    Clone,
-    Serialize,
-    Deserialize,
-    Encode,
-    Decode,
-    TreeHash,
-    Default,
-    TestRandom,
-    Derivative,
-    arbitrary::Arbitrary,
-)]
-#[serde(bound = "T: EthSpec")]
-#[arbitrary(bound = "T: EthSpec")]
-#[derivative(PartialEq, Hash(bound = "T: EthSpec"))]
-pub struct BlobsSidecar<T: EthSpec> {
-    pub beacon_block_root: Hash256,
-    pub beacon_block_slot: Slot,
-    #[serde(with = "ssz_types::serde_utils::list_of_hex_fixed_vec")]
-    pub blobs: Blobs<T>,
-    pub kzg_aggregated_proof: KzgProof,
-}
-
-impl<T: EthSpec> SignedRoot for BlobsSidecar<T> {}
-
-impl<T: EthSpec> BlobsSidecar<T> {
-    pub fn empty() -> Self {
-        Self::default()
-    }
-
-    pub fn empty_from_parts(beacon_block_root: Hash256, beacon_block_slot: Slot) -> Self {
-        Self {
-            beacon_block_root,
-            beacon_block_slot,
-            blobs: VariableList::empty(),
-            kzg_aggregated_proof: KzgProof::empty(),
-        }
-    }
-
-    #[allow(clippy::integer_arithmetic)]
-    pub fn max_size() -> usize {
-        // Fixed part
-        Self::empty().as_ssz_bytes().len()
-            // Max size of variable length `blobs` field
-            + (T::max_blobs_per_block() * <Blob<T> as Encode>::ssz_fixed_len())
-    }
-}

consensus/types/src/lib.rs

@@ -97,11 +97,8 @@ pub mod slot_data;
 #[cfg(feature = "sqlite")]
 pub mod sqlite;
 
-pub mod beacon_block_and_blob_sidecars;
 pub mod blob_sidecar;
-pub mod blobs_sidecar;
 pub mod signed_blob;
-pub mod signed_block_and_blobs;
 pub mod transaction;
 
 use ethereum_types::{H160, H256};
@@ -115,7 +112,6 @@ pub use crate::beacon_block::{
     BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockCapella, BeaconBlockEip4844,
     BeaconBlockMerge, BeaconBlockRef, BeaconBlockRefMut, BlindedBeaconBlock, EmptyBlock,
 };
-pub use crate::beacon_block_and_blob_sidecars::BeaconBlockAndBlobSidecars;
 pub use crate::beacon_block_body::{
     BeaconBlockBody, BeaconBlockBodyAltair, BeaconBlockBodyBase, BeaconBlockBodyCapella,
     BeaconBlockBodyEip4844, BeaconBlockBodyMerge, BeaconBlockBodyRef, BeaconBlockBodyRefMut,
@@ -124,7 +120,6 @@ pub use crate::beacon_block_header::BeaconBlockHeader;
 pub use crate::beacon_committee::{BeaconCommittee, OwnedBeaconCommittee};
 pub use crate::beacon_state::{BeaconTreeHashCache, Error as BeaconStateError, *};
 pub use crate::blob_sidecar::{BlobSidecar, BlobSidecarList};
-pub use crate::blobs_sidecar::{Blobs, BlobsSidecar};
 pub use crate::bls_to_execution_change::BlsToExecutionChange;
 pub use crate::chain_spec::{ChainSpec, Config, Domain};
 pub use crate::checkpoint::Checkpoint;
@@ -183,9 +178,6 @@ pub use crate::signed_beacon_block::{
 };
 pub use crate::signed_beacon_block_header::SignedBeaconBlockHeader;
 pub use crate::signed_blob::*;
-pub use crate::signed_block_and_blobs::{
-    SignedBeaconBlockAndBlobsSidecar, SignedBeaconBlockAndBlobsSidecarDecode,
-};
 pub use crate::signed_bls_to_execution_change::SignedBlsToExecutionChange;
 pub use crate::signed_contribution_and_proof::SignedContributionAndProof;
 pub use crate::signed_voluntary_exit::SignedVoluntaryExit;

consensus/types/src/signed_beacon_block.rs

@@ -35,14 +35,6 @@ impl From<SignedBeaconBlockHash> for Hash256 {
     }
 }
 
-#[derive(Debug)]
-pub enum BlobReconstructionError {
-    /// No blobs for the specified block where we would expect blobs.
-    UnavailableBlobs,
-    /// Blobs provided for a pre-Eip4844 fork.
-    InconsistentFork,
-}
-
 /// A `BeaconBlock` and a signature from its proposer.
 #[superstruct(
     variants(Base, Altair, Merge, Capella, Eip4844),
@@ -250,28 +242,6 @@ impl<E: EthSpec, Payload: AbstractExecPayload<E>> SignedBeaconBlock<E, Payload>
     pub fn canonical_root(&self) -> Hash256 {
         self.message().tree_hash_root()
     }
-
-    /// Reconstructs an empty `BlobsSidecar`, using the given block root if provided, else calculates it.
-    /// If this block has kzg commitments, an error will be returned. If this block is from prior to the
-    /// Eip4844 fork, this will error.
-    pub fn reconstruct_empty_blobs(
-        &self,
-        block_root_opt: Option<Hash256>,
-    ) -> Result<BlobsSidecar<E>, BlobReconstructionError> {
-        let kzg_commitments = self
-            .message()
-            .body()
-            .blob_kzg_commitments()
-            .map_err(|_| BlobReconstructionError::InconsistentFork)?;
-        if kzg_commitments.is_empty() {
-            Ok(BlobsSidecar::empty_from_parts(
-                block_root_opt.unwrap_or_else(|| self.canonical_root()),
-                self.slot(),
-            ))
-        } else {
-            Err(BlobReconstructionError::UnavailableBlobs)
-        }
-    }
 }
 
 // We can convert pre-Bellatrix blocks without payloads into blocks with payloads.

consensus/types/src/signed_blob.rs

@@ -1,10 +1,15 @@
-use crate::{test_utils::TestRandom, BlobSidecar, EthSpec, Signature};
+use crate::{
+    test_utils::TestRandom, BlobSidecar, ChainSpec, Domain, EthSpec, Fork, Hash256, Signature,
+    SignedRoot, SigningData,
+};
+use bls::PublicKey;
 use derivative::Derivative;
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use ssz_types::VariableList;
 use std::sync::Arc;
 use test_random_derive::TestRandom;
+use tree_hash::TreeHash;
 use tree_hash_derive::TreeHash;
 
 #[derive(
@@ -30,3 +35,37 @@ pub struct SignedBlobSidecar<T: EthSpec> {
 
 pub type SignedBlobSidecarList<T> =
     VariableList<SignedBlobSidecar<T>, <T as EthSpec>::MaxBlobsPerBlock>;
+
+impl<T: EthSpec> SignedBlobSidecar<T> {
+    /// Verify `self.signature`.
+    ///
+    /// If the root of `block.message` is already known it can be passed in via `object_root_opt`.
+    /// Otherwise, it will be computed locally.
+    pub fn verify_signature(
+        &self,
+        object_root_opt: Option<Hash256>,
+        pubkey: &PublicKey,
+        fork: &Fork,
+        genesis_validators_root: Hash256,
+        spec: &ChainSpec,
+    ) -> bool {
+        let domain = spec.get_domain(
+            self.message.slot.epoch(T::slots_per_epoch()),
+            Domain::BlobSidecar,
+            fork,
+            genesis_validators_root,
+        );
+
+        let message = if let Some(object_root) = object_root_opt {
+            SigningData {
+                object_root,
+                domain,
+            }
+            .tree_hash_root()
+        } else {
+            self.message.signing_root(domain)
+        };
+
+        self.signature.verify(pubkey, message)
+    }
+}

consensus/types/src/signed_block_and_blobs.rs

@@ -1,35 +0,0 @@
-use crate::{BlobsSidecar, EthSpec, SignedBeaconBlock, SignedBeaconBlockEip4844};
-use derivative::Derivative;
-use serde_derive::{Deserialize, Serialize};
-use ssz::{Decode, DecodeError};
-use ssz_derive::{Decode, Encode};
-use std::sync::Arc;
-use tree_hash_derive::TreeHash;
-
-#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, PartialEq)]
-#[serde(bound = "T: EthSpec")]
-pub struct SignedBeaconBlockAndBlobsSidecarDecode<T: EthSpec> {
-    pub beacon_block: SignedBeaconBlockEip4844<T>,
-    pub blobs_sidecar: BlobsSidecar<T>,
-}
-
-// TODO: will be removed once we decouple blobs in Gossip
-#[derive(Debug, Clone, Serialize, Deserialize, Encode, TreeHash, Derivative)]
-#[derivative(PartialEq, Hash(bound = "T: EthSpec"))]
-pub struct SignedBeaconBlockAndBlobsSidecar<T: EthSpec> {
-    pub beacon_block: Arc<SignedBeaconBlock<T>>,
-    pub blobs_sidecar: Arc<BlobsSidecar<T>>,
-}
-
-impl<T: EthSpec> SignedBeaconBlockAndBlobsSidecar<T> {
-    pub fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
-        let SignedBeaconBlockAndBlobsSidecarDecode {
-            beacon_block,
-            blobs_sidecar,
-        } = SignedBeaconBlockAndBlobsSidecarDecode::from_ssz_bytes(bytes)?;
-        Ok(SignedBeaconBlockAndBlobsSidecar {
-            beacon_block: Arc::new(SignedBeaconBlock::Eip4844(beacon_block)),
-            blobs_sidecar: Arc::new(blobs_sidecar),
-        })
-    }
-}