add random blobs

beacon_node/beacon_chain/src/data_availability_checker.rs

@@ -32,7 +32,7 @@ pub enum AvailabilityCheckError {
         num_kzg_commitments: usize,
         num_blobs: usize,
     },
-    TxKzgCommitmentMismatch,
+    TxKzgCommitmentMismatch(String),
     KzgCommitmentMismatch {
         blob_index: u64,
     },
@@ -489,9 +489,11 @@ impl<T: EthSpec, S: SlotClock> DataAvailabilityChecker<T, S> {
                     transactions,
                     block_kzg_commitments,
                 )
-                .map_err(|_| AvailabilityCheckError::TxKzgCommitmentMismatch)?;
+                .map_err(|e| AvailabilityCheckError::TxKzgCommitmentMismatch(format!("{e:?}")))?;
                 if !verified {
-                    return Err(AvailabilityCheckError::TxKzgCommitmentMismatch);
+                    return Err(AvailabilityCheckError::TxKzgCommitmentMismatch(
+                        "a commitment and version didn't match".to_string(),
+                    ));
                 }
             }
 

beacon_node/execution_layer/src/test_utils/execution_block_generator.rs

@@ -16,6 +16,7 @@ use std::collections::HashMap;
 use std::sync::Arc;
 use tree_hash::TreeHash;
 use tree_hash_derive::TreeHash;
+use types::consts::deneb::BLOB_TX_TYPE;
 use types::transaction::{BlobTransaction, EcdsaSignature, SignedBlobTransaction};
 use types::{
     Blob, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella,
@@ -585,7 +586,8 @@ impl<T: EthSpec> ExecutionBlockGenerator<T> {
                     ForkName::Deneb => {
                         // get random number between 0 and Max Blobs
                         let num_blobs = rand::random::<usize>() % T::max_blobs_per_block();
-                        let (bundle, transactions) = self.generate_random_blobs(num_blobs)?;
+                        let kzg = self.kzg.as_ref().ok_or("kzg not initialized")?;
+                        let (bundle, transactions) = generate_random_blobs(num_blobs, &kzg)?;
                         for tx in Vec::from(transactions) {
                             execution_payload
                                 .transactions_mut()
@@ -625,88 +627,82 @@ impl<T: EthSpec> ExecutionBlockGenerator<T> {
             payload_id: id.map(Into::into),
         })
     }
+}
 
-    fn generate_random_blobs(
+pub fn generate_random_blobs<T: EthSpec>(
-        &self,
+    n_blobs: usize,
-        n_blobs: usize,
+    kzg: &Kzg,
-    ) -> Result<(BlobsBundleV1<T>, Transactions<T>), String> {
+) -> Result<(BlobsBundleV1<T>, Transactions<T>), String> {
-        let mut bundle = BlobsBundleV1::<T>::default();
+    let mut bundle = BlobsBundleV1::<T>::default();
-        let mut transactions = vec![];
+    let mut transactions = vec![];
-        for blob_index in 0..n_blobs {
+    for blob_index in 0..n_blobs {
-            // fill a vector with random bytes
+        // fill a vector with random bytes
-            let mut blob_bytes = [0u8; BYTES_PER_BLOB];
+        let mut blob_bytes = [0u8; BYTES_PER_BLOB];
-            rand::thread_rng().fill_bytes(&mut blob_bytes);
+        rand::thread_rng().fill_bytes(&mut blob_bytes);
-            // Ensure that the blob is canonical by ensuring that
+        // Ensure that the blob is canonical by ensuring that
-            // each field element contained in the blob is < BLS_MODULUS
+        // each field element contained in the blob is < BLS_MODULUS
-            for i in 0..FIELD_ELEMENTS_PER_BLOB {
+        for i in 0..FIELD_ELEMENTS_PER_BLOB {
-                blob_bytes[i * BYTES_PER_FIELD_ELEMENT + BYTES_PER_FIELD_ELEMENT - 1] = 0;
+            blob_bytes[i * BYTES_PER_FIELD_ELEMENT + BYTES_PER_FIELD_ELEMENT - 1] = 0;
-            }
-
-            let blob = Blob::<T>::new(Vec::from(blob_bytes))
-                .map_err(|e| format!("error constructing random blob: {:?}", e))?;
-
-            let commitment = self
-                .kzg
-                .as_ref()
-                .ok_or("kzg not initialized")?
-                .blob_to_kzg_commitment(blob_bytes.into())
-                .map_err(|e| format!("error computing kzg commitment: {:?}", e))?;
-
-            let proof = self
-                .kzg
-                .as_ref()
-                .ok_or("kzg not initialized")?
-                .compute_blob_kzg_proof(blob_bytes.into(), commitment)
-                .map_err(|e| format!("error computing kzg proof: {:?}", e))?;
-
-            let versioned_hash = commitment.calculate_versioned_hash();
-
-            let blob_transaction = BlobTransaction {
-                chain_id: Default::default(),
-                nonce: 0,
-                max_priority_fee_per_gas: Default::default(),
-                max_fee_per_gas: Default::default(),
-                gas: 100000,
-                to: None,
-                value: Default::default(),
-                data: Default::default(),
-                access_list: Default::default(),
-                max_fee_per_data_gas: Default::default(),
-                versioned_hashes: vec![versioned_hash].into(),
-            };
-            let bad_signature = EcdsaSignature {
-                y_parity: false,
-                r: Uint256::from(0),
-                s: Uint256::from(0),
-            };
-            let signed_blob_transaction = SignedBlobTransaction {
-                message: blob_transaction,
-                signature: bad_signature,
-            };
-            // calculate transaction bytes
-            let tx_bytes = [0x05u8]
-                .into_iter()
-                .chain(signed_blob_transaction.as_ssz_bytes().into_iter())
-                .collect::<Vec<_>>();
-            let tx = Transaction::<T::MaxBytesPerTransaction>::from(tx_bytes);
-
-            transactions.push(tx);
-            bundle
-                .blobs
-                .push(blob)
-                .map_err(|_| format!("blobs are full, blob index: {:?}", blob_index))?;
-            bundle
-                .commitments
-                .push(commitment)
-                .map_err(|_| format!("blobs are full, blob index: {:?}", blob_index))?;
-            bundle
-                .proofs
-                .push(proof)
-                .map_err(|_| format!("blobs are full, blob index: {:?}", blob_index))?;
         }
 
-        Ok((bundle, transactions.into()))
+        let blob = Blob::<T>::new(Vec::from(blob_bytes))
+            .map_err(|e| format!("error constructing random blob: {:?}", e))?;
+
+        let commitment = kzg
+            .blob_to_kzg_commitment(blob_bytes.into())
+            .map_err(|e| format!("error computing kzg commitment: {:?}", e))?;
+
+        let proof = kzg
+            .compute_blob_kzg_proof(blob_bytes.into(), commitment)
+            .map_err(|e| format!("error computing kzg proof: {:?}", e))?;
+
+        let versioned_hash = commitment.calculate_versioned_hash();
+
+        let blob_transaction = BlobTransaction {
+            chain_id: Default::default(),
+            nonce: 0,
+            max_priority_fee_per_gas: Default::default(),
+            max_fee_per_gas: Default::default(),
+            gas: 100000,
+            to: None,
+            value: Default::default(),
+            data: Default::default(),
+            access_list: Default::default(),
+            max_fee_per_data_gas: Default::default(),
+            versioned_hashes: vec![versioned_hash].into(),
+        };
+        let bad_signature = EcdsaSignature {
+            y_parity: false,
+            r: Uint256::from(0),
+            s: Uint256::from(0),
+        };
+        let signed_blob_transaction = SignedBlobTransaction {
+            message: blob_transaction,
+            signature: bad_signature,
+        };
+        // calculate transaction bytes
+        let tx_bytes = [BLOB_TX_TYPE]
+            .into_iter()
+            .chain(signed_blob_transaction.as_ssz_bytes().into_iter())
+            .collect::<Vec<_>>();
+        let tx = Transaction::<T::MaxBytesPerTransaction>::from(tx_bytes);
+
+        transactions.push(tx);
+        bundle
+            .blobs
+            .push(blob)
+            .map_err(|_| format!("blobs are full, blob index: {:?}", blob_index))?;
+        bundle
+            .commitments
+            .push(commitment)
+            .map_err(|_| format!("blobs are full, blob index: {:?}", blob_index))?;
+        bundle
+            .proofs
+            .push(proof)
+            .map_err(|_| format!("blobs are full, blob index: {:?}", blob_index))?;
     }
+
+    Ok((bundle, transactions.into()))
 }
 
 fn payload_id_from_u64(n: u64) -> PayloadId {

beacon_node/execution_layer/src/test_utils/mod.rs

@@ -24,7 +24,9 @@ use types::{EthSpec, ExecutionBlockHash, Uint256};
 use warp::{http::StatusCode, Filter, Rejection};
 
 use crate::EngineCapabilities;
-pub use execution_block_generator::{generate_pow_block, Block, ExecutionBlockGenerator};
+pub use execution_block_generator::{
+    generate_pow_block, generate_random_blobs, Block, ExecutionBlockGenerator,
+};
 pub use hook::Hook;
 pub use mock_builder::{Context as MockBuilderContext, MockBuilder, Operation, TestingBuilder};
 pub use mock_execution_layer::MockExecutionLayer;

beacon_node/network/Cargo.toml

@@ -50,4 +50,5 @@ execution_layer =  { path = "../execution_layer" }
 
 [features]
 deterministic_long_lived_attnets = [ "ethereum-types" ]
+spec-minimal = ["beacon_chain/spec-minimal"]
 # default = ["deterministic_long_lived_attnets"]

beacon_node/network/src/sync/block_lookups/tests.rs

@@ -21,7 +21,7 @@ use tokio::sync::mpsc;
 use types::{
     map_fork_name, map_fork_name_with,
     test_utils::{SeedableRng, TestRandom, XorShiftRng},
-    BeaconBlock, ForkName, MinimalEthSpec as E, SignedBeaconBlock,
+    BeaconBlock, EthSpec, ForkName, FullPayloadDeneb, MinimalEthSpec as E, SignedBeaconBlock,
 };
 
 type T = Witness<TestingSlotClock, CachingEth1Backend<E>, E, MemoryStore<E>, MemoryStore<E>>;
@@ -30,6 +30,7 @@ struct TestRig {
     beacon_processor_rx: mpsc::Receiver<WorkEvent<T>>,
     network_rx: mpsc::UnboundedReceiver<NetworkMessage<E>>,
     rng: XorShiftRng,
+    harness: BeaconChainHarness<T>,
 }
 
 const D: Duration = Duration::new(0, 0);
@@ -46,7 +47,7 @@ impl TestRig {
             .fresh_ephemeral_store()
             .build();
 
-        let chain = harness.chain;
+        let chain = harness.chain.clone();
 
         let (beacon_processor_tx, beacon_processor_rx) = mpsc::channel(100);
         let (network_tx, network_rx) = mpsc::unbounded_channel();
@@ -55,17 +56,11 @@ impl TestRig {
             beacon_processor_rx,
             network_rx,
             rng,
+            harness,
         };
 
-        //TODO(sean) add a data availability checker to the harness and use that one
-        let da_checker = Arc::new(DataAvailabilityChecker::new(
-            chain.slot_clock.clone(),
-            None,
-            chain.spec.clone(),
-        ));
-
         let bl = BlockLookups::new(
-            da_checker,
+            chain.data_availability_checker.clone(),
             log.new(slog::o!("component" => "block_lookups")),
         );
         let cx = {
@@ -84,7 +79,25 @@ impl TestRig {
 
     fn rand_block(&mut self, fork_name: ForkName) -> SignedBeaconBlock<E> {
         let inner = map_fork_name!(fork_name, BeaconBlock, <_>::random_for_test(&mut self.rng));
-        SignedBeaconBlock::from_block(inner, types::Signature::random_for_test(&mut self.rng))
+        let mut block =
+            SignedBeaconBlock::from_block(inner, types::Signature::random_for_test(&mut self.rng));
+        if let Ok(message) = block.message_deneb_mut() {
+            // get random number between 0 and Max Blobs
+            let mut payload: &mut FullPayloadDeneb<E> = &mut message.body.execution_payload;
+            let num_blobs = rand::random::<usize>() % E::max_blobs_per_block();
+            let (bundle, transactions) = execution_layer::test_utils::generate_random_blobs::<E>(
+                num_blobs,
+                &self.harness.chain.kzg.as_ref().unwrap(),
+            )
+            .unwrap();
+            payload.execution_payload.transactions = <_>::default();
+            for tx in Vec::from(transactions) {
+                payload.execution_payload.transactions.push(tx).unwrap();
+            }
+            message.body.blob_kzg_commitments = bundle.commitments.clone();
+        }
+
+        block
     }
 
     #[track_caller]
@@ -184,10 +197,9 @@ impl TestRig {
         parent_root: Hash256,
         fork_name: ForkName,
     ) -> SignedBeaconBlock<E> {
-        let mut inner = map_fork_name!(fork_name, BeaconBlock, <_>::random_for_test(&mut self.rng));
+        let mut block = self.rand_block(fork_name);
-
+        *block.message_mut().parent_root_mut() = parent_root;
-        *inner.parent_root_mut() = parent_root;
+        block
-        SignedBeaconBlock::from_block(inner, types::Signature::random_for_test(&mut self.rng))
     }
 }
 
@@ -914,7 +926,9 @@ macro_rules! common_tests {
         }
     };
 }
-use crate::sync::manager::ResponseType::Block;
+use crate::sync::manager::ResponseType::{Blob, Block};
-common_tests!(base, Base, Block);
+// common_tests!(base, Base, Block);
-common_tests!(capella, Capella, Block);
+// common_tests!(capella, Capella, Block);
-common_tests!(deneb, Deneb, Block);
+// common_tests!(deneb, Deneb, Block);
+
+common_tests!(deneb, Deneb, Blob);

consensus/types/src/eth_spec.rs

@@ -255,6 +255,11 @@ pub trait EthSpec:
     fn max_blobs_per_block() -> usize {
         Self::MaxBlobsPerBlock::to_usize()
     }
+
+    /// Returns the `BYTES_PER_BLOB` constant for this specification.
+    fn bytes_per_blob() -> usize {
+        Self::BytesPerBlob::to_usize()
+    }
 }
 
 /// Macro to inherit some type values from another EthSpec.

consensus/types/src/test_utils/test_random/bitfield.rs

@@ -4,8 +4,24 @@ use smallvec::smallvec;
 
 impl<N: Unsigned + Clone> TestRandom for BitList<N> {
     fn random_for_test(rng: &mut impl RngCore) -> Self {
-        let mut raw_bytes = smallvec![0; std::cmp::max(1, (N::to_usize() + 7) / 8)];
+        let initial_len = std::cmp::max(1, (N::to_usize() + 7) / 8);
+        let mut raw_bytes = smallvec![0; initial_len];
         rng.fill_bytes(&mut raw_bytes);
+
+        let highest_set_bit = raw_bytes
+            .iter()
+            .enumerate()
+            .rev()
+            .find(|(_, byte)| **byte > 0)
+            .map(|(i, byte)| i * 8 + 7 - byte.leading_zeros() as usize)
+            .unwrap_or(0);
+
+        let actual_len = highest_set_bit / 8 + 1;
+
+        if actual_len < initial_len {
+            raw_bytes.truncate(actual_len);
+        }
+
         Self::from_bytes(raw_bytes).expect("we generate a valid BitList")
     }
 }