Merge remote-tracking branch 'origin/deneb-free-blobs' into tree-states

beacon_node/beacon_chain/src/test_utils.rs

@@ -1,3 +1,4 @@
+use crate::block_verification_types::{AsBlock, RpcBlock};
 use crate::observed_operations::ObservationOutcome;
 pub use crate::persisted_beacon_chain::PersistedBeaconChain;
 pub use crate::{
@@ -14,6 +15,8 @@ use crate::{
     StateSkipConfig,
 };
 use bls::get_withdrawal_credentials;
+use eth2::types::SignedBlockContentsTuple;
+use execution_layer::test_utils::generate_genesis_header;
 use execution_layer::{
     auth::JwtKey,
     test_utils::{
@@ -25,26 +28,31 @@ use execution_layer::{
 use futures::channel::mpsc::Receiver;
 pub use genesis::{interop_genesis_state_with_eth1, DEFAULT_ETH1_BLOCK_HASH};
 use int_to_bytes::int_to_bytes32;
+use kzg::{Kzg, TrustedSetup};
 use merkle_proof::MerkleTree;
 use operation_pool::ReceivedPreCapella;
-use parking_lot::Mutex;
 use parking_lot::RwLockWriteGuard;
+use parking_lot::{Mutex, RwLock};
 use rand::rngs::StdRng;
 use rand::Rng;
 use rand::SeedableRng;
 use rayon::prelude::*;
 use sensitive_url::SensitiveUrl;
-use slog::Logger;
+use slog::{o, Drain, Logger};
+use slog_async::Async;
+use slog_term::{FullFormat, TermDecorator};
 use slot_clock::{SlotClock, TestingSlotClock};
 use state_processing::per_block_processing::compute_timestamp_at_slot;
 use state_processing::state_advance::complete_state_advance;
 use std::borrow::Cow;
 use std::collections::{HashMap, HashSet};
 use std::fmt;
+use std::marker::PhantomData;
 use std::str::FromStr;
 use std::sync::Arc;
 use std::time::Duration;
 use store::{config::StoreConfig, HotColdDB, ItemStore, LevelDB, MemoryStore};
+use task_executor::TaskExecutor;
 use task_executor::{test_utils::TestRuntime, ShutdownReason};
 use tree_hash::TreeHash;
 use types::sync_selection_proof::SyncSelectionProof;
@@ -54,7 +62,7 @@ use types::{typenum::U4294967296, *};
 // 4th September 2019
 pub const HARNESS_GENESIS_TIME: u64 = 1_567_552_690;
 // Environment variable to read if `fork_from_env` feature is enabled.
-const FORK_NAME_ENV_VAR: &str = "FORK_NAME";
+pub const FORK_NAME_ENV_VAR: &str = "FORK_NAME";
 
 // Default target aggregators to set during testing, this ensures an aggregator at each slot.
 //
@@ -186,11 +194,12 @@ impl<E: EthSpec> Builder<EphemeralHarnessType<E>> {
             .unwrap(),
         );
         let mutator = move |builder: BeaconChainBuilder<_>| {
+            let header = generate_genesis_header::<E>(builder.get_spec(), false);
             let genesis_state = interop_genesis_state_with_eth1::<E>(
                 &validator_keypairs,
                 HARNESS_GENESIS_TIME,
                 Hash256::from_slice(DEFAULT_ETH1_BLOCK_HASH),
-                None,
+                header,
                 builder.get_spec(),
             )
             .expect("should generate interop state");
@@ -247,11 +256,12 @@ impl<E: EthSpec> Builder<DiskHarnessType<E>> {
             .expect("cannot build without validator keypairs");
 
         let mutator = move |builder: BeaconChainBuilder<_>| {
+            let header = generate_genesis_header::<E>(builder.get_spec(), false);
             let genesis_state = interop_genesis_state_with_eth1::<E>(
                 &validator_keypairs,
                 HARNESS_GENESIS_TIME,
                 Hash256::from_slice(DEFAULT_ETH1_BLOCK_HASH),
-                None,
+                header,
                 builder.get_spec(),
             )
             .expect("should generate interop state");
@@ -314,6 +324,11 @@ where
         self
     }
 
+    pub fn withdrawal_keypairs(mut self, withdrawal_keypairs: Vec<Option<Keypair>>) -> Self {
+        self.withdrawal_keypairs = withdrawal_keypairs;
+        self
+    }
+
     /// Initializes the BLS withdrawal keypairs for `num_keypairs` validators to
     /// the "determistic" values, regardless of wether or not the validator has
     /// a BLS or execution address in the genesis deposits.
@@ -329,11 +344,6 @@ where
         )
     }
 
-    pub fn withdrawal_keypairs(mut self, withdrawal_keypairs: Vec<Option<Keypair>>) -> Self {
-        self.withdrawal_keypairs = withdrawal_keypairs;
-        self
-    }
-
     pub fn default_spec(self) -> Self {
         self.spec_or_default(None)
     }
@@ -382,7 +392,7 @@ where
         self
     }
 
-    pub fn execution_layer(mut self, urls: &[&str]) -> Self {
+    pub fn execution_layer_from_urls(mut self, urls: &[&str]) -> Self {
         assert!(
             self.execution_layer.is_none(),
             "execution layer already defined"
@@ -411,6 +421,11 @@ where
         self
     }
 
+    pub fn execution_layer(mut self, el: Option<ExecutionLayer<E>>) -> Self {
+        self.execution_layer = el;
+        self
+    }
+
     pub fn recalculate_fork_times_with_genesis(mut self, genesis_time: u64) -> Self {
         let mock = self
             .mock_execution_layer
@@ -424,6 +439,9 @@ where
             spec.capella_fork_epoch.map(|epoch| {
                 genesis_time + spec.seconds_per_slot * E::slots_per_epoch() * epoch.as_u64()
             });
+        mock.server.execution_block_generator().cancun_time = spec.deneb_fork_epoch.map(|epoch| {
+            genesis_time + spec.seconds_per_slot * E::slots_per_epoch() * epoch.as_u64()
+        });
 
         self
     }
@@ -433,17 +451,10 @@ where
     }
 
     pub fn mock_execution_layer_with_config(mut self, builder_threshold: Option<u128>) -> Self {
-        let spec = self.spec.clone().expect("cannot build without spec");
-        let shanghai_time = spec.capella_fork_epoch.map(|epoch| {
-            HARNESS_GENESIS_TIME + spec.seconds_per_slot * E::slots_per_epoch() * epoch.as_u64()
-        });
-        let mock = MockExecutionLayer::new(
+        let mock = mock_execution_layer_from_parts::<E>(
+            self.spec.as_ref().expect("cannot build without spec"),
             self.runtime.task_executor.clone(),
-            DEFAULT_TERMINAL_BLOCK,
-            shanghai_time,
             builder_threshold,
-            Some(JwtKey::from_slice(&DEFAULT_JWT_SECRET).unwrap()),
-            spec,
         );
         self.execution_layer = Some(mock.el.clone());
         self.mock_execution_layer = Some(mock);
@@ -476,6 +487,10 @@ where
             .validator_keypairs
             .expect("cannot build without validator keypairs");
         let chain_config = self.chain_config.unwrap_or_default();
+        let trusted_setup: TrustedSetup =
+            serde_json::from_reader(eth2_network_config::get_trusted_setup::<E::Kzg>())
+                .map_err(|e| format!("Unable to read trusted setup file: {}", e))
+                .unwrap();
 
         let mut builder = BeaconChainBuilder::new(self.eth_spec_instance)
             .logger(log.clone())
@@ -496,7 +511,8 @@ where
                 log.clone(),
                 5,
             )))
-            .monitor_validators(true, vec![], DEFAULT_INDIVIDUAL_TRACKING_THRESHOLD, log);
+            .monitor_validators(true, vec![], DEFAULT_INDIVIDUAL_TRACKING_THRESHOLD, log)
+            .trusted_setup(trusted_setup);
 
         builder = if let Some(mutator) = self.initial_mutator {
             mutator(builder)
@@ -532,11 +548,42 @@ where
             runtime: self.runtime,
             mock_execution_layer: self.mock_execution_layer,
             mock_builder: None,
+            blob_signature_cache: <_>::default(),
             rng: make_rng(),
         }
     }
 }
 
+pub fn mock_execution_layer_from_parts<T: EthSpec>(
+    spec: &ChainSpec,
+    task_executor: TaskExecutor,
+    builder_threshold: Option<u128>,
+) -> MockExecutionLayer<T> {
+    let shanghai_time = spec.capella_fork_epoch.map(|epoch| {
+        HARNESS_GENESIS_TIME + spec.seconds_per_slot * T::slots_per_epoch() * epoch.as_u64()
+    });
+    let cancun_time = spec.deneb_fork_epoch.map(|epoch| {
+        HARNESS_GENESIS_TIME + spec.seconds_per_slot * T::slots_per_epoch() * epoch.as_u64()
+    });
+
+    let trusted_setup: TrustedSetup =
+        serde_json::from_reader(eth2_network_config::get_trusted_setup::<T::Kzg>())
+            .map_err(|e| format!("Unable to read trusted setup file: {}", e))
+            .expect("should have trusted setup");
+    let kzg = Kzg::new_from_trusted_setup(trusted_setup).expect("should create kzg");
+
+    MockExecutionLayer::new(
+        task_executor,
+        DEFAULT_TERMINAL_BLOCK,
+        shanghai_time,
+        cancun_time,
+        builder_threshold,
+        Some(JwtKey::from_slice(&DEFAULT_JWT_SECRET).unwrap()),
+        spec.clone(),
+        Some(kzg),
+    )
+}
+
 /// A testing harness which can instantiate a `BeaconChain` and populate it with blocks and
 /// attestations.
 ///
@@ -558,9 +605,29 @@ pub struct BeaconChainHarness<T: BeaconChainTypes> {
     pub mock_execution_layer: Option<MockExecutionLayer<T::EthSpec>>,
     pub mock_builder: Option<Arc<MockBuilder<T::EthSpec>>>,
 
+    /// Cache for blob signature because we don't need them for import, but we do need them
+    /// to test gossip validation. We always make them during block production but drop them
+    /// before storing them in the db.
+    pub blob_signature_cache: Arc<RwLock<HashMap<BlobSignatureKey, Signature>>>,
+
     pub rng: Mutex<StdRng>,
 }
 
+#[derive(Clone, Debug, Hash, Eq, PartialEq)]
+pub struct BlobSignatureKey {
+    block_root: Hash256,
+    blob_index: u64,
+}
+
+impl BlobSignatureKey {
+    pub fn new(block_root: Hash256, blob_index: u64) -> Self {
+        Self {
+            block_root,
+            blob_index,
+        }
+    }
+}
+
 pub type CommitteeAttestations<E> = Vec<(Attestation<E>, SubnetId)>;
 pub type HarnessAttestations<E> =
     Vec<(CommitteeAttestations<E>, Option<SignedAggregateAndProof<E>>)>;
@@ -635,6 +702,20 @@ where
         mock_builder_server
     }
 
+    pub fn get_head_block(&self) -> RpcBlock<E> {
+        let block = self.chain.head_beacon_block();
+        let block_root = block.canonical_root();
+        let blobs = self.chain.get_blobs(&block_root).unwrap();
+        RpcBlock::new(block, Some(blobs)).unwrap()
+    }
+
+    pub fn get_full_block(&self, block_root: &Hash256) -> RpcBlock<E> {
+        let block = self.chain.get_blinded_block(block_root).unwrap().unwrap();
+        let full_block = self.chain.store.make_full_block(block_root, block).unwrap();
+        let blobs = self.chain.get_blobs(block_root).unwrap();
+        RpcBlock::new(Arc::new(full_block), Some(blobs)).unwrap()
+    }
+
     pub fn get_all_validators(&self) -> Vec<usize> {
         (0..self.validator_keypairs.len()).collect()
     }
@@ -742,9 +823,28 @@ where
         &self,
         state: BeaconState<E>,
         slot: Slot,
-    ) -> (SignedBlindedBeaconBlock<E>, BeaconState<E>) {
+    ) -> (
+        SignedBlockContentsTuple<E, BlindedPayload<E>>,
+        BeaconState<E>,
+    ) {
         let (unblinded, new_state) = self.make_block(state, slot).await;
-        (unblinded.into(), new_state)
+        let maybe_blinded_blob_sidecars = unblinded.1.map(|blob_sidecar_list| {
+            VariableList::new(
+                blob_sidecar_list
+                    .into_iter()
+                    .map(|blob_sidecar| {
+                        let blinded_sidecar: BlindedBlobSidecar = blob_sidecar.message.into();
+                        SignedSidecar {
+                            message: Arc::new(blinded_sidecar),
+                            signature: blob_sidecar.signature,
+                            _phantom: PhantomData,
+                        }
+                    })
+                    .collect(),
+            )
+            .unwrap()
+        });
+        ((unblinded.0.into(), maybe_blinded_blob_sidecars), new_state)
     }
 
     /// Returns a newly created block, signed by the proposer for the given slot.
@@ -752,7 +852,7 @@ where
         &self,
         mut state: BeaconState<E>,
         slot: Slot,
-    ) -> (SignedBeaconBlock<E>, BeaconState<E>) {
+    ) -> (SignedBlockContentsTuple<E, FullPayload<E>>, BeaconState<E>) {
         assert_ne!(slot, 0, "can't produce a block at slot 0");
         assert!(slot >= state.slot());
 
@@ -770,7 +870,7 @@ where
 
         let randao_reveal = self.sign_randao_reveal(&state, proposer_index, slot);
 
-        let (block, state) = self
+        let (block, state, maybe_blob_sidecars) = self
             .chain
             .produce_block_on_state(
                 state,
@@ -790,7 +890,18 @@ where
             &self.spec,
         );
 
-        (signed_block, state)
+        let block_contents: SignedBlockContentsTuple<E, FullPayload<E>> = match &signed_block {
+            SignedBeaconBlock::Base(_)
+            | SignedBeaconBlock::Altair(_)
+            | SignedBeaconBlock::Merge(_)
+            | SignedBeaconBlock::Capella(_) => (signed_block, None),
+            SignedBeaconBlock::Deneb(_) => (
+                signed_block,
+                maybe_blob_sidecars.map(|blobs| self.sign_blobs(blobs, &state, proposer_index)),
+            ),
+        };
+
+        (block_contents, state)
     }
 
     /// Useful for the `per_block_processing` tests. Creates a block, and returns the state after
@@ -799,7 +910,7 @@ where
         &self,
         mut state: BeaconState<E>,
         slot: Slot,
-    ) -> (SignedBeaconBlock<E>, BeaconState<E>) {
+    ) -> (SignedBlockContentsTuple<E, FullPayload<E>>, BeaconState<E>) {
         assert_ne!(slot, 0, "can't produce a block at slot 0");
         assert!(slot >= state.slot());
 
@@ -819,7 +930,7 @@ where
 
         let pre_state = state.clone();
 
-        let (block, state) = self
+        let (block, state, maybe_blob_sidecars) = self
             .chain
             .produce_block_on_state(
                 state,
@@ -839,7 +950,40 @@ where
             &self.spec,
         );
 
-        (signed_block, pre_state)
+        let block_contents: SignedBlockContentsTuple<E, FullPayload<E>> = match &signed_block {
+            SignedBeaconBlock::Base(_)
+            | SignedBeaconBlock::Altair(_)
+            | SignedBeaconBlock::Merge(_)
+            | SignedBeaconBlock::Capella(_) => (signed_block, None),
+            SignedBeaconBlock::Deneb(_) => {
+                if let Some(blobs) = maybe_blob_sidecars {
+                    let signed_blobs: SignedSidecarList<E, BlobSidecar<E>> = Vec::from(blobs)
+                        .into_iter()
+                        .map(|blob| {
+                            blob.sign(
+                                &self.validator_keypairs[proposer_index].sk,
+                                &state.fork(),
+                                state.genesis_validators_root(),
+                                &self.spec,
+                            )
+                        })
+                        .collect::<Vec<_>>()
+                        .into();
+                    let mut guard = self.blob_signature_cache.write();
+                    for blob in &signed_blobs {
+                        guard.insert(
+                            BlobSignatureKey::new(blob.message.block_root, blob.message.index),
+                            blob.signature.clone(),
+                        );
+                    }
+                    (signed_block, Some(signed_blobs))
+                } else {
+                    (signed_block, None)
+                }
+            }
+        };
+
+        (block_contents, pre_state)
     }
 
     /// Create a randao reveal for a block at `slot`.
@@ -876,6 +1020,35 @@ where
         )
     }
 
+    /// Sign blobs, and cache their signatures.
+    pub fn sign_blobs(
+        &self,
+        blobs: BlobSidecarList<E>,
+        state: &BeaconState<E>,
+        proposer_index: usize,
+    ) -> SignedSidecarList<E, BlobSidecar<E>> {
+        let signed_blobs: SignedSidecarList<E, BlobSidecar<E>> = Vec::from(blobs)
+            .into_iter()
+            .map(|blob| {
+                blob.sign(
+                    &self.validator_keypairs[proposer_index].sk,
+                    &state.fork(),
+                    state.genesis_validators_root(),
+                    &self.spec,
+                )
+            })
+            .collect::<Vec<_>>()
+            .into();
+        let mut guard = self.blob_signature_cache.write();
+        for blob in &signed_blobs {
+            guard.insert(
+                BlobSignatureKey::new(blob.message.block_root, blob.message.index),
+                blob.signature.clone(),
+            );
+        }
+        signed_blobs
+    }
+
     /// Produces an "unaggregated" attestation for the given `slot` and `index` that attests to
     /// `beacon_block_root`. The provided `state` should match the `block.state_root` for the
     /// `block` identified by `beacon_block_root`.
@@ -1512,14 +1685,13 @@ where
 
     pub fn make_voluntary_exit(&self, validator_index: u64, epoch: Epoch) -> SignedVoluntaryExit {
         let sk = &self.validator_keypairs[validator_index as usize].sk;
-        let fork = self.chain.canonical_head.cached_head().head_fork();
         let genesis_validators_root = self.chain.genesis_validators_root;
 
         VoluntaryExit {
             epoch,
             validator_index,
         }
-        .sign(sk, &fork, genesis_validators_root, &self.chain.spec)
+        .sign(sk, genesis_validators_root, &self.chain.spec)
     }
 
     pub fn add_proposer_slashing(&self, validator_index: u64) -> Result<(), String> {
@@ -1628,11 +1800,12 @@ where
         state: BeaconState<E>,
         slot: Slot,
         block_modifier: impl FnOnce(&mut BeaconBlock<E>),
-    ) -> (SignedBeaconBlock<E>, BeaconState<E>) {
+    ) -> (SignedBlockContentsTuple<E, FullPayload<E>>, BeaconState<E>) {
         assert_ne!(slot, 0, "can't produce a block at slot 0");
         assert!(slot >= state.slot());
 
-        let (block, state) = self.make_block_return_pre_state(state, slot).await;
+        let ((block, blobs), state) = self.make_block_return_pre_state(state, slot).await;
+
         let (mut block, _) = block.deconstruct();
 
         block_modifier(&mut block);
@@ -1645,7 +1818,7 @@ where
             state.genesis_validators_root(),
             &self.spec,
         );
-        (signed_block, state)
+        ((signed_block, blobs), state)
     }
 
     pub fn make_deposits<'a>(
@@ -1725,37 +1898,59 @@ where
         &self,
         slot: Slot,
         block_root: Hash256,
-        block: SignedBeaconBlock<E>,
+        block_contents: SignedBlockContentsTuple<E, FullPayload<E>>,
     ) -> Result<SignedBeaconBlockHash, BlockError<E>> {
         self.set_current_slot(slot);
+        let (block, blobs) = block_contents;
+        // Note: we are just dropping signatures here and skipping signature verification.
+        let blobs_without_signatures = blobs.map(|blobs| {
+            VariableList::from(
+                blobs
+                    .into_iter()
+                    .map(|blob| blob.message)
+                    .collect::<Vec<_>>(),
+            )
+        });
         let block_hash: SignedBeaconBlockHash = self
             .chain
             .process_block(
                 block_root,
-                Arc::new(block),
+                RpcBlock::new(Arc::new(block), blobs_without_signatures).unwrap(),
                 NotifyExecutionLayer::Yes,
                 || Ok(()),
             )
             .await?
-            .into();
+            .try_into()
+            .unwrap();
         self.chain.recompute_head_at_current_slot().await;
         Ok(block_hash)
     }
 
     pub async fn process_block_result(
         &self,
-        block: SignedBeaconBlock<E>,
+        block_contents: SignedBlockContentsTuple<E, FullPayload<E>>,
     ) -> Result<SignedBeaconBlockHash, BlockError<E>> {
+        let (block, blobs) = block_contents;
+        // Note: we are just dropping signatures here and skipping signature verification.
+        let blobs_without_signatures = blobs.map(|blobs| {
+            VariableList::from(
+                blobs
+                    .into_iter()
+                    .map(|blob| blob.message)
+                    .collect::<Vec<_>>(),
+            )
+        });
         let block_hash: SignedBeaconBlockHash = self
             .chain
             .process_block(
                 block.canonical_root(),
-                Arc::new(block),
+                RpcBlock::new(Arc::new(block), blobs_without_signatures).unwrap(),
                 NotifyExecutionLayer::Yes,
                 || Ok(()),
             )
             .await?
-            .into();
+            .try_into()
+            .expect("block blobs are available");
         self.chain.recompute_head_at_current_slot().await;
         Ok(block_hash)
     }
@@ -1815,13 +2010,25 @@ where
         &self,
         slot: Slot,
         state: BeaconState<E>,
-    ) -> Result<(SignedBeaconBlockHash, SignedBeaconBlock<E>, BeaconState<E>), BlockError<E>> {
+    ) -> Result<
+        (
+            SignedBeaconBlockHash,
+            SignedBlockContentsTuple<E, FullPayload<E>>,
+            BeaconState<E>,
+        ),
+        BlockError<E>,
+    > {
         self.set_current_slot(slot);
-        let (block, new_state) = self.make_block(state, slot).await;
+        let (block_contents, new_state) = self.make_block(state, slot).await;
+
         let block_hash = self
-            .process_block(slot, block.canonical_root(), block.clone())
+            .process_block(
+                slot,
+                block_contents.0.canonical_root(),
+                block_contents.clone(),
+            )
             .await?;
-        Ok((block_hash, block, new_state))
+        Ok((block_hash, block_contents, new_state))
     }
 
     pub fn attest_block(
@@ -1875,7 +2082,7 @@ where
         sync_committee_strategy: SyncCommitteeStrategy,
     ) -> Result<(SignedBeaconBlockHash, BeaconState<E>), BlockError<E>> {
         let (block_hash, block, state) = self.add_block_at_slot(slot, state).await?;
-        self.attest_block(&state, state_root, block_hash, &block, validators);
+        self.attest_block(&state, state_root, block_hash, &block.0, validators);
 
         if sync_committee_strategy == SyncCommitteeStrategy::AllValidators
             && state.current_sync_committee().is_ok()
@@ -2073,8 +2280,9 @@ where
         chain_dump
             .iter()
             .cloned()
-            .map(|checkpoint| checkpoint.beacon_state.finalized_checkpoint().root.into())
-            .filter(|block_hash| *block_hash != Hash256::zero().into())
+            .map(|checkpoint| checkpoint.beacon_state.finalized_checkpoint().root)
+            .filter(|block_hash| *block_hash != Hash256::zero())
+            .map(|hash| hash.into())
             .collect()
     }
 
@@ -2286,3 +2494,15 @@ pub struct MakeAttestationOptions {
     /// Fork to use for signing attestations.
     pub fork: Fork,
 }
+
+pub fn build_log(level: slog::Level, enabled: bool) -> Logger {
+    let decorator = TermDecorator::new().build();
+    let drain = FullFormat::new(decorator).build().fuse();
+    let drain = Async::new(drain).build().fuse();
+
+    if enabled {
+        Logger::root(drain.filter_level(level).fuse(), o!())
+    } else {
+        Logger::root(drain.filter(|_| false).fuse(), o!())
+    }
+}