Merge branch 'deneb-free-blobs' of https://github.com/sigp/lighthouse into some-blob-reprocessing-work

.github/workflows/local-testnet.yml

@@ -26,7 +26,16 @@ jobs:
             repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Install ganache
         run: npm install ganache@latest --global
-
+      - name: Install geth
+        run: |
+          sudo add-apt-repository -y ppa:ethereum/ethereum
+          sudo apt-get update
+          sudo apt-get install ethereum
+        if: matrix.os == 'ubuntu-22.04'
+        run: |
+          brew tap ethereum/ethereum
+          brew install ethereum
+        if: matrix.os == 'macos-12'
       - name: Install GNU sed & GNU grep
         run: |
           brew install gnu-sed grep

.github/workflows/test-suite.yml

@@ -260,8 +260,11 @@ jobs:
       uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-    - name: Install ganache
+    - name: Install geth
-      run: sudo npm install -g ganache
+      run: |
+          sudo add-apt-repository -y ppa:ethereum/ethereum
+          sudo apt-get update
+          sudo apt-get install ethereum
     - name: Install lighthouse and lcli
       run: |
           make
@@ -269,11 +272,11 @@ jobs:
     - name: Run the doppelganger protection success test script
       run: |
           cd scripts/tests
-          ./doppelganger_protection.sh success
+          ./doppelganger_protection.sh success genesis.json
     - name: Run the doppelganger protection failure test script
       run: |
           cd scripts/tests
-          ./doppelganger_protection.sh failure
+          ./doppelganger_protection.sh failure genesis.json
   execution-engine-integration-ubuntu:
     name: execution-engine-integration-ubuntu
     runs-on: ubuntu-latest

beacon_node/beacon_chain/src/blob_verification.rs

@@ -13,6 +13,7 @@ use crate::data_availability_checker::{
 use crate::kzg_utils::{validate_blob, validate_blobs};
 use crate::BeaconChainError;
 use kzg::Kzg;
+use slog::{debug, warn};
 use std::borrow::Cow;
 use types::blob_sidecar::{BlobIdentifier, FixedBlobSidecarList};
 use types::{
@@ -217,10 +218,7 @@ pub fn validate_blob_sidecar_for_gossip<T: BeaconChainTypes>(
         return Err(BlobError::BlobParentUnknown(signed_blob_sidecar.message));
     }
 
-    // Note: The spec checks the signature directly against `blob_sidecar.message.proposer_index`
+    // Note: We check that the proposer_index matches against the shuffling first to avoid
-    // before checking that the provided proposer index is valid w.r.t the current shuffling.
-    //
-    // However, we check that the proposer_index matches against the shuffling first to avoid
     // signature verification against an invalid proposer_index.
     let proposer_shuffling_root = signed_blob_sidecar.message.block_parent_root;
 
@@ -232,6 +230,12 @@ pub fn validate_blob_sidecar_for_gossip<T: BeaconChainTypes>(
     let (proposer_index, fork) = if let Some(proposer) = proposer_opt {
         (proposer.index, proposer.fork)
     } else {
+        debug!(
+            chain.log,
+            "Proposer shuffling cache miss for blob verification";
+            "block_root" => %block_root,
+            "index" => %blob_index,
+        );
         // The cached head state is in the same epoch as the blob or the state has already been
         // advanced to the blob's epoch
         let snapshot = &chain.canonical_head.cached_head().snapshot;
@@ -245,6 +249,18 @@ pub fn validate_blob_sidecar_for_gossip<T: BeaconChainTypes>(
         }
         // Need to advance the state to get the proposer index
         else {
+            // Reaching this condition too often might be an issue since we could theoretically have
+            // 5 threads (4 blob indices + 1 block) cloning the state.
+            // We shouldn't be seeing this condition a lot because we try to advance the state
+            // 3 seconds before the start of a slot. However, if this becomes an issue during testing, we should
+            // consider sending a blob for reprocessing to reduce the number of state clones.
+            warn!(
+                chain.log,
+                "Cached head not advanced for blob verification";
+                "block_root" => %block_root,
+                "index" => %blob_index,
+                "action" => "contact the devs if you see this msg too often"
+            );
             // The state produced is only valid for determining proposer/attester shuffling indices.
             let mut cloned_state = snapshot.clone_with(CloneConfig::committee_caches_only());
             let state = cheap_state_advance_to_obtain_committees(

beacon_node/beacon_chain/src/observed_blob_sidecars.rs

@@ -97,3 +97,293 @@ impl<T: EthSpec> ObservedBlobSidecars<T> {
         self.items.retain(|k, _| k.1 > finalized_slot);
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use types::{BlobSidecar, Hash256, MainnetEthSpec};
+
+    type E = MainnetEthSpec;
+
+    fn get_blob_sidecar(slot: u64, block_root: Hash256, index: u64) -> Arc<BlobSidecar<E>> {
+        let mut blob_sidecar = BlobSidecar::empty();
+        blob_sidecar.block_root = block_root;
+        blob_sidecar.slot = slot.into();
+        blob_sidecar.index = index;
+        Arc::new(blob_sidecar)
+    }
+
+    #[test]
+    fn pruning() {
+        let mut cache = ObservedBlobSidecars::default();
+
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 0, "no slots should be present");
+
+        // Slot 0, index 0
+        let block_root_a = Hash256::random();
+        let sidecar_a = get_blob_sidecar(0, block_root_a, 0);
+
+        assert_eq!(
+            cache.observe_sidecar(&sidecar_a),
+            Ok(false),
+            "can observe proposer, indicates proposer unobserved"
+        );
+
+        /*
+         * Preconditions.
+         */
+
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(
+            cache.items.len(),
+            1,
+            "only one (slot, root) tuple should be present"
+        );
+        assert_eq!(
+            cache
+                .items
+                .get(&(block_root_a, Slot::new(0)))
+                .expect("slot zero should be present")
+                .len(),
+            1,
+            "only one item should be present"
+        );
+
+        /*
+         * Check that a prune at the genesis slot does nothing.
+         */
+
+        cache.prune(Slot::new(0));
+
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 1, "only one slot should be present");
+        assert_eq!(
+            cache
+                .items
+                .get(&(block_root_a, Slot::new(0)))
+                .expect("slot zero should be present")
+                .len(),
+            1,
+            "only one item should be present"
+        );
+
+        /*
+         * Check that a prune empties the cache
+         */
+
+        cache.prune(E::slots_per_epoch().into());
+        assert_eq!(
+            cache.finalized_slot,
+            Slot::from(E::slots_per_epoch()),
+            "finalized slot is updated"
+        );
+        assert_eq!(cache.items.len(), 0, "no items left");
+
+        /*
+         * Check that we can't insert a finalized sidecar
+         */
+
+        // First slot of finalized epoch
+        let block_b = get_blob_sidecar(E::slots_per_epoch(), Hash256::random(), 0);
+
+        assert_eq!(
+            cache.observe_sidecar(&block_b),
+            Err(Error::FinalizedBlob {
+                slot: E::slots_per_epoch().into(),
+                finalized_slot: E::slots_per_epoch().into(),
+            }),
+            "cant insert finalized sidecar"
+        );
+
+        assert_eq!(cache.items.len(), 0, "sidecar was not added");
+
+        /*
+         * Check that we _can_ insert a non-finalized block
+         */
+
+        let three_epochs = E::slots_per_epoch() * 3;
+
+        // First slot of finalized epoch
+        let block_root_b = Hash256::random();
+        let block_b = get_blob_sidecar(three_epochs, block_root_b, 0);
+
+        assert_eq!(
+            cache.observe_sidecar(&block_b),
+            Ok(false),
+            "can insert non-finalized block"
+        );
+
+        assert_eq!(cache.items.len(), 1, "only one slot should be present");
+        assert_eq!(
+            cache
+                .items
+                .get(&(block_root_b, Slot::new(three_epochs)))
+                .expect("the three epochs slot should be present")
+                .len(),
+            1,
+            "only one proposer should be present"
+        );
+
+        /*
+         * Check that a prune doesnt wipe later blocks
+         */
+
+        let two_epochs = E::slots_per_epoch() * 2;
+        cache.prune(two_epochs.into());
+
+        assert_eq!(
+            cache.finalized_slot,
+            Slot::from(two_epochs),
+            "finalized slot is updated"
+        );
+
+        assert_eq!(cache.items.len(), 1, "only one slot should be present");
+        assert_eq!(
+            cache
+                .items
+                .get(&(block_root_b, Slot::new(three_epochs)))
+                .expect("the three epochs slot should be present")
+                .len(),
+            1,
+            "only one proposer should be present"
+        );
+    }
+
+    #[test]
+    fn simple_observations() {
+        let mut cache = ObservedBlobSidecars::default();
+
+        // Slot 0, index 0
+        let block_root_a = Hash256::random();
+        let sidecar_a = get_blob_sidecar(0, block_root_a, 0);
+
+        assert_eq!(
+            cache.is_known(&sidecar_a),
+            Ok(false),
+            "no observation in empty cache"
+        );
+
+        assert_eq!(
+            cache.observe_sidecar(&sidecar_a),
+            Ok(false),
+            "can observe proposer, indicates proposer unobserved"
+        );
+
+        assert_eq!(
+            cache.is_known(&sidecar_a),
+            Ok(true),
+            "observed block is indicated as true"
+        );
+
+        assert_eq!(
+            cache.observe_sidecar(&sidecar_a),
+            Ok(true),
+            "observing again indicates true"
+        );
+
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 1, "only one slot should be present");
+        assert_eq!(
+            cache
+                .items
+                .get(&(block_root_a, Slot::new(0)))
+                .expect("slot zero should be present")
+                .len(),
+            1,
+            "only one proposer should be present"
+        );
+
+        // Slot 1, proposer 0
+
+        let block_root_b = Hash256::random();
+        let sidecar_b = get_blob_sidecar(1, block_root_b, 0);
+
+        assert_eq!(
+            cache.is_known(&sidecar_b),
+            Ok(false),
+            "no observation for new slot"
+        );
+        assert_eq!(
+            cache.observe_sidecar(&sidecar_b),
+            Ok(false),
+            "can observe proposer for new slot, indicates proposer unobserved"
+        );
+        assert_eq!(
+            cache.is_known(&sidecar_b),
+            Ok(true),
+            "observed block in slot 1 is indicated as true"
+        );
+        assert_eq!(
+            cache.observe_sidecar(&sidecar_b),
+            Ok(true),
+            "observing slot 1 again indicates true"
+        );
+
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 2, "two slots should be present");
+        assert_eq!(
+            cache
+                .items
+                .get(&(block_root_a, Slot::new(0)))
+                .expect("slot zero should be present")
+                .len(),
+            1,
+            "only one proposer should be present in slot 0"
+        );
+        assert_eq!(
+            cache
+                .items
+                .get(&(block_root_b, Slot::new(1)))
+                .expect("slot zero should be present")
+                .len(),
+            1,
+            "only one proposer should be present in slot 1"
+        );
+
+        // Slot 0, index 1
+        let sidecar_c = get_blob_sidecar(0, block_root_a, 1);
+
+        assert_eq!(
+            cache.is_known(&sidecar_c),
+            Ok(false),
+            "no observation for new index"
+        );
+        assert_eq!(
+            cache.observe_sidecar(&sidecar_c),
+            Ok(false),
+            "can observe new index, indicates sidecar unobserved for new index"
+        );
+        assert_eq!(
+            cache.is_known(&sidecar_c),
+            Ok(true),
+            "observed new sidecar is indicated as true"
+        );
+        assert_eq!(
+            cache.observe_sidecar(&sidecar_c),
+            Ok(true),
+            "observing new sidecar again indicates true"
+        );
+
+        assert_eq!(cache.finalized_slot, 0, "finalized slot is zero");
+        assert_eq!(cache.items.len(), 2, "two slots should be present");
+        assert_eq!(
+            cache
+                .items
+                .get(&(block_root_a, Slot::new(0)))
+                .expect("slot zero should be present")
+                .len(),
+            2,
+            "two blob indices should be present in slot 0"
+        );
+
+        // Try adding an out of bounds index
+        let invalid_index = E::max_blobs_per_block() as u64;
+        let sidecar_d = get_blob_sidecar(0, block_root_a, 4);
+        assert_eq!(
+            cache.observe_sidecar(&sidecar_d),
+            Err(Error::InvalidBlobIndex(invalid_index)),
+            "cannot add an index > MaxBlobsPerBlock"
+        );
+    }
+}

scripts/local_testnet/beacon_node.sh

@@ -53,6 +53,7 @@ exec $lighthouse_binary \
 	--datadir $data_dir \
 	--testnet-dir $TESTNET_DIR \
 	--enable-private-discovery \
+  --disable-peer-scoring \
 	--staking \
 	--enr-address 127.0.0.1 \
 	--enr-udp-port $network_port \

scripts/local_testnet/el_bootnode.sh

@@ -2,4 +2,4 @@ priv_key="02fd74636e96a8ffac8e7b01b0de8dea94d6bcf4989513b38cf59eb32163ff91"
 
 source ./vars.env
 
-$BOOTNODE_BINARY --nodekeyhex $priv_key
+$EL_BOOTNODE_BINARY --nodekeyhex $priv_key

scripts/local_testnet/setup.sh

@@ -56,5 +56,7 @@ GENESIS_TIME=$(lcli pretty-ssz state_merge ~/.lighthouse/local-testnet/testnet/g
 CAPELLA_TIME=$((GENESIS_TIME + (CAPELLA_FORK_EPOCH * 32 * SECONDS_PER_SLOT)))
 DENEB_TIME=$((GENESIS_TIME + (DENEB_FORK_EPOCH * 32 * SECONDS_PER_SLOT)))
 
-sed -i 's/"shanghaiTime".*$/"shanghaiTime": '"$CAPELLA_TIME"',/g' genesis.json
+CURR_DIR=`pwd`
-sed -i 's/"shardingForkTime".*$/"shardingForkTime": '"$DENEB_TIME"',/g' genesis.json
+
+sed -i 's/"shanghaiTime".*$/"shanghaiTime": '"$CAPELLA_TIME"',/g' $CURR_DIR/genesis.json
+sed -i 's/"shardingForkTime".*$/"shardingForkTime": '"$DENEB_TIME"',/g' $CURR_DIR/genesis.json

scripts/local_testnet/vars.env

@@ -1,5 +1,5 @@
 GETH_BINARY=geth
-BOOTNODE_BINARY=bootnode
+EL_BOOTNODE_BINARY=bootnode
 
 # Base directories for the validator keys and secrets
 DATADIR=~/.lighthouse/local-testnet

scripts/tests/doppelganger_protection.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Requires `lighthouse`, ``lcli`, `ganache`, `curl`, `jq`
+# Requires `lighthouse`, ``lcli`, `geth`, `curl`, `jq`
 
 
 BEHAVIOR=$1
@@ -15,21 +15,15 @@ exit_if_fails() {
     $@
     EXIT_CODE=$?
     if [[ $EXIT_CODE -eq 1 ]]; then
-        exit 111
+        exit 1
     fi
 }
+genesis_file=$2
 
 source ./vars.env
 
 exit_if_fails ../local_testnet/clean.sh
 
-echo "Starting ganache"
-
-exit_if_fails ../local_testnet/ganache_test_node.sh &> /dev/null &
-GANACHE_PID=$!
-
-# Wait for ganache to start
-sleep 5
 
 echo "Setting up local testnet"
 
@@ -41,28 +35,33 @@ exit_if_fails cp -R $HOME/.lighthouse/local-testnet/node_1 $HOME/.lighthouse/loc
 echo "Starting bootnode"
 
 exit_if_fails ../local_testnet/bootnode.sh &> /dev/null &
-BOOT_PID=$!
+
+exit_if_fails ../local_testnet/el_bootnode.sh &> /dev/null &
 
 # wait for the bootnode to start
 sleep 10
 
-echo "Starting local beacon nodes"
+echo "Starting local execution nodes"
 
-exit_if_fails ../local_testnet/beacon_node.sh $HOME/.lighthouse/local-testnet/node_1 9000 8000 &> /dev/null &
+exit_if_fails ../local_testnet/geth.sh $HOME/.lighthouse/local-testnet/geth_datadir1 7000 6000 5000 $genesis_file &> geth.log &
-BEACON_PID=$!
+exit_if_fails ../local_testnet/geth.sh $HOME/.lighthouse/local-testnet/geth_datadir2 7100 6100 5100 $genesis_file &> /dev/null &
-exit_if_fails ../local_testnet/beacon_node.sh $HOME/.lighthouse/local-testnet/node_2 9100 8100 &> /dev/null &
+exit_if_fails ../local_testnet/geth.sh $HOME/.lighthouse/local-testnet/geth_datadir3 7200 6200 5200 $genesis_file &> /dev/null &
-BEACON_PID2=$!
+
-exit_if_fails ../local_testnet/beacon_node.sh $HOME/.lighthouse/local-testnet/node_3 9200 8200 &> /dev/null &
+sleep 20
-BEACON_PID3=$!
+
+# Reset the `genesis.json` config file fork times.
+sed -i 's/"shanghaiTime".*$/"shanghaiTime": 0,/g' genesis.json
+sed -i 's/"shardingForkTime".*$/"shardingForkTime": 0,/g' genesis.json
+
+exit_if_fails ../local_testnet/beacon_node.sh $HOME/.lighthouse/local-testnet/node_1 9000 8000 http://localhost:5000 $HOME/.lighthouse/local-testnet/geth_datadir1/geth/jwtsecret &> /dev/null &
+exit_if_fails ../local_testnet/beacon_node.sh $HOME/.lighthouse/local-testnet/node_2 9100 8100 http://localhost:5100 $HOME/.lighthouse/local-testnet/geth_datadir2/geth/jwtsecret &> beacon1.log &
+exit_if_fails ../local_testnet/beacon_node.sh $HOME/.lighthouse/local-testnet/node_3 9200 8200 http://localhost:5200 $HOME/.lighthouse/local-testnet/geth_datadir3/geth/jwtsecret &> /dev/null &
 
 echo "Starting local validator clients"
 
 exit_if_fails ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_1 http://localhost:8000 &> /dev/null &
-VALIDATOR_1_PID=$!
 exit_if_fails ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_2 http://localhost:8100 &> /dev/null &
-VALIDATOR_2_PID=$!
 exit_if_fails ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_3 http://localhost:8200 &> /dev/null &
-VALIDATOR_3_PID=$!
 
 echo "Waiting an epoch before starting the next validator client"
 sleep $(( $SECONDS_PER_SLOT * 32 ))
@@ -71,7 +70,7 @@ if [[ "$BEHAVIOR" == "failure" ]]; then
 
     echo "Starting the doppelganger validator client"
 
-    # Use same keys as keys from VC1, but connect to BN2
+    # Use same keys as keys from VC1 and connect to BN2
     # This process should not last longer than 2 epochs
     timeout $(( $SECONDS_PER_SLOT * 32 * 2 )) ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_1_doppelganger http://localhost:8100
     DOPPELGANGER_EXIT=$?
@@ -79,7 +78,9 @@ if [[ "$BEHAVIOR" == "failure" ]]; then
     echo "Shutting down"
 
     # Cleanup
-    kill $BOOT_PID $BEACON_PID $BEACON_PID2 $BEACON_PID3 $GANACHE_PID $VALIDATOR_1_PID $VALIDATOR_2_PID $VALIDATOR_3_PID
+    killall geth
+    killall lighthouse
+    killall bootnode
 
     echo "Done"
 
@@ -98,7 +99,6 @@ if [[ "$BEHAVIOR" == "success" ]]; then
     echo "Starting the last validator client"
 
     ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_4 http://localhost:8100 &
-    VALIDATOR_4_PID=$!
     DOPPELGANGER_FAILURE=0
 
     # Sleep three epochs, then make sure all validators were active in epoch 2. Use
@@ -144,7 +144,10 @@ if [[ "$BEHAVIOR" == "success" ]]; then
 
     # Cleanup
     cd $PREVIOUS_DIR
-    kill $BOOT_PID $BEACON_PID $BEACON_PID2 $BEACON_PID3 $GANACHE_PID $VALIDATOR_1_PID $VALIDATOR_2_PID $VALIDATOR_3_PID $VALIDATOR_4_PID
+
+    killall geth
+    killall lighthouse
+    killall bootnode
 
     echo "Done"
 

scripts/tests/vars.env

@@ -1,17 +1,23 @@
+# Path to the geth binary
+GETH_BINARY=geth
+EL_BOOTNODE_BINARY=bootnode
+
 # Base directories for the validator keys and secrets
 DATADIR=~/.lighthouse/local-testnet
 
 # Directory for the eth2 config
 TESTNET_DIR=$DATADIR/testnet
 
-# Mnemonic for the ganache test network
+EL_BOOTNODE_ENODE="enode://51ea9bb34d31efc3491a842ed13b8cab70e753af108526b57916d716978b380ed713f4336a80cdb85ec2a115d5a8c0ae9f3247bed3c84d3cb025c6bab311062c@127.0.0.1:0?discport=30301"
-ETH1_NETWORK_MNEMONIC="vast thought differ pull jewel broom cook wrist tribe word before omit"
 
-# Hardcoded deposit contract based on ETH1_NETWORK_MNEMONIC
+# Hardcoded deposit contract
-DEPOSIT_CONTRACT_ADDRESS=8c594691c0e592ffa21f153a16ae41db5befcaaa
+DEPOSIT_CONTRACT_ADDRESS=4242424242424242424242424242424242424242
 
 GENESIS_FORK_VERSION=0x42424242
 
+# Block hash generated from genesis.json in directory
+ETH1_BLOCK_HASH=4c2221e15760fd06c8c7a5202258c67e3d9e4aedf6db3a886ce9dc36938ad8d0
+
 VALIDATOR_COUNT=80
 GENESIS_VALIDATOR_COUNT=80
 
@@ -33,11 +39,13 @@ BOOTNODE_PORT=4242
 CHAIN_ID=4242
 
 # Hard fork configuration
-ALTAIR_FORK_EPOCH=18446744073709551615
+ALTAIR_FORK_EPOCH=0
-BELLATRIX_FORK_EPOCH=18446744073709551615
+BELLATRIX_FORK_EPOCH=0
-CAPELLA_FORK_EPOCH=18446744073709551615
+CAPELLA_FORK_EPOCH=1
 DENEB_FORK_EPOCH=18446744073709551615
 
+TTD=0
+
 # Spec version (mainnet or minimal)
 SPEC_PRESET=mainnet
 
@@ -52,9 +60,3 @@ PROPOSER_SCORE_BOOST=40
 
 # Enable doppelganger detection
 VC_ARGS=" --enable-doppelganger-protection "
-
-# Using value of DEFAULT_TERMINAL_DIFFICULTY.
-TTD=6400
-
-# Using value of DEFAULT_ETH1_BLOCK_HASH.
-ETH1_BLOCK_HASH="0x4242424242424242424242424242424242424242424242424242424242424242"