Fix electra light client types (#6361)

* persist light client updates

* update beacon chain to serve light client updates

* resolve todos

* cache best update

* extend cache parts

* is better light client update

* resolve merge conflict

* initial api changes

* add lc update db column

* fmt

* added tests

* add sim

* Merge branch 'unstable' of https://github.com/sigp/lighthouse into persist-light-client-updates

* fix some weird issues with the simulator

* tests

* Merge branch 'unstable' of https://github.com/sigp/lighthouse into persist-light-client-updates

* test changes

* merge conflict

* testing

* started work on ef tests and some code clean up

* update tests

* linting

* noop pre altair, were still failing on electra though

* allow for zeroed light client header

* Merge branch 'unstable' of https://github.com/sigp/lighthouse into persist-light-client-updates

* merge unstable

* remove unwraps

* remove unwraps

* fetch bootstrap without always querying for state

* storing bootstrap parts in db

* mroe code cleanup

* test

* prune sync committee branches from dropped chains

* Update light_client_update.rs

* merge unstable

* move functionality to helper methods

* refactor is best update fn

* refactor is best update fn

* improve organization of light client server cache logic

* fork diget calc, and only spawn as many blcoks as we need for the lc update test

* resovle merge conflict

* add electra bootstrap logic, add logic to cache current sync committee

* add latest sync committe branch cache

* fetch lc update from the cache if it exists

* fmt

* Fix beacon_chain tests

* Add debug code to update ranking_order ef test

* Fix compare code

* merge conflicts

* merge conflict

* add better error messaging

* resolve merge conflicts

* remove lc update from basicsim

* rename sync comittte variable and fix persist condition

* refactor get_light_client_update logic

* add better comments, return helpful error messages over http and rpc

* pruning canonical non checkpoint slots

* fix test

* rerun test

* update pruning logic, add tests

* fix tests

* fix imports

* fmt

* refactor db code

* Refactor db method

* Refactor db method

* lc electra changes

* Merge branch 'unstable' of https://github.com/sigp/lighthouse into light-client-electra

* add additional comments

* testing lc merkle changes

* lc electra

* update struct defs

* Merge branch 'unstable' of https://github.com/sigp/lighthouse into light-client-electra

* fix merge

* Merge branch 'unstable' of https://github.com/sigp/lighthouse into persist-light-client-bootstrap

* fix merge

* linting

* merge conflict

* prevent overflow

* enable lc server for http api tests

* Merge branch 'unstable' of https://github.com/sigp/lighthouse into light-client-electra

* get tests working:

* remove related TODOs

* fix test lint

* Merge branch 'persist-light-client-bootstrap' of https://github.com/eserilev/lighthouse into light-client-electra

* fix tests

* fix conflicts

* remove prints

* Merge branch 'persist-light-client-bootstrap' of https://github.com/eserilev/lighthouse into light-client-electra

* remove warning

* resolve conflicts

* merge conflicts

* linting

* remove comments

* cleanup

* linting

* Merge branch 'unstable' of https://github.com/sigp/lighthouse into light-client-electra

* pre/post electra light client cached data

* add proof type alias

* move is_empty_branch method out of impl

* add ssz tests for all forks

* refactor beacon state proof codepaths

* rename method

* fmt

* clean up proof logic

* refactor merkle proof api

* fmt

* Merge branch 'unstable' into light-client-electra

* Use superstruct mapping macros

* Merge branch 'unstable' of https://github.com/sigp/lighthouse into light-client-electra

* rename proof to merkleproof

* fmt

* Resolve merge conflicts

* merge conflicts

testing/ef_tests/src/cases/merkle_proof_validity.rs

@@ -3,8 +3,8 @@ use crate::decode::{ssz_decode_file, ssz_decode_state, yaml_decode_file};
 use serde::Deserialize;
 use tree_hash::Hash256;
 use types::{
-    BeaconBlockBody, BeaconBlockBodyDeneb, BeaconBlockBodyElectra, BeaconState, FixedVector,
-    FullPayload, Unsigned,
+    light_client_update, BeaconBlockBody, BeaconBlockBodyCapella, BeaconBlockBodyDeneb,
+    BeaconBlockBodyElectra, BeaconState, FixedVector, FullPayload, Unsigned,
 };
 
 #[derive(Debug, Clone, Deserialize)]
@@ -22,13 +22,13 @@ pub struct MerkleProof {
 
 #[derive(Debug, Clone, Deserialize)]
 #[serde(bound = "E: EthSpec")]
-pub struct MerkleProofValidity<E: EthSpec> {
+pub struct BeaconStateMerkleProofValidity<E: EthSpec> {
     pub metadata: Option<Metadata>,
     pub state: BeaconState<E>,
     pub merkle_proof: MerkleProof,
 }
 
-impl<E: EthSpec> LoadCase for MerkleProofValidity<E> {
+impl<E: EthSpec> LoadCase for BeaconStateMerkleProofValidity<E> {
     fn load_from_dir(path: &Path, fork_name: ForkName) -> Result<Self, Error> {
         let spec = &testing_spec::<E>(fork_name);
         let state = ssz_decode_state(&path.join("object.ssz_snappy"), spec)?;
@@ -49,11 +49,30 @@ impl<E: EthSpec> LoadCase for MerkleProofValidity<E> {
     }
 }
 
-impl<E: EthSpec> Case for MerkleProofValidity<E> {
+impl<E: EthSpec> Case for BeaconStateMerkleProofValidity<E> {
     fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
         let mut state = self.state.clone();
         state.update_tree_hash_cache().unwrap();
-        let Ok(proof) = state.compute_merkle_proof(self.merkle_proof.leaf_index) else {
+
+        let proof = match self.merkle_proof.leaf_index {
+            light_client_update::CURRENT_SYNC_COMMITTEE_INDEX_ELECTRA
+            | light_client_update::CURRENT_SYNC_COMMITTEE_INDEX => {
+                state.compute_current_sync_committee_proof()
+            }
+            light_client_update::NEXT_SYNC_COMMITTEE_INDEX_ELECTRA
+            | light_client_update::NEXT_SYNC_COMMITTEE_INDEX => {
+                state.compute_next_sync_committee_proof()
+            }
+            light_client_update::FINALIZED_ROOT_INDEX_ELECTRA
+            | light_client_update::FINALIZED_ROOT_INDEX => state.compute_finalized_root_proof(),
+            _ => {
+                return Err(Error::FailedToParseTest(
+                    "Could not retrieve merkle proof, invalid index".to_string(),
+                ));
+            }
+        };
+
+        let Ok(proof) = proof else {
             return Err(Error::FailedToParseTest(
                 "Could not retrieve merkle proof".to_string(),
             ));
@@ -198,3 +217,81 @@ impl<E: EthSpec> Case for KzgInclusionMerkleProofValidity<E> {
         }
     }
 }
+
+#[derive(Debug, Clone, Deserialize)]
+#[serde(bound = "E: EthSpec")]
+pub struct BeaconBlockBodyMerkleProofValidity<E: EthSpec> {
+    pub metadata: Option<Metadata>,
+    pub block_body: BeaconBlockBody<E, FullPayload<E>>,
+    pub merkle_proof: MerkleProof,
+}
+
+impl<E: EthSpec> LoadCase for BeaconBlockBodyMerkleProofValidity<E> {
+    fn load_from_dir(path: &Path, fork_name: ForkName) -> Result<Self, Error> {
+        let block_body: BeaconBlockBody<E, FullPayload<E>> = match fork_name {
+            ForkName::Base | ForkName::Altair | ForkName::Bellatrix => {
+                return Err(Error::InternalError(format!(
+                    "Beacon block body merkle proof validity test skipped for {:?}",
+                    fork_name
+                )))
+            }
+            ForkName::Capella => {
+                ssz_decode_file::<BeaconBlockBodyCapella<E>>(&path.join("object.ssz_snappy"))?
+                    .into()
+            }
+            ForkName::Deneb => {
+                ssz_decode_file::<BeaconBlockBodyDeneb<E>>(&path.join("object.ssz_snappy"))?.into()
+            }
+            ForkName::Electra => {
+                ssz_decode_file::<BeaconBlockBodyElectra<E>>(&path.join("object.ssz_snappy"))?
+                    .into()
+            }
+        };
+        let merkle_proof = yaml_decode_file(&path.join("proof.yaml"))?;
+        // Metadata does not exist in these tests but it is left like this just in case.
+        let meta_path = path.join("meta.yaml");
+        let metadata = if meta_path.exists() {
+            Some(yaml_decode_file(&meta_path)?)
+        } else {
+            None
+        };
+        Ok(Self {
+            metadata,
+            block_body,
+            merkle_proof,
+        })
+    }
+}
+
+impl<E: EthSpec> Case for BeaconBlockBodyMerkleProofValidity<E> {
+    fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
+        let binding = self.block_body.clone();
+        let block_body = binding.to_ref();
+        let Ok(proof) = block_body.block_body_merkle_proof(self.merkle_proof.leaf_index) else {
+            return Err(Error::FailedToParseTest(
+                "Could not retrieve merkle proof".to_string(),
+            ));
+        };
+        let proof_len = proof.len();
+        let branch_len = self.merkle_proof.branch.len();
+        if proof_len != branch_len {
+            return Err(Error::NotEqual(format!(
+                "Branches not equal in length computed: {}, expected {}",
+                proof_len, branch_len
+            )));
+        }
+
+        for (i, proof_leaf) in proof.iter().enumerate().take(proof_len) {
+            let expected_leaf = self.merkle_proof.branch[i];
+            if *proof_leaf != expected_leaf {
+                return Err(Error::NotEqual(format!(
+                    "Leaves not equal in merke proof computed: {}, expected: {}",
+                    hex::encode(proof_leaf),
+                    hex::encode(expected_leaf)
+                )));
+            }
+        }
+
+        Ok(())
+    }
+}

testing/ef_tests/src/handler.rs

@@ -921,10 +921,10 @@ impl<E: EthSpec> Handler for KZGRecoverCellsAndKZGProofHandler<E> {
 
 #[derive(Derivative)]
 #[derivative(Default(bound = ""))]
-pub struct MerkleProofValidityHandler<E>(PhantomData<E>);
+pub struct BeaconStateMerkleProofValidityHandler<E>(PhantomData<E>);
 
-impl<E: EthSpec + TypeName> Handler for MerkleProofValidityHandler<E> {
-    type Case = cases::MerkleProofValidity<E>;
+impl<E: EthSpec + TypeName> Handler for BeaconStateMerkleProofValidityHandler<E> {
+    type Case = cases::BeaconStateMerkleProofValidity<E>;
 
     fn config_name() -> &'static str {
         E::name()
@@ -935,15 +935,11 @@ impl<E: EthSpec + TypeName> Handler for MerkleProofValidityHandler<E> {
     }
 
     fn handler_name(&self) -> String {
-        "single_merkle_proof".into()
+        "single_merkle_proof/BeaconState".into()
     }
 
-    fn is_enabled_for_fork(&self, _fork_name: ForkName) -> bool {
-        // Test is skipped due to some changes in the Capella light client
-        // spec.
-        //
-        // https://github.com/sigp/lighthouse/issues/4022
-        false
+    fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool {
+        fork_name.altair_enabled()
     }
 }
 
@@ -967,8 +963,32 @@ impl<E: EthSpec + TypeName> Handler for KzgInclusionMerkleProofValidityHandler<E
     }
 
     fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool {
-        // TODO(electra) re-enable for electra once merkle proof issues for electra are resolved
-        fork_name.deneb_enabled() && !fork_name.electra_enabled()
+        // Enabled in Deneb
+        fork_name.deneb_enabled()
+    }
+}
+
+#[derive(Derivative)]
+#[derivative(Default(bound = ""))]
+pub struct BeaconBlockBodyMerkleProofValidityHandler<E>(PhantomData<E>);
+
+impl<E: EthSpec + TypeName> Handler for BeaconBlockBodyMerkleProofValidityHandler<E> {
+    type Case = cases::BeaconBlockBodyMerkleProofValidity<E>;
+
+    fn config_name() -> &'static str {
+        E::name()
+    }
+
+    fn runner_name() -> &'static str {
+        "light_client"
+    }
+
+    fn handler_name(&self) -> String {
+        "single_merkle_proof/BeaconBlockBody".into()
+    }
+
+    fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool {
+        fork_name.capella_enabled()
     }
 }
 
@@ -993,8 +1013,7 @@ impl<E: EthSpec + TypeName> Handler for LightClientUpdateHandler<E> {
 
     fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool {
         // Enabled in Altair
-        // TODO(electra) re-enable once https://github.com/sigp/lighthouse/issues/6002 is resolved
-        fork_name.altair_enabled() && fork_name != ForkName::Electra
+        fork_name.altair_enabled()
     }
 }