Fix light client merkle proofs (#7007)

Fix a regression introduced in this PR:

- https://github.com/sigp/lighthouse/pull/6361

We were indexing into the `MerkleTree` with raw generalized indices, which was incorrect and triggering `debug_assert` failures, as described here:

- https://github.com/sigp/lighthouse/issues/7005


  - Convert `generalized_index` to the correct leaf index prior to proof generation.
- Add sanity checks on indices used in `BeaconState::generate_proof`.
- Remove debug asserts from `MerkleTree::generate_proof` in favour of actual errors. This would have caught the bug earlier.
- Refactor the EF tests so that the merkle validity tests are actually run. They were misconfigured in a way that resulted in them running silently with 0 test cases, and the `check_all_files_accessed.py` script still had an ignore that covered the test files, so this omission wasn't detected.

testing/ef_tests/check_all_files_accessed.py

@@ -27,10 +27,8 @@ excluded_paths = [
     "tests/.*/.*/ssz_static/PowBlock/",
     # We no longer implement merge logic.
     "tests/.*/bellatrix/fork_choice/on_merge_block",
-    # light_client
-    "tests/.*/.*/light_client/single_merkle_proof",
+    # Light client sync is not implemented
     "tests/.*/.*/light_client/sync",
-    "tests/.*/electra/light_client/update_ranking",
     # LightClientStore
     "tests/.*/.*/ssz_static/LightClientStore",
     # LightClientSnapshot

testing/ef_tests/src/cases/merkle_proof_validity.rs

@@ -20,6 +20,12 @@ pub struct MerkleProof {
     pub branch: Vec<Hash256>,
 }
 
+#[derive(Debug)]
+pub enum GenericMerkleProofValidity<E: EthSpec> {
+    BeaconState(BeaconStateMerkleProofValidity<E>),
+    BeaconBlockBody(Box<BeaconBlockBodyMerkleProofValidity<E>>),
+}
+
 #[derive(Debug, Clone, Deserialize)]
 #[serde(bound = "E: EthSpec")]
 pub struct BeaconStateMerkleProofValidity<E: EthSpec> {
@@ -28,6 +34,39 @@ pub struct BeaconStateMerkleProofValidity<E: EthSpec> {
     pub merkle_proof: MerkleProof,
 }
 
+impl<E: EthSpec> LoadCase for GenericMerkleProofValidity<E> {
+    fn load_from_dir(path: &Path, fork_name: ForkName) -> Result<Self, Error> {
+        let path_components = path.iter().collect::<Vec<_>>();
+
+        // The "suite" name is the 2nd last directory in the path.
+        assert!(
+            path_components.len() >= 2,
+            "path should have at least 2 components"
+        );
+        let suite_name = path_components[path_components.len() - 2];
+
+        if suite_name == "BeaconState" {
+            BeaconStateMerkleProofValidity::load_from_dir(path, fork_name)
+                .map(GenericMerkleProofValidity::BeaconState)
+        } else if suite_name == "BeaconBlockBody" {
+            BeaconBlockBodyMerkleProofValidity::load_from_dir(path, fork_name)
+                .map(Box::new)
+                .map(GenericMerkleProofValidity::BeaconBlockBody)
+        } else {
+            panic!("unsupported type for merkle proof test: {:?}", suite_name)
+        }
+    }
+}
+
+impl<E: EthSpec> Case for GenericMerkleProofValidity<E> {
+    fn result(&self, case_index: usize, fork_name: ForkName) -> Result<(), Error> {
+        match self {
+            Self::BeaconState(test) => test.result(case_index, fork_name),
+            Self::BeaconBlockBody(test) => test.result(case_index, fork_name),
+        }
+    }
+}
+
 impl<E: EthSpec> LoadCase for BeaconStateMerkleProofValidity<E> {
     fn load_from_dir(path: &Path, fork_name: ForkName) -> Result<Self, Error> {
         let spec = &testing_spec::<E>(fork_name);
@@ -72,11 +111,9 @@ impl<E: EthSpec> Case for BeaconStateMerkleProofValidity<E> {
             }
         };
 
-        let Ok(proof) = proof else {
-            return Err(Error::FailedToParseTest(
-                "Could not retrieve merkle proof".to_string(),
-            ));
-        };
+        let proof = proof.map_err(|e| {
+            Error::FailedToParseTest(format!("Could not retrieve merkle proof: {e:?}"))
+        })?;
         let proof_len = proof.len();
         let branch_len = self.merkle_proof.branch.len();
         if proof_len != branch_len {
@@ -273,11 +310,11 @@ impl<E: EthSpec> Case for BeaconBlockBodyMerkleProofValidity<E> {
     fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
         let binding = self.block_body.clone();
         let block_body = binding.to_ref();
-        let Ok(proof) = block_body.block_body_merkle_proof(self.merkle_proof.leaf_index) else {
-            return Err(Error::FailedToParseTest(
-                "Could not retrieve merkle proof".to_string(),
-            ));
-        };
+        let proof = block_body
+            .block_body_merkle_proof(self.merkle_proof.leaf_index)
+            .map_err(|e| {
+                Error::FailedToParseTest(format!("Could not retrieve merkle proof: {e:?}"))
+            })?;
         let proof_len = proof.len();
         let branch_len = self.merkle_proof.branch.len();
         if proof_len != branch_len {

testing/ef_tests/src/handler.rs

@@ -1000,30 +1000,6 @@ impl<E: EthSpec> Handler for KZGRecoverCellsAndKZGProofHandler<E> {
     }
 }
 
-#[derive(Derivative)]
-#[derivative(Default(bound = ""))]
-pub struct BeaconStateMerkleProofValidityHandler<E>(PhantomData<E>);
-
-impl<E: EthSpec + TypeName> Handler for BeaconStateMerkleProofValidityHandler<E> {
-    type Case = cases::BeaconStateMerkleProofValidity<E>;
-
-    fn config_name() -> &'static str {
-        E::name()
-    }
-
-    fn runner_name() -> &'static str {
-        "light_client"
-    }
-
-    fn handler_name(&self) -> String {
-        "single_merkle_proof/BeaconState".into()
-    }
-
-    fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool {
-        fork_name.altair_enabled()
-    }
-}
-
 #[derive(Derivative)]
 #[derivative(Default(bound = ""))]
 pub struct KzgInclusionMerkleProofValidityHandler<E>(PhantomData<E>);
@@ -1054,10 +1030,10 @@ impl<E: EthSpec + TypeName> Handler for KzgInclusionMerkleProofValidityHandler<E
 
 #[derive(Derivative)]
 #[derivative(Default(bound = ""))]
-pub struct BeaconBlockBodyMerkleProofValidityHandler<E>(PhantomData<E>);
+pub struct MerkleProofValidityHandler<E>(PhantomData<E>);
 
-impl<E: EthSpec + TypeName> Handler for BeaconBlockBodyMerkleProofValidityHandler<E> {
-    type Case = cases::BeaconBlockBodyMerkleProofValidity<E>;
+impl<E: EthSpec + TypeName> Handler for MerkleProofValidityHandler<E> {
+    type Case = cases::GenericMerkleProofValidity<E>;
 
     fn config_name() -> &'static str {
         E::name()
@@ -1068,11 +1044,11 @@ impl<E: EthSpec + TypeName> Handler for BeaconBlockBodyMerkleProofValidityHandle
     }
 
     fn handler_name(&self) -> String {
-        "single_merkle_proof/BeaconBlockBody".into()
+        "single_merkle_proof".into()
     }
 
     fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool {
-        fork_name.capella_enabled()
+        fork_name.altair_enabled()
     }
 }
 

testing/ef_tests/tests/tests.rs

@@ -955,13 +955,9 @@ fn kzg_recover_cells_and_proofs() {
 }
 
 #[test]
-fn beacon_state_merkle_proof_validity() {
-    BeaconStateMerkleProofValidityHandler::<MainnetEthSpec>::default().run();
-}
-
-#[test]
-fn beacon_block_body_merkle_proof_validity() {
-    BeaconBlockBodyMerkleProofValidityHandler::<MainnetEthSpec>::default().run();
+fn light_client_merkle_proof_validity() {
+    MerkleProofValidityHandler::<MinimalEthSpec>::default().run();
+    MerkleProofValidityHandler::<MainnetEthSpec>::default().run();
 }
 
 #[test]