Fix light client merkle proofs (#7007)

Fix a regression introduced in this PR:

- https://github.com/sigp/lighthouse/pull/6361

We were indexing into the `MerkleTree` with raw generalized indices, which was incorrect and triggering `debug_assert` failures, as described here:

- https://github.com/sigp/lighthouse/issues/7005


  - Convert `generalized_index` to the correct leaf index prior to proof generation.
- Add sanity checks on indices used in `BeaconState::generate_proof`.
- Remove debug asserts from `MerkleTree::generate_proof` in favour of actual errors. This would have caught the bug earlier.
- Refactor the EF tests so that the merkle validity tests are actually run. They were misconfigured in a way that resulted in them running silently with 0 test cases, and the `check_all_files_accessed.py` script still had an ignore that covered the test files, so this omission wasn't detected.

testing/ef_tests/src/cases/merkle_proof_validity.rs

@@ -20,6 +20,12 @@ pub struct MerkleProof {
     pub branch: Vec<Hash256>,
 }
 
+#[derive(Debug)]
+pub enum GenericMerkleProofValidity<E: EthSpec> {
+    BeaconState(BeaconStateMerkleProofValidity<E>),
+    BeaconBlockBody(Box<BeaconBlockBodyMerkleProofValidity<E>>),
+}
+
 #[derive(Debug, Clone, Deserialize)]
 #[serde(bound = "E: EthSpec")]
 pub struct BeaconStateMerkleProofValidity<E: EthSpec> {
@@ -28,6 +34,39 @@ pub struct BeaconStateMerkleProofValidity<E: EthSpec> {
     pub merkle_proof: MerkleProof,
 }
 
+impl<E: EthSpec> LoadCase for GenericMerkleProofValidity<E> {
+    fn load_from_dir(path: &Path, fork_name: ForkName) -> Result<Self, Error> {
+        let path_components = path.iter().collect::<Vec<_>>();
+
+        // The "suite" name is the 2nd last directory in the path.
+        assert!(
+            path_components.len() >= 2,
+            "path should have at least 2 components"
+        );
+        let suite_name = path_components[path_components.len() - 2];
+
+        if suite_name == "BeaconState" {
+            BeaconStateMerkleProofValidity::load_from_dir(path, fork_name)
+                .map(GenericMerkleProofValidity::BeaconState)
+        } else if suite_name == "BeaconBlockBody" {
+            BeaconBlockBodyMerkleProofValidity::load_from_dir(path, fork_name)
+                .map(Box::new)
+                .map(GenericMerkleProofValidity::BeaconBlockBody)
+        } else {
+            panic!("unsupported type for merkle proof test: {:?}", suite_name)
+        }
+    }
+}
+
+impl<E: EthSpec> Case for GenericMerkleProofValidity<E> {
+    fn result(&self, case_index: usize, fork_name: ForkName) -> Result<(), Error> {
+        match self {
+            Self::BeaconState(test) => test.result(case_index, fork_name),
+            Self::BeaconBlockBody(test) => test.result(case_index, fork_name),
+        }
+    }
+}
+
 impl<E: EthSpec> LoadCase for BeaconStateMerkleProofValidity<E> {
     fn load_from_dir(path: &Path, fork_name: ForkName) -> Result<Self, Error> {
         let spec = &testing_spec::<E>(fork_name);
@@ -72,11 +111,9 @@ impl<E: EthSpec> Case for BeaconStateMerkleProofValidity<E> {
             }
         };
 
-        let Ok(proof) = proof else {
-            return Err(Error::FailedToParseTest(
-                "Could not retrieve merkle proof".to_string(),
-            ));
-        };
+        let proof = proof.map_err(|e| {
+            Error::FailedToParseTest(format!("Could not retrieve merkle proof: {e:?}"))
+        })?;
         let proof_len = proof.len();
         let branch_len = self.merkle_proof.branch.len();
         if proof_len != branch_len {
@@ -273,11 +310,11 @@ impl<E: EthSpec> Case for BeaconBlockBodyMerkleProofValidity<E> {
     fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
         let binding = self.block_body.clone();
         let block_body = binding.to_ref();
-        let Ok(proof) = block_body.block_body_merkle_proof(self.merkle_proof.leaf_index) else {
-            return Err(Error::FailedToParseTest(
-                "Could not retrieve merkle proof".to_string(),
-            ));
-        };
+        let proof = block_body
+            .block_body_merkle_proof(self.merkle_proof.leaf_index)
+            .map_err(|e| {
+                Error::FailedToParseTest(format!("Could not retrieve merkle proof: {e:?}"))
+            })?;
         let proof_len = proof.len();
         let branch_len = self.merkle_proof.branch.len();
         if proof_len != branch_len {