Fix light client merkle proofs (#7007)

Fix a regression introduced in this PR:

- https://github.com/sigp/lighthouse/pull/6361

We were indexing into the `MerkleTree` with raw generalized indices, which was incorrect and triggering `debug_assert` failures, as described here:

- https://github.com/sigp/lighthouse/issues/7005


  - Convert `generalized_index` to the correct leaf index prior to proof generation.
- Add sanity checks on indices used in `BeaconState::generate_proof`.
- Remove debug asserts from `MerkleTree::generate_proof` in favour of actual errors. This would have caught the bug earlier.
- Refactor the EF tests so that the merkle validity tests are actually run. They were misconfigured in a way that resulted in them running silently with 0 test cases, and the `check_all_files_accessed.py` script still had an ignore that covered the test files, so this omission wasn't detected.

consensus/types/src/beacon_block_body.rs

@@ -277,9 +277,9 @@ impl<'a, E: EthSpec, Payload: AbstractExecPayload<E>> BeaconBlockBodyRef<'a, E,
                 // https://github.com/ethereum/consensus-specs/blob/dev/specs/deneb/beacon-chain.md#beaconblockbody
                 generalized_index
                     .checked_sub(NUM_BEACON_BLOCK_BODY_HASH_TREE_ROOT_LEAVES)
-                    .ok_or(Error::IndexNotSupported(generalized_index))?
+                    .ok_or(Error::GeneralizedIndexNotSupported(generalized_index))?
             }
-            _ => return Err(Error::IndexNotSupported(generalized_index)),
+            _ => return Err(Error::GeneralizedIndexNotSupported(generalized_index)),
         };
 
         let leaves = self.body_merkle_leaves();

consensus/types/src/beacon_state.rs

@@ -157,6 +157,7 @@ pub enum Error {
         current_fork: ForkName,
     },
     TotalActiveBalanceDiffUninitialized,
+    GeneralizedIndexNotSupported(usize),
     IndexNotSupported(usize),
     InvalidFlagIndex(usize),
     MerkleTreeError(merkle_proof::MerkleTreeError),
@@ -2580,11 +2581,12 @@ impl<E: EthSpec> BeaconState<E> {
         // for the internal nodes. Result should be 22 or 23, the field offset of the committee
         // in the `BeaconState`:
         // https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/beacon-chain.md#beaconstate
-        let field_index = if self.fork_name_unchecked().electra_enabled() {
+        let field_gindex = if self.fork_name_unchecked().electra_enabled() {
             light_client_update::CURRENT_SYNC_COMMITTEE_INDEX_ELECTRA
         } else {
             light_client_update::CURRENT_SYNC_COMMITTEE_INDEX
         };
+        let field_index = field_gindex.safe_sub(self.num_fields_pow2())?;
         let leaves = self.get_beacon_state_leaves();
         self.generate_proof(field_index, &leaves)
     }
@@ -2594,11 +2596,12 @@ impl<E: EthSpec> BeaconState<E> {
         // for the internal nodes. Result should be 22 or 23, the field offset of the committee
         // in the `BeaconState`:
         // https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/beacon-chain.md#beaconstate
-        let field_index = if self.fork_name_unchecked().electra_enabled() {
+        let field_gindex = if self.fork_name_unchecked().electra_enabled() {
             light_client_update::NEXT_SYNC_COMMITTEE_INDEX_ELECTRA
         } else {
             light_client_update::NEXT_SYNC_COMMITTEE_INDEX
         };
+        let field_index = field_gindex.safe_sub(self.num_fields_pow2())?;
         let leaves = self.get_beacon_state_leaves();
         self.generate_proof(field_index, &leaves)
     }
@@ -2606,17 +2609,24 @@ impl<E: EthSpec> BeaconState<E> {
     pub fn compute_finalized_root_proof(&self) -> Result<Vec<Hash256>, Error> {
         // Finalized root is the right child of `finalized_checkpoint`, divide by two to get
         // the generalized index of `state.finalized_checkpoint`.
-        let field_index = if self.fork_name_unchecked().electra_enabled() {
-            // Index should be 169/2 - 64 = 20 which matches the position
-            // of `finalized_checkpoint` in `BeaconState`
+        let checkpoint_root_gindex = if self.fork_name_unchecked().electra_enabled() {
             light_client_update::FINALIZED_ROOT_INDEX_ELECTRA
         } else {
-            // Index should be 105/2 - 32 = 20 which matches the position
-            // of `finalized_checkpoint` in `BeaconState`
             light_client_update::FINALIZED_ROOT_INDEX
         };
+        let checkpoint_gindex = checkpoint_root_gindex / 2;
+
+        // Convert gindex to index by subtracting 2**depth (gindex = 2**depth + index).
+        //
+        // After Electra, the index should be 169/2 - 64 = 20 which matches the position
+        // of `finalized_checkpoint` in `BeaconState`.
+        //
+        // Prior to Electra, the index should be 105/2 - 32 = 20 which matches the position
+        // of `finalized_checkpoint` in `BeaconState`.
+        let checkpoint_index = checkpoint_gindex.safe_sub(self.num_fields_pow2())?;
+
         let leaves = self.get_beacon_state_leaves();
-        let mut proof = self.generate_proof(field_index, &leaves)?;
+        let mut proof = self.generate_proof(checkpoint_index, &leaves)?;
         proof.insert(0, self.finalized_checkpoint().epoch.tree_hash_root());
         Ok(proof)
     }
@@ -2626,6 +2636,10 @@ impl<E: EthSpec> BeaconState<E> {
         field_index: usize,
         leaves: &[Hash256],
     ) -> Result<Vec<Hash256>, Error> {
+        if field_index >= leaves.len() {
+            return Err(Error::IndexNotSupported(field_index));
+        }
+
         let depth = self.num_fields_pow2().ilog2() as usize;
         let tree = merkle_proof::MerkleTree::create(leaves, depth);
         let (_, proof) = tree.generate_proof(field_index, depth)?;