Fix failing attestation tests and misc electra attestation cleanup (#5810)

* - get attestation related beacon chain tests to pass
- observed attestations are now keyed off of data + committee index
- rename op pool attestationref to compactattestationref
- remove unwraps in agg pool and use options instead
- cherry pick some changes from ef-tests-electra

* cargo fmt

* fix failing test

* Revert dockerfile changes

* make committee_index return option

* function args shouldnt be a ref to attestation ref

* fmt

* fix dup imports

---------

Co-authored-by: realbigsean <seananderson33@GMAIL.com>

consensus/state_processing/src/common/get_attesting_indices.rs

@@ -13,7 +13,6 @@ pub mod attesting_indices_base {
     ) -> Result<IndexedAttestation<E>, BlockOperationError<Invalid>> {
         let attesting_indices =
             get_attesting_indices::<E>(committee, &attestation.aggregation_bits)?;
-
         Ok(IndexedAttestation::Base(IndexedAttestationBase {
             attesting_indices: VariableList::new(attesting_indices)?,
             data: attestation.data.clone(),
@@ -52,6 +51,100 @@ pub mod attesting_indices_electra {
     use safe_arith::SafeArith;
     use types::*;
 
+    // TODO(electra) remove duplicate code
+    // get_indexed_attestation is almost an exact duplicate
+    // the only differences are the invalid selection proof
+    // and aggregator not in committee checks
+    pub fn get_indexed_attestation_from_signed_aggregate<E: EthSpec>(
+        committees: &[BeaconCommittee],
+        signed_aggregate: &SignedAggregateAndProofElectra<E>,
+        spec: &ChainSpec,
+    ) -> Result<IndexedAttestation<E>, BeaconStateError> {
+        let mut output: HashSet<u64> = HashSet::new();
+
+        let committee_bits = &signed_aggregate.message.aggregate.committee_bits;
+        let aggregation_bits = &signed_aggregate.message.aggregate.aggregation_bits;
+        let aggregator_index = signed_aggregate.message.aggregator_index;
+        let attestation = &signed_aggregate.message.aggregate;
+
+        let committee_indices = get_committee_indices::<E>(committee_bits);
+
+        let mut committee_offset = 0;
+
+        let committees_map: HashMap<u64, &BeaconCommittee> = committees
+            .iter()
+            .map(|committee| (committee.index, committee))
+            .collect();
+
+        let committee_count_per_slot = committees.len() as u64;
+        let mut participant_count = 0;
+
+        // TODO(electra):
+        // Note: this clones the signature which is known to be a relatively slow operation.
+        //
+        // Future optimizations should remove this clone.
+        let selection_proof =
+            SelectionProof::from(signed_aggregate.message.selection_proof.clone());
+
+        for index in committee_indices {
+            if let Some(&beacon_committee) = committees_map.get(&index) {
+                if !selection_proof
+                    .is_aggregator(beacon_committee.committee.len(), spec)
+                    .map_err(BeaconStateError::ArithError)?
+                {
+                    return Err(BeaconStateError::InvalidSelectionProof { aggregator_index });
+                }
+
+                if !beacon_committee
+                    .committee
+                    .contains(&(aggregator_index as usize))
+                {
+                    return Err(BeaconStateError::AggregatorNotInCommittee { aggregator_index });
+                }
+
+                // This check is new to the spec's `process_attestation` in Electra.
+                if index >= committee_count_per_slot {
+                    return Err(BeaconStateError::InvalidCommitteeIndex(index));
+                }
+
+                participant_count.safe_add_assign(beacon_committee.committee.len() as u64)?;
+                let committee_attesters = beacon_committee
+                    .committee
+                    .iter()
+                    .enumerate()
+                    .filter_map(|(i, &index)| {
+                        if let Ok(aggregation_bit_index) = committee_offset.safe_add(i) {
+                            if aggregation_bits.get(aggregation_bit_index).unwrap_or(false) {
+                                return Some(index as u64);
+                            }
+                        }
+                        None
+                    })
+                    .collect::<HashSet<u64>>();
+
+                output.extend(committee_attesters);
+
+                committee_offset.safe_add_assign(beacon_committee.committee.len())?;
+            } else {
+                return Err(Error::NoCommitteeFound(index));
+            }
+        }
+
+        // This check is new to the spec's `process_attestation` in Electra.
+        if participant_count as usize != aggregation_bits.len() {
+            return Err(Error::InvalidBitfield);
+        }
+
+        let mut indices = output.into_iter().collect_vec();
+        indices.sort_unstable();
+
+        Ok(IndexedAttestation::Electra(IndexedAttestationElectra {
+            attesting_indices: VariableList::new(indices)?,
+            data: attestation.data.clone(),
+            signature: attestation.signature.clone(),
+        }))
+    }
+
     pub fn get_indexed_attestation<E: EthSpec>(
         committees: &[BeaconCommittee],
         attestation: &AttestationElectra<E>,
@@ -155,7 +248,7 @@ pub mod attesting_indices_electra {
         Ok(indices)
     }
 
-    fn get_committee_indices<E: EthSpec>(
+    pub fn get_committee_indices<E: EthSpec>(
         committee_bits: &BitVector<E::MaxCommitteesPerSlot>,
     ) -> Vec<CommitteeIndex> {
         committee_bits

consensus/state_processing/src/per_block_processing/signature_sets.rs

@@ -326,6 +326,7 @@ where
         genesis_validators_root,
     );
 
+    // TODO(electra), signing root isnt unique in the case of electra
     let message = indexed_attestation.data().signing_root(domain);
 
     Ok(SignatureSet::multiple_pubkeys(signature, pubkeys, message))
@@ -436,7 +437,6 @@ where
     let message = slot.signing_root(domain);
     let signature = signed_aggregate_and_proof.message().selection_proof();
     let validator_index = signed_aggregate_and_proof.message().aggregator_index();
-
     Ok(SignatureSet::single_pubkey(
         signature,
         get_pubkey(validator_index as usize).ok_or(Error::ValidatorUnknown(validator_index))?,