Call beacon_committee_selections only once per epoch (#8699)

* #8381


  Add another if in the if/else branch for selections_endpoint case, and leaving the current code for non-selections_endpoint case mostly untouched. Now the if/else branch in `fill_in_selection_proofs` has 3 branches:
- 1 for DVT with selections_endpoint (added in this PR)
- 1 for DVT without selections_endpoint (Anchor) (untouched)
- Non-DVT (untouched)

Tested and it's working, also thanks very much to @KaloyanTanev for helping to test and confirmed that the calling has reduced and aggregated attestations are still working


Co-Authored-By: Tan Chee Keong <tanck@sigmaprime.io>

validator_client/validator_services/src/duties_service.rs

@@ -17,7 +17,7 @@ use eth2::types::{
 };
 use futures::{
     StreamExt,
-    stream::{self, FuturesUnordered},
+    stream::{self, FuturesUnordered, TryStreamExt},
 };
 use parking_lot::{RwLock, RwLockWriteGuard};
 use safe_arith::{ArithError, SafeArith};
@@ -141,71 +141,15 @@ impl Default for SelectionProofConfig {
 /// Create a selection proof for `duty`.
 ///
 /// Return `Ok(None)` if the attesting validator is not an aggregator.
-async fn make_selection_proof<S: ValidatorStore + 'static, T: SlotClock>(
+async fn make_selection_proof<S: ValidatorStore>(
     duty: &AttesterData,
     validator_store: &S,
     spec: &ChainSpec,
-    beacon_nodes: &Arc<BeaconNodeFallback<T>>,
-    config: &SelectionProofConfig,
 ) -> Result<Option<SelectionProof>, Error<S::Error>> {
-    let selection_proof = if config.selections_endpoint {
+    let selection_proof = validator_store
-        let beacon_committee_selection = BeaconCommitteeSelection {
-            validator_index: duty.validator_index,
-            slot: duty.slot,
-            // This is partial selection proof
-            selection_proof: validator_store
         .produce_selection_proof(duty.pubkey, duty.slot)
         .await
-                .map_err(Error::FailedToProduceSelectionProof)?
+        .map_err(Error::FailedToProduceSelectionProof)?;
-                .into(),
-        };
-        // Call the endpoint /eth/v1/validator/beacon_committee_selections
-        // by sending the BeaconCommitteeSelection that contains partial selection proof
-        // The middleware should return BeaconCommitteeSelection that contains full selection proof
-        let middleware_response = beacon_nodes
-            .first_success(|beacon_node| {
-                let selection_data = beacon_committee_selection.clone();
-                debug!(
-                    "validator_index" = duty.validator_index,
-                    "slot" = %duty.slot,
-                    "partial selection proof" = ?beacon_committee_selection.selection_proof,
-                    "Sending selection to middleware"
-                );
-                async move {
-                    beacon_node
-                        .post_validator_beacon_committee_selections(&[selection_data])
-                        .await
-                }
-            })
-            .await;
-
-        let response_data = middleware_response
-            .map_err(|e| {
-                Error::FailedToProduceSelectionProof(ValidatorStoreError::Middleware(e.to_string()))
-            })?
-            .data
-            .pop()
-            .ok_or_else(|| {
-                Error::FailedToProduceSelectionProof(ValidatorStoreError::Middleware(format!(
-                    "attestation selection proof - empty response for validator {}",
-                    duty.validator_index
-                )))
-            })?;
-
-        debug!(
-            "validator_index" = response_data.validator_index,
-            "slot" = %response_data.slot,
-            // The selection proof from middleware response will be a full selection proof
-            "full selection proof" = ?response_data.selection_proof,
-            "Received selection from middleware"
-        );
-        SelectionProof::from(response_data.selection_proof)
-    } else {
-        validator_store
-            .produce_selection_proof(duty.pubkey, duty.slot)
-            .await
-            .map_err(Error::FailedToProduceSelectionProof)?
-    };
 
     selection_proof
         .is_aggregator(duty.committee_length as usize, spec)
@@ -221,6 +165,69 @@ async fn make_selection_proof<S: ValidatorStore + 'static, T: SlotClock>(
         })
 }
 
+/// Create a Vec<BeaconCommitteeSelection> for every epoch
+/// so that when calling the selections_endpoint later, it calls once per epoch with duties of all slots in that epoch
+async fn make_beacon_committee_selection<S: ValidatorStore, T: SlotClock>(
+    duties_service: &Arc<DutiesService<S, T>>,
+    duties: &[AttesterData],
+) -> Result<Vec<BeaconCommitteeSelection>, Error<S::Error>> {
+    // collect the BeaconCommitteeSelection in duties
+    let beacon_committee_selections = duties
+        .iter()
+        .map(|duty| {
+            let validator_store = &duties_service.validator_store;
+            async move {
+                let partial_selection_proof = validator_store
+                    .produce_selection_proof(duty.pubkey, duty.slot)
+                    .await
+                    .map_err(Error::FailedToProduceSelectionProof)?;
+                Ok::<BeaconCommitteeSelection, Error<S::Error>>(BeaconCommitteeSelection {
+                    validator_index: duty.validator_index,
+                    slot: duty.slot,
+                    selection_proof: partial_selection_proof.into(),
+                })
+            }
+        })
+        .collect::<FuturesUnordered<_>>()
+        .try_collect::<Vec<_>>()
+        .await?;
+
+    let epoch = duties
+        .first()
+        .map(|attester_data| attester_data.slot.epoch(S::E::slots_per_epoch()))
+        .unwrap_or_default();
+
+    debug!(
+        %epoch,
+        count = beacon_committee_selections.len(),
+        "Sending selections to middleware"
+    );
+
+    let selections = duties_service
+        .beacon_nodes
+        .first_success(|beacon_node| {
+            let selections = beacon_committee_selections.clone();
+            async move {
+                beacon_node
+                    .post_validator_beacon_committee_selections(&selections)
+                    .await
+            }
+        })
+        .await
+        .map_err(|e| {
+            Error::FailedToProduceSelectionProof(ValidatorStoreError::Middleware(e.to_string()))
+        })?
+        .data;
+
+    debug!(
+        %epoch,
+        count = beacon_committee_selections.len(),
+        "Received selections from middleware"
+    );
+
+    Ok(selections)
+}
+
 impl DutyAndProof {
     /// Create a new `DutyAndProof` with the selection proof waiting to be filled in.
     pub fn new_without_selection_proof(duty: AttesterData, current_slot: Slot) -> Self {
@@ -1287,14 +1294,21 @@ async fn fill_in_selection_proofs<S: ValidatorStore + 'static, T: SlotClock + 's
     // Sort duties by slot in a BTreeMap.
     let mut duties_by_slot: BTreeMap<Slot, Vec<_>> = BTreeMap::new();
 
-    for duty in duties {
+    for duty in &duties {
-        duties_by_slot.entry(duty.slot).or_default().push(duty);
+        duties_by_slot
+            .entry(duty.slot)
+            .or_default()
+            .push(duty.clone());
     }
 
     // At halfway through each slot when nothing else is likely to be getting signed, sign a batch
     // of selection proofs and insert them into the duties service `attesters` map.
     let slot_clock = &duties_service.slot_clock;
 
+    // Create a HashMap for BeaconCommitteeSelection to match the duty later for distributed case involving middleware
+    let mut selection_hashmap = HashMap::new();
+    let mut call_selection_endpoint = false;
+
     while !duties_by_slot.is_empty() {
         if let Some(duration) = slot_clock.duration_to_next_slot() {
             sleep(
@@ -1333,10 +1347,77 @@ async fn fill_in_selection_proofs<S: ValidatorStore + 'static, T: SlotClock + 's
                 &[validator_metrics::ATTESTATION_SELECTION_PROOFS],
             );
 
-            // In distributed case, we want to send all partial selection proofs to the middleware to determine aggregation duties,
+            // for distributed case that uses the selections_endpoint
-            // as the middleware will need to have a threshold of partial selection proofs to be able to return the full selection proof
+            if duties_service.selection_proof_config.selections_endpoint {
-            // Thus, sign selection proofs in parallel in distributed case; Otherwise, sign them serially in non-distributed (normal) case
+                // Using lookahead_slot to determine if it is the first slot of an epoch
-            if duties_service.selection_proof_config.parallel_sign {
+                let is_lookahead_slot_epoch_start = lookahead_slot % S::E::slots_per_epoch() == 0;
+
+                // Call the selection endpoint only at the first slot of an epoch or when it errors
+                if is_lookahead_slot_epoch_start || call_selection_endpoint {
+                    let beacon_committee_selections =
+                        make_beacon_committee_selection(&duties_service, &duties).await;
+
+                    let selections = match beacon_committee_selections {
+                        Ok(selections) => selections,
+                        Err(e) => {
+                            error!(
+                                error = ?e,
+                                "Failed to fetch selection proofs"
+                            );
+                            // If calling the endpoint fails, change to true so that it will retry the next slot
+                            call_selection_endpoint = true;
+                            continue;
+                        }
+                    };
+
+                    for selection in &selections {
+                        // This is a full_selection_proof returned by middleware
+                        let selection_proof =
+                            SelectionProof::from(selection.selection_proof.clone());
+                        selection_hashmap
+                            .insert((selection.validator_index, selection.slot), selection_proof);
+                    }
+                    // Once we have the selection_proof, we don't call the selections_endpoint again
+                    call_selection_endpoint = false;
+                }
+
+                for duty in relevant_duties.into_values().flatten() {
+                    let key = (duty.validator_index, duty.slot);
+
+                    let result = if let Some(selection_proof) = selection_hashmap.remove(&key) {
+                        match selection_proof
+                            .is_aggregator(duty.committee_length as usize, &duties_service.spec)
+                            .map_err(Error::<S::Error>::InvalidModulo)
+                        {
+                            // Aggregator, return the result
+                            Ok(true) => Ok((duty, Some(selection_proof))),
+                            // Not an aggregator, do nothing and continue
+                            Ok(false) => continue,
+                            Err(_) => return,
+                        }
+                    } else {
+                        Err(Error::FailedToProduceSelectionProof(
+                            ValidatorStoreError::Middleware(format!(
+                                "Missing selection proof for validator {} slot {}",
+                                duty.validator_index, duty.slot
+                            )),
+                        ))
+                    };
+
+                    let mut attesters = duties_service.attesters.write();
+                    // if process_duty_and_proof returns false, exit the loop
+                    if !process_duty_and_proof::<S>(
+                        &mut attesters,
+                        result,
+                        dependent_root,
+                        current_slot,
+                    ) {
+                        return;
+                    }
+                }
+            }
+            // For distributed case that uses parallel_sign
+            else if duties_service.selection_proof_config.parallel_sign {
                 let mut duty_and_proof_results = relevant_duties
                     .into_values()
                     .flatten()
@@ -1345,8 +1426,6 @@ async fn fill_in_selection_proofs<S: ValidatorStore + 'static, T: SlotClock + 's
                             &duty,
                             duties_service.validator_store.as_ref(),
                             &duties_service.spec,
-                            &duties_service.beacon_nodes,
-                            &duties_service.selection_proof_config,
                         )
                         .await?;
                         Ok((duty, opt_selection_proof))
@@ -1373,8 +1452,6 @@ async fn fill_in_selection_proofs<S: ValidatorStore + 'static, T: SlotClock + 's
                             &duty,
                             duties_service.validator_store.as_ref(),
                             &duties_service.spec,
-                            &duties_service.beacon_nodes,
-                            &duties_service.selection_proof_config,
                         )
                         .await?;
                         Ok((duty, opt_selection_proof))