Standard beacon api updates (#1831)

## Issue Addressed

Resolves #1809 
Resolves #1824
Resolves #1818
Resolves #1828 (hopefully)

## Proposed Changes

- add `validator_index` to the proposer duties endpoint
- add the ability to query for historical proposer duties
- `StateId` deserialization now fails with a 400 warp rejection
- add the `validator_balances` endpoint
- update the `aggregate_and_proofs` endpoint to accept an array
- updates the attester duties endpoint from a `GET` to a `POST`
- reduces the number of times we query for proposer duties from once per slot per validator to only once per slot 


Co-authored-by: realbigsean <seananderson33@gmail.com>
Co-authored-by: Paul Hauner <paul@paulhauner.com>

validator_client/src/attestation_service.rs

@@ -437,6 +437,8 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
             .ok_or_else(|| format!("No aggregate available for {:?}", attestation_data))?
             .data;
 
+        let mut signed_aggregate_and_proofs = Vec::new();
+
         for duty_and_proof in validator_duties {
             let selection_proof = if let Some(proof) = duty_and_proof.selection_proof.as_ref() {
                 proof
@@ -462,44 +464,53 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
                 continue;
             }
 
-            let signed_aggregate_and_proof = if let Some(aggregate) =
-                self.validator_store.produce_signed_aggregate_and_proof(
-                    pubkey,
-                    validator_index,
-                    aggregated_attestation.clone(),
-                    selection_proof.clone(),
-                ) {
-                aggregate
+            if let Some(aggregate) = self.validator_store.produce_signed_aggregate_and_proof(
+                pubkey,
+                validator_index,
+                aggregated_attestation.clone(),
+                selection_proof.clone(),
+            ) {
+                signed_aggregate_and_proofs.push(aggregate);
             } else {
                 crit!(log, "Failed to sign attestation");
                 continue;
             };
+        }
 
-            let attestation = &signed_aggregate_and_proof.message.aggregate;
-
+        if !signed_aggregate_and_proofs.is_empty() {
             match self
                 .beacon_node
-                .post_validator_aggregate_and_proof(&signed_aggregate_and_proof)
+                .post_validator_aggregate_and_proof(signed_aggregate_and_proofs.as_slice())
                 .await
             {
-                Ok(()) => info!(
-                    log,
-                    "Successfully published attestation";
-                    "aggregator" => signed_aggregate_and_proof.message.aggregator_index,
-                    "signatures" => attestation.aggregation_bits.num_set_bits(),
-                    "head_block" => format!("{:?}", attestation.data.beacon_block_root),
-                    "committee_index" => attestation.data.index,
-                    "slot" => attestation.data.slot.as_u64(),
-                    "type" => "aggregated",
-                ),
-                Err(e) => crit!(
-                    log,
-                    "Failed to publish attestation";
-                    "error" => e.to_string(),
-                    "committee_index" => attestation.data.index,
-                    "slot" => attestation.data.slot.as_u64(),
-                    "type" => "aggregated",
-                ),
+                Ok(()) => {
+                    for signed_aggregate_and_proof in signed_aggregate_and_proofs {
+                        let attestation = &signed_aggregate_and_proof.message.aggregate;
+                        info!(
+                            log,
+                            "Successfully published attestations";
+                            "aggregator" => signed_aggregate_and_proof.message.aggregator_index,
+                            "signatures" => attestation.aggregation_bits.num_set_bits(),
+                            "head_block" => format!("{:?}", attestation.data.beacon_block_root),
+                            "committee_index" => attestation.data.index,
+                            "slot" => attestation.data.slot.as_u64(),
+                            "type" => "aggregated",
+                        );
+                    }
+                }
+                Err(e) => {
+                    for signed_aggregate_and_proof in signed_aggregate_and_proofs {
+                        let attestation = &signed_aggregate_and_proof.message.aggregate;
+                        crit!(
+                            log,
+                            "Failed to publish attestation";
+                            "error" => e.to_string(),
+                            "committee_index" => attestation.data.index,
+                            "slot" => attestation.data.slot.as_u64(),
+                            "type" => "aggregated",
+                        );
+                    }
+                }
             }
         }
 

validator_client/src/duties_service.rs

@@ -106,6 +106,10 @@ impl DutyAndProof {
     pub fn validator_pubkey(&self) -> &PublicKey {
         &self.duty.validator_pubkey
     }
+
+    pub fn validator_index(&self) -> Option<u64> {
+        self.duty.validator_index
+    }
 }
 
 impl Into<DutyAndProof> for ValidatorDuty {
@@ -229,6 +233,14 @@ impl DutiesStore {
             .collect()
     }
 
+    fn get_index(&self, pubkey: &PublicKey, epoch: Epoch) -> Option<u64> {
+        self.store
+            .read()
+            .get(pubkey)?
+            .get(&epoch)?
+            .validator_index()
+    }
+
     fn is_aggregator(&self, validator_pubkey: &PublicKey, epoch: Epoch) -> Option<bool> {
         Some(
             self.store
@@ -588,29 +600,42 @@ impl<T: SlotClock + 'static, E: EthSpec> DutiesService<T, E> {
         let mut replaced = 0;
         let mut invalid = 0;
 
-        let mut validator_subscriptions = vec![];
-        for pubkey in self.validator_store.voting_pubkeys() {
-            let remote_duties = match ValidatorDuty::download(
-                &self.beacon_node,
-                current_epoch,
-                request_epoch,
-                pubkey,
-            )
-            .await
-            {
-                Ok(duties) => duties,
-                Err(e) => {
-                    error!(
-                        log,
-                        "Failed to download validator duties";
-                        "error" => e
-                    );
-                    continue;
-                }
-            };
+        // Determine which pubkeys we already know the index of by checking the duties store for
+        // the current epoch.
+        let pubkeys: Vec<(PublicKey, Option<u64>)> = self
+            .validator_store
+            .voting_pubkeys()
+            .into_iter()
+            .map(|pubkey| {
+                let index = self.store.get_index(&pubkey, current_epoch);
+                (pubkey, index)
+            })
+            .collect();
 
+        let mut validator_subscriptions = vec![];
+        let remote_duties: Vec<ValidatorDuty> = match ValidatorDuty::download(
+            &self.beacon_node,
+            current_epoch,
+            request_epoch,
+            pubkeys,
+            &log,
+        )
+        .await
+        {
+            Ok(duties) => duties,
+            Err(e) => {
+                error!(
+                    log,
+                    "Failed to download validator duties";
+                    "error" => e
+                );
+                vec![]
+            }
+        };
+
+        remote_duties.iter().for_each(|remote_duty| {
             // Convert the remote duties into our local representation.
-            let duties: DutyAndProof = remote_duties.clone().into();
+            let duties: DutyAndProof = remote_duty.clone().into();
 
             let validator_pubkey = duties.duty.validator_pubkey.clone();
 
@@ -628,9 +653,9 @@ impl<T: SlotClock + 'static, E: EthSpec> DutiesService<T, E> {
                             debug!(
                                 log,
                                 "First duty assignment for validator";
-                                "proposal_slots" => format!("{:?}", &remote_duties.block_proposal_slots),
-                                "attestation_slot" => format!("{:?}", &remote_duties.attestation_slot),
-                                "validator" => format!("{:?}", &remote_duties.validator_pubkey)
+                                "proposal_slots" => format!("{:?}", &remote_duty.block_proposal_slots),
+                                "attestation_slot" => format!("{:?}", &remote_duty.attestation_slot),
+                                "validator" => format!("{:?}", &remote_duty.validator_pubkey)
                             );
                             new_validator += 1;
                         }
@@ -642,10 +667,10 @@ impl<T: SlotClock + 'static, E: EthSpec> DutiesService<T, E> {
                     }
 
                     if let Some(is_aggregator) =
-                        self.store.is_aggregator(&validator_pubkey, request_epoch)
+                    self.store.is_aggregator(&validator_pubkey, request_epoch)
                     {
                         if outcome.is_subscription_candidate() {
-                            if let Some(subscription) = remote_duties.subscription(is_aggregator) {
+                            if let Some(subscription) = remote_duty.subscription(is_aggregator) {
                                 validator_subscriptions.push(subscription)
                             }
                         }
@@ -657,7 +682,7 @@ impl<T: SlotClock + 'static, E: EthSpec> DutiesService<T, E> {
                     "error" => e
                 ),
             }
-        }
+        });
 
         if invalid > 0 {
             error!(

validator_client/src/validator_duty.rs

@@ -3,6 +3,8 @@ use eth2::{
     BeaconNodeHttpClient,
 };
 use serde::{Deserialize, Serialize};
+use slog::{error, Logger};
+use std::collections::HashMap;
 use types::{CommitteeIndex, Epoch, PublicKey, PublicKeyBytes, Slot};
 
 /// This struct is being used as a shim since we deprecated the `rest_api` in favour of `http_api`.
@@ -33,10 +35,10 @@ pub struct ValidatorDuty {
 
 impl ValidatorDuty {
     /// Instantiate `Self` as if there are no known dutes for `validator_pubkey`.
-    fn no_duties(validator_pubkey: PublicKey) -> Self {
+    fn no_duties(validator_pubkey: PublicKey, validator_index: Option<u64>) -> Self {
         ValidatorDuty {
             validator_pubkey,
-            validator_index: None,
+            validator_index,
             attestation_slot: None,
             attestation_committee_index: None,
             attestation_committee_position: None,
@@ -53,58 +55,113 @@ impl ValidatorDuty {
         beacon_node: &BeaconNodeHttpClient,
         current_epoch: Epoch,
         request_epoch: Epoch,
-        pubkey: PublicKey,
-    ) -> Result<ValidatorDuty, String> {
-        let pubkey_bytes = PublicKeyBytes::from(&pubkey);
+        mut pubkeys: Vec<(PublicKey, Option<u64>)>,
+        log: &Logger,
+    ) -> Result<Vec<ValidatorDuty>, String> {
+        for (pubkey, index_opt) in &mut pubkeys {
+            if index_opt.is_none() {
+                *index_opt = beacon_node
+                    .get_beacon_states_validator_id(
+                        StateId::Head,
+                        &ValidatorId::PublicKey(PublicKeyBytes::from(&*pubkey)),
+                    )
+                    .await
+                    .map_err(|e| {
+                        error!(
+                            log,
+                            "Failed to obtain validator index";
+                            "pubkey" => ?pubkey,
+                            "error" => ?e
+                        )
+                    })
+                    // Supress the error since we've already logged an error and we don't want to
+                    // stop the rest of the code.
+                    .ok()
+                    .and_then(|body_opt| body_opt.map(|body| body.data.index));
+            }
+        }
 
-        let validator_index = if let Some(index) = beacon_node
-            .get_beacon_states_validator_id(
-                StateId::Head,
-                &ValidatorId::PublicKey(pubkey_bytes.clone()),
-            )
-            .await
-            .map_err(|e| format!("Failed to get validator index: {}", e))?
-            .map(|body| body.data.index)
-        {
-            index
+        // Query for all block proposer duties in the current epoch and map the response by index.
+        let proposal_slots_by_index: HashMap<u64, Vec<Slot>> = if current_epoch == request_epoch {
+            beacon_node
+                .get_validator_duties_proposer(current_epoch)
+                .await
+                .map(|resp| resp.data)
+                // Exit early if there's an error.
+                .map_err(|e| format!("Failed to get proposer indices: {:?}", e))?
+                .into_iter()
+                .fold(
+                    HashMap::with_capacity(pubkeys.len()),
+                    |mut map, proposer_data| {
+                        map.entry(proposer_data.validator_index)
+                            .or_insert_with(Vec::new)
+                            .push(proposer_data.slot);
+                        map
+                    },
+                )
         } else {
-            return Ok(Self::no_duties(pubkey));
+            HashMap::new()
         };
 
-        if let Some(attester) = beacon_node
-            .get_validator_duties_attester(request_epoch, Some(&[validator_index]))
+        let query_indices = pubkeys
+            .iter()
+            .filter_map(|(_, index_opt)| *index_opt)
+            .collect::<Vec<_>>();
+        let attester_data_map = beacon_node
+            .post_validator_duties_attester(request_epoch, query_indices.as_slice())
             .await
-            .map_err(|e| format!("Failed to get attester duties: {}", e))?
-            .data
-            .first()
-        {
-            let block_proposal_slots = if current_epoch == request_epoch {
-                beacon_node
-                    .get_validator_duties_proposer(current_epoch)
-                    .await
-                    .map_err(|e| format!("Failed to get proposer indices: {}", e))?
-                    .data
-                    .into_iter()
-                    .filter(|data| data.pubkey == pubkey_bytes)
-                    .map(|data| data.slot)
-                    .collect()
-            } else {
-                vec![]
-            };
+            .map(|resp| resp.data)
+            // Exit early if there's an error.
+            .map_err(|e| format!("Failed to get attester duties: {:?}", e))?
+            .into_iter()
+            .fold(
+                HashMap::with_capacity(pubkeys.len()),
+                |mut map, attester_data| {
+                    map.insert(attester_data.validator_index, attester_data);
+                    map
+                },
+            );
 
-            Ok(ValidatorDuty {
-                validator_pubkey: pubkey,
-                validator_index: Some(attester.validator_index),
-                attestation_slot: Some(attester.slot),
-                attestation_committee_index: Some(attester.committee_index),
-                attestation_committee_position: Some(attester.validator_committee_index as usize),
-                committee_count_at_slot: Some(attester.committees_at_slot),
-                committee_length: Some(attester.committee_length),
-                block_proposal_slots: Some(block_proposal_slots),
+        let duties = pubkeys
+            .into_iter()
+            .map(|(pubkey, index_opt)| {
+                if let Some(index) = index_opt {
+                    if let Some(attester_data) = attester_data_map.get(&index) {
+                        match attester_data.pubkey.decompress() {
+                            Ok(pubkey) => ValidatorDuty {
+                                validator_pubkey: pubkey,
+                                validator_index: Some(attester_data.validator_index),
+                                attestation_slot: Some(attester_data.slot),
+                                attestation_committee_index: Some(attester_data.committee_index),
+                                attestation_committee_position: Some(
+                                    attester_data.validator_committee_index as usize,
+                                ),
+                                committee_count_at_slot: Some(attester_data.committees_at_slot),
+                                committee_length: Some(attester_data.committee_length),
+                                block_proposal_slots: proposal_slots_by_index
+                                    .get(&attester_data.validator_index)
+                                    .cloned(),
+                            },
+                            Err(e) => {
+                                error!(
+                                    log,
+                                    "Could not deserialize validator public key";
+                                    "error" => format!("{:?}", e),
+                                    "validator_index" => attester_data.validator_index
+                                );
+                                Self::no_duties(pubkey, Some(index))
+                            }
+                        }
+                    } else {
+                        Self::no_duties(pubkey, Some(index))
+                    }
+                } else {
+                    Self::no_duties(pubkey, None)
+                }
             })
-        } else {
-            Ok(Self::no_duties(pubkey))
-        }
+            .collect();
+
+        Ok(duties)
     }
 
     /// Return `true` if these validator duties are equal, ignoring their `block_proposal_slots`.