Reduce frequency of polling unknown validators to avoid overwhelming the Beacon Node (#5628)

* Reduce frequency of polling unknown validators.

* Move slot calculation into for loop.

* Simplify logic.

Co-authored-by: Michael Sproul <micsproul@gmail.com>

* Fix formatting

validator_client/src/duties_service.rs

@@ -215,6 +215,8 @@ pub struct DutiesService<T, E: EthSpec> {
     pub sync_duties: SyncDutiesMap<E>,
     /// Provides the canonical list of locally-managed validators.
     pub validator_store: Arc<ValidatorStore<T, E>>,
+    /// Maps unknown validator pubkeys to the next slot time when a poll should be conducted again.
+    pub unknown_validator_next_poll_slots: RwLock<HashMap<PublicKeyBytes, Slot>>,
     /// Tracks the current slot.
     pub slot_clock: T,
     /// Provides HTTP access to remote beacon nodes.
@@ -489,6 +491,24 @@ async fn poll_validator_indices<T: SlotClock + 'static, E: EthSpec>(
             .is_some();
 
         if !is_known {
+            let current_slot_opt = duties_service.slot_clock.now();
+
+            if let Some(current_slot) = current_slot_opt {
+                let is_first_slot_of_epoch = current_slot % E::slots_per_epoch() == 0;
+
+                // Query an unknown validator later if it was queried within the last epoch, or if
+                // the current slot is the first slot of an epoch.
+                let poll_later = duties_service
+                    .unknown_validator_next_poll_slots
+                    .read()
+                    .get(&pubkey)
+                    .map(|&poll_slot| poll_slot > current_slot || is_first_slot_of_epoch)
+                    .unwrap_or(false);
+                if poll_later {
+                    continue;
+                }
+            }
+
             // Query the remote BN to resolve a pubkey to a validator index.
             let download_result = duties_service
                 .beacon_nodes
@@ -533,10 +553,23 @@ async fn poll_validator_indices<T: SlotClock + 'static, E: EthSpec>(
                         .initialized_validators()
                         .write()
                         .set_index(&pubkey, response.data.index);
+
+                    duties_service
+                        .unknown_validator_next_poll_slots
+                        .write()
+                        .remove(&pubkey);
                 }
                 // This is not necessarily an error, it just means the validator is not yet known to
                 // the beacon chain.
                 Ok(None) => {
+                    if let Some(current_slot) = current_slot_opt {
+                        let next_poll_slot = current_slot.saturating_add(E::slots_per_epoch());
+                        duties_service
+                            .unknown_validator_next_poll_slots
+                            .write()
+                            .insert(pubkey, next_poll_slot);
+                    }
+
                     debug!(
                         log,
                         "Validator without index";

validator_client/src/lib.rs

@@ -476,6 +476,7 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
             slot_clock: slot_clock.clone(),
             beacon_nodes: beacon_nodes.clone(),
             validator_store: validator_store.clone(),
+            unknown_validator_next_poll_slots: <_>::default(),
             spec: context.eth2_config.spec.clone(),
             context: duties_context,
             enable_high_validator_count_metrics: config.enable_high_validator_count_metrics,