Reduce loglevel for UnknownPubkey in the VC

validator_client/src/attestation_service.rs

@@ -2,12 +2,12 @@ use crate::beacon_node_fallback::{BeaconNodeFallback, RequireSynced};
 use crate::{
     duties_service::{DutiesService, DutyAndProof},
     http_metrics::metrics,
-    validator_store::ValidatorStore,
+    validator_store::{Error as ValidatorStoreError, ValidatorStore},
     OfflineOnFailure,
 };
 use environment::RuntimeContext;
 use futures::future::join_all;
-use slog::{crit, error, info, trace};
+use slog::{crit, debug, error, info, trace, warn};
 use slot_clock::SlotClock;
 use std::collections::HashMap;
 use std::ops::Deref;
@@ -395,6 +395,20 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
                 .await
             {
                 Ok(()) => Some((attestation, duty.validator_index)),
+                Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
+                    // A pubkey can be missing when a validator was recently
+                    // removed via the API.
+                    warn!(
+                        log,
+                        "Missing pubkey for attestation";
+                        "info" => "a validator may have recently been removed from this VC",
+                        "pubkey" => ?pubkey,
+                        "validator" => ?duty.pubkey,
+                        "committee_index" => committee_index,
+                        "slot" => slot.as_u64(),
+                    );
+                    None
+                }
                 Err(e) => {
                     crit!(
                         log,
@@ -527,10 +541,20 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
                 .await
             {
                 Ok(aggregate) => Some(aggregate),
+                Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
+                    // A pubkey can be missing when a validator was recently
+                    // removed via the API.
+                    debug!(
+                        log,
+                        "Missing pubkey for aggregate";
+                        "pubkey" => ?pubkey,
+                    );
+                    None
+                }
                 Err(e) => {
                     crit!(
                         log,
-                        "Failed to sign attestation";
+                        "Failed to sign aggregate";
                         "error" => ?e,
                         "pubkey" => ?duty.pubkey,
                     );

validator_client/src/block_service.rs

@@ -5,7 +5,10 @@ use crate::{
     graffiti_file::GraffitiFile,
     OfflineOnFailure,
 };
-use crate::{http_metrics::metrics, validator_store::ValidatorStore};
+use crate::{
+    http_metrics::metrics,
+    validator_store::{Error as ValidatorStoreError, ValidatorStore},
+};
 use environment::RuntimeContext;
 use eth2::BeaconNodeHttpClient;
 use slog::{crit, debug, error, info, trace, warn};
@@ -417,17 +420,31 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
             BlockError::Recoverable("Unable to determine current slot from clock".to_string())
         })?;
 
-        let randao_reveal = self
+        let randao_reveal = match self
             .validator_store
             .randao_reveal(validator_pubkey, slot.epoch(E::slots_per_epoch()))
             .await
-            .map_err(|e| {
+        {
-                BlockError::Recoverable(format!(
+            Ok(signature) => signature.into(),
+            Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
+                // A pubkey can be missing when a validator was recently removed
+                // via the API.
+                warn!(
+                    log,
+                    "Missing pubkey for block randao";
+                    "info" => "a validator may have recently been removed from this VC",
+                    "pubkey" => ?pubkey,
+                    "slot" => ?slot
+                );
+                return Ok(());
+            }
+            Err(e) => {
+                return Err(BlockError::Recoverable(format!(
                     "Unable to produce randao reveal signature: {:?}",
                     e
-                ))
+                )))
-            })?
+            }
-            .into();
+        };
 
         let graffiti = determine_graffiti(
             &validator_pubkey,
@@ -522,11 +539,31 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
             .await?;
 
         let signing_timer = metrics::start_timer(&metrics::BLOCK_SIGNING_TIMES);
-        let signed_block = self_ref
+        let signed_block = match self_ref
             .validator_store
             .sign_block::<Payload>(*validator_pubkey_ref, block, current_slot)
             .await
-            .map_err(|e| BlockError::Recoverable(format!("Unable to sign block: {:?}", e)))?;
+        {
+            Ok(block) => block,
+            Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
+                // A pubkey can be missing when a validator was recently removed
+                // via the API.
+                warn!(
+                    log,
+                    "Missing pubkey for block";
+                    "info" => "a validator may have recently been removed from this VC",
+                    "pubkey" => ?pubkey,
+                    "slot" => ?slot
+                );
+                return Ok(());
+            }
+            Err(e) => {
+                return Err(BlockError::Recoverable(format!(
+                    "Unable to sign block: {:?}",
+                    e
+                )))
+            }
+        };
         let signing_time_ms =
             Duration::from_secs_f64(signing_timer.map_or(0.0, |t| t.stop_and_record())).as_millis();
 

validator_client/src/duties_service.rs

@@ -932,6 +932,20 @@ async fn fill_in_selection_proofs<T: SlotClock + 'static, E: EthSpec>(
             for result in duty_and_proof_results {
                 let duty_and_proof = match result {
                     Ok(duty_and_proof) => duty_and_proof,
+                    Err(Error::FailedToProduceSelectionProof(
+                        ValidatorStoreError::UnknownPubkey(pubkey),
+                    )) => {
+                        // A pubkey can be missing when a validator was recently
+                        // removed via the API.
+                        warn!(
+                            log,
+                            "Missing pubkey for duty and proof";
+                            "info" => "a validator may have recently been removed from this VC",
+                            "pubkey" => ?pubkey,
+                        );
+                        // Do not abort the entire batch for a single failure.
+                        continue;
+                    }
                     Err(e) => {
                         error!(
                             log,

validator_client/src/duties_service/sync.rs

@@ -2,6 +2,7 @@ use crate::beacon_node_fallback::{OfflineOnFailure, RequireSynced};
 use crate::{
     doppelganger_service::DoppelgangerStatus,
     duties_service::{DutiesService, Error},
+    validator_store::Error as ValidatorStoreError,
 };
 use futures::future::join_all;
 use itertools::Itertools;
@@ -539,6 +540,18 @@ pub async fn fill_in_aggregation_proofs<T: SlotClock + 'static, E: EthSpec>(
                         .await
                     {
                         Ok(proof) => proof,
+                        Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
+                            // A pubkey can be missing when a validator was recently
+                            // removed via the API.
+                            debug!(
+                                log,
+                                "Missing pubkey for sync selection proof";
+                                "pubkey" => ?pubkey,
+                                "pubkey" => ?duty.pubkey,
+                                "slot" => slot,
+                            );
+                            return None;
+                        }
                         Err(e) => {
                             warn!(
                                 log,

validator_client/src/preparation_service.rs

@@ -1,5 +1,5 @@
 use crate::beacon_node_fallback::{BeaconNodeFallback, RequireSynced};
-use crate::validator_store::{DoppelgangerStatus, ValidatorStore};
+use crate::validator_store::{DoppelgangerStatus, Error as ValidatorStoreError, ValidatorStore};
 use crate::OfflineOnFailure;
 use bls::PublicKeyBytes;
 use environment::RuntimeContext;
@@ -442,8 +442,23 @@ impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
                     .await
                 {
                     Ok(data) => data,
+                    Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
+                        // A pubkey can be missing when a validator was recently
+                        // removed via the API.
+                        debug!(
+                            log,
+                            "Missing pubkey for registration data";
+                            "pubkey" => ?pubkey,
+                        );
+                        continue;
+                    }
                     Err(e) => {
-                        error!(log, "Unable to sign validator registration data"; "error" => ?e, "pubkey" => ?pubkey);
+                        error!(
+                            log,
+                            "Unable to sign validator registration data";
+                            "error" => ?e,
+                            "pubkey" => ?pubkey
+                        );
                         continue;
                     }
                 };

validator_client/src/sync_committee_service.rs

@@ -1,5 +1,9 @@
 use crate::beacon_node_fallback::{BeaconNodeFallback, RequireSynced};
-use crate::{duties_service::DutiesService, validator_store::ValidatorStore, OfflineOnFailure};
+use crate::{
+    duties_service::DutiesService,
+    validator_store::{Error as ValidatorStoreError, ValidatorStore},
+    OfflineOnFailure,
+};
 use environment::RuntimeContext;
 use eth2::types::BlockId;
 use futures::future::join_all;
@@ -264,6 +268,18 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
                 .await
             {
                 Ok(signature) => Some(signature),
+                Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
+                    // A pubkey can be missing when a validator was recently
+                    // removed via the API.
+                    debug!(
+                        log,
+                        "Missing pubkey for sync committee signature";
+                        "pubkey" => ?pubkey,
+                        "validator_index" => duty.validator_index,
+                        "slot" => slot,
+                    );
+                    None
+                }
                 Err(e) => {
                     crit!(
                         log,
@@ -405,6 +421,17 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
                     .await
                 {
                     Ok(signed_contribution) => Some(signed_contribution),
+                    Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
+                        // A pubkey can be missing when a validator was recently
+                        // removed via the API.
+                        debug!(
+                            log,
+                            "Missing pubkey for sync contribution";
+                            "pubkey" => ?pubkey,
+                            "slot" => slot,
+                        );
+                        None
+                    }
                     Err(e) => {
                         crit!(
                             log,