gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-15 02:42:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Integrate tracing (#6339)

Browse Source 
Tracing Integration
- [reference](5bbf1859e9/projects/project-ideas.md (L297))


  - [x] replace slog & log with tracing throughout the codebase
- [x] implement custom crit log
- [x] make relevant changes in the formatter
- [x] replace sloggers
- [x] re-write SSE logging components

cc: @macladson @eserilev
This commit is contained in:
ThreeHrSleep
2025-03-13 04:01:05 +05:30
committed by GitHub
parent f23f984f85
commit d60c24ef1c
241 changed files with 9485 additions and 9328 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
validator_client/validator_services/Cargo.toml
View File

@@ -13,11 +13,12 @@ environment = { workspace = true }
eth2 = { workspace = true }
futures = { workspace = true }
graffiti_file = { workspace = true }
logging = { workspace = true }
parking_lot = { workspace = true }
safe_arith = { workspace = true }
slog = { workspace = true }
slot_clock = { workspace = true }
tokio = { workspace = true }
tracing = { workspace = true }
tree_hash = { workspace = true }
types = { workspace = true }
validator_metrics = { workspace = true }

176
validator_client/validator_services/src/attestation_service.rs
View File

@@ -3,12 +3,13 @@ use beacon_node_fallback::{ApiTopic, BeaconNodeFallback};
use either::Either;
use environment::RuntimeContext;
use futures::future::join_all;
use slog::{crit, debug, error, info, trace, warn};
use logging::crit;
use slot_clock::SlotClock;
use std::collections::HashMap;
use std::ops::Deref;
use std::sync::Arc;
use tokio::time::{sleep, sleep_until, Duration, Instant};
use tracing::{debug, error, info, trace, warn};
use tree_hash::TreeHash;
use types::{Attestation, AttestationData, ChainSpec, CommitteeIndex, EthSpec, Slot};
use validator_store::{Error as ValidatorStoreError, ValidatorStore};
@@ -128,9 +129,8 @@ impl<T, E: EthSpec> Deref for AttestationService<T, E> {
impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
/// Starts the service which periodically produces attestations.
pub fn start_update_service(self, spec: &ChainSpec) -> Result<(), String> {
let log = self.context.log().clone();
if self.disable {
info!(log, "Attestation service disabled");
info!("Attestation service disabled");
return Ok(());
}
@@ -141,9 +141,8 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
.ok_or("Unable to determine duration to next slot")?;
info!(
log,
"Attestation production service started";
"next_update_millis" => duration_to_next_slot.as_millis()
next_update_millis = duration_to_next_slot.as_millis(),
"Attestation production service started"
);
let executor = self.context.executor.clone();
@@ -152,22 +151,14 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
loop {
if let Some(duration_to_next_slot) = self.slot_clock.duration_to_next_slot() {
sleep(duration_to_next_slot + slot_duration / 3).await;
let log = self.context.log();
if let Err(e) = self.spawn_attestation_tasks(slot_duration) {
crit!(
log,
"Failed to spawn attestation tasks";
"error" => e
)
crit!(error = e, "Failed to spawn attestation tasks")
} else {
trace!(
log,
"Spawned attestation tasks";
)
trace!("Spawned attestation tasks");
}
} else {
error!(log, "Failed to read slot clock");
error!("Failed to read slot clock");
// If we can't read the slot clock, just wait another slot.
sleep(slot_duration).await;
continue;
@@ -249,7 +240,6 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
validator_duties: Vec<DutyAndProof>,
aggregate_production_instant: Instant,
) -> Result<(), ()> {
let log = self.context.log();
let attestations_timer = validator_metrics::start_timer_vec(
&validator_metrics::ATTESTATION_SERVICE_TIMES,
&[validator_metrics::ATTESTATIONS],
@@ -269,11 +259,10 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
.await
.map_err(move |e| {
crit!(
log,
"Error during attestation routine";
"error" => format!("{:?}", e),
"committee_index" => committee_index,
"slot" => slot.as_u64(),
error = format!("{:?}", e),
committee_index,
slot = slot.as_u64(),
"Error during attestation routine"
)
})?;
@@ -306,11 +295,10 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
.await
.map_err(move |e| {
crit!(
log,
"Error during attestation routine";
"error" => format!("{:?}", e),
"committee_index" => committee_index,
"slot" => slot.as_u64(),
error = format!("{:?}", e),
committee_index,
slot = slot.as_u64(),
"Error during attestation routine"
)
})?;
}
@@ -336,8 +324,6 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
committee_index: CommitteeIndex,
validator_duties: &[DutyAndProof],
) -> Result<Option<AttestationData>, String> {
let log = self.context.log();
if validator_duties.is_empty() {
return Ok(None);
}
@@ -373,13 +359,12 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
// Ensure that the attestation matches the duties.
if !duty.match_attestation_data::<E>(attestation_data, &self.context.eth2_config.spec) {
crit!(
log,
"Inconsistent validator duties during signing";
"validator" => ?duty.pubkey,
"duty_slot" => duty.slot,
"attestation_slot" => attestation_data.slot,
"duty_index" => duty.committee_index,
"attestation_index" => attestation_data.index,
validator = ?duty.pubkey,
duty_slot = %duty.slot,
attestation_slot = %attestation_data.slot,
duty_index = duty.committee_index,
attestation_index = attestation_data.index,
"Inconsistent validator duties during signing"
);
return None;
}
@@ -396,11 +381,10 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
Ok(attestation) => attestation,
Err(err) => {
crit!(
log,
"Invalid validator duties during signing";
"validator" => ?duty.pubkey,
"duty" => ?duty,
"err" => ?err,
validator = ?duty.pubkey,
?duty,
?err,
"Invalid validator duties during signing"
);
return None;
}
@@ -421,24 +405,22 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
// A pubkey can be missing when a validator was recently
// removed via the API.
warn!(
log,
"Missing pubkey for attestation";
"info" => "a validator may have recently been removed from this VC",
"pubkey" => ?pubkey,
"validator" => ?duty.pubkey,
"committee_index" => committee_index,
"slot" => slot.as_u64(),
info = "a validator may have recently been removed from this VC",
pubkey = ?pubkey,
validator = ?duty.pubkey,
committee_index = committee_index,
slot = slot.as_u64(),
"Missing pubkey for attestation"
);
None
}
Err(e) => {
crit!(
log,
"Failed to sign attestation";
"error" => ?e,
"validator" => ?duty.pubkey,
"committee_index" => committee_index,
"slot" => slot.as_u64(),
error = ?e,
validator = ?duty.pubkey,
committee_index,
slot = slot.as_u64(),
"Failed to sign attestation"
);
None
}
@@ -453,7 +435,7 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
.unzip();
if attestations.is_empty() {
warn!(log, "No attestations were published");
warn!("No attestations were published");
return Ok(None);
}
let fork_name = self
@@ -481,12 +463,11 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
// This shouldn't happen unless BN and VC are out of sync with
// respect to the Electra fork.
error!(
log,
"Unable to convert to SingleAttestation";
"error" => ?e,
"committee_index" => attestation_data.index,
"slot" => slot.as_u64(),
"type" => "unaggregated",
error = ?e,
committee_index = attestation_data.index,
slot = slot.as_u64(),
"type" = "unaggregated",
"Unable to convert to SingleAttestation"
);
None
}
@@ -509,22 +490,20 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
.await
{
Ok(()) => info!(
log,
"Successfully published attestations";
"count" => attestations.len(),
"validator_indices" => ?validator_indices,
"head_block" => ?attestation_data.beacon_block_root,
"committee_index" => attestation_data.index,
"slot" => attestation_data.slot.as_u64(),
"type" => "unaggregated",
count = attestations.len(),
validator_indices = ?validator_indices,
head_block = ?attestation_data.beacon_block_root,
committee_index = attestation_data.index,
slot = attestation_data.slot.as_u64(),
"type" = "unaggregated",
"Successfully published attestations"
),
Err(e) => error!(
log,
"Unable to publish attestations";
"error" => %e,
"committee_index" => attestation_data.index,
"slot" => slot.as_u64(),
"type" => "unaggregated",
error = %e,
committee_index = attestation_data.index,
slot = slot.as_u64(),
"type" = "unaggregated",
"Unable to publish attestations"
),
}
@@ -550,8 +529,6 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
committee_index: CommitteeIndex,
validator_duties: &[DutyAndProof],
) -> Result<(), String> {
let log = self.context.log();
if !validator_duties
.iter()
.any(|duty_and_proof| duty_and_proof.selection_proof.is_some())
@@ -609,7 +586,7 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
let selection_proof = duty_and_proof.selection_proof.as_ref()?;
if !duty.match_attestation_data::<E>(attestation_data, &self.context.eth2_config.spec) {
crit!(log, "Inconsistent validator duties during signing");
crit!("Inconsistent validator duties during signing");
return None;
}
@@ -627,19 +604,14 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
// A pubkey can be missing when a validator was recently
// removed via the API.
debug!(
log,
"Missing pubkey for aggregate";
"pubkey" => ?pubkey,
);
debug!(?pubkey, "Missing pubkey for aggregate");
None
}
Err(e) => {
crit!(
log,
"Failed to sign aggregate";
"error" => ?e,
"pubkey" => ?duty.pubkey,
error = ?e,
pubkey = ?duty.pubkey,
"Failed to sign aggregate"
);
None
}
@@ -683,14 +655,13 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
for signed_aggregate_and_proof in signed_aggregate_and_proofs {
let attestation = signed_aggregate_and_proof.message().aggregate();
info!(
log,
"Successfully published attestation";
"aggregator" => signed_aggregate_and_proof.message().aggregator_index(),
"signatures" => attestation.num_set_aggregation_bits(),
"head_block" => format!("{:?}", attestation.data().beacon_block_root),
"committee_index" => attestation.committee_index(),
"slot" => attestation.data().slot.as_u64(),
"type" => "aggregated",
aggregator = signed_aggregate_and_proof.message().aggregator_index(),
signatures = attestation.num_set_aggregation_bits(),
head_block = format!("{:?}", attestation.data().beacon_block_root),
committee_index = attestation.committee_index(),
slot = attestation.data().slot.as_u64(),
"type" = "aggregated",
"Successfully published attestation"
);
}
}
@@ -698,13 +669,12 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
for signed_aggregate_and_proof in signed_aggregate_and_proofs {
let attestation = &signed_aggregate_and_proof.message().aggregate();
crit!(
log,
"Failed to publish attestation";
"error" => %e,
"aggregator" => signed_aggregate_and_proof.message().aggregator_index(),
"committee_index" => attestation.committee_index(),
"slot" => attestation.data().slot.as_u64(),
"type" => "aggregated",
error = %e,
aggregator = signed_aggregate_and_proof.message().aggregator_index(),
committee_index = attestation.committee_index(),
slot = attestation.data().slot.as_u64(),
"type" = "aggregated",
"Failed to publish attestation"
);
}
}

132
validator_client/validator_services/src/block_service.rs
View File

@@ -4,7 +4,7 @@ use environment::RuntimeContext;
use eth2::types::{FullBlockContents, PublishBlockRequest};
use eth2::{BeaconNodeHttpClient, StatusCode};
use graffiti_file::{determine_graffiti, GraffitiFile};
use slog::{crit, debug, error, info, trace, warn, Logger};
use logging::crit;
use slot_clock::SlotClock;
use std::fmt::Debug;
use std::future::Future;
@@ -12,6 +12,7 @@ use std::ops::Deref;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::mpsc;
use tracing::{debug, error, info, trace, warn};
use types::{
BlindedBeaconBlock, BlockType, EthSpec, Graffiti, PublicKeyBytes, SignedBlindedBeaconBlock,
Slot,
@@ -219,9 +220,7 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
self,
mut notification_rx: mpsc::Receiver<BlockServiceNotification>,
) -> Result<(), String> {
let log = self.context.log().clone();
info!(log, "Block production service started");
info!("Block production service started");
let executor = self.inner.context.executor.clone();
@@ -230,7 +229,7 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
while let Some(notif) = notification_rx.recv().await {
self.do_update(notif).await.ok();
}
debug!(log, "Block service shutting down");
debug!("Block service shutting down");
},
"block_service",
);
@@ -240,64 +239,54 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
/// Attempt to produce a block for any block producers in the `ValidatorStore`.
async fn do_update(&self, notification: BlockServiceNotification) -> Result<(), ()> {
let log = self.context.log();
let _timer = validator_metrics::start_timer_vec(
&validator_metrics::BLOCK_SERVICE_TIMES,
&[validator_metrics::FULL_UPDATE],
);
let slot = self.slot_clock.now().ok_or_else(move || {
crit!(log, "Duties manager failed to read slot clock");
crit!("Duties manager failed to read slot clock");
})?;
if notification.slot != slot {
warn!(
log,
"Skipping block production for expired slot";
"current_slot" => slot.as_u64(),
"notification_slot" => notification.slot.as_u64(),
"info" => "Your machine could be overloaded"
current_slot = slot.as_u64(),
notification_slot = notification.slot.as_u64(),
info = "Your machine could be overloaded",
"Skipping block production for expired slot"
);
return Ok(());
}
if slot == self.context.eth2_config.spec.genesis_slot {
debug!(
log,
"Not producing block at genesis slot";
"proposers" => format!("{:?}", notification.block_proposers),
proposers = format!("{:?}", notification.block_proposers),
"Not producing block at genesis slot"
);
return Ok(());
}
trace!(
log,
"Block service update started";
"slot" => slot.as_u64()
);
trace!(slot = slot.as_u64(), "Block service update started");
let proposers = notification.block_proposers;
if proposers.is_empty() {
trace!(
log,
"No local block proposers for this slot";
"slot" => slot.as_u64()
slot = slot.as_u64(),
"No local block proposers for this slot"
)
} else if proposers.len() > 1 {
error!(
log,
"Multiple block proposers for this slot";
"action" => "producing blocks for all proposers",
"num_proposers" => proposers.len(),
"slot" => slot.as_u64(),
action = "producing blocks for all proposers",
num_proposers = proposers.len(),
slot = slot.as_u64(),
"Multiple block proposers for this slot"
)
}
for validator_pubkey in proposers {
let builder_boost_factor = self.get_builder_boost_factor(&validator_pubkey);
let service = self.clone();
let log = log.clone();
self.inner.context.executor.spawn(
async move {
let result = service
@@ -308,11 +297,10 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
Ok(_) => {}
Err(BlockError::Recoverable(e)) | Err(BlockError::Irrecoverable(e)) => {
error!(
log,
"Error whilst producing block";
"error" => ?e,
"block_slot" => ?slot,
"info" => "block v3 proposal failed, this error may or may not result in a missed block"
error = ?e,
block_slot = ?slot,
info = "block v3 proposal failed, this error may or may not result in a missed block",
"Error whilst producing block"
);
}
}
@@ -332,7 +320,6 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
validator_pubkey: &PublicKeyBytes,
unsigned_block: UnsignedBlock<E>,
) -> Result<(), BlockError> {
let log = self.context.log();
let signing_timer = validator_metrics::start_timer(&validator_metrics::BLOCK_SIGNING_TIMES);
let res = match unsigned_block {
@@ -357,11 +344,10 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
// A pubkey can be missing when a validator was recently removed
// via the API.
warn!(
log,
"Missing pubkey for block";
"info" => "a validator may have recently been removed from this VC",
"pubkey" => ?pubkey,
"slot" => ?slot
info = "a validator may have recently been removed from this VC",
?pubkey,
?slot,
"Missing pubkey for block"
);
return Ok(());
}
@@ -377,10 +363,9 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
Duration::from_secs_f64(signing_timer.map_or(0.0, |t| t.stop_and_record())).as_millis();
info!(
log,
"Publishing signed block";
"slot" => slot.as_u64(),
"signing_time_ms" => signing_time_ms,
slot = slot.as_u64(),
signing_time_ms = signing_time_ms,
"Publishing signed block"
);
// Publish block with first available beacon node.
@@ -396,13 +381,12 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
.await?;
info!(
log,
"Successfully published block";
"block_type" => ?signed_block.block_type(),
"deposits" => signed_block.num_deposits(),
"attestations" => signed_block.num_attestations(),
"graffiti" => ?graffiti.map(|g| g.as_utf8_lossy()),
"slot" => signed_block.slot().as_u64(),
block_type = ?signed_block.block_type(),
deposits = signed_block.num_deposits(),
attestations = signed_block.num_attestations(),
graffiti = ?graffiti.map(|g| g.as_utf8_lossy()),
slot = signed_block.slot().as_u64(),
"Successfully published block"
);
Ok(())
}
@@ -413,7 +397,6 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
validator_pubkey: PublicKeyBytes,
builder_boost_factor: Option<u64>,
) -> Result<(), BlockError> {
let log = self.context.log();
let _timer = validator_metrics::start_timer_vec(
&validator_metrics::BLOCK_SERVICE_TIMES,
&[validator_metrics::BEACON_BLOCK],
@@ -429,11 +412,10 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
// A pubkey can be missing when a validator was recently removed
// via the API.
warn!(
log,
"Missing pubkey for block randao";
"info" => "a validator may have recently been removed from this VC",
"pubkey" => ?pubkey,
"slot" => ?slot
info = "a validator may have recently been removed from this VC",
?pubkey,
?slot,
"Missing pubkey for block randao"
);
return Ok(());
}
@@ -447,7 +429,6 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
let graffiti = determine_graffiti(
&validator_pubkey,
log,
self.graffiti_file.clone(),
self.validator_store.graffiti(&validator_pubkey),
self.graffiti,
@@ -461,11 +442,7 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
proposer_nodes: self.proposer_nodes.clone(),
};
info!(
log,
"Requesting unsigned block";
"slot" => slot.as_u64(),
);
info!(slot = slot.as_u64(), "Requesting unsigned block");
// Request block from first responsive beacon node.
//
@@ -484,7 +461,6 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
graffiti,
proposer_index,
builder_boost_factor,
log,
)
.await
.map_err(|e| {
@@ -514,7 +490,6 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
signed_block: &SignedBlock<E>,
beacon_node: BeaconNodeHttpClient,
) -> Result<(), BlockError> {
let log = self.context.log();
let slot = signed_block.slot();
match signed_block {
SignedBlock::Full(signed_block) => {
@@ -525,7 +500,7 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
beacon_node
.post_beacon_blocks_v2_ssz(signed_block, None)
.await
.or_else(|e| handle_block_post_error(e, slot, log))?
.or_else(|e| handle_block_post_error(e, slot))?
}
SignedBlock::Blinded(signed_block) => {
let _post_timer = validator_metrics::start_timer_vec(
@@ -535,7 +510,7 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
beacon_node
.post_beacon_blinded_blocks_v2_ssz(signed_block, None)
.await
.or_else(|e| handle_block_post_error(e, slot, log))?
.or_else(|e| handle_block_post_error(e, slot))?
}
}
Ok::<_, BlockError>(())
@@ -548,7 +523,6 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
graffiti: Option<Graffiti>,
proposer_index: Option<u64>,
builder_boost_factor: Option<u64>,
log: &Logger,
) -> Result<UnsignedBlock<E>, BlockError> {
let (block_response, _) = beacon_node
.get_validator_blocks_v3::<E>(
@@ -570,11 +544,7 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
eth2::types::ProduceBlockV3Response::Blinded(block) => UnsignedBlock::Blinded(block),
};
info!(
log,
"Received unsigned block";
"slot" => slot.as_u64(),
);
info!(slot = slot.as_u64(), "Received unsigned block");
if proposer_index != Some(unsigned_block.proposer_index()) {
return Err(BlockError::Recoverable(
"Proposer index does not match block proposer. Beacon chain re-orged".to_string(),
@@ -662,23 +632,21 @@ impl<E: EthSpec> SignedBlock<E> {
}
}
fn handle_block_post_error(err: eth2::Error, slot: Slot, log: &Logger) -> Result<(), BlockError> {
fn handle_block_post_error(err: eth2::Error, slot: Slot) -> Result<(), BlockError> {
// Handle non-200 success codes.
if let Some(status) = err.status() {
if status == StatusCode::ACCEPTED {
info!(
log,
"Block is already known to BN or might be invalid";
"slot" => slot,
"status_code" => status.as_u16(),
%slot,
status_code = status.as_u16(),
"Block is already known to BN or might be invalid"
);
return Ok(());
} else if status.is_success() {
debug!(
log,
"Block published with non-standard success code";
"slot" => slot,
"status_code" => status.as_u16(),
%slot,
status_code = status.as_u16(),
"Block published with non-standard success code"
);
return Ok(());
}

180
validator_client/validator_services/src/duties_service.rs
View File

@@ -18,7 +18,6 @@ use eth2::types::{
use futures::{stream, StreamExt};
use parking_lot::RwLock;
use safe_arith::{ArithError, SafeArith};
use slog::{debug, error, info, warn, Logger};
use slot_clock::SlotClock;
use std::cmp::min;
use std::collections::{hash_map, BTreeMap, HashMap, HashSet};
@@ -26,6 +25,7 @@ use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::time::Duration;
use tokio::{sync::mpsc::Sender, time::sleep};
use tracing::{debug, error, info, warn};
use types::{ChainSpec, Epoch, EthSpec, Hash256, PublicKeyBytes, SelectionProof, Slot};
use validator_metrics::{get_int_gauge, set_int_gauge, ATTESTATION_DUTY};
use validator_store::{Error as ValidatorStoreError, ValidatorStore};
@@ -378,7 +378,6 @@ pub fn start_update_service<T: SlotClock + 'static, E: EthSpec>(
* Spawn the task which keeps track of local block proposal duties.
*/
let duties_service = core_duties_service.clone();
let log = core_duties_service.context.log().clone();
core_duties_service.context.executor.spawn(
async move {
loop {
@@ -394,9 +393,8 @@ pub fn start_update_service<T: SlotClock + 'static, E: EthSpec>(
if let Err(e) = poll_beacon_proposers(&duties_service, &mut block_service_tx).await
{
error!(
log,
"Failed to poll beacon proposers";
"error" => ?e
error = ?e,
"Failed to poll beacon proposers"
)
}
}
@@ -413,7 +411,6 @@ pub fn start_update_service<T: SlotClock + 'static, E: EthSpec>(
* Spawn the task which keeps track of local attestation duties.
*/
let duties_service = core_duties_service.clone();
let log = core_duties_service.context.log().clone();
core_duties_service.context.executor.spawn(
async move {
loop {
@@ -428,9 +425,8 @@ pub fn start_update_service<T: SlotClock + 'static, E: EthSpec>(
if let Err(e) = poll_beacon_attesters(&duties_service).await {
error!(
log,
"Failed to poll beacon attesters";
"error" => ?e
error = ?e,
"Failed to poll beacon attesters"
);
}
}
@@ -440,15 +436,13 @@ pub fn start_update_service<T: SlotClock + 'static, E: EthSpec>(
// Spawn the task which keeps track of local sync committee duties.
let duties_service = core_duties_service.clone();
let log = core_duties_service.context.log().clone();
core_duties_service.context.executor.spawn(
async move {
loop {
if let Err(e) = poll_sync_committee_duties(&duties_service).await {
error!(
log,
"Failed to poll sync committee duties";
"error" => ?e
error = ?e,
"Failed to poll sync committee duties"
);
}
@@ -480,8 +474,6 @@ async fn poll_validator_indices<T: SlotClock + 'static, E: EthSpec>(
&[validator_metrics::UPDATE_INDICES],
);
let log = duties_service.context.log();
// Collect *all* pubkeys for resolving indices, even those undergoing doppelganger protection.
//
// Since doppelganger protection queries rely on validator indices it is important to ensure we
@@ -547,11 +539,10 @@ async fn poll_validator_indices<T: SlotClock + 'static, E: EthSpec>(
match download_result {
Ok(Some(response)) => {
info!(
log,
"Validator exists in beacon chain";
"pubkey" => ?pubkey,
"validator_index" => response.data.index,
"fee_recipient" => fee_recipient
?pubkey,
validator_index = response.data.index,
fee_recipient,
"Validator exists in beacon chain"
);
duties_service
.validator_store
@@ -575,21 +566,15 @@ async fn poll_validator_indices<T: SlotClock + 'static, E: EthSpec>(
.insert(pubkey, next_poll_slot);
}
debug!(
log,
"Validator without index";
"pubkey" => ?pubkey,
"fee_recipient" => fee_recipient
)
debug!(?pubkey, fee_recipient, "Validator without index")
}
// Don't exit early on an error, keep attempting to resolve other indices.
Err(e) => {
error!(
log,
"Failed to resolve pubkey to index";
"error" => %e,
"pubkey" => ?pubkey,
"fee_recipient" => fee_recipient
error = %e,
?pubkey,
fee_recipient,
"Failed to resolve pubkey to index"
)
}
}
@@ -613,8 +598,6 @@ async fn poll_beacon_attesters<T: SlotClock + 'static, E: EthSpec>(
&[validator_metrics::UPDATE_ATTESTERS_CURRENT_EPOCH],
);
let log = duties_service.context.log();
let current_slot = duties_service
.slot_clock
.now()
@@ -653,11 +636,10 @@ async fn poll_beacon_attesters<T: SlotClock + 'static, E: EthSpec>(
.await
{
error!(
log,
"Failed to download attester duties";
"current_epoch" => current_epoch,
"request_epoch" => current_epoch,
"err" => ?e,
%current_epoch,
request_epoch = %current_epoch,
err = ?e,
"Failed to download attester duties"
)
}
@@ -675,11 +657,10 @@ async fn poll_beacon_attesters<T: SlotClock + 'static, E: EthSpec>(
.await
{
error!(
log,
"Failed to download attester duties";
"current_epoch" => current_epoch,
"request_epoch" => next_epoch,
"err" => ?e,
%current_epoch,
request_epoch = %next_epoch,
err = ?e,
"Failed to download attester duties"
)
}
@@ -758,9 +739,8 @@ async fn poll_beacon_attesters<T: SlotClock + 'static, E: EthSpec>(
.await;
if subscription_result.as_ref().is_ok() {
debug!(
log,
"Broadcast attestation subscriptions";
"count" => subscriptions.len(),
count = subscriptions.len(),
"Broadcast attestation subscriptions"
);
for subscription_slots in subscription_slots_to_confirm {
subscription_slots.record_successful_subscription_at(current_slot);
@@ -768,9 +748,8 @@ async fn poll_beacon_attesters<T: SlotClock + 'static, E: EthSpec>(
} else if let Err(e) = subscription_result {
if e.num_errors() < duties_service.beacon_nodes.num_total().await {
warn!(
log,
"Some subscriptions failed";
"error" => %e,
error = %e,
"Some subscriptions failed"
);
// If subscriptions were sent to at least one node, regard that as a success.
// There is some redundancy built into the subscription schedule to handle failures.
@@ -779,9 +758,8 @@ async fn poll_beacon_attesters<T: SlotClock + 'static, E: EthSpec>(
}
} else {
error!(
log,
"All subscriptions failed";
"error" => %e
error = %e,
"All subscriptions failed"
);
}
}
@@ -809,14 +787,11 @@ async fn poll_beacon_attesters_for_epoch<T: SlotClock + 'static, E: EthSpec>(
local_indices: &[u64],
local_pubkeys: &HashSet<PublicKeyBytes>,
) -> Result<(), Error> {
let log = duties_service.context.log();
// No need to bother the BN if we don't have any validators.
if local_indices.is_empty() {
debug!(
duties_service.context.log(),
"No validators, not downloading duties";
"epoch" => epoch,
%epoch,
"No validators, not downloading duties"
);
return Ok(());
}
@@ -895,10 +870,9 @@ async fn poll_beacon_attesters_for_epoch<T: SlotClock + 'static, E: EthSpec>(
);
debug!(
log,
"Downloaded attester duties";
"dependent_root" => %dependent_root,
"num_new_duties" => new_duties.len(),
%dependent_root,
num_new_duties = new_duties.len(),
"Downloaded attester duties"
);
// Update the duties service with the new `DutyAndProof` messages.
@@ -929,10 +903,9 @@ async fn poll_beacon_attesters_for_epoch<T: SlotClock + 'static, E: EthSpec>(
&& prior_duty_and_proof.duty == duty_and_proof.duty
{
warn!(
log,
"Redundant attester duty update";
"dependent_root" => %dependent_root,
"validator_index" => duty.validator_index,
%dependent_root,
validator_index = duty.validator_index,
"Redundant attester duty update"
);
continue;
}
@@ -940,11 +913,10 @@ async fn poll_beacon_attesters_for_epoch<T: SlotClock + 'static, E: EthSpec>(
// Using `already_warned` avoids excessive logs.
if dependent_root != *prior_dependent_root && already_warned.take().is_some() {
warn!(
log,
"Attester duties re-org";
"prior_dependent_root" => %prior_dependent_root,
"dependent_root" => %dependent_root,
"note" => "this may happen from time to time"
%prior_dependent_root,
%dependent_root,
note = "this may happen from time to time",
"Attester duties re-org"
)
}
*mut_value = (dependent_root, duty_and_proof);
@@ -1056,8 +1028,6 @@ async fn fill_in_selection_proofs<T: SlotClock + 'static, E: EthSpec>(
duties: Vec<AttesterData>,
dependent_root: Hash256,
) {
let log = duties_service.context.log();
// Sort duties by slot in a BTreeMap.
let mut duties_by_slot: BTreeMap<Slot, Vec<_>> = BTreeMap::new();
@@ -1125,20 +1095,18 @@ async fn fill_in_selection_proofs<T: SlotClock + 'static, E: EthSpec>(
// A pubkey can be missing when a validator was recently
// removed via the API.
warn!(
log,
"Missing pubkey for duty and proof";
"info" => "a validator may have recently been removed from this VC",
"pubkey" => ?pubkey,
info = "a validator may have recently been removed from this VC",
?pubkey,
"Missing pubkey for duty and proof"
);
// Do not abort the entire batch for a single failure.
continue;
}
Err(e) => {
error!(
log,
"Failed to produce duty and proof";
"error" => ?e,
"msg" => "may impair attestation duties"
error = ?e,
msg = "may impair attestation duties",
"Failed to produce duty and proof"
);
// Do not abort the entire batch for a single failure.
continue;
@@ -1163,9 +1131,8 @@ async fn fill_in_selection_proofs<T: SlotClock + 'static, E: EthSpec>(
// Our selection proofs are no longer relevant due to a reorg, abandon
// this entire background process.
debug!(
log,
"Stopping selection proof background task";
"reason" => "re-org"
reason = "re-org",
"Stopping selection proof background task"
);
return;
}
@@ -1188,11 +1155,10 @@ async fn fill_in_selection_proofs<T: SlotClock + 'static, E: EthSpec>(
let time_taken_ms =
Duration::from_secs_f64(timer.map_or(0.0, |t| t.stop_and_record())).as_millis();
debug!(
log,
"Computed attestation selection proofs";
"batch_size" => batch_size,
"lookahead_slot" => lookahead_slot,
"time_taken_ms" => time_taken_ms
batch_size,
%lookahead_slot,
time_taken_ms,
"Computed attestation selection proofs"
);
} else {
// Just sleep for one slot if we are unable to read the system clock, this gives
@@ -1234,8 +1200,6 @@ async fn poll_beacon_proposers<T: SlotClock + 'static, E: EthSpec>(
&[validator_metrics::UPDATE_PROPOSERS],
);
let log = duties_service.context.log();
let current_slot = duties_service
.slot_clock
.now()
@@ -1251,7 +1215,6 @@ async fn poll_beacon_proposers<T: SlotClock + 'static, E: EthSpec>(
&initial_block_proposers,
block_service_tx,
&duties_service.validator_store,
log,
)
.await;
@@ -1290,10 +1253,9 @@ async fn poll_beacon_proposers<T: SlotClock + 'static, E: EthSpec>(
.collect::<Vec<_>>();
debug!(
log,
"Downloaded proposer duties";
"dependent_root" => %dependent_root,
"num_relevant_duties" => relevant_duties.len(),
%dependent_root,
num_relevant_duties = relevant_duties.len(),
"Downloaded proposer duties"
);
if let Some((prior_dependent_root, _)) = duties_service
@@ -1303,20 +1265,18 @@ async fn poll_beacon_proposers<T: SlotClock + 'static, E: EthSpec>(
{
if dependent_root != prior_dependent_root {
warn!(
log,
"Proposer duties re-org";
"prior_dependent_root" => %prior_dependent_root,
"dependent_root" => %dependent_root,
"msg" => "this may happen from time to time"
%prior_dependent_root,
%dependent_root,
msg = "this may happen from time to time",
"Proposer duties re-org"
)
}
}
}
// Don't return early here, we still want to try and produce blocks using the cached values.
Err(e) => error!(
log,
"Failed to download proposer duties";
"err" => %e,
err = %e,
"Failed to download proposer duties"
),
}
@@ -1341,13 +1301,11 @@ async fn poll_beacon_proposers<T: SlotClock + 'static, E: EthSpec>(
&additional_block_producers,
block_service_tx,
&duties_service.validator_store,
log,
)
.await;
debug!(
log,
"Detected new block proposer";
"current_slot" => current_slot,
%current_slot,
"Detected new block proposer"
);
validator_metrics::inc_counter(&validator_metrics::PROPOSAL_CHANGED);
}
@@ -1368,7 +1326,6 @@ async fn notify_block_production_service<T: SlotClock + 'static, E: EthSpec>(
block_proposers: &HashSet<PublicKeyBytes>,
block_service_tx: &mut Sender<BlockServiceNotification>,
validator_store: &ValidatorStore<T, E>,
log: &Logger,
) {
let non_doppelganger_proposers = block_proposers
.iter()
@@ -1385,10 +1342,9 @@ async fn notify_block_production_service<T: SlotClock + 'static, E: EthSpec>(
.await
{
error!(
log,
"Failed to notify block service";
"current_slot" => current_slot,
"error" => %e
%current_slot,
error = %e,
"Failed to notify block service"
);
};
}

72
validator_client/validator_services/src/preparation_service.rs
View File

@@ -3,7 +3,6 @@ use bls::PublicKeyBytes;
use doppelganger_service::DoppelgangerStatus;
use environment::RuntimeContext;
use parking_lot::RwLock;
use slog::{debug, error, info, warn};
use slot_clock::SlotClock;
use std::collections::HashMap;
use std::hash::Hash;
@@ -11,6 +10,7 @@ use std::ops::Deref;
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use tokio::time::{sleep, Duration};
use tracing::{debug, error, info, warn};
use types::{
Address, ChainSpec, EthSpec, ProposerPreparationData, SignedValidatorRegistrationData,
ValidatorRegistrationData,
@@ -173,13 +173,8 @@ impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
/// Starts the service which periodically produces proposer preparations.
pub fn start_proposer_prepare_service(self, spec: &ChainSpec) -> Result<(), String> {
let log = self.context.log().clone();
let slot_duration = Duration::from_secs(spec.seconds_per_slot);
info!(
log,
"Proposer preparation service started";
);
info!("Proposer preparation service started");
let executor = self.context.executor.clone();
let spec = spec.clone();
@@ -192,9 +187,8 @@ impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
.await
.map_err(|e| {
error!(
log,
"Error during proposer preparation";
"error" => ?e,
error = ?e,
"Error during proposer preparation"
)
})
.unwrap_or(());
@@ -203,7 +197,7 @@ impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
if let Some(duration_to_next_slot) = self.slot_clock.duration_to_next_slot() {
sleep(duration_to_next_slot).await;
} else {
error!(log, "Failed to read slot clock");
error!("Failed to read slot clock");
// If we can't read the slot clock, just wait another slot.
sleep(slot_duration).await;
}
@@ -216,12 +210,7 @@ impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
/// Starts the service which periodically sends connected beacon nodes validator registration information.
pub fn start_validator_registration_service(self, spec: &ChainSpec) -> Result<(), String> {
let log = self.context.log().clone();
info!(
log,
"Validator registration service started";
);
info!("Validator registration service started");
let spec = spec.clone();
let slot_duration = Duration::from_secs(spec.seconds_per_slot);
@@ -232,14 +221,14 @@ impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
loop {
// Poll the endpoint immediately to ensure fee recipients are received.
if let Err(e) = self.register_validators().await {
error!(log,"Error during validator registration";"error" => ?e);
error!(error = ?e, "Error during validator registration");
}
// Wait one slot if the register validator request fails or if we should not publish at the current slot.
if let Some(duration_to_next_slot) = self.slot_clock.duration_to_next_slot() {
sleep(duration_to_next_slot).await;
} else {
error!(log, "Failed to read slot clock");
error!("Failed to read slot clock");
// If we can't read the slot clock, just wait another slot.
sleep(slot_duration).await;
}
@@ -274,7 +263,6 @@ impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
}
fn collect_preparation_data(&self, spec: &ChainSpec) -> Vec<ProposerPreparationData> {
let log = self.context.log();
self.collect_proposal_data(|pubkey, proposal_data| {
if let Some(fee_recipient) = proposal_data.fee_recipient {
Some(ProposerPreparationData {
@@ -285,10 +273,9 @@ impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
} else {
if spec.bellatrix_fork_epoch.is_some() {
error!(
log,
"Validator is missing fee recipient";
"msg" => "update validator_definitions.yml",
"pubkey" => ?pubkey
msg = "update validator_definitions.yml",
?pubkey,
"Validator is missing fee recipient"
);
}
None
@@ -336,8 +323,6 @@ impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
&self,
preparation_data: Vec<ProposerPreparationData>,
) -> Result<(), String> {
let log = self.context.log();
// Post the proposer preparations to the BN.
let preparation_data_len = preparation_data.len();
let preparation_entries = preparation_data.as_slice();
@@ -351,14 +336,12 @@ impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
.await
{
Ok(()) => debug!(
log,
"Published proposer preparation";
"count" => preparation_data_len,
count = preparation_data_len,
"Published proposer preparation"
),
Err(e) => error!(
log,
"Unable to publish proposer preparation to all beacon nodes";
"error" => %e,
error = %e,
"Unable to publish proposer preparation to all beacon nodes"
),
}
Ok(())
@@ -400,8 +383,6 @@ impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
&self,
registration_keys: Vec<ValidatorRegistrationKey>,
) -> Result<(), String> {
let log = self.context.log();
let registration_data_len = registration_keys.len();
let mut signed = Vec::with_capacity(registration_data_len);
@@ -442,19 +423,14 @@ impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
// A pubkey can be missing when a validator was recently
// removed via the API.
debug!(
log,
"Missing pubkey for registration data";
"pubkey" => ?pubkey,
);
debug!(?pubkey, "Missing pubkey for registration data");
continue;
}
Err(e) => {
error!(
log,
"Unable to sign validator registration data";
"error" => ?e,
"pubkey" => ?pubkey
error = ?e,
?pubkey,
"Unable to sign validator registration data"
);
continue;
}
@@ -474,9 +450,8 @@ impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
{
Ok(()) => {
info!(
log,
"Published validator registrations to the builder network";
"count" => batch.len(),
count = batch.len(),
"Published validator registrations to the builder network"
);
let mut guard = self.validator_registration_cache.write();
for signed_data in batch {
@@ -487,9 +462,8 @@ impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
}
}
Err(e) => warn!(
log,
"Unable to publish validator registrations to the builder network";
"error" => %e,
error = %e,
"Unable to publish validator registrations to the builder network"
),
}
}

108
validator_client/validator_services/src/sync.rs
View File

@@ -1,12 +1,13 @@
use crate::duties_service::{DutiesService, Error};
use doppelganger_service::DoppelgangerStatus;
use futures::future::join_all;
use logging::crit;
use parking_lot::{MappedRwLockReadGuard, RwLock, RwLockReadGuard, RwLockWriteGuard};
use slog::{crit, debug, info, warn};
use slot_clock::SlotClock;
use std::collections::{HashMap, HashSet};
use std::marker::PhantomData;
use std::sync::Arc;
use tracing::{debug, info, warn};
use types::{ChainSpec, EthSpec, PublicKeyBytes, Slot, SyncDuty, SyncSelectionProof, SyncSubnetId};
use validator_store::Error as ValidatorStoreError;
@@ -414,23 +415,20 @@ pub async fn poll_sync_committee_duties_for_period<T: SlotClock + 'static, E: Et
sync_committee_period: u64,
) -> Result<(), Error> {
let spec = &duties_service.spec;
let log = duties_service.context.log();
// no local validators don't need to poll for sync committee
if local_indices.is_empty() {
debug!(
duties_service.context.log(),
"No validators, not polling for sync committee duties";
"sync_committee_period" => sync_committee_period,
sync_committee_period,
"No validators, not polling for sync committee duties"
);
return Ok(());
}
debug!(
log,
"Fetching sync committee duties";
"sync_committee_period" => sync_committee_period,
"num_validators" => local_indices.len(),
sync_committee_period,
num_validators = local_indices.len(),
"Fetching sync committee duties"
);
let period_start_epoch = spec.epochs_per_sync_committee_period * sync_committee_period;
@@ -452,16 +450,15 @@ pub async fn poll_sync_committee_duties_for_period<T: SlotClock + 'static, E: Et
Ok(res) => res.data,
Err(e) => {
warn!(
log,
"Failed to download sync committee duties";
"sync_committee_period" => sync_committee_period,
"error" => %e,
sync_committee_period,
error = %e,
"Failed to download sync committee duties"
);
return Ok(());
}
};
debug!(log, "Fetched sync duties from BN"; "count" => duties.len());
debug!(count = duties.len(), "Fetched sync duties from BN");
// Add duties to map.
let committee_duties = duties_service
@@ -479,9 +476,8 @@ pub async fn poll_sync_committee_duties_for_period<T: SlotClock + 'static, E: Et
!= duty.validator_sync_committee_indices;
if updated_due_to_reorg {
warn!(
log,
"Sync committee duties changed";
"message" => "this could be due to a really long re-org, or a bug"
message = "this could be due to a really long re-org, or a bug",
"Sync committee duties changed"
);
}
updated_due_to_reorg
@@ -489,10 +485,8 @@ pub async fn poll_sync_committee_duties_for_period<T: SlotClock + 'static, E: Et
if updated {
info!(
log,
"Validator in sync committee";
"validator_index" => duty.validator_index,
"sync_committee_period" => sync_committee_period,
validator_index = duty.validator_index,
sync_committee_period, "Validator in sync committee"
);
*validator_duties = Some(ValidatorDuties::new(duty));
@@ -509,14 +503,11 @@ pub async fn fill_in_aggregation_proofs<T: SlotClock + 'static, E: EthSpec>(
current_slot: Slot,
pre_compute_slot: Slot,
) {
let log = duties_service.context.log();
debug!(
log,
"Calculating sync selection proofs";
"period" => sync_committee_period,
"current_slot" => current_slot,
"pre_compute_slot" => pre_compute_slot
period = sync_committee_period,
%current_slot,
%pre_compute_slot,
"Calculating sync selection proofs"
);
// Generate selection proofs for each validator at each slot, one slot at a time.
@@ -532,9 +523,8 @@ pub async fn fill_in_aggregation_proofs<T: SlotClock + 'static, E: EthSpec>(
Ok(subnet_ids) => subnet_ids,
Err(e) => {
crit!(
log,
"Arithmetic error computing subnet IDs";
"error" => ?e,
error = ?e,
"Arithmetic error computing subnet IDs"
);
continue;
}
@@ -556,21 +546,19 @@ pub async fn fill_in_aggregation_proofs<T: SlotClock + 'static, E: EthSpec>(
// A pubkey can be missing when a validator was recently
// removed via the API.
debug!(
log,
"Missing pubkey for sync selection proof";
"pubkey" => ?pubkey,
"pubkey" => ?duty.pubkey,
"slot" => proof_slot,
?pubkey,
pubkey = ?duty.pubkey,
slot = %proof_slot,
"Missing pubkey for sync selection proof"
);
return None;
}
Err(e) => {
warn!(
log,
"Unable to sign selection proof";
"error" => ?e,
"pubkey" => ?duty.pubkey,
"slot" => proof_slot,
error = ?e,
pubkey = ?duty.pubkey,
slot = %proof_slot,
"Unable to sign selection proof"
);
return None;
}
@@ -579,22 +567,20 @@ pub async fn fill_in_aggregation_proofs<T: SlotClock + 'static, E: EthSpec>(
match proof.is_aggregator::<E>() {
Ok(true) => {
debug!(
log,
"Validator is sync aggregator";
"validator_index" => duty.validator_index,
"slot" => proof_slot,
"subnet_id" => %subnet_id,
validator_index = duty.validator_index,
slot = %proof_slot,
%subnet_id,
"Validator is sync aggregator"
);
Some(((proof_slot, *subnet_id), proof))
}
Ok(false) => None,
Err(e) => {
warn!(
log,
"Error determining is_aggregator";
"pubkey" => ?duty.pubkey,
"slot" => proof_slot,
"error" => ?e,
pubkey = ?duty.pubkey,
slot = %proof_slot,
error = ?e,
"Error determining is_aggregator"
);
None
}
@@ -614,11 +600,7 @@ pub async fn fill_in_aggregation_proofs<T: SlotClock + 'static, E: EthSpec>(
// Add to global storage (we add regularly so the proofs can be used ASAP).
let sync_map = duties_service.sync_duties.committees.read();
let Some(committee_duties) = sync_map.get(&sync_committee_period) else {
debug!(
log,
"Missing sync duties";
"period" => sync_committee_period,
);
debug!(period = sync_committee_period, "Missing sync duties");
continue;
};
let validators = committee_duties.validators.read();
@@ -629,20 +611,18 @@ pub async fn fill_in_aggregation_proofs<T: SlotClock + 'static, E: EthSpec>(
duty.aggregation_duties.proofs.write().extend(proofs);
} else {
debug!(
log,
"Missing sync duty to update";
"validator_index" => validator_index,
"period" => sync_committee_period,
validator_index,
period = sync_committee_period,
"Missing sync duty to update"
);
}
}
if num_validators_updated > 0 {
debug!(
log,
"Finished computing sync selection proofs";
"slot" => slot,
"updated_validators" => num_validators_updated,
%slot,
updated_validators = num_validators_updated,
"Finished computing sync selection proofs"
);
}
}

173
validator_client/validator_services/src/sync_committee_service.rs
View File

@@ -4,13 +4,14 @@ use environment::RuntimeContext;
use eth2::types::BlockId;
use futures::future::join_all;
use futures::future::FutureExt;
use slog::{crit, debug, error, info, trace, warn};
use logging::crit;
use slot_clock::SlotClock;
use std::collections::HashMap;
use std::ops::Deref;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use tokio::time::{sleep, sleep_until, Duration, Instant};
use tracing::{debug, error, info, trace, warn};
use types::{
ChainSpec, EthSpec, Hash256, PublicKeyBytes, Slot, SyncCommitteeSubscription,
SyncContributionData, SyncDuty, SyncSelectionProof, SyncSubnetId,
@@ -86,9 +87,8 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
}
pub fn start_update_service(self, spec: &ChainSpec) -> Result<(), String> {
let log = self.context.log().clone();
if self.duties_service.disable_attesting {
info!(log, "Sync committee service disabled");
info!("Sync committee service disabled");
return Ok(());
}
@@ -99,9 +99,8 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
.ok_or("Unable to determine duration to next slot")?;
info!(
log,
"Sync committee service started";
"next_update_millis" => duration_to_next_slot.as_millis()
next_update_millis = duration_to_next_slot.as_millis(),
"Sync committee service started"
);
let executor = self.context.executor.clone();
@@ -110,7 +109,6 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
loop {
if let Some(duration_to_next_slot) = self.slot_clock.duration_to_next_slot() {
// Wait for contribution broadcast interval 1/3 of the way through the slot.
let log = self.context.log();
sleep(duration_to_next_slot + slot_duration / 3).await;
// Do nothing if the Altair fork has not yet occurred.
@@ -120,21 +118,17 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
if let Err(e) = self.spawn_contribution_tasks(slot_duration).await {
crit!(
log,
"Failed to spawn sync contribution tasks";
"error" => e
error = ?e,
"Failed to spawn sync contribution tasks"
)
} else {
trace!(
log,
"Spawned sync contribution tasks";
)
trace!("Spawned sync contribution tasks")
}
// Do subscriptions for future slots/epochs.
self.spawn_subscription_tasks();
} else {
error!(log, "Failed to read slot clock");
error!("Failed to read slot clock");
// If we can't read the slot clock, just wait another slot.
sleep(slot_duration).await;
}
@@ -146,7 +140,6 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
}
async fn spawn_contribution_tasks(&self, slot_duration: Duration) -> Result<(), String> {
let log = self.context.log().clone();
let slot = self.slot_clock.now().ok_or("Failed to read slot clock")?;
let duration_to_next_slot = self
.slot_clock
@@ -165,16 +158,12 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
.sync_duties
.get_duties_for_slot(slot, &self.duties_service.spec)
else {
debug!(log, "No duties known for slot {}", slot);
debug!("No duties known for slot {}", slot);
return Ok(());
};
if slot_duties.duties.is_empty() {
debug!(
log,
"No local validators in current sync committee";
"slot" => slot,
);
debug!(%slot, "No local validators in current sync committee");
return Ok(());
}
@@ -201,11 +190,10 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
Ok(block) => block.data.root,
Err(errs) => {
warn!(
log,
errors = errs.to_string(),
%slot,
"Refusing to sign sync committee messages for an optimistic head block or \
a block head with unknown optimistic status";
"errors" => errs.to_string(),
"slot" => slot,
a block head with unknown optimistic status"
);
return Ok(());
}
@@ -251,8 +239,6 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
beacon_block_root: Hash256,
validator_duties: Vec<SyncDuty>,
) -> Result<(), ()> {
let log = self.context.log();
// Create futures to produce sync committee signatures.
let signature_futures = validator_duties.iter().map(|duty| async move {
match self
@@ -270,21 +256,19 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
// A pubkey can be missing when a validator was recently
// removed via the API.
debug!(
log,
"Missing pubkey for sync committee signature";
"pubkey" => ?pubkey,
"validator_index" => duty.validator_index,
"slot" => slot,
?pubkey,
validator_index = duty.validator_index,
%slot,
"Missing pubkey for sync committee signature"
);
None
}
Err(e) => {
crit!(
log,
"Failed to sign sync committee signature";
"validator_index" => duty.validator_index,
"slot" => slot,
"error" => ?e,
validator_index = duty.validator_index,
%slot,
error = ?e,
"Failed to sign sync committee signature"
);
None
}
@@ -307,19 +291,17 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
.await
.map_err(|e| {
error!(
log,
"Unable to publish sync committee messages";
"slot" => slot,
"error" => %e,
%slot,
error = %e,
"Unable to publish sync committee messages"
);
})?;
info!(
log,
"Successfully published sync committee messages";
"count" => committee_signatures.len(),
"head_block" => ?beacon_block_root,
"slot" => slot,
count = committee_signatures.len(),
head_block = ?beacon_block_root,
%slot,
"Successfully published sync committee messages"
);
Ok(())
@@ -362,8 +344,6 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
) -> Result<(), ()> {
sleep_until(aggregate_instant).await;
let log = self.context.log();
let contribution = &self
.beacon_nodes
.first_success(|beacon_node| async move {
@@ -380,20 +360,14 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
.await
.map_err(|e| {
crit!(
log,
"Failed to produce sync contribution";
"slot" => slot,
"beacon_block_root" => ?beacon_block_root,
"error" => %e,
%slot,
?beacon_block_root,
error = %e,
"Failed to produce sync contribution"
)
})?
.ok_or_else(|| {
crit!(
log,
"No aggregate contribution found";
"slot" => slot,
"beacon_block_root" => ?beacon_block_root,
);
crit!(%slot, ?beacon_block_root, "No aggregate contribution found");
})?
.data;
@@ -414,20 +388,14 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
// A pubkey can be missing when a validator was recently
// removed via the API.
debug!(
log,
"Missing pubkey for sync contribution";
"pubkey" => ?pubkey,
"slot" => slot,
);
debug!(?pubkey, %slot, "Missing pubkey for sync contribution");
None
}
Err(e) => {
crit!(
log,
"Unable to sign sync committee contribution";
"slot" => slot,
"error" => ?e,
%slot,
error = ?e,
"Unable to sign sync committee contribution"
);
None
}
@@ -452,20 +420,18 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
.await
.map_err(|e| {
error!(
log,
"Unable to publish signed contributions and proofs";
"slot" => slot,
"error" => %e,
%slot,
error = %e,
"Unable to publish signed contributions and proofs"
);
})?;
info!(
log,
"Successfully published sync contributions";
"subnet" => %subnet_id,
"beacon_block_root" => %beacon_block_root,
"num_signers" => contribution.aggregation_bits.num_set_bits(),
"slot" => slot,
subnet = %subnet_id,
beacon_block_root = %beacon_block_root,
num_signers = contribution.aggregation_bits.num_set_bits(),
%slot,
"Successfully published sync contributions"
);
Ok(())
@@ -473,14 +439,13 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
fn spawn_subscription_tasks(&self) {
let service = self.clone();
let log = self.context.log().clone();
self.inner.context.executor.spawn(
async move {
service.publish_subscriptions().await.unwrap_or_else(|e| {
error!(
log,
"Error publishing subscriptions";
"error" => ?e,
error = ?e,
"Error publishing subscriptions"
)
});
},
@@ -489,7 +454,6 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
}
async fn publish_subscriptions(self) -> Result<(), String> {
let log = self.context.log().clone();
let spec = &self.duties_service.spec;
let slot = self.slot_clock.now().ok_or("Failed to read slot clock")?;
@@ -526,12 +490,7 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
let mut subscriptions = vec![];
for (duty_slot, sync_committee_period) in duty_slots {
debug!(
log,
"Fetching subscription duties";
"duty_slot" => duty_slot,
"current_slot" => slot,
);
debug!(%duty_slot, %slot, "Fetching subscription duties");
match self
.duties_service
.sync_duties
@@ -544,9 +503,8 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
)),
None => {
debug!(
log,
"No duties for subscription";
"slot" => duty_slot,
slot = %duty_slot,
"No duties for subscription"
);
all_succeeded = false;
}
@@ -554,29 +512,23 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
}
if subscriptions.is_empty() {
debug!(
log,
"No sync subscriptions to send";
"slot" => slot,
);
debug!(%slot, "No sync subscriptions to send");
return Ok(());
}
// Post subscriptions to BN.
debug!(
log,
"Posting sync subscriptions to BN";
"count" => subscriptions.len(),
count = subscriptions.len(),
"Posting sync subscriptions to BN"
);
let subscriptions_slice = &subscriptions;
for subscription in subscriptions_slice {
debug!(
log,
"Subscription";
"validator_index" => subscription.validator_index,
"validator_sync_committee_indices" => ?subscription.sync_committee_indices,
"until_epoch" => subscription.until_epoch,
validator_index = subscription.validator_index,
validator_sync_committee_indices = ?subscription.sync_committee_indices,
until_epoch = %subscription.until_epoch,
"Subscription"
);
}
@@ -590,10 +542,9 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
.await
{
error!(
log,
"Unable to post sync committee subscriptions";
"slot" => slot,
"error" => %e,
%slot,
error = %e,
"Unable to post sync committee subscriptions"
);
all_succeeded = false;
}