gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-03 16:51:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Integrate tracing (#6339)

Browse Source 
Tracing Integration
- [reference](5bbf1859e9/projects/project-ideas.md (L297))


  - [x] replace slog & log with tracing throughout the codebase
- [x] implement custom crit log
- [x] make relevant changes in the formatter
- [x] replace sloggers
- [x] re-write SSE logging components

cc: @macladson @eserilev
This commit is contained in:
ThreeHrSleep
2025-03-13 04:01:05 +05:30
committed by GitHub
parent f23f984f85
commit d60c24ef1c
241 changed files with 9485 additions and 9328 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
validator_client/Cargo.toml
View File

@@ -29,9 +29,9 @@ reqwest = { workspace = true }
sensitive_url = { workspace = true }
serde = { workspace = true }
slashing_protection = { workspace = true }
slog = { workspace = true }
slot_clock = { workspace = true }
tokio = { workspace = true }
tracing = { workspace = true }
types = { workspace = true }
validator_http_api = { workspace = true }
validator_http_metrics = { workspace = true }

2
validator_client/beacon_node_fallback/Cargo.toml
View File

@@ -15,10 +15,10 @@ eth2 = { workspace = true }
futures = { workspace = true }
itertools = { workspace = true }
serde = { workspace = true }
slog = { workspace = true }
slot_clock = { workspace = true }
strum = { workspace = true }
tokio = { workspace = true }
tracing = { workspace = true }
types = { workspace = true }
validator_metrics = { workspace = true }

8
validator_client/beacon_node_fallback/src/beacon_node_health.rs
View File

@@ -1,9 +1,9 @@
use super::CandidateError;
use eth2::BeaconNodeHttpClient;
use serde::{Deserialize, Serialize};
use slog::{warn, Logger};
use std::cmp::Ordering;
use std::fmt::{Debug, Display, Formatter};
use tracing::warn;
use types::Slot;
/// Sync distances between 0 and DEFAULT_SYNC_TOLERANCE are considered `synced`.
@@ -276,15 +276,13 @@ impl BeaconNodeHealth {
pub async fn check_node_health(
beacon_node: &BeaconNodeHttpClient,
log: &Logger,
) -> Result<(Slot, bool, bool), CandidateError> {
let resp = match beacon_node.get_node_syncing().await {
Ok(resp) => resp,
Err(e) => {
warn!(
log,
"Unable connect to beacon node";
"error" => %e
error = %e,
"Unable connect to beacon node"
);
return Err(CandidateError::Offline);

131
validator_client/beacon_node_fallback/src/lib.rs
View File

@@ -12,7 +12,6 @@ use environment::RuntimeContext;
use eth2::BeaconNodeHttpClient;
use futures::future;
use serde::{ser::SerializeStruct, Deserialize, Serialize, Serializer};
use slog::{debug, error, warn, Logger};
use slot_clock::SlotClock;
use std::cmp::Ordering;
use std::fmt;
@@ -24,6 +23,7 @@ use std::time::{Duration, Instant};
use std::vec::Vec;
use strum::EnumVariantNames;
use tokio::{sync::RwLock, time::sleep};
use tracing::{debug, error, warn};
use types::{ChainSpec, Config as ConfigSpec, EthSpec, Slot};
use validator_metrics::{inc_counter_vec, ENDPOINT_ERRORS, ENDPOINT_REQUESTS};
@@ -222,15 +222,14 @@ impl<E: EthSpec> CandidateBeaconNode<E> {
distance_tiers: &BeaconNodeSyncDistanceTiers,
slot_clock: Option<&T>,
spec: &ChainSpec,
log: &Logger,
) -> Result<(), CandidateError> {
if let Err(e) = self.is_compatible(spec, log).await {
if let Err(e) = self.is_compatible(spec).await {
*self.health.write().await = Err(e);
return Err(e);
}
if let Some(slot_clock) = slot_clock {
match check_node_health(&self.beacon_node, log).await {
match check_node_health(&self.beacon_node).await {
Ok((head, is_optimistic, el_offline)) => {
let Some(slot_clock_head) = slot_clock.now() else {
let e = match slot_clock.is_prior_to_genesis() {
@@ -288,17 +287,16 @@ impl<E: EthSpec> CandidateBeaconNode<E> {
}
/// Checks if the node has the correct specification.
async fn is_compatible(&self, spec: &ChainSpec, log: &Logger) -> Result<(), CandidateError> {
async fn is_compatible(&self, spec: &ChainSpec) -> Result<(), CandidateError> {
let config = self
.beacon_node
.get_config_spec::<ConfigSpec>()
.await
.map_err(|e| {
error!(
log,
"Unable to read spec from beacon node";
"error" => %e,
"endpoint" => %self.beacon_node,
error = %e,
endpoint = %self.beacon_node,
"Unable to read spec from beacon node"
);
CandidateError::Offline
})?
@@ -306,71 +304,64 @@ impl<E: EthSpec> CandidateBeaconNode<E> {
let beacon_node_spec = ChainSpec::from_config::<E>(&config).ok_or_else(|| {
error!(
log,
endpoint = %self.beacon_node,
"The minimal/mainnet spec type of the beacon node does not match the validator \
client. See the --network command.";
"endpoint" => %self.beacon_node,
client. See the --network command."
);
CandidateError::Incompatible
})?;
if beacon_node_spec.genesis_fork_version != spec.genesis_fork_version {
error!(
log,
"Beacon node is configured for a different network";
"endpoint" => %self.beacon_node,
"bn_genesis_fork" => ?beacon_node_spec.genesis_fork_version,
"our_genesis_fork" => ?spec.genesis_fork_version,
endpoint = %self.beacon_node,
bn_genesis_fork = ?beacon_node_spec.genesis_fork_version,
our_genesis_fork = ?spec.genesis_fork_version,
"Beacon node is configured for a different network"
);
return Err(CandidateError::Incompatible);
} else if beacon_node_spec.altair_fork_epoch != spec.altair_fork_epoch {
warn!(
log,
"Beacon node has mismatched Altair fork epoch";
"endpoint" => %self.beacon_node,
"endpoint_altair_fork_epoch" => ?beacon_node_spec.altair_fork_epoch,
"hint" => UPDATE_REQUIRED_LOG_HINT,
endpoint = %self.beacon_node,
endpoint_altair_fork_epoch = ?beacon_node_spec.altair_fork_epoch,
hint = UPDATE_REQUIRED_LOG_HINT,
"Beacon node has mismatched Altair fork epoch"
);
} else if beacon_node_spec.bellatrix_fork_epoch != spec.bellatrix_fork_epoch {
warn!(
log,
"Beacon node has mismatched Bellatrix fork epoch";
"endpoint" => %self.beacon_node,
"endpoint_bellatrix_fork_epoch" => ?beacon_node_spec.bellatrix_fork_epoch,
"hint" => UPDATE_REQUIRED_LOG_HINT,
endpoint = %self.beacon_node,
endpoint_bellatrix_fork_epoch = ?beacon_node_spec.bellatrix_fork_epoch,
hint = UPDATE_REQUIRED_LOG_HINT,
"Beacon node has mismatched Bellatrix fork epoch"
);
} else if beacon_node_spec.capella_fork_epoch != spec.capella_fork_epoch {
warn!(
log,
"Beacon node has mismatched Capella fork epoch";
"endpoint" => %self.beacon_node,
"endpoint_capella_fork_epoch" => ?beacon_node_spec.capella_fork_epoch,
"hint" => UPDATE_REQUIRED_LOG_HINT,
endpoint = %self.beacon_node,
endpoint_capella_fork_epoch = ?beacon_node_spec.capella_fork_epoch,
hint = UPDATE_REQUIRED_LOG_HINT,
"Beacon node has mismatched Capella fork epoch"
);
} else if beacon_node_spec.deneb_fork_epoch != spec.deneb_fork_epoch {
warn!(
log,
"Beacon node has mismatched Deneb fork epoch";
"endpoint" => %self.beacon_node,
"endpoint_deneb_fork_epoch" => ?beacon_node_spec.deneb_fork_epoch,
"hint" => UPDATE_REQUIRED_LOG_HINT,
endpoint = %self.beacon_node,
endpoint_deneb_fork_epoch = ?beacon_node_spec.deneb_fork_epoch,
hint = UPDATE_REQUIRED_LOG_HINT,
"Beacon node has mismatched Deneb fork epoch"
);
} else if beacon_node_spec.electra_fork_epoch != spec.electra_fork_epoch {
warn!(
log,
"Beacon node has mismatched Electra fork epoch";
"endpoint" => %self.beacon_node,
"endpoint_electra_fork_epoch" => ?beacon_node_spec.electra_fork_epoch,
"hint" => UPDATE_REQUIRED_LOG_HINT,
endpoint = %self.beacon_node,
endpoint_electra_fork_epoch = ?beacon_node_spec.electra_fork_epoch,
hint = UPDATE_REQUIRED_LOG_HINT,
"Beacon node has mismatched Electra fork epoch"
);
} else if beacon_node_spec.fulu_fork_epoch != spec.fulu_fork_epoch {
warn!(
log,
"Beacon node has mismatched Fulu fork epoch";
"endpoint" => %self.beacon_node,
"endpoint_fulu_fork_epoch" => ?beacon_node_spec.fulu_fork_epoch,
"hint" => UPDATE_REQUIRED_LOG_HINT,
);
endpoint = %self.beacon_node,
endpoint_fulu_fork_epoch = ?beacon_node_spec.fulu_fork_epoch,
hint = UPDATE_REQUIRED_LOG_HINT,
"Beacon node has mismatched Fulu fork epoch"
);
}
Ok(())
@@ -387,7 +378,6 @@ pub struct BeaconNodeFallback<T, E> {
slot_clock: Option<T>,
broadcast_topics: Vec<ApiTopic>,
spec: Arc<ChainSpec>,
log: Logger,
}
impl<T: SlotClock, E: EthSpec> BeaconNodeFallback<T, E> {
@@ -396,7 +386,6 @@ impl<T: SlotClock, E: EthSpec> BeaconNodeFallback<T, E> {
config: Config,
broadcast_topics: Vec<ApiTopic>,
spec: Arc<ChainSpec>,
log: Logger,
) -> Self {
let distance_tiers = config.sync_tolerances;
Self {
@@ -405,7 +394,6 @@ impl<T: SlotClock, E: EthSpec> BeaconNodeFallback<T, E> {
slot_clock: None,
broadcast_topics,
spec,
log,
}
}
@@ -488,7 +476,6 @@ impl<T: SlotClock, E: EthSpec> BeaconNodeFallback<T, E> {
&self.distance_tiers,
self.slot_clock.as_ref(),
&self.spec,
&self.log,
));
nodes.push(candidate.beacon_node.to_string());
}
@@ -501,10 +488,9 @@ impl<T: SlotClock, E: EthSpec> BeaconNodeFallback<T, E> {
if let Err(e) = result {
if *e != CandidateError::PreGenesis {
warn!(
self.log,
"A connected beacon node errored during routine health check";
"error" => ?e,
"endpoint" => node,
error = ?e,
endpoint = %node,
"A connected beacon node errored during routine health check"
);
}
}
@@ -576,11 +562,7 @@ impl<T: SlotClock, E: EthSpec> BeaconNodeFallback<T, E> {
// Run `func` using a `candidate`, returning the value or capturing errors.
for candidate in candidates.iter() {
futures.push(Self::run_on_candidate(
candidate.beacon_node.clone(),
&func,
&self.log,
));
futures.push(Self::run_on_candidate(candidate.beacon_node.clone(), &func));
}
drop(candidates);
@@ -598,11 +580,7 @@ impl<T: SlotClock, E: EthSpec> BeaconNodeFallback<T, E> {
// Run `func` using a `candidate`, returning the value or capturing errors.
for candidate in candidates.iter() {
futures.push(Self::run_on_candidate(
candidate.beacon_node.clone(),
&func,
&self.log,
));
futures.push(Self::run_on_candidate(candidate.beacon_node.clone(), &func));
}
drop(candidates);
@@ -621,7 +599,6 @@ impl<T: SlotClock, E: EthSpec> BeaconNodeFallback<T, E> {
async fn run_on_candidate<F, R, Err, O>(
candidate: BeaconNodeHttpClient,
func: F,
log: &Logger,
) -> Result<O, (String, Error<Err>)>
where
F: Fn(BeaconNodeHttpClient) -> R,
@@ -636,10 +613,9 @@ impl<T: SlotClock, E: EthSpec> BeaconNodeFallback<T, E> {
Ok(val) => Ok(val),
Err(e) => {
debug!(
log,
"Request to beacon node failed";
"node" => %candidate,
"error" => ?e,
node = %candidate,
error = ?e,
"Request to beacon node failed"
);
inc_counter_vec(&ENDPOINT_ERRORS, &[candidate.as_ref()]);
Err((candidate.to_string(), Error::RequestFailed(e)))
@@ -666,11 +642,7 @@ impl<T: SlotClock, E: EthSpec> BeaconNodeFallback<T, E> {
// Run `func` using a `candidate`, returning the value or capturing errors.
for candidate in candidates.iter() {
futures.push(Self::run_on_candidate(
candidate.beacon_node.clone(),
&func,
&self.log,
));
futures.push(Self::run_on_candidate(candidate.beacon_node.clone(), &func));
}
drop(candidates);
@@ -752,7 +724,6 @@ mod tests {
use crate::beacon_node_health::BeaconNodeHealthTier;
use eth2::SensitiveUrl;
use eth2::Timeouts;
use logging::test_logger;
use slot_clock::TestingSlotClock;
use strum::VariantNames;
use types::{BeaconBlockDeneb, MainnetEthSpec, Slot};
@@ -902,10 +873,9 @@ mod tests {
candidates: Vec<CandidateBeaconNode<E>>,
topics: Vec<ApiTopic>,
spec: Arc<ChainSpec>,
log: Logger,
) -> BeaconNodeFallback<TestingSlotClock, E> {
let mut beacon_node_fallback =
BeaconNodeFallback::new(candidates, Config::default(), topics, spec, log);
BeaconNodeFallback::new(candidates, Config::default(), topics, spec);
beacon_node_fallback.set_slot_clock(TestingSlotClock::new(
Slot::new(1),
@@ -932,7 +902,6 @@ mod tests {
],
vec![],
spec.clone(),
test_logger(),
);
// BeaconNodeHealthTier 1
@@ -979,7 +948,6 @@ mod tests {
vec![beacon_node_1, beacon_node_2],
vec![ApiTopic::Blocks],
spec.clone(),
test_logger(),
);
mock_beacon_node_1.mock_post_beacon_blinded_blocks_v2_ssz(Duration::from_secs(0));
@@ -1021,7 +989,6 @@ mod tests {
vec![beacon_node_1, beacon_node_2, beacon_node_3],
vec![],
spec.clone(),
test_logger(),
);
let mock1 = mock_beacon_node_1.mock_offline_node();

3
validator_client/doppelganger_service/Cargo.toml
View File

@@ -8,11 +8,12 @@ authors = ["Sigma Prime <contact@sigmaprime.io>"]
beacon_node_fallback = { workspace = true }
environment = { workspace = true }
eth2 = { workspace = true }
logging = { workspace = true }
parking_lot = { workspace = true }
slog = { workspace = true }
slot_clock = { workspace = true }
task_executor = { workspace = true }
tokio = { workspace = true }
tracing = { workspace = true }
types = { workspace = true }
[dev-dependencies]

118
validator_client/doppelganger_service/src/lib.rs
View File

@@ -32,14 +32,15 @@
use beacon_node_fallback::BeaconNodeFallback;
use environment::RuntimeContext;
use eth2::types::LivenessResponseData;
use logging::crit;
use parking_lot::RwLock;
use slog::{crit, error, info, Logger};
use slot_clock::SlotClock;
use std::collections::{HashMap, HashSet};
use std::future::Future;
use std::sync::Arc;
use task_executor::ShutdownReason;
use tokio::time::sleep;
use tracing::{error, info};
use types::{Epoch, EthSpec, PublicKeyBytes, Slot};
/// A wrapper around `PublicKeyBytes` which encodes information about the status of a validator
@@ -164,7 +165,6 @@ impl DoppelgangerState {
/// doppelganger progression.
async fn beacon_node_liveness<T: 'static + SlotClock, E: EthSpec>(
beacon_nodes: Arc<BeaconNodeFallback<T, E>>,
log: Logger,
current_epoch: Epoch,
validator_indices: Vec<u64>,
) -> LivenessResponses {
@@ -203,10 +203,9 @@ async fn beacon_node_liveness<T: 'static + SlotClock, E: EthSpec>(
.await
.unwrap_or_else(|e| {
crit!(
log,
"Failed previous epoch liveness query";
"error" => %e,
"previous_epoch" => %previous_epoch,
error = %e,
previous_epoch = %previous_epoch,
"Failed previous epoch liveness query"
);
// Return an empty vec. In effect, this means to keep trying to make doppelganger
// progress even if some of the calls are failing.
@@ -239,10 +238,9 @@ async fn beacon_node_liveness<T: 'static + SlotClock, E: EthSpec>(
.await
.unwrap_or_else(|e| {
crit!(
log,
"Failed current epoch liveness query";
"error" => %e,
"current_epoch" => %current_epoch,
error = %e,
current_epoch = %current_epoch,
"Failed current epoch liveness query"
);
// Return an empty vec. In effect, this means to keep trying to make doppelganger
// progress even if some of the calls are failing.
@@ -257,11 +255,10 @@ async fn beacon_node_liveness<T: 'static + SlotClock, E: EthSpec>(
|| current_epoch_responses.len() != previous_epoch_responses.len()
{
error!(
log,
"Liveness query omitted validators";
"previous_epoch_response" => previous_epoch_responses.len(),
"current_epoch_response" => current_epoch_responses.len(),
"requested" => validator_indices.len(),
previous_epoch_response = previous_epoch_responses.len(),
current_epoch_response = current_epoch_responses.len(),
requested = validator_indices.len(),
"Liveness query omitted validators"
)
}
@@ -271,19 +268,12 @@ async fn beacon_node_liveness<T: 'static + SlotClock, E: EthSpec>(
}
}
#[derive(Default)]
pub struct DoppelgangerService {
doppelganger_states: RwLock<HashMap<PublicKeyBytes, DoppelgangerState>>,
log: Logger,
}
impl DoppelgangerService {
pub fn new(log: Logger) -> Self {
Self {
doppelganger_states: <_>::default(),
log,
}
}
/// Starts a reoccurring future which will try to keep the doppelganger service updated each
/// slot.
pub fn start_update_service<E, T, V>(
@@ -302,35 +292,25 @@ impl DoppelgangerService {
let get_index = move |pubkey| validator_store.get_validator_index(&pubkey);
// Define the `get_liveness` function as one that queries the beacon node API.
let log = service.log.clone();
let get_liveness = move |current_epoch, validator_indices| {
beacon_node_liveness(
beacon_nodes.clone(),
log.clone(),
current_epoch,
validator_indices,
)
beacon_node_liveness(beacon_nodes.clone(), current_epoch, validator_indices)
};
let mut shutdown_sender = context.executor.shutdown_sender();
let log = service.log.clone();
let mut shutdown_func = move || {
if let Err(e) =
shutdown_sender.try_send(ShutdownReason::Failure("Doppelganger detected."))
{
crit!(
log,
"Failed to send shutdown signal";
"msg" => "terminate this process immediately",
"error" => ?e
msg = "terminate this process immediately",
error = ?e,
"Failed to send shutdown signal"
);
}
};
info!(
service.log,
"Doppelganger detection service started";
);
info!("Doppelganger detection service started");
context.executor.spawn(
async move {
@@ -360,9 +340,8 @@ impl DoppelgangerService {
.await
{
error!(
service.log,
"Error during doppelganger detection";
"error" => ?e
error = ?e,
"Error during doppelganger detection"
);
}
}
@@ -387,10 +366,9 @@ impl DoppelgangerService {
})
.unwrap_or_else(|| {
crit!(
self.log,
"Validator unknown to doppelganger service";
"msg" => "preventing validator from performing duties",
"pubkey" => ?validator
msg = "preventing validator from performing duties",
pubkey = ?validator,
"Validator unknown to doppelganger service"
);
DoppelgangerStatus::UnknownToDoppelganger(validator)
})
@@ -552,11 +530,7 @@ impl DoppelgangerService {
// Resolve the index from the server response back to a public key.
let Some(pubkey) = indices_map.get(&response.index) else {
crit!(
self.log,
"Inconsistent indices map";
"validator_index" => response.index,
);
crit!(validator_index = response.index, "Inconsistent indices map");
// Skip this result if an inconsistency is detected.
continue;
};
@@ -566,9 +540,8 @@ impl DoppelgangerService {
state.next_check_epoch
} else {
crit!(
self.log,
"Inconsistent doppelganger state";
"validator_pubkey" => ?pubkey,
validator_pubkey = ?pubkey,
"Inconsistent doppelganger state"
);
// Skip this result if an inconsistency is detected.
continue;
@@ -582,15 +555,14 @@ impl DoppelgangerService {
let violators_exist = !violators.is_empty();
if violators_exist {
crit!(
self.log,
"Doppelganger(s) detected";
"msg" => "A doppelganger occurs when two different validator clients run the \
same public key. This validator client detected another instance of a local \
validator on the network and is shutting down to prevent potential slashable \
offences. Ensure that you are not running a duplicate or overlapping \
validator client",
"doppelganger_indices" => ?violators
)
msg = "A doppelganger occurs when two different validator clients run the \
same public key. This validator client detected another instance of a local \
validator on the network and is shutting down to prevent potential slashable \
offences. Ensure that you are not running a duplicate or overlapping \
validator client",
doppelganger_indices = ?violators,
"Doppelganger(s) detected"
);
}
// The concept of "epoch satisfaction" is that for some epoch `e` we are *satisfied* that
@@ -665,19 +637,17 @@ impl DoppelgangerService {
doppelganger_state.complete_detection_in_epoch(previous_epoch);
info!(
self.log,
"Found no doppelganger";
"further_checks_remaining" => doppelganger_state.remaining_epochs,
"epoch" => response.epoch,
"validator_index" => response.index
further_checks_remaining = doppelganger_state.remaining_epochs,
epoch = %response.epoch,
validator_index = response.index,
"Found no doppelganger"
);
if doppelganger_state.remaining_epochs == 0 {
info!(
self.log,
"Doppelganger detection complete";
"msg" => "starting validator",
"validator_index" => response.index
msg = "starting validator",
validator_index = response.index,
"Doppelganger detection complete"
);
}
}
@@ -696,7 +666,6 @@ impl DoppelgangerService {
mod test {
use super::*;
use futures::executor::block_on;
use logging::test_logger;
use slot_clock::TestingSlotClock;
use std::future;
use std::time::Duration;
@@ -740,13 +709,12 @@ mod test {
fn build(self) -> TestScenario {
let mut rng = XorShiftRng::from_seed([42; 16]);
let slot_clock = TestingSlotClock::new(Slot::new(0), GENESIS_TIME, SLOT_DURATION);
let log = test_logger();
TestScenario {
validators: (0..self.validator_count)
.map(|_| PublicKeyBytes::random_for_test(&mut rng))
.collect(),
doppelganger: DoppelgangerService::new(log),
doppelganger: DoppelgangerService::default(),
slot_clock,
}
}

2
validator_client/graffiti_file/Cargo.toml
View File

@@ -11,7 +11,7 @@ path = "src/lib.rs"
[dependencies]
bls = { workspace = true }
serde = { workspace = true }
slog = { workspace = true }
tracing = { workspace = true }
types = { workspace = true }
[dev-dependencies]

8
validator_client/graffiti_file/src/lib.rs
View File

@@ -1,12 +1,11 @@
use bls::PublicKeyBytes;
use serde::{Deserialize, Serialize};
use slog::warn;
use std::collections::HashMap;
use std::fs::File;
use std::io::{prelude::*, BufReader};
use std::path::PathBuf;
use std::str::FromStr;
use bls::PublicKeyBytes;
use tracing::warn;
use types::{graffiti::GraffitiString, Graffiti};
#[derive(Debug)]
@@ -108,7 +107,6 @@ fn read_line(line: &str) -> Result<(Option<PublicKeyBytes>, Graffiti), Error> {
// the next block produced by the validator with the given public key.
pub fn determine_graffiti(
validator_pubkey: &PublicKeyBytes,
log: &slog::Logger,
graffiti_file: Option<GraffitiFile>,
validator_definition_graffiti: Option<Graffiti>,
graffiti_flag: Option<Graffiti>,
@@ -117,7 +115,7 @@ pub fn determine_graffiti(
.and_then(|mut g| match g.load_graffiti(validator_pubkey) {
Ok(g) => g,
Err(e) => {
warn!(log, "Failed to read graffiti file"; "error" => ?e);
warn!(error = ?e, "Failed to read graffiti file");
None
}
})

3
validator_client/http_api/Cargo.toml
View File

@@ -29,9 +29,9 @@ parking_lot = { workspace = true }
rand = { workspace = true }
sensitive_url = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
signing_method = { workspace = true }
slashing_protection = { workspace = true }
slog = { workspace = true }
slot_clock = { workspace = true }
sysinfo = { workspace = true }
system_health = { workspace = true }
@@ -39,6 +39,7 @@ task_executor = { workspace = true }
tempfile = { workspace = true }
tokio = { workspace = true }
tokio-stream = { workspace = true }
tracing = { workspace = true }
types = { workspace = true }
url = { workspace = true }
validator_dir = { workspace = true }

10
validator_client/http_api/src/create_signed_voluntary_exit.rs
View File

@@ -1,8 +1,8 @@
use bls::{PublicKey, PublicKeyBytes};
use eth2::types::GenericResponse;
use slog::{info, Logger};
use slot_clock::SlotClock;
use std::sync::Arc;
use tracing::info;
use types::{Epoch, EthSpec, SignedVoluntaryExit, VoluntaryExit};
use validator_store::ValidatorStore;
@@ -11,7 +11,6 @@ pub async fn create_signed_voluntary_exit<T: 'static + SlotClock + Clone, E: Eth
maybe_epoch: Option<Epoch>,
validator_store: Arc<ValidatorStore<T, E>>,
slot_clock: T,
log: Logger,
) -> Result<GenericResponse<SignedVoluntaryExit>, warp::Rejection> {
let epoch = match maybe_epoch {
Some(epoch) => epoch,
@@ -45,10 +44,9 @@ pub async fn create_signed_voluntary_exit<T: 'static + SlotClock + Clone, E: Eth
};
info!(
log,
"Signing voluntary exit";
"validator" => pubkey_bytes.as_hex_string(),
"epoch" => epoch
validator = pubkey_bytes.as_hex_string(),
%epoch,
"Signing voluntary exit"
);
let signed_voluntary_exit = validator_store

39
validator_client/http_api/src/keystores.rs
View File

@@ -11,12 +11,12 @@ use eth2::lighthouse_vc::{
use eth2_keystore::Keystore;
use initialized_validators::{Error, InitializedValidators};
use signing_method::SigningMethod;
use slog::{info, warn, Logger};
use slot_clock::SlotClock;
use std::path::PathBuf;
use std::sync::Arc;
use task_executor::TaskExecutor;
use tokio::runtime::Handle;
use tracing::{info, warn};
use types::{EthSpec, PublicKeyBytes};
use validator_dir::{keystore_password_path, Builder as ValidatorDirBuilder};
use validator_store::ValidatorStore;
@@ -64,7 +64,6 @@ pub fn import<T: SlotClock + 'static, E: EthSpec>(
secrets_dir: Option<PathBuf>,
validator_store: Arc<ValidatorStore<T, E>>,
task_executor: TaskExecutor,
log: Logger,
) -> Result<ImportKeystoresResponse, Rejection> {
// Check request validity. This is the only cases in which we should return a 4xx code.
if request.keystores.len() != request.passwords.len() {
@@ -88,18 +87,14 @@ pub fn import<T: SlotClock + 'static, E: EthSpec>(
.iter()
.any(|data| data.pubkey == pubkey_bytes)
{
warn!(
log,
"Slashing protection data not provided";
"public_key" => ?public_key,
);
warn!(?public_key, "Slashing protection data not provided");
}
}
}
validator_store.import_slashing_protection(slashing_protection)
} else {
warn!(log, "No slashing protection data provided with keystores");
warn!("No slashing protection data provided with keystores");
Ok(())
};
@@ -133,10 +128,9 @@ pub fn import<T: SlotClock + 'static, E: EthSpec>(
Ok(status) => Status::ok(status),
Err(e) => {
warn!(
log,
"Error importing keystore, skipped";
"pubkey" => pubkey_str,
"error" => ?e,
pubkey = pubkey_str,
error = ?e,
"Error importing keystore, skipped"
);
Status::error(ImportKeystoreStatus::Error, e)
}
@@ -157,9 +151,8 @@ pub fn import<T: SlotClock + 'static, E: EthSpec>(
if successful_import > 0 {
info!(
log,
"Imported keystores via standard HTTP API";
"count" => successful_import,
count = successful_import,
"Imported keystores via standard HTTP API"
);
}
@@ -243,9 +236,8 @@ pub fn delete<T: SlotClock + 'static, E: EthSpec>(
request: DeleteKeystoresRequest,
validator_store: Arc<ValidatorStore<T, E>>,
task_executor: TaskExecutor,
log: Logger,
) -> Result<DeleteKeystoresResponse, Rejection> {
let export_response = export(request, validator_store, task_executor, log.clone())?;
let export_response = export(request, validator_store, task_executor)?;
// Check the status is Deleted to confirm deletion is successful, then only display the log
let successful_deletion = export_response
@@ -256,9 +248,8 @@ pub fn delete<T: SlotClock + 'static, E: EthSpec>(
if successful_deletion > 0 {
info!(
log,
"Deleted keystore via standard HTTP API";
"count" => successful_deletion,
count = successful_deletion,
"Deleted keystore via standard HTTP API"
);
}
@@ -276,7 +267,6 @@ pub fn export<T: SlotClock + 'static, E: EthSpec>(
request: DeleteKeystoresRequest,
validator_store: Arc<ValidatorStore<T, E>>,
task_executor: TaskExecutor,
log: Logger,
) -> Result<ExportKeystoresResponse, Rejection> {
// Remove from initialized validators.
let initialized_validators_rwlock = validator_store.initialized_validators();
@@ -294,10 +284,9 @@ pub fn export<T: SlotClock + 'static, E: EthSpec>(
Ok(status) => status,
Err(error) => {
warn!(
log,
"Error deleting keystore";
"pubkey" => ?pubkey_bytes,
"error" => ?error,
pubkey = ?pubkey_bytes,
?error,
"Error deleting keystore"
);
SingleExportKeystoresResponse {
status: Status::error(DeleteKeystoreStatus::Error, error),

64
validator_client/http_api/src/lib.rs
View File

@@ -34,10 +34,10 @@ use eth2::lighthouse_vc::{
};
use health_metrics::observe::Observe;
use lighthouse_version::version_with_platform;
use logging::crit;
use logging::SSELoggingComponents;
use parking_lot::RwLock;
use serde::{Deserialize, Serialize};
use slog::{crit, info, warn, Logger};
use slot_clock::SlotClock;
use std::collections::HashMap;
use std::future::Future;
@@ -49,6 +49,7 @@ use sysinfo::{System, SystemExt};
use system_health::observe_system_health_vc;
use task_executor::TaskExecutor;
use tokio_stream::{wrappers::BroadcastStream, StreamExt};
use tracing::{info, warn};
use types::{ChainSpec, ConfigAndPreset, EthSpec};
use validator_dir::Builder as ValidatorDirBuilder;
use validator_services::block_service::BlockService;
@@ -87,7 +88,6 @@ pub struct Context<T: SlotClock, E: EthSpec> {
pub graffiti_flag: Option<Graffiti>,
pub spec: Arc<ChainSpec>,
pub config: Config,
pub log: Logger,
pub sse_logging_components: Option<SSELoggingComponents>,
pub slot_clock: T,
pub _phantom: PhantomData<E>,
@@ -148,7 +148,6 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
let config = &ctx.config;
let allow_keystore_export = config.allow_keystore_export;
let store_passwords_in_secrets_dir = config.store_passwords_in_secrets_dir;
let log = ctx.log.clone();
// Configure CORS.
let cors_builder = {
@@ -165,7 +164,7 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
// Sanity check.
if !config.enabled {
crit!(log, "Cannot start disabled metrics HTTP server");
crit!("Cannot start disabled metrics HTTP server");
return Err(Error::Other(
"A disabled metrics server should not be started".to_string(),
));
@@ -179,9 +178,8 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
Ok(abs_path) => api_token_path = abs_path,
Err(e) => {
warn!(
log,
"Error canonicalizing token path";
"error" => ?e,
error = ?e,
"Error canonicalizing token path"
);
}
};
@@ -239,9 +237,6 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
let inner_graffiti_flag = ctx.graffiti_flag;
let graffiti_flag_filter = warp::any().map(move || inner_graffiti_flag);
let inner_ctx = ctx.clone();
let log_filter = warp::any().map(move || inner_ctx.log.clone());
let inner_slot_clock = ctx.slot_clock.clone();
let slot_clock_filter = warp::any().map(move || inner_slot_clock.clone());
@@ -399,12 +394,10 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
.and(validator_store_filter.clone())
.and(graffiti_file_filter.clone())
.and(graffiti_flag_filter)
.and(log_filter.clone())
.then(
|validator_store: Arc<ValidatorStore<T, E>>,
graffiti_file: Option<GraffitiFile>,
graffiti_flag: Option<Graffiti>,
log| {
graffiti_flag: Option<Graffiti>| {
blocking_json_task(move || {
let mut result = HashMap::new();
for (key, graffiti_definition) in validator_store
@@ -414,7 +407,6 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
{
let graffiti = determine_graffiti(
key,
&log,
graffiti_file.clone(),
graffiti_definition,
graffiti_flag,
@@ -834,11 +826,10 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
.and(warp::body::json())
.and(validator_store_filter.clone())
.and(task_executor_filter.clone())
.and(log_filter.clone())
.then(move |request, validator_store, task_executor, log| {
.then(move |request, validator_store, task_executor| {
blocking_json_task(move || {
if allow_keystore_export {
keystores::export(request, validator_store, task_executor, log)
keystores::export(request, validator_store, task_executor)
} else {
Err(warp_utils::reject::custom_bad_request(
"keystore export is disabled".to_string(),
@@ -1079,14 +1070,12 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
.and(warp::path::end())
.and(validator_store_filter.clone())
.and(slot_clock_filter)
.and(log_filter.clone())
.and(task_executor_filter.clone())
.then(
|pubkey: PublicKey,
query: api_types::VoluntaryExitQuery,
validator_store: Arc<ValidatorStore<T, E>>,
slot_clock: T,
log,
task_executor: TaskExecutor| {
blocking_json_task(move || {
if let Some(handle) = task_executor.handle() {
@@ -1096,7 +1085,6 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
query.epoch,
validator_store,
slot_clock,
log,
))?;
Ok(signed_voluntary_exit)
} else {
@@ -1196,9 +1184,8 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
.and(secrets_dir_filter)
.and(validator_store_filter.clone())
.and(task_executor_filter.clone())
.and(log_filter.clone())
.then(
move |request, validator_dir, secrets_dir, validator_store, task_executor, log| {
move |request, validator_dir, secrets_dir, validator_store, task_executor| {
let secrets_dir = store_passwords_in_secrets_dir.then_some(secrets_dir);
blocking_json_task(move || {
keystores::import(
@@ -1207,7 +1194,6 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
secrets_dir,
validator_store,
task_executor,
log,
)
})
},
@@ -1218,11 +1204,8 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
.and(warp::body::json())
.and(validator_store_filter.clone())
.and(task_executor_filter.clone())
.and(log_filter.clone())
.then(|request, validator_store, task_executor, log| {
blocking_json_task(move || {
keystores::delete(request, validator_store, task_executor, log)
})
.then(|request, validator_store, task_executor| {
blocking_json_task(move || keystores::delete(request, validator_store, task_executor))
});
// GET /eth/v1/remotekeys
@@ -1237,11 +1220,8 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
.and(warp::body::json())
.and(validator_store_filter.clone())
.and(task_executor_filter.clone())
.and(log_filter.clone())
.then(|request, validator_store, task_executor, log| {
blocking_json_task(move || {
remotekeys::import(request, validator_store, task_executor, log)
})
.then(|request, validator_store, task_executor| {
blocking_json_task(move || remotekeys::import(request, validator_store, task_executor))
});
// DELETE /eth/v1/remotekeys
@@ -1249,11 +1229,8 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
.and(warp::body::json())
.and(validator_store_filter)
.and(task_executor_filter)
.and(log_filter.clone())
.then(|request, validator_store, task_executor, log| {
blocking_json_task(move || {
remotekeys::delete(request, validator_store, task_executor, log)
})
.then(|request, validator_store, task_executor| {
blocking_json_task(move || remotekeys::delete(request, validator_store, task_executor))
});
// Subscribe to get VC logs via Server side events
@@ -1271,7 +1248,9 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
match msg {
Ok(data) => {
// Serialize to json
match data.to_json_string() {
match serde_json::to_string(&data)
.map_err(|e| format!("{:?}", e))
{
// Send the json as a Server Sent Event
Ok(json) => Event::default().json_data(json).map_err(|e| {
warp_utils::reject::server_sent_event_error(format!(
@@ -1364,10 +1343,9 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
)?;
info!(
log,
"HTTP API started";
"listen_address" => listening_socket.to_string(),
"api_token_file" => ?api_token_path,
listen_address = listening_socket.to_string(),
?api_token_path,
"HTTP API started"
);
Ok((listening_socket, server))

28
validator_client/http_api/src/remotekeys.rs
View File

@@ -8,11 +8,11 @@ use eth2::lighthouse_vc::std_types::{
ListRemotekeysResponse, SingleListRemotekeysResponse, Status,
};
use initialized_validators::{Error, InitializedValidators};
use slog::{info, warn, Logger};
use slot_clock::SlotClock;
use std::sync::Arc;
use task_executor::TaskExecutor;
use tokio::runtime::Handle;
use tracing::{info, warn};
use types::{EthSpec, PublicKeyBytes};
use url::Url;
use validator_store::ValidatorStore;
@@ -52,12 +52,10 @@ pub fn import<T: SlotClock + 'static, E: EthSpec>(
request: ImportRemotekeysRequest,
validator_store: Arc<ValidatorStore<T, E>>,
task_executor: TaskExecutor,
log: Logger,
) -> Result<ImportRemotekeysResponse, Rejection> {
info!(
log,
"Importing remotekeys via standard HTTP API";
"count" => request.remote_keys.len(),
count = request.remote_keys.len(),
"Importing remotekeys via standard HTTP API"
);
// Import each remotekey. Some remotekeys may fail to be imported, so we record a status for each.
let mut statuses = Vec::with_capacity(request.remote_keys.len());
@@ -70,10 +68,9 @@ pub fn import<T: SlotClock + 'static, E: EthSpec>(
Ok(status) => Status::ok(status),
Err(e) => {
warn!(
log,
"Error importing keystore, skipped";
"pubkey" => remotekey.pubkey.to_string(),
"error" => ?e,
pubkey = remotekey.pubkey.to_string(),
error = ?e,
"Error importing keystore, skipped"
);
Status::error(ImportRemotekeyStatus::Error, e)
}
@@ -148,12 +145,10 @@ pub fn delete<T: SlotClock + 'static, E: EthSpec>(
request: DeleteRemotekeysRequest,
validator_store: Arc<ValidatorStore<T, E>>,
task_executor: TaskExecutor,
log: Logger,
) -> Result<DeleteRemotekeysResponse, Rejection> {
info!(
log,
"Deleting remotekeys via standard HTTP API";
"count" => request.pubkeys.len(),
count = request.pubkeys.len(),
"Deleting remotekeys via standard HTTP API"
);
// Remove from initialized validators.
let initialized_validators_rwlock = validator_store.initialized_validators();
@@ -171,10 +166,9 @@ pub fn delete<T: SlotClock + 'static, E: EthSpec>(
Ok(status) => Status::ok(status),
Err(error) => {
warn!(
log,
"Error deleting keystore";
"pubkey" => ?pubkey_bytes,
"error" => ?error,
pubkey = ?pubkey_bytes,
?error,
"Error deleting keystore"
);
Status::error(DeleteRemotekeyStatus::Error, error)
}

8
validator_client/http_api/src/test_utils.rs
View File

@@ -14,7 +14,6 @@ use eth2::{
use eth2_keystore::KeystoreBuilder;
use initialized_validators::key_cache::{KeyCache, CACHE_FILENAME};
use initialized_validators::{InitializedValidators, OnDecryptFailure};
use logging::test_logger;
use parking_lot::RwLock;
use sensitive_url::SensitiveUrl;
use slashing_protection::{SlashingDatabase, SLASHING_PROTECTION_FILENAME};
@@ -70,8 +69,6 @@ impl ApiTester {
}
pub async fn new_with_http_config(http_config: HttpConfig) -> Self {
let log = test_logger();
let validator_dir = tempdir().unwrap();
let secrets_dir = tempdir().unwrap();
let token_path = tempdir().unwrap().path().join(PK_FILENAME);
@@ -82,7 +79,6 @@ impl ApiTester {
validator_defs,
validator_dir.path().into(),
Default::default(),
log.clone(),
)
.await
.unwrap();
@@ -110,11 +106,10 @@ impl ApiTester {
slashing_protection,
Hash256::repeat_byte(42),
spec.clone(),
Some(Arc::new(DoppelgangerService::new(log.clone()))),
Some(Arc::new(DoppelgangerService::default())),
slot_clock.clone(),
&config,
test_runtime.task_executor.clone(),
log.clone(),
));
validator_store
@@ -134,7 +129,6 @@ impl ApiTester {
graffiti_flag: Some(Graffiti::default()),
spec,
config: http_config,
log,
sse_logging_components: None,
slot_clock,
_phantom: PhantomData,

8
validator_client/http_api/src/tests.rs
View File

@@ -18,7 +18,6 @@ use eth2::{
Error as ApiError,
};
use eth2_keystore::KeystoreBuilder;
use logging::test_logger;
use parking_lot::RwLock;
use sensitive_url::SensitiveUrl;
use slashing_protection::{SlashingDatabase, SLASHING_PROTECTION_FILENAME};
@@ -61,8 +60,6 @@ impl ApiTester {
}
pub async fn new_with_config(config: ValidatorStoreConfig) -> Self {
let log = test_logger();
let validator_dir = tempdir().unwrap();
let secrets_dir = tempdir().unwrap();
let token_path = tempdir().unwrap().path().join("api-token.txt");
@@ -73,7 +70,6 @@ impl ApiTester {
validator_defs,
validator_dir.path().into(),
InitializedValidatorsConfig::default(),
log.clone(),
)
.await
.unwrap();
@@ -100,11 +96,10 @@ impl ApiTester {
slashing_protection,
Hash256::repeat_byte(42),
spec.clone(),
Some(Arc::new(DoppelgangerService::new(log.clone()))),
Some(Arc::new(DoppelgangerService::default())),
slot_clock.clone(),
&config,
test_runtime.task_executor.clone(),
log.clone(),
));
validator_store
@@ -133,7 +128,6 @@ impl ApiTester {
http_token_path: token_path,
},
sse_logging_components: None,
log,
slot_clock: slot_clock.clone(),
_phantom: PhantomData,
});

3
validator_client/http_metrics/Cargo.toml
View File

@@ -7,12 +7,13 @@ authors = ["Sigma Prime <contact@sigmaprime.io>"]
[dependencies]
health_metrics = { workspace = true }
lighthouse_version = { workspace = true }
logging = { workspace = true }
malloc_utils = { workspace = true }
metrics = { workspace = true }
parking_lot = { workspace = true }
serde = { workspace = true }
slog = { workspace = true }
slot_clock = { workspace = true }
tracing = { workspace = true }
types = { workspace = true }
validator_metrics = { workspace = true }
validator_services = { workspace = true }

12
validator_client/http_metrics/src/lib.rs
View File

@@ -3,15 +3,16 @@
//! For other endpoints, see the `http_api` crate.
use lighthouse_version::version_with_platform;
use logging::crit;
use malloc_utils::scrape_allocator_metrics;
use parking_lot::RwLock;
use serde::{Deserialize, Serialize};
use slog::{crit, info, Logger};
use slot_clock::{SlotClock, SystemTimeSlotClock};
use std::future::Future;
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use tracing::info;
use types::EthSpec;
use validator_services::duties_service::DutiesService;
use validator_store::ValidatorStore;
@@ -48,7 +49,6 @@ pub struct Shared<E: EthSpec> {
pub struct Context<E: EthSpec> {
pub config: Config,
pub shared: RwLock<Shared<E>>,
pub log: Logger,
}
/// Configuration for the HTTP server.
@@ -93,7 +93,6 @@ pub fn serve<E: EthSpec>(
shutdown: impl Future<Output = ()> + Send + Sync + 'static,
) -> Result<(SocketAddr, impl Future<Output = ()>), Error> {
let config = &ctx.config;
let log = ctx.log.clone();
// Configure CORS.
let cors_builder = {
@@ -110,7 +109,7 @@ pub fn serve<E: EthSpec>(
// Sanity check.
if !config.enabled {
crit!(log, "Cannot start disabled metrics HTTP server");
crit!("Cannot start disabled metrics HTTP server");
return Err(Error::Other(
"A disabled metrics server should not be started".to_string(),
));
@@ -151,9 +150,8 @@ pub fn serve<E: EthSpec>(
)?;
info!(
log,
"Metrics HTTP server started";
"listen_address" => listening_socket.to_string(),
listen_address = listening_socket.to_string(),
"Metrics HTTP server started"
);
Ok((listening_socket, server))

2
validator_client/initialized_validators/Cargo.toml
View File

@@ -18,8 +18,8 @@ reqwest = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
signing_method = { workspace = true }
slog = { workspace = true }
tokio = { workspace = true }
tracing = { workspace = true }
types = { workspace = true }
url = { workspace = true }
validator_dir = { workspace = true }

70
validator_client/initialized_validators/src/lib.rs
View File

@@ -22,13 +22,13 @@ use parking_lot::{MappedMutexGuard, Mutex, MutexGuard};
use reqwest::{Certificate, Client, Error as ReqwestError, Identity};
use serde::{Deserialize, Serialize};
use signing_method::SigningMethod;
use slog::{debug, error, info, warn, Logger};
use std::collections::{HashMap, HashSet};
use std::fs::{self, File};
use std::io::{self, Read};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use tracing::{debug, error, info, warn};
use types::graffiti::GraffitiString;
use types::{Address, Graffiti, Keypair, PublicKey, PublicKeyBytes};
use url::{ParseError, Url};
@@ -503,8 +503,6 @@ pub struct InitializedValidators {
validators: HashMap<PublicKeyBytes, InitializedValidator>,
/// The clients used for communications with a remote signer.
web3_signer_client_map: Option<HashMap<Web3SignerDefinition, Client>>,
/// For logging via `slog`.
log: Logger,
config: Config,
}
@@ -514,7 +512,6 @@ impl InitializedValidators {
definitions: ValidatorDefinitions,
validators_dir: PathBuf,
config: Config,
log: Logger,
) -> Result<Self, Error> {
let mut this = Self {
validators_dir,
@@ -522,7 +519,6 @@ impl InitializedValidators {
validators: HashMap::default(),
web3_signer_client_map: None,
config,
log,
};
this.update_validators().await?;
Ok(this)
@@ -1151,10 +1147,9 @@ impl InitializedValidators {
for uuid in cache.uuids() {
if !definitions_map.contains_key(uuid) {
debug!(
self.log,
"Resetting the key cache";
"keystore_uuid" => %uuid,
"reason" => "impossible to decrypt due to missing keystore",
keystore_uuid = %uuid,
reason = "impossible to decrypt due to missing keystore",
"Resetting the key cache"
);
return Ok(KeyCache::new());
}
@@ -1281,30 +1276,27 @@ impl InitializedValidators {
self.validators
.insert(init.voting_public_key().compress(), init);
info!(
self.log,
"Enabled validator";
"signing_method" => "local_keystore",
"voting_pubkey" => format!("{:?}", def.voting_public_key),
signing_method = "local_keystore",
voting_pubkey = format!("{:?}", def.voting_public_key),
"Enabled validator"
);
if let Some(lockfile_path) = existing_lockfile_path {
warn!(
self.log,
"Ignored stale lockfile";
"path" => lockfile_path.display(),
"cause" => "Ungraceful shutdown (harmless) OR \
path = ?lockfile_path.display(),
cause = "Ungraceful shutdown (harmless) OR \
non-Lighthouse client using this keystore \
(risky)"
(risky)",
"Ignored stale lockfile"
);
}
}
Err(e) => {
error!(
self.log,
"Failed to initialize validator";
"error" => format!("{:?}", e),
"signing_method" => "local_keystore",
"validator" => format!("{:?}", def.voting_public_key)
error = format!("{:?}", e),
signing_method = "local_keystore",
validator = format!("{:?}", def.voting_public_key),
"Failed to initialize validator"
);
// Exit on an invalid validator.
@@ -1327,19 +1319,17 @@ impl InitializedValidators {
.insert(init.voting_public_key().compress(), init);
info!(
self.log,
"Enabled validator";
"signing_method" => "remote_signer",
"voting_pubkey" => format!("{:?}", def.voting_public_key),
signing_method = "remote_signer",
voting_pubkey = format!("{:?}", def.voting_public_key),
"Enabled validator"
);
}
Err(e) => {
error!(
self.log,
"Failed to initialize validator";
"error" => format!("{:?}", e),
"signing_method" => "remote_signer",
"validator" => format!("{:?}", def.voting_public_key)
error = format!("{:?}", e),
signing_method = "remote_signer",
validator = format!("{:?}", def.voting_public_key),
"Failed to initialize validator"
);
// Exit on an invalid validator.
@@ -1364,9 +1354,8 @@ impl InitializedValidators {
}
info!(
self.log,
"Disabled validator";
"voting_pubkey" => format!("{:?}", def.voting_public_key)
voting_pubkey = format!("{:?}", def.voting_public_key),
"Disabled validator"
);
}
}
@@ -1378,23 +1367,18 @@ impl InitializedValidators {
}
let validators_dir = self.validators_dir.clone();
let log = self.log.clone();
if has_local_definitions && key_cache.is_modified() {
tokio::task::spawn_blocking(move || {
match key_cache.save(validators_dir) {
Err(e) => warn!(
log,
"Error during saving of key_cache";
"err" => format!("{:?}", e)
),
Ok(true) => info!(log, "Modified key_cache saved successfully"),
Err(e) => warn!(err = format!("{:?}", e), "Error during saving of key_cache"),
Ok(true) => info!("Modified key_cache saved successfully"),
_ => {}
};
})
.await
.map_err(Error::TokioJoin)?;
} else {
debug!(log, "Key cache not modified");
debug!("Key cache not modified");
}
// Update the enabled and total validator counts

1
validator_client/slashing_protection/Cargo.toml
View File

@@ -19,6 +19,7 @@ rusqlite = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
tempfile = { workspace = true }
tracing = { workspace = true }
types = { workspace = true }
[dev-dependencies]

25
validator_client/src/check_synced.rs Normal file
View File

@@ -0,0 +1,25 @@
use crate::beacon_node_fallback::CandidateError;
use eth2::{types::Slot, BeaconNodeHttpClient};
use tracing::warn;
pub async fn check_node_health(
beacon_node: &BeaconNodeHttpClient,
) -> Result<(Slot, bool, bool), CandidateError> {
let resp = match beacon_node.get_node_syncing().await {
Ok(resp) => resp,
Err(e) => {
warn!(
error = %e,
"Unable connect to beacon node"
);
return Err(CandidateError::Offline);
}
};
Ok((
resp.data.head_slot,
resp.data.is_optimistic,
resp.data.el_offline,
))
}

15
validator_client/src/config.rs
View File

@@ -12,11 +12,11 @@ use graffiti_file::GraffitiFile;
use initialized_validators::Config as InitializedValidatorsConfig;
use sensitive_url::SensitiveUrl;
use serde::{Deserialize, Serialize};
use slog::{info, warn, Logger};
use std::fs;
use std::net::IpAddr;
use std::path::PathBuf;
use std::time::Duration;
use tracing::{info, warn};
use types::GRAFFITI_BYTES_LEN;
use validator_http_api::{self, PK_FILENAME};
use validator_http_metrics;
@@ -141,7 +141,6 @@ impl Config {
pub fn from_cli(
cli_args: &ArgMatches,
validator_client_config: &ValidatorClient,
log: &Logger,
) -> Result<Config, String> {
let mut config = Config::default();
@@ -207,7 +206,10 @@ impl Config {
.read_graffiti_file()
.map_err(|e| format!("Error reading graffiti file: {:?}", e))?;
config.graffiti_file = Some(graffiti_file);
info!(log, "Successfully loaded graffiti file"; "path" => graffiti_file_path.to_str());
info!(
path = graffiti_file_path.to_str(),
"Successfully loaded graffiti file"
);
}
if let Some(input_graffiti) = validator_client_config.graffiti.as_ref() {
@@ -375,10 +377,9 @@ impl Config {
config.validator_store.enable_web3signer_slashing_protection =
if validator_client_config.disable_slashing_protection_web3signer {
warn!(
log,
"Slashing protection for remote keys disabled";
"info" => "ensure slashing protection on web3signer is enabled or you WILL \
get slashed"
info = "ensure slashing protection on web3signer is enabled or you WILL \
get slashed",
"Slashing protection for remote keys disabled"
);
false
} else {

11
validator_client/src/latency.rs
View File

@@ -1,9 +1,9 @@
use beacon_node_fallback::BeaconNodeFallback;
use environment::RuntimeContext;
use slog::debug;
use slot_clock::SlotClock;
use std::sync::Arc;
use tokio::time::sleep;
use tracing::debug;
use types::EthSpec;
/// The latency service will run 11/12ths of the way through the slot.
@@ -17,8 +17,6 @@ pub fn start_latency_service<T: SlotClock + 'static, E: EthSpec>(
slot_clock: T,
beacon_nodes: Arc<BeaconNodeFallback<T, E>>,
) {
let log = context.log().clone();
let future = async move {
loop {
let sleep_time = slot_clock
@@ -39,10 +37,9 @@ pub fn start_latency_service<T: SlotClock + 'static, E: EthSpec>(
for (i, measurement) in beacon_nodes.measure_latency().await.iter().enumerate() {
if let Some(latency) = measurement.latency {
debug!(
log,
"Measured BN latency";
"node" => &measurement.beacon_node_id,
"latency" => latency.as_millis(),
node = &measurement.beacon_node_id,
latency = latency.as_millis(),
"Measured BN latency"
);
validator_metrics::observe_timer_vec(
&validator_metrics::VC_BEACON_NODE_LATENCY,

173
validator_client/src/lib.rs
View File

@@ -23,7 +23,6 @@ use initialized_validators::Error::UnableToOpenVotingKeystore;
use notifier::spawn_notifier;
use parking_lot::RwLock;
use reqwest::Certificate;
use slog::{debug, error, info, warn, Logger};
use slot_clock::SlotClock;
use slot_clock::SystemTimeSlotClock;
use std::fs::File;
@@ -37,6 +36,7 @@ use tokio::{
sync::mpsc,
time::{sleep, Duration},
};
use tracing::{debug, error, info, warn};
use types::{EthSpec, Hash256};
use validator_http_api::ApiSecret;
use validator_services::{
@@ -97,7 +97,7 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
cli_args: &ArgMatches,
validator_client_config: &ValidatorClient,
) -> Result<Self, String> {
let config = Config::from_cli(cli_args, validator_client_config, context.log())
let config = Config::from_cli(cli_args, validator_client_config)
.map_err(|e| format!("Unable to initialize config: {}", e))?;
Self::new(context, config).await
}
@@ -105,8 +105,6 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
/// Instantiates the validator client, _without_ starting the timers to trigger block
/// and attestation production.
pub async fn new(context: RuntimeContext<E>, config: Config) -> Result<Self, String> {
let log = context.log().clone();
// Attempt to raise soft fd limit. The behavior is OS specific:
// `linux` - raise soft fd limit to hard
// `macos` - raise soft fd limit to `min(kernel limit, hard fd limit)`
@@ -114,25 +112,20 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
match fdlimit::raise_fd_limit().map_err(|e| format!("Unable to raise fd limit: {}", e))? {
fdlimit::Outcome::LimitRaised { from, to } => {
debug!(
log,
"Raised soft open file descriptor resource limit";
"old_limit" => from,
"new_limit" => to
old_limit = from,
new_limit = to,
"Raised soft open file descriptor resource limit"
);
}
fdlimit::Outcome::Unsupported => {
debug!(
log,
"Raising soft open file descriptor resource limit is not supported"
);
debug!("Raising soft open file descriptor resource limit is not supported");
}
};
info!(
log,
"Starting validator client";
"beacon_nodes" => format!("{:?}", &config.beacon_nodes),
"validator_dir" => format!("{:?}", config.validator_dir),
beacon_nodes = ?config.beacon_nodes,
validator_dir = ?config.validator_dir,
"Starting validator client"
);
// Optionally start the metrics server.
@@ -147,7 +140,6 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
Arc::new(validator_http_metrics::Context {
config: config.http_metrics.clone(),
shared: RwLock::new(shared),
log: log.clone(),
});
let exit = context.executor.exit();
@@ -162,15 +154,14 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
Some(ctx)
} else {
info!(log, "HTTP metrics server is disabled");
info!("HTTP metrics server is disabled");
None
};
// Start the explorer client which periodically sends validator process
// and system metrics to the configured endpoint.
if let Some(monitoring_config) = &config.monitoring_api {
let monitoring_client =
MonitoringHttpClient::new(monitoring_config, context.log().clone())?;
let monitoring_client = MonitoringHttpClient::new(monitoring_config)?;
monitoring_client.auto_update(
context.executor.clone(),
vec![ProcessType::Validator, ProcessType::System],
@@ -182,7 +173,7 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
if !config.disable_auto_discover {
let new_validators = validator_defs
.discover_local_keystores(&config.validator_dir, &config.secrets_dir, &log)
.discover_local_keystores(&config.validator_dir, &config.secrets_dir)
.map_err(|e| format!("Unable to discover local validator keystores: {:?}", e))?;
validator_defs.save(&config.validator_dir).map_err(|e| {
format!(
@@ -190,18 +181,13 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
e
)
})?;
info!(
log,
"Completed validator discovery";
"new_validators" => new_validators,
);
info!(new_validators, "Completed validator discovery");
}
let validators = InitializedValidators::from_definitions(
validator_defs,
config.validator_dir.clone(),
config.initialized_validators.clone(),
log.clone(),
)
.await
.map_err(|e| {
@@ -218,17 +204,17 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
let voting_pubkeys: Vec<_> = validators.iter_voting_pubkeys().collect();
info!(
log,
"Initialized validators";
"disabled" => validators.num_total().saturating_sub(validators.num_enabled()),
"enabled" => validators.num_enabled(),
disabled = validators
.num_total()
.saturating_sub(validators.num_enabled()),
enabled = validators.num_enabled(),
"Initialized validators"
);
if voting_pubkeys.is_empty() {
warn!(
log,
"No enabled validators";
"hint" => "create validators via the API, or the `lighthouse account` CLI command"
hint = "create validators via the API, or the `lighthouse account` CLI command",
"No enabled validators"
);
}
@@ -303,10 +289,7 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
// Use quicker timeouts if a fallback beacon node exists.
let timeouts = if i < last_beacon_node_index && !config.use_long_timeouts {
info!(
log,
"Fallback endpoints are available, using optimized timeouts.";
);
info!("Fallback endpoints are available, using optimized timeouts.");
Timeouts {
attestation: slot_duration / HTTP_ATTESTATION_TIMEOUT_QUOTIENT,
attester_duties: slot_duration / HTTP_ATTESTER_DUTIES_TIMEOUT_QUOTIENT,
@@ -389,7 +372,6 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
config.beacon_node_fallback,
config.broadcast_topics.clone(),
context.eth2_config.spec.clone(),
log.clone(),
);
let mut proposer_nodes: BeaconNodeFallback<_, E> = BeaconNodeFallback::new(
@@ -397,12 +379,11 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
config.beacon_node_fallback,
config.broadcast_topics.clone(),
context.eth2_config.spec.clone(),
log.clone(),
);
// Perform some potentially long-running initialization tasks.
let (genesis_time, genesis_validators_root) = tokio::select! {
tuple = init_from_beacon_node(&beacon_nodes, &proposer_nodes, &context) => tuple?,
tuple = init_from_beacon_node(&beacon_nodes, &proposer_nodes) => tuple?,
() = context.executor.exit() => return Err("Shutting down".to_string())
};
@@ -427,12 +408,7 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
start_fallback_updater_service(context.clone(), proposer_nodes.clone())?;
let doppelganger_service = if config.enable_doppelganger_protection {
Some(Arc::new(DoppelgangerService::new(
context
.service_context(DOPPELGANGER_SERVICE_NAME.into())
.log()
.clone(),
)))
Some(Arc::new(DoppelgangerService::default()))
} else {
None
};
@@ -446,16 +422,14 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
slot_clock.clone(),
&config.validator_store,
context.executor.clone(),
log.clone(),
));
// Ensure all validators are registered in doppelganger protection.
validator_store.register_all_in_doppelganger_protection_if_enabled()?;
info!(
log,
"Loaded validator keypair store";
"voting_validators" => validator_store.num_voting_validators()
voting_validators = validator_store.num_voting_validators(),
"Loaded validator keypair store"
);
// Perform pruning of the slashing protection database on start-up. In case the database is
@@ -551,7 +525,6 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
// whole epoch!
let channel_capacity = E::slots_per_epoch() as usize;
let (block_service_tx, block_service_rx) = mpsc::channel(channel_capacity);
let log = self.context.log();
let api_secret = ApiSecret::create_or_open(&self.config.http_api.http_token_path)?;
@@ -569,7 +542,6 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
config: self.config.http_api.clone(),
sse_logging_components: self.context.sse_logging_components.clone(),
slot_clock: self.slot_clock.clone(),
log: log.clone(),
_phantom: PhantomData,
});
@@ -585,12 +557,12 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
Some(listen_addr)
} else {
info!(log, "HTTP API server is disabled");
info!("HTTP API server is disabled");
None
};
// Wait until genesis has occurred.
wait_for_genesis(&self.beacon_nodes, self.genesis_time, &self.context).await?;
wait_for_genesis(&self.beacon_nodes, self.genesis_time).await?;
duties_service::start_update_service(self.duties_service.clone(), block_service_tx);
@@ -625,7 +597,7 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
)
.map_err(|e| format!("Unable to start doppelganger service: {}", e))?
} else {
info!(log, "Doppelganger protection disabled.")
info!("Doppelganger protection disabled.")
}
spawn_notifier(self).map_err(|e| format!("Failed to start notifier: {}", e))?;
@@ -645,7 +617,6 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
async fn init_from_beacon_node<E: EthSpec>(
beacon_nodes: &BeaconNodeFallback<SystemTimeSlotClock, E>,
proposer_nodes: &BeaconNodeFallback<SystemTimeSlotClock, E>,
context: &RuntimeContext<E>,
) -> Result<(u64, Hash256), String> {
loop {
beacon_nodes.update_all_candidates().await;
@@ -659,41 +630,37 @@ async fn init_from_beacon_node<E: EthSpec>(
if proposer_total > 0 && proposer_available == 0 {
warn!(
context.log(),
"Unable to connect to a proposer node";
"retry in" => format!("{} seconds", RETRY_DELAY.as_secs()),
"total_proposers" => proposer_total,
"available_proposers" => proposer_available,
"total_beacon_nodes" => num_total,
"available_beacon_nodes" => num_available,
retry_in = format!("{} seconds", RETRY_DELAY.as_secs()),
total_proposers = proposer_total,
available_proposers = proposer_available,
total_beacon_nodes = num_total,
available_beacon_nodes = num_available,
"Unable to connect to a proposer node"
);
}
if num_available > 0 && proposer_available == 0 {
info!(
context.log(),
"Initialized beacon node connections";
"total" => num_total,
"available" => num_available,
total = num_total,
available = num_available,
"Initialized beacon node connections"
);
break;
} else if num_available > 0 {
info!(
context.log(),
"Initialized beacon node connections";
"total" => num_total,
"available" => num_available,
"proposers_available" => proposer_available,
"proposers_total" => proposer_total,
total = num_total,
available = num_available,
proposer_available,
proposer_total,
"Initialized beacon node connections"
);
break;
} else {
warn!(
context.log(),
"Unable to connect to a beacon node";
"retry in" => format!("{} seconds", RETRY_DELAY.as_secs()),
"total" => num_total,
"available" => num_available,
retry_in = format!("{} seconds", RETRY_DELAY.as_secs()),
total = num_total,
available = num_available,
"Unable to connect to a beacon node"
);
sleep(RETRY_DELAY).await;
}
@@ -714,15 +681,11 @@ async fn init_from_beacon_node<E: EthSpec>(
.filter_map(|(_, e)| e.request_failure())
.any(|e| e.status() == Some(StatusCode::NOT_FOUND))
{
info!(
context.log(),
"Waiting for genesis";
);
info!("Waiting for genesis");
} else {
error!(
context.log(),
"Errors polling beacon node";
"error" => %errors
%errors,
"Errors polling beacon node"
);
}
}
@@ -737,7 +700,6 @@ async fn init_from_beacon_node<E: EthSpec>(
async fn wait_for_genesis<E: EthSpec>(
beacon_nodes: &BeaconNodeFallback<SystemTimeSlotClock, E>,
genesis_time: u64,
context: &RuntimeContext<E>,
) -> Result<(), String> {
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
@@ -751,28 +713,25 @@ async fn wait_for_genesis<E: EthSpec>(
// the slot clock.
if now < genesis_time {
info!(
context.log(),
"Starting node prior to genesis";
"seconds_to_wait" => (genesis_time - now).as_secs()
seconds_to_wait = (genesis_time - now).as_secs(),
"Starting node prior to genesis"
);
// Start polling the node for pre-genesis information, cancelling the polling as soon as the
// timer runs out.
tokio::select! {
result = poll_whilst_waiting_for_genesis(beacon_nodes, genesis_time, context.log()) => result?,
result = poll_whilst_waiting_for_genesis(beacon_nodes, genesis_time) => result?,
() = sleep(genesis_time - now) => ()
};
info!(
context.log(),
"Genesis has occurred";
"ms_since_genesis" => (genesis_time - now).as_millis()
ms_since_genesis = (genesis_time - now).as_millis(),
"Genesis has occurred"
);
} else {
info!(
context.log(),
"Genesis has already occurred";
"seconds_ago" => (now - genesis_time).as_secs()
seconds_ago = (now - genesis_time).as_secs(),
"Genesis has already occurred"
);
}
@@ -784,7 +743,6 @@ async fn wait_for_genesis<E: EthSpec>(
async fn poll_whilst_waiting_for_genesis<E: EthSpec>(
beacon_nodes: &BeaconNodeFallback<SystemTimeSlotClock, E>,
genesis_time: Duration,
log: &Logger,
) -> Result<(), String> {
loop {
match beacon_nodes
@@ -798,19 +756,17 @@ async fn poll_whilst_waiting_for_genesis<E: EthSpec>(
if !is_staking {
error!(
log,
"Staking is disabled for beacon node";
"msg" => "this will caused missed duties",
"info" => "see the --staking CLI flag on the beacon node"
msg = "this will caused missed duties",
info = "see the --staking CLI flag on the beacon node",
"Staking is disabled for beacon node"
);
}
if now < genesis_time {
info!(
log,
"Waiting for genesis";
"bn_staking_enabled" => is_staking,
"seconds_to_wait" => (genesis_time - now).as_secs()
bn_staking_enabled = is_staking,
seconds_to_wait = (genesis_time - now).as_secs(),
"Waiting for genesis"
);
} else {
break Ok(());
@@ -818,9 +774,8 @@ async fn poll_whilst_waiting_for_genesis<E: EthSpec>(
}
Err(e) => {
error!(
log,
"Error polling beacon node";
"error" => %e
error = %e,
"Error polling beacon node"
);
}
}

106
validator_client/src/notifier.rs
View File

@@ -1,8 +1,8 @@
use crate::{DutiesService, ProductionValidatorClient};
use metrics::set_gauge;
use slog::{debug, error, info, Logger};
use slot_clock::SlotClock;
use tokio::time::{sleep, Duration};
use tracing::{debug, error, info};
use types::EthSpec;
/// Spawns a notifier service which periodically logs information about the node.
@@ -14,14 +14,12 @@ pub fn spawn_notifier<E: EthSpec>(client: &ProductionValidatorClient<E>) -> Resu
let slot_duration = Duration::from_secs(context.eth2_config.spec.seconds_per_slot);
let interval_fut = async move {
let log = context.log();
loop {
if let Some(duration_to_next_slot) = duties_service.slot_clock.duration_to_next_slot() {
sleep(duration_to_next_slot + slot_duration / 2).await;
notify(&duties_service, log).await;
notify(&duties_service).await;
} else {
error!(log, "Failed to read slot clock");
error!("Failed to read slot clock");
// If we can't read the slot clock, just wait another slot.
sleep(slot_duration).await;
continue;
@@ -34,10 +32,7 @@ pub fn spawn_notifier<E: EthSpec>(client: &ProductionValidatorClient<E>) -> Resu
}
/// Performs a single notification routine.
async fn notify<T: SlotClock + 'static, E: EthSpec>(
duties_service: &DutiesService<T, E>,
log: &Logger,
) {
async fn notify<T: SlotClock + 'static, E: EthSpec>(duties_service: &DutiesService<T, E>) {
let (candidate_info, num_available, num_synced) =
duties_service.beacon_nodes.get_notifier_info().await;
let num_total = candidate_info.len();
@@ -61,20 +56,18 @@ async fn notify<T: SlotClock + 'static, E: EthSpec>(
.map(|candidate| candidate.endpoint.as_str())
.unwrap_or("None");
info!(
log,
"Connected to beacon node(s)";
"primary" => primary,
"total" => num_total,
"available" => num_available,
"synced" => num_synced,
primary,
total = num_total,
available = num_available,
synced = num_synced,
"Connected to beacon node(s)"
)
} else {
error!(
log,
"No synced beacon nodes";
"total" => num_total,
"available" => num_available,
"synced" => num_synced,
total = num_total,
available = num_available,
synced = num_synced,
"No synced beacon nodes"
)
}
if num_synced_fallback > 0 {
@@ -86,23 +79,21 @@ async fn notify<T: SlotClock + 'static, E: EthSpec>(
for info in candidate_info {
if let Ok(health) = info.health {
debug!(
log,
"Beacon node info";
"status" => "Connected",
"index" => info.index,
"endpoint" => info.endpoint,
"head_slot" => %health.head,
"is_optimistic" => ?health.optimistic_status,
"execution_engine_status" => ?health.execution_status,
"health_tier" => %health.health_tier,
status = "Connected",
index = info.index,
endpoint = info.endpoint,
head_slot = %health.head,
is_optimistic = ?health.optimistic_status,
execution_engine_status = ?health.execution_status,
health_tier = %health.health_tier,
"Beacon node info"
);
} else {
debug!(
log,
"Beacon node info";
"status" => "Disconnected",
"index" => info.index,
"endpoint" => info.endpoint,
status = "Disconnected",
index = info.index,
endpoint = info.endpoint,
"Beacon node info"
);
}
}
@@ -116,45 +107,44 @@ async fn notify<T: SlotClock + 'static, E: EthSpec>(
let doppelganger_detecting_validators = duties_service.doppelganger_detecting_count();
if doppelganger_detecting_validators > 0 {
info!(log, "Listening for doppelgangers"; "doppelganger_detecting_validators" => doppelganger_detecting_validators)
info!(
doppelganger_detecting_validators,
"Listening for doppelgangers"
)
}
if total_validators == 0 {
info!(
log,
"No validators present";
"msg" => "see `lighthouse vm create --help` or the HTTP API documentation"
msg = "see `lighthouse vm create --help` or the HTTP API documentation",
"No validators present"
)
} else if total_validators == attesting_validators {
info!(
log,
"All validators active";
"current_epoch_proposers" => proposing_validators,
"active_validators" => attesting_validators,
"total_validators" => total_validators,
"epoch" => format!("{}", epoch),
"slot" => format!("{}", slot),
current_epoch_proposers = proposing_validators,
active_validators = attesting_validators,
total_validators = total_validators,
%epoch,
%slot,
"All validators active"
);
} else if attesting_validators > 0 {
info!(
log,
"Some validators active";
"current_epoch_proposers" => proposing_validators,
"active_validators" => attesting_validators,
"total_validators" => total_validators,
"epoch" => format!("{}", epoch),
"slot" => format!("{}", slot),
current_epoch_proposers = proposing_validators,
active_validators = attesting_validators,
total_validators = total_validators,
%epoch,
%slot,
"Some validators active"
);
} else {
info!(
log,
"Awaiting activation";
"validators" => total_validators,
"epoch" => format!("{}", epoch),
"slot" => format!("{}", slot),
validators = total_validators,
%epoch,
%slot,
"Awaiting activation"
);
}
} else {
error!(log, "Unable to read slot clock");
error!("Unable to read slot clock");
}
}

3
validator_client/validator_services/Cargo.toml
View File

@@ -13,11 +13,12 @@ environment = { workspace = true }
eth2 = { workspace = true }
futures = { workspace = true }
graffiti_file = { workspace = true }
logging = { workspace = true }
parking_lot = { workspace = true }
safe_arith = { workspace = true }
slog = { workspace = true }
slot_clock = { workspace = true }
tokio = { workspace = true }
tracing = { workspace = true }
tree_hash = { workspace = true }
types = { workspace = true }
validator_metrics = { workspace = true }

176
validator_client/validator_services/src/attestation_service.rs
View File

@@ -3,12 +3,13 @@ use beacon_node_fallback::{ApiTopic, BeaconNodeFallback};
use either::Either;
use environment::RuntimeContext;
use futures::future::join_all;
use slog::{crit, debug, error, info, trace, warn};
use logging::crit;
use slot_clock::SlotClock;
use std::collections::HashMap;
use std::ops::Deref;
use std::sync::Arc;
use tokio::time::{sleep, sleep_until, Duration, Instant};
use tracing::{debug, error, info, trace, warn};
use tree_hash::TreeHash;
use types::{Attestation, AttestationData, ChainSpec, CommitteeIndex, EthSpec, Slot};
use validator_store::{Error as ValidatorStoreError, ValidatorStore};
@@ -128,9 +129,8 @@ impl<T, E: EthSpec> Deref for AttestationService<T, E> {
impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
/// Starts the service which periodically produces attestations.
pub fn start_update_service(self, spec: &ChainSpec) -> Result<(), String> {
let log = self.context.log().clone();
if self.disable {
info!(log, "Attestation service disabled");
info!("Attestation service disabled");
return Ok(());
}
@@ -141,9 +141,8 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
.ok_or("Unable to determine duration to next slot")?;
info!(
log,
"Attestation production service started";
"next_update_millis" => duration_to_next_slot.as_millis()
next_update_millis = duration_to_next_slot.as_millis(),
"Attestation production service started"
);
let executor = self.context.executor.clone();
@@ -152,22 +151,14 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
loop {
if let Some(duration_to_next_slot) = self.slot_clock.duration_to_next_slot() {
sleep(duration_to_next_slot + slot_duration / 3).await;
let log = self.context.log();
if let Err(e) = self.spawn_attestation_tasks(slot_duration) {
crit!(
log,
"Failed to spawn attestation tasks";
"error" => e
)
crit!(error = e, "Failed to spawn attestation tasks")
} else {
trace!(
log,
"Spawned attestation tasks";
)
trace!("Spawned attestation tasks");
}
} else {
error!(log, "Failed to read slot clock");
error!("Failed to read slot clock");
// If we can't read the slot clock, just wait another slot.
sleep(slot_duration).await;
continue;
@@ -249,7 +240,6 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
validator_duties: Vec<DutyAndProof>,
aggregate_production_instant: Instant,
) -> Result<(), ()> {
let log = self.context.log();
let attestations_timer = validator_metrics::start_timer_vec(
&validator_metrics::ATTESTATION_SERVICE_TIMES,
&[validator_metrics::ATTESTATIONS],
@@ -269,11 +259,10 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
.await
.map_err(move |e| {
crit!(
log,
"Error during attestation routine";
"error" => format!("{:?}", e),
"committee_index" => committee_index,
"slot" => slot.as_u64(),
error = format!("{:?}", e),
committee_index,
slot = slot.as_u64(),
"Error during attestation routine"
)
})?;
@@ -306,11 +295,10 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
.await
.map_err(move |e| {
crit!(
log,
"Error during attestation routine";
"error" => format!("{:?}", e),
"committee_index" => committee_index,
"slot" => slot.as_u64(),
error = format!("{:?}", e),
committee_index,
slot = slot.as_u64(),
"Error during attestation routine"
)
})?;
}
@@ -336,8 +324,6 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
committee_index: CommitteeIndex,
validator_duties: &[DutyAndProof],
) -> Result<Option<AttestationData>, String> {
let log = self.context.log();
if validator_duties.is_empty() {
return Ok(None);
}
@@ -373,13 +359,12 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
// Ensure that the attestation matches the duties.
if !duty.match_attestation_data::<E>(attestation_data, &self.context.eth2_config.spec) {
crit!(
log,
"Inconsistent validator duties during signing";
"validator" => ?duty.pubkey,
"duty_slot" => duty.slot,
"attestation_slot" => attestation_data.slot,
"duty_index" => duty.committee_index,
"attestation_index" => attestation_data.index,
validator = ?duty.pubkey,
duty_slot = %duty.slot,
attestation_slot = %attestation_data.slot,
duty_index = duty.committee_index,
attestation_index = attestation_data.index,
"Inconsistent validator duties during signing"
);
return None;
}
@@ -396,11 +381,10 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
Ok(attestation) => attestation,
Err(err) => {
crit!(
log,
"Invalid validator duties during signing";
"validator" => ?duty.pubkey,
"duty" => ?duty,
"err" => ?err,
validator = ?duty.pubkey,
?duty,
?err,
"Invalid validator duties during signing"
);
return None;
}
@@ -421,24 +405,22 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
// A pubkey can be missing when a validator was recently
// removed via the API.
warn!(
log,
"Missing pubkey for attestation";
"info" => "a validator may have recently been removed from this VC",
"pubkey" => ?pubkey,
"validator" => ?duty.pubkey,
"committee_index" => committee_index,
"slot" => slot.as_u64(),
info = "a validator may have recently been removed from this VC",
pubkey = ?pubkey,
validator = ?duty.pubkey,
committee_index = committee_index,
slot = slot.as_u64(),
"Missing pubkey for attestation"
);
None
}
Err(e) => {
crit!(
log,
"Failed to sign attestation";
"error" => ?e,
"validator" => ?duty.pubkey,
"committee_index" => committee_index,
"slot" => slot.as_u64(),
error = ?e,
validator = ?duty.pubkey,
committee_index,
slot = slot.as_u64(),
"Failed to sign attestation"
);
None
}
@@ -453,7 +435,7 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
.unzip();
if attestations.is_empty() {
warn!(log, "No attestations were published");
warn!("No attestations were published");
return Ok(None);
}
let fork_name = self
@@ -481,12 +463,11 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
// This shouldn't happen unless BN and VC are out of sync with
// respect to the Electra fork.
error!(
log,
"Unable to convert to SingleAttestation";
"error" => ?e,
"committee_index" => attestation_data.index,
"slot" => slot.as_u64(),
"type" => "unaggregated",
error = ?e,
committee_index = attestation_data.index,
slot = slot.as_u64(),
"type" = "unaggregated",
"Unable to convert to SingleAttestation"
);
None
}
@@ -509,22 +490,20 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
.await
{
Ok(()) => info!(
log,
"Successfully published attestations";
"count" => attestations.len(),
"validator_indices" => ?validator_indices,
"head_block" => ?attestation_data.beacon_block_root,
"committee_index" => attestation_data.index,
"slot" => attestation_data.slot.as_u64(),
"type" => "unaggregated",
count = attestations.len(),
validator_indices = ?validator_indices,
head_block = ?attestation_data.beacon_block_root,
committee_index = attestation_data.index,
slot = attestation_data.slot.as_u64(),
"type" = "unaggregated",
"Successfully published attestations"
),
Err(e) => error!(
log,
"Unable to publish attestations";
"error" => %e,
"committee_index" => attestation_data.index,
"slot" => slot.as_u64(),
"type" => "unaggregated",
error = %e,
committee_index = attestation_data.index,
slot = slot.as_u64(),
"type" = "unaggregated",
"Unable to publish attestations"
),
}
@@ -550,8 +529,6 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
committee_index: CommitteeIndex,
validator_duties: &[DutyAndProof],
) -> Result<(), String> {
let log = self.context.log();
if !validator_duties
.iter()
.any(|duty_and_proof| duty_and_proof.selection_proof.is_some())
@@ -609,7 +586,7 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
let selection_proof = duty_and_proof.selection_proof.as_ref()?;
if !duty.match_attestation_data::<E>(attestation_data, &self.context.eth2_config.spec) {
crit!(log, "Inconsistent validator duties during signing");
crit!("Inconsistent validator duties during signing");
return None;
}
@@ -627,19 +604,14 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
// A pubkey can be missing when a validator was recently
// removed via the API.
debug!(
log,
"Missing pubkey for aggregate";
"pubkey" => ?pubkey,
);
debug!(?pubkey, "Missing pubkey for aggregate");
None
}
Err(e) => {
crit!(
log,
"Failed to sign aggregate";
"error" => ?e,
"pubkey" => ?duty.pubkey,
error = ?e,
pubkey = ?duty.pubkey,
"Failed to sign aggregate"
);
None
}
@@ -683,14 +655,13 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
for signed_aggregate_and_proof in signed_aggregate_and_proofs {
let attestation = signed_aggregate_and_proof.message().aggregate();
info!(
log,
"Successfully published attestation";
"aggregator" => signed_aggregate_and_proof.message().aggregator_index(),
"signatures" => attestation.num_set_aggregation_bits(),
"head_block" => format!("{:?}", attestation.data().beacon_block_root),
"committee_index" => attestation.committee_index(),
"slot" => attestation.data().slot.as_u64(),
"type" => "aggregated",
aggregator = signed_aggregate_and_proof.message().aggregator_index(),
signatures = attestation.num_set_aggregation_bits(),
head_block = format!("{:?}", attestation.data().beacon_block_root),
committee_index = attestation.committee_index(),
slot = attestation.data().slot.as_u64(),
"type" = "aggregated",
"Successfully published attestation"
);
}
}
@@ -698,13 +669,12 @@ impl<T: SlotClock + 'static, E: EthSpec> AttestationService<T, E> {
for signed_aggregate_and_proof in signed_aggregate_and_proofs {
let attestation = &signed_aggregate_and_proof.message().aggregate();
crit!(
log,
"Failed to publish attestation";
"error" => %e,
"aggregator" => signed_aggregate_and_proof.message().aggregator_index(),
"committee_index" => attestation.committee_index(),
"slot" => attestation.data().slot.as_u64(),
"type" => "aggregated",
error = %e,
aggregator = signed_aggregate_and_proof.message().aggregator_index(),
committee_index = attestation.committee_index(),
slot = attestation.data().slot.as_u64(),
"type" = "aggregated",
"Failed to publish attestation"
);
}
}

132
validator_client/validator_services/src/block_service.rs
View File

@@ -4,7 +4,7 @@ use environment::RuntimeContext;
use eth2::types::{FullBlockContents, PublishBlockRequest};
use eth2::{BeaconNodeHttpClient, StatusCode};
use graffiti_file::{determine_graffiti, GraffitiFile};
use slog::{crit, debug, error, info, trace, warn, Logger};
use logging::crit;
use slot_clock::SlotClock;
use std::fmt::Debug;
use std::future::Future;
@@ -12,6 +12,7 @@ use std::ops::Deref;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::mpsc;
use tracing::{debug, error, info, trace, warn};
use types::{
BlindedBeaconBlock, BlockType, EthSpec, Graffiti, PublicKeyBytes, SignedBlindedBeaconBlock,
Slot,
@@ -219,9 +220,7 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
self,
mut notification_rx: mpsc::Receiver<BlockServiceNotification>,
) -> Result<(), String> {
let log = self.context.log().clone();
info!(log, "Block production service started");
info!("Block production service started");
let executor = self.inner.context.executor.clone();
@@ -230,7 +229,7 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
while let Some(notif) = notification_rx.recv().await {
self.do_update(notif).await.ok();
}
debug!(log, "Block service shutting down");
debug!("Block service shutting down");
},
"block_service",
);
@@ -240,64 +239,54 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
/// Attempt to produce a block for any block producers in the `ValidatorStore`.
async fn do_update(&self, notification: BlockServiceNotification) -> Result<(), ()> {
let log = self.context.log();
let _timer = validator_metrics::start_timer_vec(
&validator_metrics::BLOCK_SERVICE_TIMES,
&[validator_metrics::FULL_UPDATE],
);
let slot = self.slot_clock.now().ok_or_else(move || {
crit!(log, "Duties manager failed to read slot clock");
crit!("Duties manager failed to read slot clock");
})?;
if notification.slot != slot {
warn!(
log,
"Skipping block production for expired slot";
"current_slot" => slot.as_u64(),
"notification_slot" => notification.slot.as_u64(),
"info" => "Your machine could be overloaded"
current_slot = slot.as_u64(),
notification_slot = notification.slot.as_u64(),
info = "Your machine could be overloaded",
"Skipping block production for expired slot"
);
return Ok(());
}
if slot == self.context.eth2_config.spec.genesis_slot {
debug!(
log,
"Not producing block at genesis slot";
"proposers" => format!("{:?}", notification.block_proposers),
proposers = format!("{:?}", notification.block_proposers),
"Not producing block at genesis slot"
);
return Ok(());
}
trace!(
log,
"Block service update started";
"slot" => slot.as_u64()
);
trace!(slot = slot.as_u64(), "Block service update started");
let proposers = notification.block_proposers;
if proposers.is_empty() {
trace!(
log,
"No local block proposers for this slot";
"slot" => slot.as_u64()
slot = slot.as_u64(),
"No local block proposers for this slot"
)
} else if proposers.len() > 1 {
error!(
log,
"Multiple block proposers for this slot";
"action" => "producing blocks for all proposers",
"num_proposers" => proposers.len(),
"slot" => slot.as_u64(),
action = "producing blocks for all proposers",
num_proposers = proposers.len(),
slot = slot.as_u64(),
"Multiple block proposers for this slot"
)
}
for validator_pubkey in proposers {
let builder_boost_factor = self.get_builder_boost_factor(&validator_pubkey);
let service = self.clone();
let log = log.clone();
self.inner.context.executor.spawn(
async move {
let result = service
@@ -308,11 +297,10 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
Ok(_) => {}
Err(BlockError::Recoverable(e)) | Err(BlockError::Irrecoverable(e)) => {
error!(
log,
"Error whilst producing block";
"error" => ?e,
"block_slot" => ?slot,
"info" => "block v3 proposal failed, this error may or may not result in a missed block"
error = ?e,
block_slot = ?slot,
info = "block v3 proposal failed, this error may or may not result in a missed block",
"Error whilst producing block"
);
}
}
@@ -332,7 +320,6 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
validator_pubkey: &PublicKeyBytes,
unsigned_block: UnsignedBlock<E>,
) -> Result<(), BlockError> {
let log = self.context.log();
let signing_timer = validator_metrics::start_timer(&validator_metrics::BLOCK_SIGNING_TIMES);
let res = match unsigned_block {
@@ -357,11 +344,10 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
// A pubkey can be missing when a validator was recently removed
// via the API.
warn!(
log,
"Missing pubkey for block";
"info" => "a validator may have recently been removed from this VC",
"pubkey" => ?pubkey,
"slot" => ?slot
info = "a validator may have recently been removed from this VC",
?pubkey,
?slot,
"Missing pubkey for block"
);
return Ok(());
}
@@ -377,10 +363,9 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
Duration::from_secs_f64(signing_timer.map_or(0.0, |t| t.stop_and_record())).as_millis();
info!(
log,
"Publishing signed block";
"slot" => slot.as_u64(),
"signing_time_ms" => signing_time_ms,
slot = slot.as_u64(),
signing_time_ms = signing_time_ms,
"Publishing signed block"
);
// Publish block with first available beacon node.
@@ -396,13 +381,12 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
.await?;
info!(
log,
"Successfully published block";
"block_type" => ?signed_block.block_type(),
"deposits" => signed_block.num_deposits(),
"attestations" => signed_block.num_attestations(),
"graffiti" => ?graffiti.map(|g| g.as_utf8_lossy()),
"slot" => signed_block.slot().as_u64(),
block_type = ?signed_block.block_type(),
deposits = signed_block.num_deposits(),
attestations = signed_block.num_attestations(),
graffiti = ?graffiti.map(|g| g.as_utf8_lossy()),
slot = signed_block.slot().as_u64(),
"Successfully published block"
);
Ok(())
}
@@ -413,7 +397,6 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
validator_pubkey: PublicKeyBytes,
builder_boost_factor: Option<u64>,
) -> Result<(), BlockError> {
let log = self.context.log();
let _timer = validator_metrics::start_timer_vec(
&validator_metrics::BLOCK_SERVICE_TIMES,
&[validator_metrics::BEACON_BLOCK],
@@ -429,11 +412,10 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
// A pubkey can be missing when a validator was recently removed
// via the API.
warn!(
log,
"Missing pubkey for block randao";
"info" => "a validator may have recently been removed from this VC",
"pubkey" => ?pubkey,
"slot" => ?slot
info = "a validator may have recently been removed from this VC",
?pubkey,
?slot,
"Missing pubkey for block randao"
);
return Ok(());
}
@@ -447,7 +429,6 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
let graffiti = determine_graffiti(
&validator_pubkey,
log,
self.graffiti_file.clone(),
self.validator_store.graffiti(&validator_pubkey),
self.graffiti,
@@ -461,11 +442,7 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
proposer_nodes: self.proposer_nodes.clone(),
};
info!(
log,
"Requesting unsigned block";
"slot" => slot.as_u64(),
);
info!(slot = slot.as_u64(), "Requesting unsigned block");
// Request block from first responsive beacon node.
//
@@ -484,7 +461,6 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
graffiti,
proposer_index,
builder_boost_factor,
log,
)
.await
.map_err(|e| {
@@ -514,7 +490,6 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
signed_block: &SignedBlock<E>,
beacon_node: BeaconNodeHttpClient,
) -> Result<(), BlockError> {
let log = self.context.log();
let slot = signed_block.slot();
match signed_block {
SignedBlock::Full(signed_block) => {
@@ -525,7 +500,7 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
beacon_node
.post_beacon_blocks_v2_ssz(signed_block, None)
.await
.or_else(|e| handle_block_post_error(e, slot, log))?
.or_else(|e| handle_block_post_error(e, slot))?
}
SignedBlock::Blinded(signed_block) => {
let _post_timer = validator_metrics::start_timer_vec(
@@ -535,7 +510,7 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
beacon_node
.post_beacon_blinded_blocks_v2_ssz(signed_block, None)
.await
.or_else(|e| handle_block_post_error(e, slot, log))?
.or_else(|e| handle_block_post_error(e, slot))?
}
}
Ok::<_, BlockError>(())
@@ -548,7 +523,6 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
graffiti: Option<Graffiti>,
proposer_index: Option<u64>,
builder_boost_factor: Option<u64>,
log: &Logger,
) -> Result<UnsignedBlock<E>, BlockError> {
let (block_response, _) = beacon_node
.get_validator_blocks_v3::<E>(
@@ -570,11 +544,7 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
eth2::types::ProduceBlockV3Response::Blinded(block) => UnsignedBlock::Blinded(block),
};
info!(
log,
"Received unsigned block";
"slot" => slot.as_u64(),
);
info!(slot = slot.as_u64(), "Received unsigned block");
if proposer_index != Some(unsigned_block.proposer_index()) {
return Err(BlockError::Recoverable(
"Proposer index does not match block proposer. Beacon chain re-orged".to_string(),
@@ -662,23 +632,21 @@ impl<E: EthSpec> SignedBlock<E> {
}
}
fn handle_block_post_error(err: eth2::Error, slot: Slot, log: &Logger) -> Result<(), BlockError> {
fn handle_block_post_error(err: eth2::Error, slot: Slot) -> Result<(), BlockError> {
// Handle non-200 success codes.
if let Some(status) = err.status() {
if status == StatusCode::ACCEPTED {
info!(
log,
"Block is already known to BN or might be invalid";
"slot" => slot,
"status_code" => status.as_u16(),
%slot,
status_code = status.as_u16(),
"Block is already known to BN or might be invalid"
);
return Ok(());
} else if status.is_success() {
debug!(
log,
"Block published with non-standard success code";
"slot" => slot,
"status_code" => status.as_u16(),
%slot,
status_code = status.as_u16(),
"Block published with non-standard success code"
);
return Ok(());
}

180
validator_client/validator_services/src/duties_service.rs
View File

@@ -18,7 +18,6 @@ use eth2::types::{
use futures::{stream, StreamExt};
use parking_lot::RwLock;
use safe_arith::{ArithError, SafeArith};
use slog::{debug, error, info, warn, Logger};
use slot_clock::SlotClock;
use std::cmp::min;
use std::collections::{hash_map, BTreeMap, HashMap, HashSet};
@@ -26,6 +25,7 @@ use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::time::Duration;
use tokio::{sync::mpsc::Sender, time::sleep};
use tracing::{debug, error, info, warn};
use types::{ChainSpec, Epoch, EthSpec, Hash256, PublicKeyBytes, SelectionProof, Slot};
use validator_metrics::{get_int_gauge, set_int_gauge, ATTESTATION_DUTY};
use validator_store::{Error as ValidatorStoreError, ValidatorStore};
@@ -378,7 +378,6 @@ pub fn start_update_service<T: SlotClock + 'static, E: EthSpec>(
* Spawn the task which keeps track of local block proposal duties.
*/
let duties_service = core_duties_service.clone();
let log = core_duties_service.context.log().clone();
core_duties_service.context.executor.spawn(
async move {
loop {
@@ -394,9 +393,8 @@ pub fn start_update_service<T: SlotClock + 'static, E: EthSpec>(
if let Err(e) = poll_beacon_proposers(&duties_service, &mut block_service_tx).await
{
error!(
log,
"Failed to poll beacon proposers";
"error" => ?e
error = ?e,
"Failed to poll beacon proposers"
)
}
}
@@ -413,7 +411,6 @@ pub fn start_update_service<T: SlotClock + 'static, E: EthSpec>(
* Spawn the task which keeps track of local attestation duties.
*/
let duties_service = core_duties_service.clone();
let log = core_duties_service.context.log().clone();
core_duties_service.context.executor.spawn(
async move {
loop {
@@ -428,9 +425,8 @@ pub fn start_update_service<T: SlotClock + 'static, E: EthSpec>(
if let Err(e) = poll_beacon_attesters(&duties_service).await {
error!(
log,
"Failed to poll beacon attesters";
"error" => ?e
error = ?e,
"Failed to poll beacon attesters"
);
}
}
@@ -440,15 +436,13 @@ pub fn start_update_service<T: SlotClock + 'static, E: EthSpec>(
// Spawn the task which keeps track of local sync committee duties.
let duties_service = core_duties_service.clone();
let log = core_duties_service.context.log().clone();
core_duties_service.context.executor.spawn(
async move {
loop {
if let Err(e) = poll_sync_committee_duties(&duties_service).await {
error!(
log,
"Failed to poll sync committee duties";
"error" => ?e
error = ?e,
"Failed to poll sync committee duties"
);
}
@@ -480,8 +474,6 @@ async fn poll_validator_indices<T: SlotClock + 'static, E: EthSpec>(
&[validator_metrics::UPDATE_INDICES],
);
let log = duties_service.context.log();
// Collect *all* pubkeys for resolving indices, even those undergoing doppelganger protection.
//
// Since doppelganger protection queries rely on validator indices it is important to ensure we
@@ -547,11 +539,10 @@ async fn poll_validator_indices<T: SlotClock + 'static, E: EthSpec>(
match download_result {
Ok(Some(response)) => {
info!(
log,
"Validator exists in beacon chain";
"pubkey" => ?pubkey,
"validator_index" => response.data.index,
"fee_recipient" => fee_recipient
?pubkey,
validator_index = response.data.index,
fee_recipient,
"Validator exists in beacon chain"
);
duties_service
.validator_store
@@ -575,21 +566,15 @@ async fn poll_validator_indices<T: SlotClock + 'static, E: EthSpec>(
.insert(pubkey, next_poll_slot);
}
debug!(
log,
"Validator without index";
"pubkey" => ?pubkey,
"fee_recipient" => fee_recipient
)
debug!(?pubkey, fee_recipient, "Validator without index")
}
// Don't exit early on an error, keep attempting to resolve other indices.
Err(e) => {
error!(
log,
"Failed to resolve pubkey to index";
"error" => %e,
"pubkey" => ?pubkey,
"fee_recipient" => fee_recipient
error = %e,
?pubkey,
fee_recipient,
"Failed to resolve pubkey to index"
)
}
}
@@ -613,8 +598,6 @@ async fn poll_beacon_attesters<T: SlotClock + 'static, E: EthSpec>(
&[validator_metrics::UPDATE_ATTESTERS_CURRENT_EPOCH],
);
let log = duties_service.context.log();
let current_slot = duties_service
.slot_clock
.now()
@@ -653,11 +636,10 @@ async fn poll_beacon_attesters<T: SlotClock + 'static, E: EthSpec>(
.await
{
error!(
log,
"Failed to download attester duties";
"current_epoch" => current_epoch,
"request_epoch" => current_epoch,
"err" => ?e,
%current_epoch,
request_epoch = %current_epoch,
err = ?e,
"Failed to download attester duties"
)
}
@@ -675,11 +657,10 @@ async fn poll_beacon_attesters<T: SlotClock + 'static, E: EthSpec>(
.await
{
error!(
log,
"Failed to download attester duties";
"current_epoch" => current_epoch,
"request_epoch" => next_epoch,
"err" => ?e,
%current_epoch,
request_epoch = %next_epoch,
err = ?e,
"Failed to download attester duties"
)
}
@@ -758,9 +739,8 @@ async fn poll_beacon_attesters<T: SlotClock + 'static, E: EthSpec>(
.await;
if subscription_result.as_ref().is_ok() {
debug!(
log,
"Broadcast attestation subscriptions";
"count" => subscriptions.len(),
count = subscriptions.len(),
"Broadcast attestation subscriptions"
);
for subscription_slots in subscription_slots_to_confirm {
subscription_slots.record_successful_subscription_at(current_slot);
@@ -768,9 +748,8 @@ async fn poll_beacon_attesters<T: SlotClock + 'static, E: EthSpec>(
} else if let Err(e) = subscription_result {
if e.num_errors() < duties_service.beacon_nodes.num_total().await {
warn!(
log,
"Some subscriptions failed";
"error" => %e,
error = %e,
"Some subscriptions failed"
);
// If subscriptions were sent to at least one node, regard that as a success.
// There is some redundancy built into the subscription schedule to handle failures.
@@ -779,9 +758,8 @@ async fn poll_beacon_attesters<T: SlotClock + 'static, E: EthSpec>(
}
} else {
error!(
log,
"All subscriptions failed";
"error" => %e
error = %e,
"All subscriptions failed"
);
}
}
@@ -809,14 +787,11 @@ async fn poll_beacon_attesters_for_epoch<T: SlotClock + 'static, E: EthSpec>(
local_indices: &[u64],
local_pubkeys: &HashSet<PublicKeyBytes>,
) -> Result<(), Error> {
let log = duties_service.context.log();
// No need to bother the BN if we don't have any validators.
if local_indices.is_empty() {
debug!(
duties_service.context.log(),
"No validators, not downloading duties";
"epoch" => epoch,
%epoch,
"No validators, not downloading duties"
);
return Ok(());
}
@@ -895,10 +870,9 @@ async fn poll_beacon_attesters_for_epoch<T: SlotClock + 'static, E: EthSpec>(
);
debug!(
log,
"Downloaded attester duties";
"dependent_root" => %dependent_root,
"num_new_duties" => new_duties.len(),
%dependent_root,
num_new_duties = new_duties.len(),
"Downloaded attester duties"
);
// Update the duties service with the new `DutyAndProof` messages.
@@ -929,10 +903,9 @@ async fn poll_beacon_attesters_for_epoch<T: SlotClock + 'static, E: EthSpec>(
&& prior_duty_and_proof.duty == duty_and_proof.duty
{
warn!(
log,
"Redundant attester duty update";
"dependent_root" => %dependent_root,
"validator_index" => duty.validator_index,
%dependent_root,
validator_index = duty.validator_index,
"Redundant attester duty update"
);
continue;
}
@@ -940,11 +913,10 @@ async fn poll_beacon_attesters_for_epoch<T: SlotClock + 'static, E: EthSpec>(
// Using `already_warned` avoids excessive logs.
if dependent_root != *prior_dependent_root && already_warned.take().is_some() {
warn!(
log,
"Attester duties re-org";
"prior_dependent_root" => %prior_dependent_root,
"dependent_root" => %dependent_root,
"note" => "this may happen from time to time"
%prior_dependent_root,
%dependent_root,
note = "this may happen from time to time",
"Attester duties re-org"
)
}
*mut_value = (dependent_root, duty_and_proof);
@@ -1056,8 +1028,6 @@ async fn fill_in_selection_proofs<T: SlotClock + 'static, E: EthSpec>(
duties: Vec<AttesterData>,
dependent_root: Hash256,
) {
let log = duties_service.context.log();
// Sort duties by slot in a BTreeMap.
let mut duties_by_slot: BTreeMap<Slot, Vec<_>> = BTreeMap::new();
@@ -1125,20 +1095,18 @@ async fn fill_in_selection_proofs<T: SlotClock + 'static, E: EthSpec>(
// A pubkey can be missing when a validator was recently
// removed via the API.
warn!(
log,
"Missing pubkey for duty and proof";
"info" => "a validator may have recently been removed from this VC",
"pubkey" => ?pubkey,
info = "a validator may have recently been removed from this VC",
?pubkey,
"Missing pubkey for duty and proof"
);
// Do not abort the entire batch for a single failure.
continue;
}
Err(e) => {
error!(
log,
"Failed to produce duty and proof";
"error" => ?e,
"msg" => "may impair attestation duties"
error = ?e,
msg = "may impair attestation duties",
"Failed to produce duty and proof"
);
// Do not abort the entire batch for a single failure.
continue;
@@ -1163,9 +1131,8 @@ async fn fill_in_selection_proofs<T: SlotClock + 'static, E: EthSpec>(
// Our selection proofs are no longer relevant due to a reorg, abandon
// this entire background process.
debug!(
log,
"Stopping selection proof background task";
"reason" => "re-org"
reason = "re-org",
"Stopping selection proof background task"
);
return;
}
@@ -1188,11 +1155,10 @@ async fn fill_in_selection_proofs<T: SlotClock + 'static, E: EthSpec>(
let time_taken_ms =
Duration::from_secs_f64(timer.map_or(0.0, |t| t.stop_and_record())).as_millis();
debug!(
log,
"Computed attestation selection proofs";
"batch_size" => batch_size,
"lookahead_slot" => lookahead_slot,
"time_taken_ms" => time_taken_ms
batch_size,
%lookahead_slot,
time_taken_ms,
"Computed attestation selection proofs"
);
} else {
// Just sleep for one slot if we are unable to read the system clock, this gives
@@ -1234,8 +1200,6 @@ async fn poll_beacon_proposers<T: SlotClock + 'static, E: EthSpec>(
&[validator_metrics::UPDATE_PROPOSERS],
);
let log = duties_service.context.log();
let current_slot = duties_service
.slot_clock
.now()
@@ -1251,7 +1215,6 @@ async fn poll_beacon_proposers<T: SlotClock + 'static, E: EthSpec>(
&initial_block_proposers,
block_service_tx,
&duties_service.validator_store,
log,
)
.await;
@@ -1290,10 +1253,9 @@ async fn poll_beacon_proposers<T: SlotClock + 'static, E: EthSpec>(
.collect::<Vec<_>>();
debug!(
log,
"Downloaded proposer duties";
"dependent_root" => %dependent_root,
"num_relevant_duties" => relevant_duties.len(),
%dependent_root,
num_relevant_duties = relevant_duties.len(),
"Downloaded proposer duties"
);
if let Some((prior_dependent_root, _)) = duties_service
@@ -1303,20 +1265,18 @@ async fn poll_beacon_proposers<T: SlotClock + 'static, E: EthSpec>(
{
if dependent_root != prior_dependent_root {
warn!(
log,
"Proposer duties re-org";
"prior_dependent_root" => %prior_dependent_root,
"dependent_root" => %dependent_root,
"msg" => "this may happen from time to time"
%prior_dependent_root,
%dependent_root,
msg = "this may happen from time to time",
"Proposer duties re-org"
)
}
}
}
// Don't return early here, we still want to try and produce blocks using the cached values.
Err(e) => error!(
log,
"Failed to download proposer duties";
"err" => %e,
err = %e,
"Failed to download proposer duties"
),
}
@@ -1341,13 +1301,11 @@ async fn poll_beacon_proposers<T: SlotClock + 'static, E: EthSpec>(
&additional_block_producers,
block_service_tx,
&duties_service.validator_store,
log,
)
.await;
debug!(
log,
"Detected new block proposer";
"current_slot" => current_slot,
%current_slot,
"Detected new block proposer"
);
validator_metrics::inc_counter(&validator_metrics::PROPOSAL_CHANGED);
}
@@ -1368,7 +1326,6 @@ async fn notify_block_production_service<T: SlotClock + 'static, E: EthSpec>(
block_proposers: &HashSet<PublicKeyBytes>,
block_service_tx: &mut Sender<BlockServiceNotification>,
validator_store: &ValidatorStore<T, E>,
log: &Logger,
) {
let non_doppelganger_proposers = block_proposers
.iter()
@@ -1385,10 +1342,9 @@ async fn notify_block_production_service<T: SlotClock + 'static, E: EthSpec>(
.await
{
error!(
log,
"Failed to notify block service";
"current_slot" => current_slot,
"error" => %e
%current_slot,
error = %e,
"Failed to notify block service"
);
};
}

72
validator_client/validator_services/src/preparation_service.rs
View File

@@ -3,7 +3,6 @@ use bls::PublicKeyBytes;
use doppelganger_service::DoppelgangerStatus;
use environment::RuntimeContext;
use parking_lot::RwLock;
use slog::{debug, error, info, warn};
use slot_clock::SlotClock;
use std::collections::HashMap;
use std::hash::Hash;
@@ -11,6 +10,7 @@ use std::ops::Deref;
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use tokio::time::{sleep, Duration};
use tracing::{debug, error, info, warn};
use types::{
Address, ChainSpec, EthSpec, ProposerPreparationData, SignedValidatorRegistrationData,
ValidatorRegistrationData,
@@ -173,13 +173,8 @@ impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
/// Starts the service which periodically produces proposer preparations.
pub fn start_proposer_prepare_service(self, spec: &ChainSpec) -> Result<(), String> {
let log = self.context.log().clone();
let slot_duration = Duration::from_secs(spec.seconds_per_slot);
info!(
log,
"Proposer preparation service started";
);
info!("Proposer preparation service started");
let executor = self.context.executor.clone();
let spec = spec.clone();
@@ -192,9 +187,8 @@ impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
.await
.map_err(|e| {
error!(
log,
"Error during proposer preparation";
"error" => ?e,
error = ?e,
"Error during proposer preparation"
)
})
.unwrap_or(());
@@ -203,7 +197,7 @@ impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
if let Some(duration_to_next_slot) = self.slot_clock.duration_to_next_slot() {
sleep(duration_to_next_slot).await;
} else {
error!(log, "Failed to read slot clock");
error!("Failed to read slot clock");
// If we can't read the slot clock, just wait another slot.
sleep(slot_duration).await;
}
@@ -216,12 +210,7 @@ impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
/// Starts the service which periodically sends connected beacon nodes validator registration information.
pub fn start_validator_registration_service(self, spec: &ChainSpec) -> Result<(), String> {
let log = self.context.log().clone();
info!(
log,
"Validator registration service started";
);
info!("Validator registration service started");
let spec = spec.clone();
let slot_duration = Duration::from_secs(spec.seconds_per_slot);
@@ -232,14 +221,14 @@ impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
loop {
// Poll the endpoint immediately to ensure fee recipients are received.
if let Err(e) = self.register_validators().await {
error!(log,"Error during validator registration";"error" => ?e);
error!(error = ?e, "Error during validator registration");
}
// Wait one slot if the register validator request fails or if we should not publish at the current slot.
if let Some(duration_to_next_slot) = self.slot_clock.duration_to_next_slot() {
sleep(duration_to_next_slot).await;
} else {
error!(log, "Failed to read slot clock");
error!("Failed to read slot clock");
// If we can't read the slot clock, just wait another slot.
sleep(slot_duration).await;
}
@@ -274,7 +263,6 @@ impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
}
fn collect_preparation_data(&self, spec: &ChainSpec) -> Vec<ProposerPreparationData> {
let log = self.context.log();
self.collect_proposal_data(|pubkey, proposal_data| {
if let Some(fee_recipient) = proposal_data.fee_recipient {
Some(ProposerPreparationData {
@@ -285,10 +273,9 @@ impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
} else {
if spec.bellatrix_fork_epoch.is_some() {
error!(
log,
"Validator is missing fee recipient";
"msg" => "update validator_definitions.yml",
"pubkey" => ?pubkey
msg = "update validator_definitions.yml",
?pubkey,
"Validator is missing fee recipient"
);
}
None
@@ -336,8 +323,6 @@ impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
&self,
preparation_data: Vec<ProposerPreparationData>,
) -> Result<(), String> {
let log = self.context.log();
// Post the proposer preparations to the BN.
let preparation_data_len = preparation_data.len();
let preparation_entries = preparation_data.as_slice();
@@ -351,14 +336,12 @@ impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
.await
{
Ok(()) => debug!(
log,
"Published proposer preparation";
"count" => preparation_data_len,
count = preparation_data_len,
"Published proposer preparation"
),
Err(e) => error!(
log,
"Unable to publish proposer preparation to all beacon nodes";
"error" => %e,
error = %e,
"Unable to publish proposer preparation to all beacon nodes"
),
}
Ok(())
@@ -400,8 +383,6 @@ impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
&self,
registration_keys: Vec<ValidatorRegistrationKey>,
) -> Result<(), String> {
let log = self.context.log();
let registration_data_len = registration_keys.len();
let mut signed = Vec::with_capacity(registration_data_len);
@@ -442,19 +423,14 @@ impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
// A pubkey can be missing when a validator was recently
// removed via the API.
debug!(
log,
"Missing pubkey for registration data";
"pubkey" => ?pubkey,
);
debug!(?pubkey, "Missing pubkey for registration data");
continue;
}
Err(e) => {
error!(
log,
"Unable to sign validator registration data";
"error" => ?e,
"pubkey" => ?pubkey
error = ?e,
?pubkey,
"Unable to sign validator registration data"
);
continue;
}
@@ -474,9 +450,8 @@ impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
{
Ok(()) => {
info!(
log,
"Published validator registrations to the builder network";
"count" => batch.len(),
count = batch.len(),
"Published validator registrations to the builder network"
);
let mut guard = self.validator_registration_cache.write();
for signed_data in batch {
@@ -487,9 +462,8 @@ impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
}
}
Err(e) => warn!(
log,
"Unable to publish validator registrations to the builder network";
"error" => %e,
error = %e,
"Unable to publish validator registrations to the builder network"
),
}
}

108
validator_client/validator_services/src/sync.rs
View File

@@ -1,12 +1,13 @@
use crate::duties_service::{DutiesService, Error};
use doppelganger_service::DoppelgangerStatus;
use futures::future::join_all;
use logging::crit;
use parking_lot::{MappedRwLockReadGuard, RwLock, RwLockReadGuard, RwLockWriteGuard};
use slog::{crit, debug, info, warn};
use slot_clock::SlotClock;
use std::collections::{HashMap, HashSet};
use std::marker::PhantomData;
use std::sync::Arc;
use tracing::{debug, info, warn};
use types::{ChainSpec, EthSpec, PublicKeyBytes, Slot, SyncDuty, SyncSelectionProof, SyncSubnetId};
use validator_store::Error as ValidatorStoreError;
@@ -414,23 +415,20 @@ pub async fn poll_sync_committee_duties_for_period<T: SlotClock + 'static, E: Et
sync_committee_period: u64,
) -> Result<(), Error> {
let spec = &duties_service.spec;
let log = duties_service.context.log();
// no local validators don't need to poll for sync committee
if local_indices.is_empty() {
debug!(
duties_service.context.log(),
"No validators, not polling for sync committee duties";
"sync_committee_period" => sync_committee_period,
sync_committee_period,
"No validators, not polling for sync committee duties"
);
return Ok(());
}
debug!(
log,
"Fetching sync committee duties";
"sync_committee_period" => sync_committee_period,
"num_validators" => local_indices.len(),
sync_committee_period,
num_validators = local_indices.len(),
"Fetching sync committee duties"
);
let period_start_epoch = spec.epochs_per_sync_committee_period * sync_committee_period;
@@ -452,16 +450,15 @@ pub async fn poll_sync_committee_duties_for_period<T: SlotClock + 'static, E: Et
Ok(res) => res.data,
Err(e) => {
warn!(
log,
"Failed to download sync committee duties";
"sync_committee_period" => sync_committee_period,
"error" => %e,
sync_committee_period,
error = %e,
"Failed to download sync committee duties"
);
return Ok(());
}
};
debug!(log, "Fetched sync duties from BN"; "count" => duties.len());
debug!(count = duties.len(), "Fetched sync duties from BN");
// Add duties to map.
let committee_duties = duties_service
@@ -479,9 +476,8 @@ pub async fn poll_sync_committee_duties_for_period<T: SlotClock + 'static, E: Et
!= duty.validator_sync_committee_indices;
if updated_due_to_reorg {
warn!(
log,
"Sync committee duties changed";
"message" => "this could be due to a really long re-org, or a bug"
message = "this could be due to a really long re-org, or a bug",
"Sync committee duties changed"
);
}
updated_due_to_reorg
@@ -489,10 +485,8 @@ pub async fn poll_sync_committee_duties_for_period<T: SlotClock + 'static, E: Et
if updated {
info!(
log,
"Validator in sync committee";
"validator_index" => duty.validator_index,
"sync_committee_period" => sync_committee_period,
validator_index = duty.validator_index,
sync_committee_period, "Validator in sync committee"
);
*validator_duties = Some(ValidatorDuties::new(duty));
@@ -509,14 +503,11 @@ pub async fn fill_in_aggregation_proofs<T: SlotClock + 'static, E: EthSpec>(
current_slot: Slot,
pre_compute_slot: Slot,
) {
let log = duties_service.context.log();
debug!(
log,
"Calculating sync selection proofs";
"period" => sync_committee_period,
"current_slot" => current_slot,
"pre_compute_slot" => pre_compute_slot
period = sync_committee_period,
%current_slot,
%pre_compute_slot,
"Calculating sync selection proofs"
);
// Generate selection proofs for each validator at each slot, one slot at a time.
@@ -532,9 +523,8 @@ pub async fn fill_in_aggregation_proofs<T: SlotClock + 'static, E: EthSpec>(
Ok(subnet_ids) => subnet_ids,
Err(e) => {
crit!(
log,
"Arithmetic error computing subnet IDs";
"error" => ?e,
error = ?e,
"Arithmetic error computing subnet IDs"
);
continue;
}
@@ -556,21 +546,19 @@ pub async fn fill_in_aggregation_proofs<T: SlotClock + 'static, E: EthSpec>(
// A pubkey can be missing when a validator was recently
// removed via the API.
debug!(
log,
"Missing pubkey for sync selection proof";
"pubkey" => ?pubkey,
"pubkey" => ?duty.pubkey,
"slot" => proof_slot,
?pubkey,
pubkey = ?duty.pubkey,
slot = %proof_slot,
"Missing pubkey for sync selection proof"
);
return None;
}
Err(e) => {
warn!(
log,
"Unable to sign selection proof";
"error" => ?e,
"pubkey" => ?duty.pubkey,
"slot" => proof_slot,
error = ?e,
pubkey = ?duty.pubkey,
slot = %proof_slot,
"Unable to sign selection proof"
);
return None;
}
@@ -579,22 +567,20 @@ pub async fn fill_in_aggregation_proofs<T: SlotClock + 'static, E: EthSpec>(
match proof.is_aggregator::<E>() {
Ok(true) => {
debug!(
log,
"Validator is sync aggregator";
"validator_index" => duty.validator_index,
"slot" => proof_slot,
"subnet_id" => %subnet_id,
validator_index = duty.validator_index,
slot = %proof_slot,
%subnet_id,
"Validator is sync aggregator"
);
Some(((proof_slot, *subnet_id), proof))
}
Ok(false) => None,
Err(e) => {
warn!(
log,
"Error determining is_aggregator";
"pubkey" => ?duty.pubkey,
"slot" => proof_slot,
"error" => ?e,
pubkey = ?duty.pubkey,
slot = %proof_slot,
error = ?e,
"Error determining is_aggregator"
);
None
}
@@ -614,11 +600,7 @@ pub async fn fill_in_aggregation_proofs<T: SlotClock + 'static, E: EthSpec>(
// Add to global storage (we add regularly so the proofs can be used ASAP).
let sync_map = duties_service.sync_duties.committees.read();
let Some(committee_duties) = sync_map.get(&sync_committee_period) else {
debug!(
log,
"Missing sync duties";
"period" => sync_committee_period,
);
debug!(period = sync_committee_period, "Missing sync duties");
continue;
};
let validators = committee_duties.validators.read();
@@ -629,20 +611,18 @@ pub async fn fill_in_aggregation_proofs<T: SlotClock + 'static, E: EthSpec>(
duty.aggregation_duties.proofs.write().extend(proofs);
} else {
debug!(
log,
"Missing sync duty to update";
"validator_index" => validator_index,
"period" => sync_committee_period,
validator_index,
period = sync_committee_period,
"Missing sync duty to update"
);
}
}
if num_validators_updated > 0 {
debug!(
log,
"Finished computing sync selection proofs";
"slot" => slot,
"updated_validators" => num_validators_updated,
%slot,
updated_validators = num_validators_updated,
"Finished computing sync selection proofs"
);
}
}

173
validator_client/validator_services/src/sync_committee_service.rs
View File

@@ -4,13 +4,14 @@ use environment::RuntimeContext;
use eth2::types::BlockId;
use futures::future::join_all;
use futures::future::FutureExt;
use slog::{crit, debug, error, info, trace, warn};
use logging::crit;
use slot_clock::SlotClock;
use std::collections::HashMap;
use std::ops::Deref;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use tokio::time::{sleep, sleep_until, Duration, Instant};
use tracing::{debug, error, info, trace, warn};
use types::{
ChainSpec, EthSpec, Hash256, PublicKeyBytes, Slot, SyncCommitteeSubscription,
SyncContributionData, SyncDuty, SyncSelectionProof, SyncSubnetId,
@@ -86,9 +87,8 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
}
pub fn start_update_service(self, spec: &ChainSpec) -> Result<(), String> {
let log = self.context.log().clone();
if self.duties_service.disable_attesting {
info!(log, "Sync committee service disabled");
info!("Sync committee service disabled");
return Ok(());
}
@@ -99,9 +99,8 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
.ok_or("Unable to determine duration to next slot")?;
info!(
log,
"Sync committee service started";
"next_update_millis" => duration_to_next_slot.as_millis()
next_update_millis = duration_to_next_slot.as_millis(),
"Sync committee service started"
);
let executor = self.context.executor.clone();
@@ -110,7 +109,6 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
loop {
if let Some(duration_to_next_slot) = self.slot_clock.duration_to_next_slot() {
// Wait for contribution broadcast interval 1/3 of the way through the slot.
let log = self.context.log();
sleep(duration_to_next_slot + slot_duration / 3).await;
// Do nothing if the Altair fork has not yet occurred.
@@ -120,21 +118,17 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
if let Err(e) = self.spawn_contribution_tasks(slot_duration).await {
crit!(
log,
"Failed to spawn sync contribution tasks";
"error" => e
error = ?e,
"Failed to spawn sync contribution tasks"
)
} else {
trace!(
log,
"Spawned sync contribution tasks";
)
trace!("Spawned sync contribution tasks")
}
// Do subscriptions for future slots/epochs.
self.spawn_subscription_tasks();
} else {
error!(log, "Failed to read slot clock");
error!("Failed to read slot clock");
// If we can't read the slot clock, just wait another slot.
sleep(slot_duration).await;
}
@@ -146,7 +140,6 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
}
async fn spawn_contribution_tasks(&self, slot_duration: Duration) -> Result<(), String> {
let log = self.context.log().clone();
let slot = self.slot_clock.now().ok_or("Failed to read slot clock")?;
let duration_to_next_slot = self
.slot_clock
@@ -165,16 +158,12 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
.sync_duties
.get_duties_for_slot(slot, &self.duties_service.spec)
else {
debug!(log, "No duties known for slot {}", slot);
debug!("No duties known for slot {}", slot);
return Ok(());
};
if slot_duties.duties.is_empty() {
debug!(
log,
"No local validators in current sync committee";
"slot" => slot,
);
debug!(%slot, "No local validators in current sync committee");
return Ok(());
}
@@ -201,11 +190,10 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
Ok(block) => block.data.root,
Err(errs) => {
warn!(
log,
errors = errs.to_string(),
%slot,
"Refusing to sign sync committee messages for an optimistic head block or \
a block head with unknown optimistic status";
"errors" => errs.to_string(),
"slot" => slot,
a block head with unknown optimistic status"
);
return Ok(());
}
@@ -251,8 +239,6 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
beacon_block_root: Hash256,
validator_duties: Vec<SyncDuty>,
) -> Result<(), ()> {
let log = self.context.log();
// Create futures to produce sync committee signatures.
let signature_futures = validator_duties.iter().map(|duty| async move {
match self
@@ -270,21 +256,19 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
// A pubkey can be missing when a validator was recently
// removed via the API.
debug!(
log,
"Missing pubkey for sync committee signature";
"pubkey" => ?pubkey,
"validator_index" => duty.validator_index,
"slot" => slot,
?pubkey,
validator_index = duty.validator_index,
%slot,
"Missing pubkey for sync committee signature"
);
None
}
Err(e) => {
crit!(
log,
"Failed to sign sync committee signature";
"validator_index" => duty.validator_index,
"slot" => slot,
"error" => ?e,
validator_index = duty.validator_index,
%slot,
error = ?e,
"Failed to sign sync committee signature"
);
None
}
@@ -307,19 +291,17 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
.await
.map_err(|e| {
error!(
log,
"Unable to publish sync committee messages";
"slot" => slot,
"error" => %e,
%slot,
error = %e,
"Unable to publish sync committee messages"
);
})?;
info!(
log,
"Successfully published sync committee messages";
"count" => committee_signatures.len(),
"head_block" => ?beacon_block_root,
"slot" => slot,
count = committee_signatures.len(),
head_block = ?beacon_block_root,
%slot,
"Successfully published sync committee messages"
);
Ok(())
@@ -362,8 +344,6 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
) -> Result<(), ()> {
sleep_until(aggregate_instant).await;
let log = self.context.log();
let contribution = &self
.beacon_nodes
.first_success(|beacon_node| async move {
@@ -380,20 +360,14 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
.await
.map_err(|e| {
crit!(
log,
"Failed to produce sync contribution";
"slot" => slot,
"beacon_block_root" => ?beacon_block_root,
"error" => %e,
%slot,
?beacon_block_root,
error = %e,
"Failed to produce sync contribution"
)
})?
.ok_or_else(|| {
crit!(
log,
"No aggregate contribution found";
"slot" => slot,
"beacon_block_root" => ?beacon_block_root,
);
crit!(%slot, ?beacon_block_root, "No aggregate contribution found");
})?
.data;
@@ -414,20 +388,14 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
// A pubkey can be missing when a validator was recently
// removed via the API.
debug!(
log,
"Missing pubkey for sync contribution";
"pubkey" => ?pubkey,
"slot" => slot,
);
debug!(?pubkey, %slot, "Missing pubkey for sync contribution");
None
}
Err(e) => {
crit!(
log,
"Unable to sign sync committee contribution";
"slot" => slot,
"error" => ?e,
%slot,
error = ?e,
"Unable to sign sync committee contribution"
);
None
}
@@ -452,20 +420,18 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
.await
.map_err(|e| {
error!(
log,
"Unable to publish signed contributions and proofs";
"slot" => slot,
"error" => %e,
%slot,
error = %e,
"Unable to publish signed contributions and proofs"
);
})?;
info!(
log,
"Successfully published sync contributions";
"subnet" => %subnet_id,
"beacon_block_root" => %beacon_block_root,
"num_signers" => contribution.aggregation_bits.num_set_bits(),
"slot" => slot,
subnet = %subnet_id,
beacon_block_root = %beacon_block_root,
num_signers = contribution.aggregation_bits.num_set_bits(),
%slot,
"Successfully published sync contributions"
);
Ok(())
@@ -473,14 +439,13 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
fn spawn_subscription_tasks(&self) {
let service = self.clone();
let log = self.context.log().clone();
self.inner.context.executor.spawn(
async move {
service.publish_subscriptions().await.unwrap_or_else(|e| {
error!(
log,
"Error publishing subscriptions";
"error" => ?e,
error = ?e,
"Error publishing subscriptions"
)
});
},
@@ -489,7 +454,6 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
}
async fn publish_subscriptions(self) -> Result<(), String> {
let log = self.context.log().clone();
let spec = &self.duties_service.spec;
let slot = self.slot_clock.now().ok_or("Failed to read slot clock")?;
@@ -526,12 +490,7 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
let mut subscriptions = vec![];
for (duty_slot, sync_committee_period) in duty_slots {
debug!(
log,
"Fetching subscription duties";
"duty_slot" => duty_slot,
"current_slot" => slot,
);
debug!(%duty_slot, %slot, "Fetching subscription duties");
match self
.duties_service
.sync_duties
@@ -544,9 +503,8 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
)),
None => {
debug!(
log,
"No duties for subscription";
"slot" => duty_slot,
slot = %duty_slot,
"No duties for subscription"
);
all_succeeded = false;
}
@@ -554,29 +512,23 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
}
if subscriptions.is_empty() {
debug!(
log,
"No sync subscriptions to send";
"slot" => slot,
);
debug!(%slot, "No sync subscriptions to send");
return Ok(());
}
// Post subscriptions to BN.
debug!(
log,
"Posting sync subscriptions to BN";
"count" => subscriptions.len(),
count = subscriptions.len(),
"Posting sync subscriptions to BN"
);
let subscriptions_slice = &subscriptions;
for subscription in subscriptions_slice {
debug!(
log,
"Subscription";
"validator_index" => subscription.validator_index,
"validator_sync_committee_indices" => ?subscription.sync_committee_indices,
"until_epoch" => subscription.until_epoch,
validator_index = subscription.validator_index,
validator_sync_committee_indices = ?subscription.sync_committee_indices,
until_epoch = %subscription.until_epoch,
"Subscription"
);
}
@@ -590,10 +542,9 @@ impl<T: SlotClock + 'static, E: EthSpec> SyncCommitteeService<T, E> {
.await
{
error!(
log,
"Unable to post sync committee subscriptions";
"slot" => slot,
"error" => %e,
%slot,
error = %e,
"Unable to post sync committee subscriptions"
);
all_succeeded = false;
}

3
validator_client/validator_store/Cargo.toml
View File

@@ -12,12 +12,13 @@ path = "src/lib.rs"
account_utils = { workspace = true }
doppelganger_service = { workspace = true }
initialized_validators = { workspace = true }
logging = { workspace = true }
parking_lot = { workspace = true }
serde = { workspace = true }
signing_method = { workspace = true }
slashing_protection = { workspace = true }
slog = { workspace = true }
slot_clock = { workspace = true }
task_executor = { workspace = true }
tracing = { workspace = true }
types = { workspace = true }
validator_metrics = { workspace = true }

71
validator_client/validator_store/src/lib.rs
View File

@@ -1,18 +1,19 @@
use account_utils::validator_definitions::{PasswordStorage, ValidatorDefinition};
use doppelganger_service::{DoppelgangerService, DoppelgangerStatus, DoppelgangerValidatorStore};
use initialized_validators::InitializedValidators;
use logging::crit;
use parking_lot::{Mutex, RwLock};
use serde::{Deserialize, Serialize};
use signing_method::{Error as SigningError, SignableMessage, SigningContext, SigningMethod};
use slashing_protection::{
interchange::Interchange, InterchangeError, NotSafe, Safe, SlashingDatabase,
};
use slog::{crit, error, info, warn, Logger};
use slot_clock::SlotClock;
use std::marker::PhantomData;
use std::path::Path;
use std::sync::Arc;
use task_executor::TaskExecutor;
use tracing::{error, info, warn};
use types::{
attestation::Error as AttestationError, graffiti::GraffitiString, AbstractExecPayload, Address,
AggregateAndProof, Attestation, BeaconBlock, BlindedPayload, ChainSpec, ContributionAndProof,
@@ -82,7 +83,6 @@ pub struct ValidatorStore<T, E: EthSpec> {
slashing_protection_last_prune: Arc<Mutex<Epoch>>,
genesis_validators_root: Hash256,
spec: Arc<ChainSpec>,
log: Logger,
doppelganger_service: Option<Arc<DoppelgangerService>>,
slot_clock: T,
fee_recipient_process: Option<Address>,
@@ -114,7 +114,6 @@ impl<T: SlotClock + 'static, E: EthSpec> ValidatorStore<T, E> {
slot_clock: T,
config: &Config,
task_executor: TaskExecutor,
log: Logger,
) -> Self {
Self {
validators: Arc::new(RwLock::new(validators)),
@@ -122,7 +121,6 @@ impl<T: SlotClock + 'static, E: EthSpec> ValidatorStore<T, E> {
slashing_protection_last_prune: Arc::new(Mutex::new(Epoch::new(0))),
genesis_validators_root,
spec,
log,
doppelganger_service,
slot_clock,
fee_recipient_process: config.fee_recipient,
@@ -581,10 +579,9 @@ impl<T: SlotClock + 'static, E: EthSpec> ValidatorStore<T, E> {
// Make sure the block slot is not higher than the current slot to avoid potential attacks.
if block.slot() > current_slot {
warn!(
self.log,
"Not signing block with slot greater than current slot";
"block_slot" => block.slot().as_u64(),
"current_slot" => current_slot.as_u64()
block_slot = block.slot().as_u64(),
current_slot = current_slot.as_u64(),
"Not signing block with slot greater than current slot"
);
return Err(Error::GreaterThanCurrentSlot {
slot: block.slot(),
@@ -630,10 +627,7 @@ impl<T: SlotClock + 'static, E: EthSpec> ValidatorStore<T, E> {
Ok(SignedBeaconBlock::from_block(block, signature))
}
Ok(Safe::SameData) => {
warn!(
self.log,
"Skipping signing of previously signed block";
);
warn!("Skipping signing of previously signed block");
validator_metrics::inc_counter_vec(
&validator_metrics::SIGNED_BLOCKS_TOTAL,
&[validator_metrics::SAME_DATA],
@@ -642,10 +636,9 @@ impl<T: SlotClock + 'static, E: EthSpec> ValidatorStore<T, E> {
}
Err(NotSafe::UnregisteredValidator(pk)) => {
warn!(
self.log,
"Not signing block for unregistered validator";
"msg" => "Carefully consider running with --init-slashing-protection (see --help)",
"public_key" => format!("{:?}", pk)
msg = "Carefully consider running with --init-slashing-protection (see --help)",
public_key = format!("{:?}", pk),
"Not signing block for unregistered validator"
);
validator_metrics::inc_counter_vec(
&validator_metrics::SIGNED_BLOCKS_TOTAL,
@@ -654,11 +647,7 @@ impl<T: SlotClock + 'static, E: EthSpec> ValidatorStore<T, E> {
Err(Error::Slashable(NotSafe::UnregisteredValidator(pk)))
}
Err(e) => {
crit!(
self.log,
"Not signing slashable block";
"error" => format!("{:?}", e)
);
crit!(error = format!("{:?}", e), "Not signing slashable block");
validator_metrics::inc_counter_vec(
&validator_metrics::SIGNED_BLOCKS_TOTAL,
&[validator_metrics::SLASHABLE],
@@ -725,10 +714,7 @@ impl<T: SlotClock + 'static, E: EthSpec> ValidatorStore<T, E> {
Ok(())
}
Ok(Safe::SameData) => {
warn!(
self.log,
"Skipping signing of previously signed attestation"
);
warn!("Skipping signing of previously signed attestation");
validator_metrics::inc_counter_vec(
&validator_metrics::SIGNED_ATTESTATIONS_TOTAL,
&[validator_metrics::SAME_DATA],
@@ -737,10 +723,9 @@ impl<T: SlotClock + 'static, E: EthSpec> ValidatorStore<T, E> {
}
Err(NotSafe::UnregisteredValidator(pk)) => {
warn!(
self.log,
"Not signing attestation for unregistered validator";
"msg" => "Carefully consider running with --init-slashing-protection (see --help)",
"public_key" => format!("{:?}", pk)
msg = "Carefully consider running with --init-slashing-protection (see --help)",
public_key = format!("{:?}", pk),
"Not signing attestation for unregistered validator"
);
validator_metrics::inc_counter_vec(
&validator_metrics::SIGNED_ATTESTATIONS_TOTAL,
@@ -750,10 +735,9 @@ impl<T: SlotClock + 'static, E: EthSpec> ValidatorStore<T, E> {
}
Err(e) => {
crit!(
self.log,
"Not signing slashable attestation";
"attestation" => format!("{:?}", attestation.data()),
"error" => format!("{:?}", e)
attestation = format!("{:?}", attestation.data()),
error = format!("{:?}", e),
"Not signing slashable attestation"
);
validator_metrics::inc_counter_vec(
&validator_metrics::SIGNED_ATTESTATIONS_TOTAL,
@@ -1068,13 +1052,12 @@ impl<T: SlotClock + 'static, E: EthSpec> ValidatorStore<T, E> {
if first_run {
info!(
self.log,
"Pruning slashing protection DB";
"epoch" => current_epoch,
"msg" => "pruning may take several minutes the first time it runs"
epoch = %current_epoch,
msg = "pruning may take several minutes the first time it runs",
"Pruning slashing protection DB"
);
} else {
info!(self.log, "Pruning slashing protection DB"; "epoch" => current_epoch);
info!(epoch = %current_epoch, "Pruning slashing protection DB");
}
let _timer =
@@ -1090,9 +1073,8 @@ impl<T: SlotClock + 'static, E: EthSpec> ValidatorStore<T, E> {
.prune_all_signed_attestations(all_pubkeys.iter(), new_min_target_epoch)
{
error!(
self.log,
"Error during pruning of signed attestations";
"error" => ?e,
error = ?e,
"Error during pruning of signed attestations"
);
return;
}
@@ -1102,15 +1084,14 @@ impl<T: SlotClock + 'static, E: EthSpec> ValidatorStore<T, E> {
.prune_all_signed_blocks(all_pubkeys.iter(), new_min_slot)
{
error!(
self.log,
"Error during pruning of signed blocks";
"error" => ?e,
error = ?e,
"Error during pruning of signed blocks"
);
return;
}
*last_prune = current_epoch;
info!(self.log, "Completed pruning of slashing protection DB");
info!("Completed pruning of slashing protection DB");
}
}