From fc657db5ea865253b8f5468312060bb2aa78baa7 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Tue, 18 Feb 2025 16:10:10 +0800 Subject: [PATCH 01/75] Define type and modify function --- common/eth2/src/types.rs | 7 +++ .../validator_services/src/duties_service.rs | 49 +++++++++++++------ 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 59374f629d..d258b6a23d 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -948,6 +948,13 @@ pub struct PeerCount { pub disconnecting: u64, } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct BeaconCommitteeSelection { + #[serde(with = "serde_utils::quoted_u64")] + pub validator_index: u64, + pub slot: Slot, + pub selection_proof: Signature, +} // --------- Server Sent Event Types ----------- #[derive(PartialEq, Debug, Serialize, Deserialize, Clone)] diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index 187eb4feb5..f9bbd4897d 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -129,24 +129,39 @@ async fn make_selection_proof( duty: &AttesterData, validator_store: &ValidatorStore, spec: &ChainSpec, + distributed: bool, + beacon_nodes: &Arc>, ) -> Result, Error> { - let selection_proof = validator_store - .produce_selection_proof(duty.pubkey, duty.slot) - .await - .map_err(Error::FailedToProduceSelectionProof)?; + if distributed { + // call the middleware for the endpoint /eth/v1/validator/beacon_committee_subscriptions + beacon_nodes + .first_success(|beacon_node| async move { + beacon_node + .post_validator_beacon_committee_subscriptions() + .await + }) + .await + .map_err(|e| Error::FailedToProduceSelectionProof(e.to_string())) + .map(|_| None) + } else { + let selection_proof = validator_store + .produce_selection_proof(duty.pubkey, duty.slot) + .await + .map_err(Error::FailedToProduceSelectionProof)?; - selection_proof - .is_aggregator(duty.committee_length as usize, spec) - .map_err(Error::InvalidModulo) - .map(|is_aggregator| { - if is_aggregator { - Some(selection_proof) - } else { - // Don't bother storing the selection proof if the validator isn't an - // aggregator, we won't need it. - None - } - }) + selection_proof + .is_aggregator(duty.committee_length as usize, spec) + .map_err(Error::InvalidModulo) + .map(|is_aggregator| { + if is_aggregator { + Some(selection_proof) + } else { + // Don't bother storing the selection proof if the validator isn't an + // aggregator, we won't need it. + None + } + }) + } } impl DutyAndProof { @@ -1101,6 +1116,8 @@ async fn fill_in_selection_proofs( &duty, &duties_service.validator_store, &duties_service.spec, + duties_service.distributed, + &duties_service.beacon_nodes, ) .await?; Ok((duty, opt_selection_proof)) From 8c7a995c91da16d428b92e66a4e2a9d55c1f8387 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Wed, 19 Feb 2025 21:46:12 +0800 Subject: [PATCH 02/75] Add client side endpoint --- common/eth2/src/lib.rs | 16 ++++++ .../validator_services/src/duties_service.rs | 52 ++++++++++++------- 2 files changed, 48 insertions(+), 20 deletions(-) diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 73e9d57abc..7a87eeb90c 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -2664,6 +2664,22 @@ impl BeaconNodeHttpClient { ) .await } + + /// `POST validator/beacon_committee_selections` + pub async fn post_validator_beacon_committee_selections( + &self, + selections: &[BeaconCommitteeSelection], + ) -> Result<(), Error> { + let mut path = self.eth_path(V1)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("validator") + .push("beacon_committee_selections"); + + self.post_with_timeout(path, &selections, self.timeouts.attester_duties) + .await + } } /// Returns `Ok(response)` if the response is a `200 OK` response. Otherwise, creates an diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index f9bbd4897d..de695ec0eb 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -13,7 +13,8 @@ use beacon_node_fallback::{ApiTopic, BeaconNodeFallback}; use doppelganger_service::DoppelgangerStatus; use environment::RuntimeContext; use eth2::types::{ - AttesterData, BeaconCommitteeSubscription, DutiesResponse, ProposerData, StateId, ValidatorId, + AttesterData, BeaconCommitteeSelection, BeaconCommitteeSubscription, DutiesResponse, + ProposerData, StateId, ValidatorId, }; use futures::{stream, StreamExt}; use parking_lot::RwLock; @@ -132,36 +133,47 @@ async fn make_selection_proof( distributed: bool, beacon_nodes: &Arc>, ) -> Result, Error> { - if distributed { - // call the middleware for the endpoint /eth/v1/validator/beacon_committee_subscriptions + let selection_proof = if distributed { + // Call the endpoint /eth/v1/validator/beacon_committee_selections + // During the call, we submit a partial selection proof in the data field of the POST HTTP endpoint + // The end point (middleware) should return a full selection proof + + let selections = BeaconCommitteeSelection { + validator_index: duty.validator_idnex, + slot: duty.slot, + selection_proof: validator_store + .produce_selection_proof(duty.pubkey, duty.slot) + .await + .map_err(Error::FailedToProduceSelectionProof)?; + }; + beacon_nodes .first_success(|beacon_node| async move { beacon_node - .post_validator_beacon_committee_subscriptions() + .post_validator_beacon_committee_selections(selections) .await }) .await .map_err(|e| Error::FailedToProduceSelectionProof(e.to_string())) - .map(|_| None) } else { - let selection_proof = validator_store + validator_store .produce_selection_proof(duty.pubkey, duty.slot) .await .map_err(Error::FailedToProduceSelectionProof)?; - - selection_proof - .is_aggregator(duty.committee_length as usize, spec) - .map_err(Error::InvalidModulo) - .map(|is_aggregator| { - if is_aggregator { - Some(selection_proof) - } else { - // Don't bother storing the selection proof if the validator isn't an - // aggregator, we won't need it. - None - } - }) - } + }; + + selection_proof + .is_aggregator(duty.committee_length as usize, spec) + .map_err(Error::InvalidModulo) + .map(|is_aggregator| { + if is_aggregator { + Some(selection_proof) + } else { + // Don't bother storing the selection proof if the validator isn't an + // aggregator, we won't need it. + None + } + }) } impl DutyAndProof { From 358646cf9bffbd5dcfec44804043a2ea7eacd7b2 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Wed, 19 Feb 2025 22:38:11 +0800 Subject: [PATCH 03/75] update if loop --- .../validator_services/src/duties_service.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index de695ec0eb..de4eeb2d62 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -137,31 +137,30 @@ async fn make_selection_proof( // Call the endpoint /eth/v1/validator/beacon_committee_selections // During the call, we submit a partial selection proof in the data field of the POST HTTP endpoint // The end point (middleware) should return a full selection proof - let selections = BeaconCommitteeSelection { - validator_index: duty.validator_idnex, + validator_index: duty.validator_index, slot: duty.slot, selection_proof: validator_store .produce_selection_proof(duty.pubkey, duty.slot) .await - .map_err(Error::FailedToProduceSelectionProof)?; + .map_err(Error::FailedToProduceSelectionProof)? + .into(), }; - beacon_nodes .first_success(|beacon_node| async move { beacon_node - .post_validator_beacon_committee_selections(selections) + .post_validator_beacon_committee_selections(&[selections]) .await }) .await - .map_err(|e| Error::FailedToProduceSelectionProof(e.to_string())) + .map_err(|e| Error::FailedToDownloadAttesters(e.to_string()))? } else { validator_store .produce_selection_proof(duty.pubkey, duty.slot) .await .map_err(Error::FailedToProduceSelectionProof)?; }; - + selection_proof .is_aggregator(duty.committee_length as usize, spec) .map_err(Error::InvalidModulo) From 83db29a0dab960fc4699ad504baf498a4568cdec Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Thu, 20 Feb 2025 09:53:15 +0800 Subject: [PATCH 04/75] Revise a bit --- .../validator_services/src/duties_service.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index de4eeb2d62..2e1174db67 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -134,9 +134,7 @@ async fn make_selection_proof( beacon_nodes: &Arc>, ) -> Result, Error> { let selection_proof = if distributed { - // Call the endpoint /eth/v1/validator/beacon_committee_selections - // During the call, we submit a partial selection proof in the data field of the POST HTTP endpoint - // The end point (middleware) should return a full selection proof + // Submit a partial selection proof in the data field of the POST HTTP endpoint let selections = BeaconCommitteeSelection { validator_index: duty.validator_index, slot: duty.slot, @@ -146,6 +144,8 @@ async fn make_selection_proof( .map_err(Error::FailedToProduceSelectionProof)? .into(), }; + // Call the endpoint /eth/v1/validator/beacon_committee_selections + // The middleware should return a full selection proof here beacon_nodes .first_success(|beacon_node| async move { beacon_node @@ -153,12 +153,12 @@ async fn make_selection_proof( .await }) .await - .map_err(|e| Error::FailedToDownloadAttesters(e.to_string()))? + .map_err(Error::FailedToProduceSelectionProof)? } else { validator_store .produce_selection_proof(duty.pubkey, duty.slot) .await - .map_err(Error::FailedToProduceSelectionProof)?; + .map_err(Error::FailedToProduceSelectionProof)? }; selection_proof From 20ecb35351af53f4f08d049101d6eff43b001d1c Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Thu, 20 Feb 2025 10:13:13 +0800 Subject: [PATCH 05/75] post endpoint return selection proof --- common/eth2/src/lib.rs | 4 ++-- consensus/types/src/selection_proof.rs | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 7a87eeb90c..1bcf52192d 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -2669,7 +2669,7 @@ impl BeaconNodeHttpClient { pub async fn post_validator_beacon_committee_selections( &self, selections: &[BeaconCommitteeSelection], - ) -> Result<(), Error> { + ) -> Result, Error> { let mut path = self.eth_path(V1)?; path.path_segments_mut() @@ -2677,7 +2677,7 @@ impl BeaconNodeHttpClient { .push("validator") .push("beacon_committee_selections"); - self.post_with_timeout(path, &selections, self.timeouts.attester_duties) + self.post_with_timeout_and_response(path, &selections, self.timeouts.attester_duties) .await } } diff --git a/consensus/types/src/selection_proof.rs b/consensus/types/src/selection_proof.rs index c80a00c3d1..47362717bd 100644 --- a/consensus/types/src/selection_proof.rs +++ b/consensus/types/src/selection_proof.rs @@ -3,10 +3,11 @@ use crate::{ }; use ethereum_hashing::hash; use safe_arith::{ArithError, SafeArith}; +use serde::{Deserialize, Serialize}; use ssz::Encode; use std::cmp; -#[derive(arbitrary::Arbitrary, PartialEq, Debug, Clone)] +#[derive(arbitrary::Arbitrary, PartialEq, Debug, Clone, Serialize, Deserialize)] pub struct SelectionProof(Signature); impl SelectionProof { From 7ff3ffe8d35adf88948a88b57c3ae3071a0b9416 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Thu, 20 Feb 2025 10:44:30 +0800 Subject: [PATCH 06/75] Fix error --- .../validator_services/src/duties_service.rs | 16 +++++++++++----- validator_client/validator_store/src/lib.rs | 1 + 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index 2e1174db67..54af3dc772 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -147,13 +147,19 @@ async fn make_selection_proof( // Call the endpoint /eth/v1/validator/beacon_committee_selections // The middleware should return a full selection proof here beacon_nodes - .first_success(|beacon_node| async move { - beacon_node - .post_validator_beacon_committee_selections(&[selections]) - .await + .first_success(|beacon_node| { + let value = selections.clone(); + async move { + beacon_node + .post_validator_beacon_committee_selections(&[value]) + .await + } }) .await - .map_err(Error::FailedToProduceSelectionProof)? + .map_err(|e| { + Error::FailedToProduceSelectionProof(ValidatorStoreError::Middleware(e.to_string())) + })? + .data } else { validator_store .produce_selection_proof(duty.pubkey, duty.slot) diff --git a/validator_client/validator_store/src/lib.rs b/validator_client/validator_store/src/lib.rs index 837af5b51d..712127abb5 100644 --- a/validator_client/validator_store/src/lib.rs +++ b/validator_client/validator_store/src/lib.rs @@ -34,6 +34,7 @@ pub enum Error { GreaterThanCurrentEpoch { epoch: Epoch, current_epoch: Epoch }, UnableToSignAttestation(AttestationError), UnableToSign(SigningError), + Middleware(String), } impl From for Error { From d8d31b7b0110edd0be42cafebfbe4fa66f3994ae Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Thu, 20 Feb 2025 13:37:40 +0800 Subject: [PATCH 07/75] Add Vec --- common/eth2/src/lib.rs | 2 +- consensus/types/src/selection_proof.rs | 1 + validator_client/validator_services/src/duties_service.rs | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 1bcf52192d..e665a6c1d1 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -2669,7 +2669,7 @@ impl BeaconNodeHttpClient { pub async fn post_validator_beacon_committee_selections( &self, selections: &[BeaconCommitteeSelection], - ) -> Result, Error> { + ) -> Result>, Error> { let mut path = self.eth_path(V1)?; path.path_segments_mut() diff --git a/consensus/types/src/selection_proof.rs b/consensus/types/src/selection_proof.rs index 47362717bd..60446ee3c5 100644 --- a/consensus/types/src/selection_proof.rs +++ b/consensus/types/src/selection_proof.rs @@ -8,6 +8,7 @@ use ssz::Encode; use std::cmp; #[derive(arbitrary::Arbitrary, PartialEq, Debug, Clone, Serialize, Deserialize)] +#[serde(transparent)] pub struct SelectionProof(Signature); impl SelectionProof { diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index 54af3dc772..498369a87e 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -159,7 +159,8 @@ async fn make_selection_proof( .map_err(|e| { Error::FailedToProduceSelectionProof(ValidatorStoreError::Middleware(e.to_string())) })? - .data + .data[0] + .clone() } else { validator_store .produce_selection_proof(duty.pubkey, duty.slot) From cba2adbfcca9fdcea81bd3e8d9c1b5584ab4b689 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Thu, 6 Mar 2025 15:16:48 +0800 Subject: [PATCH 08/75] println logs --- .../validator_services/src/duties_service.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index 498369a87e..26066d5ec5 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -135,7 +135,7 @@ async fn make_selection_proof( ) -> Result, Error> { let selection_proof = if distributed { // Submit a partial selection proof in the data field of the POST HTTP endpoint - let selections = BeaconCommitteeSelection { + let selection = BeaconCommitteeSelection { validator_index: duty.validator_index, slot: duty.slot, selection_proof: validator_store @@ -146,12 +146,14 @@ async fn make_selection_proof( }; // Call the endpoint /eth/v1/validator/beacon_committee_selections // The middleware should return a full selection proof here - beacon_nodes + + let response = beacon_nodes .first_success(|beacon_node| { - let value = selections.clone(); + let selections = selection.clone(); + println!("Selection proof: {:?}", selections); async move { beacon_node - .post_validator_beacon_committee_selections(&[value]) + .post_validator_beacon_committee_selections(&[selections]) .await } }) @@ -160,7 +162,10 @@ async fn make_selection_proof( Error::FailedToProduceSelectionProof(ValidatorStoreError::Middleware(e.to_string())) })? .data[0] - .clone() + .clone(); + + println!("Response: {:?}", response); + response } else { validator_store .produce_selection_proof(duty.pubkey, duty.slot) From 61e79b6901b3a328048e2a94ccdd78406557a441 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Thu, 6 Mar 2025 21:31:15 +0800 Subject: [PATCH 09/75] println response --- .../validator_services/src/duties_service.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index 26066d5ec5..2786633248 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -152,20 +152,22 @@ async fn make_selection_proof( let selections = selection.clone(); println!("Selection proof: {:?}", selections); async move { - beacon_node + let response = beacon_node .post_validator_beacon_committee_selections(&[selections]) - .await + .await; + + println!("Response from middleware {:?}", response); + + response } }) - .await + .await; + response .map_err(|e| { Error::FailedToProduceSelectionProof(ValidatorStoreError::Middleware(e.to_string())) })? .data[0] - .clone(); - - println!("Response: {:?}", response); - response + .clone() } else { validator_store .produce_selection_proof(duty.pubkey, duty.slot) From 4518b4600434034772ec2a7c4e899c3d31377782 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Fri, 7 Mar 2025 08:42:32 +0800 Subject: [PATCH 10/75] Testing --- common/eth2/src/types.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index d258b6a23d..0a056ce671 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -949,6 +949,7 @@ pub struct PeerCount { } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename = "")] pub struct BeaconCommitteeSelection { #[serde(with = "serde_utils::quoted_u64")] pub validator_index: u64, From 8e0315d635c657cfa323e11a2c1bd77170db983d Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Fri, 7 Mar 2025 11:17:46 +0800 Subject: [PATCH 11/75] remove serde --- common/eth2/src/types.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 0a056ce671..d258b6a23d 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -949,7 +949,6 @@ pub struct PeerCount { } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename = "")] pub struct BeaconCommitteeSelection { #[serde(with = "serde_utils::quoted_u64")] pub validator_index: u64, From 0d724b18a0e93b3f42e1b4dd7bbc43eb7ff1f96c Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Mon, 10 Mar 2025 13:26:59 +0800 Subject: [PATCH 12/75] Correct output type thanks Michael --- common/eth2/src/lib.rs | 2 +- .../validator_services/src/duties_service.rs | 17 +++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index e665a6c1d1..39d2260dbc 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -2669,7 +2669,7 @@ impl BeaconNodeHttpClient { pub async fn post_validator_beacon_committee_selections( &self, selections: &[BeaconCommitteeSelection], - ) -> Result>, Error> { + ) -> Result>, Error> { let mut path = self.eth_path(V1)?; path.path_segments_mut() diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index 2786633248..f70b339042 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -162,12 +162,17 @@ async fn make_selection_proof( } }) .await; - response - .map_err(|e| { - Error::FailedToProduceSelectionProof(ValidatorStoreError::Middleware(e.to_string())) - })? - .data[0] - .clone() + SelectionProof::from( + response + .map_err(|e| { + Error::FailedToProduceSelectionProof(ValidatorStoreError::Middleware( + e.to_string(), + )) + })? + .data[0] + .selection_proof + .clone(), + ) } else { validator_store .produce_selection_proof(duty.pubkey, duty.slot) From d6c7461351a27eff2544b4c0eb34c994ae22f1c0 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Mon, 10 Mar 2025 17:22:08 +0800 Subject: [PATCH 13/75] Add timeout for aggregator --- common/eth2/src/lib.rs | 4 +++- validator_client/src/lib.rs | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 39d2260dbc..065fedf1b8 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -144,6 +144,7 @@ pub struct Timeouts { pub get_debug_beacon_states: Duration, pub get_deposit_snapshot: Duration, pub get_validator_block: Duration, + pub aggregator_duties: Duration, } impl Timeouts { @@ -161,6 +162,7 @@ impl Timeouts { get_debug_beacon_states: timeout, get_deposit_snapshot: timeout, get_validator_block: timeout, + aggregator_duties: timeout, } } } @@ -2677,7 +2679,7 @@ impl BeaconNodeHttpClient { .push("validator") .push("beacon_committee_selections"); - self.post_with_timeout_and_response(path, &selections, self.timeouts.attester_duties) + self.post_with_timeout_and_response(path, &selections, self.timeouts.aggregator_duties) .await } } diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 70236d6a3c..82cd95cfd2 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -69,6 +69,7 @@ const HTTP_GET_BEACON_BLOCK_SSZ_TIMEOUT_QUOTIENT: u32 = 4; const HTTP_GET_DEBUG_BEACON_STATE_QUOTIENT: u32 = 4; const HTTP_GET_DEPOSIT_SNAPSHOT_QUOTIENT: u32 = 4; const HTTP_GET_VALIDATOR_BLOCK_TIMEOUT_QUOTIENT: u32 = 4; +const HTTP_AGGREGATOR_DUTIES_TIMEOUT_QUOTIENT: u32 = 1; const DOPPELGANGER_SERVICE_NAME: &str = "doppelganger"; @@ -323,6 +324,7 @@ impl ProductionValidatorClient { get_debug_beacon_states: slot_duration / HTTP_GET_DEBUG_BEACON_STATE_QUOTIENT, get_deposit_snapshot: slot_duration / HTTP_GET_DEPOSIT_SNAPSHOT_QUOTIENT, get_validator_block: slot_duration / HTTP_GET_VALIDATOR_BLOCK_TIMEOUT_QUOTIENT, + aggregator_duties: slot_duration / HTTP_AGGREGATOR_DUTIES_TIMEOUT_QUOTIENT, } } else { Timeouts::set_all(slot_duration) From 7ef2a48f9fc2d9bf2d6a9e1f8b4cfd198b80a7a2 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Wed, 12 Mar 2025 16:58:41 +0800 Subject: [PATCH 14/75] update aggregator timeout --- common/eth2/src/lib.rs | 12 ++++++++---- validator_client/src/lib.rs | 5 +++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 065fedf1b8..19099510c9 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -135,6 +135,7 @@ pub struct Timeouts { pub attestation: Duration, pub attester_duties: Duration, pub attestation_subscriptions: Duration, + pub attestation_aggregators: Duration, pub liveness: Duration, pub proposal: Duration, pub proposer_duties: Duration, @@ -144,7 +145,6 @@ pub struct Timeouts { pub get_debug_beacon_states: Duration, pub get_deposit_snapshot: Duration, pub get_validator_block: Duration, - pub aggregator_duties: Duration, } impl Timeouts { @@ -153,6 +153,7 @@ impl Timeouts { attestation: timeout, attester_duties: timeout, attestation_subscriptions: timeout, + attestation_aggregators: timeout, liveness: timeout, proposal: timeout, proposer_duties: timeout, @@ -162,7 +163,6 @@ impl Timeouts { get_debug_beacon_states: timeout, get_deposit_snapshot: timeout, get_validator_block: timeout, - aggregator_duties: timeout, } } } @@ -2679,8 +2679,12 @@ impl BeaconNodeHttpClient { .push("validator") .push("beacon_committee_selections"); - self.post_with_timeout_and_response(path, &selections, self.timeouts.aggregator_duties) - .await + self.post_with_timeout_and_response( + path, + &selections, + self.timeouts.attestation_aggregators, + ) + .await } } diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 82cd95cfd2..f0c084bdb2 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -60,6 +60,7 @@ const WAITING_FOR_GENESIS_POLL_TIME: Duration = Duration::from_secs(12); const HTTP_ATTESTATION_TIMEOUT_QUOTIENT: u32 = 4; const HTTP_ATTESTER_DUTIES_TIMEOUT_QUOTIENT: u32 = 4; const HTTP_ATTESTATION_SUBSCRIPTIONS_TIMEOUT_QUOTIENT: u32 = 24; +const HTTP_ATTESTATION_AGGREGATOR_TIMEOUT_QUOTIENT: u32 = 3; const HTTP_LIVENESS_TIMEOUT_QUOTIENT: u32 = 4; const HTTP_PROPOSAL_TIMEOUT_QUOTIENT: u32 = 2; const HTTP_PROPOSER_DUTIES_TIMEOUT_QUOTIENT: u32 = 4; @@ -69,7 +70,6 @@ const HTTP_GET_BEACON_BLOCK_SSZ_TIMEOUT_QUOTIENT: u32 = 4; const HTTP_GET_DEBUG_BEACON_STATE_QUOTIENT: u32 = 4; const HTTP_GET_DEPOSIT_SNAPSHOT_QUOTIENT: u32 = 4; const HTTP_GET_VALIDATOR_BLOCK_TIMEOUT_QUOTIENT: u32 = 4; -const HTTP_AGGREGATOR_DUTIES_TIMEOUT_QUOTIENT: u32 = 1; const DOPPELGANGER_SERVICE_NAME: &str = "doppelganger"; @@ -313,6 +313,8 @@ impl ProductionValidatorClient { attester_duties: slot_duration / HTTP_ATTESTER_DUTIES_TIMEOUT_QUOTIENT, attestation_subscriptions: slot_duration / HTTP_ATTESTATION_SUBSCRIPTIONS_TIMEOUT_QUOTIENT, + attestation_aggregators: 2 * slot_duration + / HTTP_ATTESTATION_AGGREGATOR_TIMEOUT_QUOTIENT, liveness: slot_duration / HTTP_LIVENESS_TIMEOUT_QUOTIENT, proposal: slot_duration / HTTP_PROPOSAL_TIMEOUT_QUOTIENT, proposer_duties: slot_duration / HTTP_PROPOSER_DUTIES_TIMEOUT_QUOTIENT, @@ -324,7 +326,6 @@ impl ProductionValidatorClient { get_debug_beacon_states: slot_duration / HTTP_GET_DEBUG_BEACON_STATE_QUOTIENT, get_deposit_snapshot: slot_duration / HTTP_GET_DEPOSIT_SNAPSHOT_QUOTIENT, get_validator_block: slot_duration / HTTP_GET_VALIDATOR_BLOCK_TIMEOUT_QUOTIENT, - aggregator_duties: slot_duration / HTTP_AGGREGATOR_DUTIES_TIMEOUT_QUOTIENT, } } else { Timeouts::set_all(slot_duration) From 81ee8e3f363ff8ed6cd501739cc86141f9aac41f Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Thu, 13 Mar 2025 17:52:23 +0800 Subject: [PATCH 15/75] Change timeout and add debug log --- validator_client/src/lib.rs | 2 +- .../validator_services/src/duties_service.rs | 22 ++++++++++++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index f0c084bdb2..08cba24805 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -60,7 +60,7 @@ const WAITING_FOR_GENESIS_POLL_TIME: Duration = Duration::from_secs(12); const HTTP_ATTESTATION_TIMEOUT_QUOTIENT: u32 = 4; const HTTP_ATTESTER_DUTIES_TIMEOUT_QUOTIENT: u32 = 4; const HTTP_ATTESTATION_SUBSCRIPTIONS_TIMEOUT_QUOTIENT: u32 = 24; -const HTTP_ATTESTATION_AGGREGATOR_TIMEOUT_QUOTIENT: u32 = 3; +const HTTP_ATTESTATION_AGGREGATOR_TIMEOUT_QUOTIENT: u32 = 24; const HTTP_LIVENESS_TIMEOUT_QUOTIENT: u32 = 4; const HTTP_PROPOSAL_TIMEOUT_QUOTIENT: u32 = 2; const HTTP_PROPOSER_DUTIES_TIMEOUT_QUOTIENT: u32 = 4; diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index f70b339042..24c941cadf 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -132,6 +132,7 @@ async fn make_selection_proof( spec: &ChainSpec, distributed: bool, beacon_nodes: &Arc>, + duties_service: &DutiesService, ) -> Result, Error> { let selection_proof = if distributed { // Submit a partial selection proof in the data field of the POST HTTP endpoint @@ -146,17 +147,31 @@ async fn make_selection_proof( }; // Call the endpoint /eth/v1/validator/beacon_committee_selections // The middleware should return a full selection proof here - + let log = duties_service.context.log(); let response = beacon_nodes .first_success(|beacon_node| { let selections = selection.clone(); - println!("Selection proof: {:?}", selections); + debug!( + log, + "Partial selection proof from VC"; + "Validator index" => selections.validator_index, + "Slot" => selections.slot, + "Selection proof" => ?selections.selection_proof, + + ); + // println!("Selection proof: {:?}", selections); async move { let response = beacon_node .post_validator_beacon_committee_selections(&[selections]) .await; - println!("Response from middleware {:?}", response); + debug!( + log, + "Response from middleware"; + "response" => ?response, + + ); + // println!("Response from middleware {:?}", response); response } @@ -1148,6 +1163,7 @@ async fn fill_in_selection_proofs( &duties_service.spec, duties_service.distributed, &duties_service.beacon_nodes, + &duties_service, ) .await?; Ok((duty, opt_selection_proof)) From 376bc70fcff25c7548e128e2f81cde8d5314e235 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Thu, 13 Mar 2025 19:05:39 +0800 Subject: [PATCH 16/75] space --- validator_client/validator_services/src/duties_service.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index 24c941cadf..c6280a59ff 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -157,7 +157,6 @@ async fn make_selection_proof( "Validator index" => selections.validator_index, "Slot" => selections.slot, "Selection proof" => ?selections.selection_proof, - ); // println!("Selection proof: {:?}", selections); async move { From 0c360c893e111ca05b8f155991b4b0223cf516cc Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Thu, 13 Mar 2025 20:38:15 +0800 Subject: [PATCH 17/75] log --- validator_client/validator_services/src/duties_service.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index c6280a59ff..e99e2c9cb2 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -154,9 +154,7 @@ async fn make_selection_proof( debug!( log, "Partial selection proof from VC"; - "Validator index" => selections.validator_index, - "Slot" => selections.slot, - "Selection proof" => ?selections.selection_proof, + "Selection proof" => ?selections, ); // println!("Selection proof: {:?}", selections); async move { From 50ec5f17a1d05b279c2b75125ccd2a498bfd8d39 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Sun, 16 Mar 2025 21:27:07 +0800 Subject: [PATCH 18/75] add distributed lookahead --- .../validator_services/src/duties_service.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index e99e2c9cb2..a6acded3ff 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -1137,8 +1137,17 @@ async fn fill_in_selection_proofs( let lookahead_slot = current_slot + selection_lookahead; - let mut relevant_duties = duties_by_slot.split_off(&lookahead_slot); - std::mem::swap(&mut relevant_duties, &mut duties_by_slot); + let relevant_duties = if duties_service.distributed { + if let Some(duties) = duties_by_slot.remove(&lookahead_slot) { + BTreeMap::from([(lookahead_slot, duties)]) + } else { + BTreeMap::new() + } + } else { + let mut duties = duties_by_slot.split_off(&lookahead_slot); + std::mem::swap(&mut duties, &mut duties_by_slot); + duties + }; let batch_size = relevant_duties.values().map(Vec::len).sum::(); From 7dd7ed240234bc57ae67ead68941ef3e567f5554 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Mon, 17 Mar 2025 12:00:22 +0800 Subject: [PATCH 19/75] simplify --- .../validator_services/src/duties_service.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index a6acded3ff..9774044efe 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -1138,11 +1138,10 @@ async fn fill_in_selection_proofs( let lookahead_slot = current_slot + selection_lookahead; let relevant_duties = if duties_service.distributed { - if let Some(duties) = duties_by_slot.remove(&lookahead_slot) { - BTreeMap::from([(lookahead_slot, duties)]) - } else { - BTreeMap::new() - } + duties_by_slot + .remove(&lookahead_slot) + .map(|duties| BTreeMap::from([(lookahead_slot, duties)])) + .unwrap_or_default() } else { let mut duties = duties_by_slot.split_off(&lookahead_slot); std::mem::swap(&mut duties, &mut duties_by_slot); From 4f3b3d5b508ade670a570596f2c359dd36f52b6b Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Mon, 17 Mar 2025 13:55:24 +0800 Subject: [PATCH 20/75] correct timeout --- validator_client/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 08cba24805..279d33b93a 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -313,7 +313,7 @@ impl ProductionValidatorClient { attester_duties: slot_duration / HTTP_ATTESTER_DUTIES_TIMEOUT_QUOTIENT, attestation_subscriptions: slot_duration / HTTP_ATTESTATION_SUBSCRIPTIONS_TIMEOUT_QUOTIENT, - attestation_aggregators: 2 * slot_duration + attestation_aggregators: slot_duration / HTTP_ATTESTATION_AGGREGATOR_TIMEOUT_QUOTIENT, liveness: slot_duration / HTTP_LIVENESS_TIMEOUT_QUOTIENT, proposal: slot_duration / HTTP_PROPOSAL_TIMEOUT_QUOTIENT, From 52854a0d12b10e9a3c3235c223aa8db8268ee3da Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Wed, 19 Mar 2025 12:28:19 +0800 Subject: [PATCH 21/75] Add public key in log --- validator_client/validator_services/src/duties_service.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index 9774044efe..95dc673792 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -155,6 +155,7 @@ async fn make_selection_proof( log, "Partial selection proof from VC"; "Selection proof" => ?selections, + "Public key" => ?duty.pubkey, ); // println!("Selection proof: {:?}", selections); async move { From 8b2f058d7be12022dc62ce26cf00e8a4deb43c85 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Thu, 20 Mar 2025 14:03:22 +0800 Subject: [PATCH 22/75] Sign selection proof in parallel --- .../validator_services/src/duties_service.rs | 51 ++++++++++++------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index 95dc673792..44d4d54d14 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -155,7 +155,6 @@ async fn make_selection_proof( log, "Partial selection proof from VC"; "Selection proof" => ?selections, - "Public key" => ?duty.pubkey, ); // println!("Selection proof: {:?}", selections); async move { @@ -1160,22 +1159,40 @@ async fn fill_in_selection_proofs( &[validator_metrics::ATTESTATION_SELECTION_PROOFS], ); - // Sign selection proofs (serially). - let duty_and_proof_results = stream::iter(relevant_duties.into_values().flatten()) - .then(|duty| async { - let opt_selection_proof = make_selection_proof( - &duty, - &duties_service.validator_store, - &duties_service.spec, - duties_service.distributed, - &duties_service.beacon_nodes, - &duties_service, - ) - .await?; - Ok((duty, opt_selection_proof)) - }) - .collect::>() - .await; + // In distributed case, sign selection proofs in parallel; otherwise, sign them serially in non-distributed case + let duty_and_proof_results = if duties_service.distributed { + futures::future::join_all(relevant_duties.into_values().flatten().map( + |duty| async { + let opt_selection_proof = make_selection_proof( + &duty, + &duties_service.validator_store, + &duties_service.spec, + duties_service.distributed, + &duties_service.beacon_nodes, + &duties_service, + ) + .await?; + Ok((duty, opt_selection_proof)) + }, + )) + .await + } else { + stream::iter(relevant_duties.into_values().flatten()) + .then(|duty| async { + let opt_selection_proof = make_selection_proof( + &duty, + &duties_service.validator_store, + &duties_service.spec, + duties_service.distributed, + &duties_service.beacon_nodes, + &duties_service, + ) + .await?; + Ok((duty, opt_selection_proof)) + }) + .collect::>() + .await + }; // Add to attesters store. let mut attesters = duties_service.attesters.write(); From 1631c860dcbbb4146d4f03b3e84c9ad86c8965b7 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Mon, 24 Mar 2025 14:12:34 +0800 Subject: [PATCH 23/75] Implement sync committee selection endpoint --- common/eth2/src/lib.rs | 18 ++ common/eth2/src/types.rs | 11 ++ consensus/types/src/sync_selection_proof.rs | 3 +- validator_client/src/lib.rs | 2 + .../validator_services/src/duties_service.rs | 4 +- .../validator_services/src/sync.rs | 186 +++++++++++++----- 6 files changed, 169 insertions(+), 55 deletions(-) diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 19099510c9..64cf9f7ce6 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -141,6 +141,7 @@ pub struct Timeouts { pub proposer_duties: Duration, pub sync_committee_contribution: Duration, pub sync_duties: Duration, + pub sync_aggregators: Duration, pub get_beacon_blocks_ssz: Duration, pub get_debug_beacon_states: Duration, pub get_deposit_snapshot: Duration, @@ -159,6 +160,7 @@ impl Timeouts { proposer_duties: timeout, sync_committee_contribution: timeout, sync_duties: timeout, + sync_aggregators: timeout, get_beacon_blocks_ssz: timeout, get_debug_beacon_states: timeout, get_deposit_snapshot: timeout, @@ -2686,6 +2688,22 @@ impl BeaconNodeHttpClient { ) .await } + + /// `POST validator/sync_committee_selections` + pub async fn post_validator_sync_committee_selections( + &self, + selections: &[SyncCommitteeSelection], + ) -> Result>, Error> { + let mut path = self.eth_path(V1)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("validator") + .push("sync_committee_selections"); + + self.post_with_timeout_and_response(path, &selections, self.timeouts.sync_aggregators) + .await + } } /// Returns `Ok(response)` if the response is a `200 OK` response. Otherwise, creates an diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index d258b6a23d..35b0de02f7 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -955,6 +955,17 @@ pub struct BeaconCommitteeSelection { pub slot: Slot, pub selection_proof: Signature, } + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] + +pub struct SyncCommitteeSelection { + #[serde(with = "serde_utils::quoted_u64")] + pub validator_index: u64, + pub slot: Slot, + #[serde(with = "serde_utils::quoted_u64")] + pub subcommittee_index: u64, + pub selection_proof: Signature, +} // --------- Server Sent Event Types ----------- #[derive(PartialEq, Debug, Serialize, Deserialize, Clone)] diff --git a/consensus/types/src/sync_selection_proof.rs b/consensus/types/src/sync_selection_proof.rs index 4adb90b26e..e4f6ce43cd 100644 --- a/consensus/types/src/sync_selection_proof.rs +++ b/consensus/types/src/sync_selection_proof.rs @@ -7,11 +7,12 @@ use crate::{ }; use ethereum_hashing::hash; use safe_arith::{ArithError, SafeArith}; +use serde::{Deserialize, Serialize}; use ssz::Encode; use ssz_types::typenum::Unsigned; use std::cmp; -#[derive(arbitrary::Arbitrary, PartialEq, Debug, Clone)] +#[derive(arbitrary::Arbitrary, PartialEq, Debug, Clone, Serialize, Deserialize)] pub struct SyncSelectionProof(Signature); impl SyncSelectionProof { diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 279d33b93a..2385ebc19a 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -66,6 +66,7 @@ const HTTP_PROPOSAL_TIMEOUT_QUOTIENT: u32 = 2; const HTTP_PROPOSER_DUTIES_TIMEOUT_QUOTIENT: u32 = 4; const HTTP_SYNC_COMMITTEE_CONTRIBUTION_TIMEOUT_QUOTIENT: u32 = 4; const HTTP_SYNC_DUTIES_TIMEOUT_QUOTIENT: u32 = 4; +const HTTP_SYNC_AGGREGATOR_TIMEOUT_QUOTIENT: u32 = 24; const HTTP_GET_BEACON_BLOCK_SSZ_TIMEOUT_QUOTIENT: u32 = 4; const HTTP_GET_DEBUG_BEACON_STATE_QUOTIENT: u32 = 4; const HTTP_GET_DEPOSIT_SNAPSHOT_QUOTIENT: u32 = 4; @@ -321,6 +322,7 @@ impl ProductionValidatorClient { sync_committee_contribution: slot_duration / HTTP_SYNC_COMMITTEE_CONTRIBUTION_TIMEOUT_QUOTIENT, sync_duties: slot_duration / HTTP_SYNC_DUTIES_TIMEOUT_QUOTIENT, + sync_aggregators: slot_duration / HTTP_SYNC_AGGREGATOR_TIMEOUT_QUOTIENT, get_beacon_blocks_ssz: slot_duration / HTTP_GET_BEACON_BLOCK_SSZ_TIMEOUT_QUOTIENT, get_debug_beacon_states: slot_duration / HTTP_GET_DEBUG_BEACON_STATE_QUOTIENT, diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index 44d4d54d14..07a87b3ea5 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -1159,7 +1159,9 @@ async fn fill_in_selection_proofs( &[validator_metrics::ATTESTATION_SELECTION_PROOFS], ); - // In distributed case, sign selection proofs in parallel; otherwise, sign them serially in non-distributed case + // In distributed case, we want to send all partial selection proofs to the middleware to determine aggregation duties, + // as the middleware will need to have a threshold of partial selection proof to be able to return the full selection proof + // Thus, sign selection proofs in parallel in distributed case; Otherwise, sign them serially in non-distributed (normal) case let duty_and_proof_results = if duties_service.distributed { futures::future::join_all(relevant_duties.into_values().flatten().map( |duty| async { diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index dd3e05088e..73e4c69d71 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -1,5 +1,6 @@ use crate::duties_service::{DutiesService, Error}; use doppelganger_service::DoppelgangerStatus; +use eth2::types::SyncCommitteeSelection; use futures::future::join_all; use parking_lot::{MappedRwLockReadGuard, RwLock, RwLockReadGuard, RwLockWriteGuard}; use slog::{crit, debug, info, warn}; @@ -542,61 +543,140 @@ pub async fn fill_in_aggregation_proofs( // Create futures to produce proofs. let duties_service_ref = &duties_service; - let futures = subnet_ids.iter().map(|subnet_id| async move { - // Construct proof for prior slot. - let proof_slot = slot - 1; + let futures = subnet_ids.iter().map(|subnet_id| { + let duties_service = duties_service.clone(); + async move { + // Construct proof for prior slot. + let proof_slot = slot - 1; - let proof = match duties_service_ref - .validator_store - .produce_sync_selection_proof(&duty.pubkey, proof_slot, *subnet_id) - .await - { - Ok(proof) => proof, - Err(ValidatorStoreError::UnknownPubkey(pubkey)) => { - // A pubkey can be missing when a validator was recently - // removed via the API. - debug!( - log, - "Missing pubkey for sync selection proof"; - "pubkey" => ?pubkey, - "pubkey" => ?duty.pubkey, - "slot" => proof_slot, - ); - return None; - } - Err(e) => { - warn!( - log, - "Unable to sign selection proof"; - "error" => ?e, - "pubkey" => ?duty.pubkey, - "slot" => proof_slot, - ); - return None; - } - }; + let proof = if duties_service.distributed { + let sync_selection_proof = SyncCommitteeSelection { + validator_index: duty.validator_index, + slot: proof_slot, + subcommittee_index: **subnet_id, + selection_proof: match duties_service_ref + .validator_store + .produce_sync_selection_proof(&duty.pubkey, proof_slot, *subnet_id) + .await + { + Ok(proof) => proof.into(), + Err(e) => { + return match e { + ValidatorStoreError::UnknownPubkey(pubkey) => { + debug!( + log, + "Missing pubkey for sync selection proof"; + "pubkey" => ?pubkey, + "slot" => proof_slot, + ); + None + } + _ => { + warn!( + log, + "Unable to sign selection proof"; + "error" => ?e, + "pubkey" => ?duty.pubkey, + "slot" => proof_slot, + ); + None + } + }; + } + }, + }; - match proof.is_aggregator::() { - Ok(true) => { - debug!( - log, - "Validator is sync aggregator"; - "validator_index" => duty.validator_index, - "slot" => proof_slot, - "subnet_id" => %subnet_id, - ); - Some(((proof_slot, *subnet_id), proof)) - } - Ok(false) => None, - Err(e) => { - warn!( - log, - "Error determining is_aggregator"; - "pubkey" => ?duty.pubkey, - "slot" => proof_slot, - "error" => ?e, - ); - None + let response = match duties_service + .beacon_nodes + .first_success(|beacon_node| { + let selection = sync_selection_proof.clone(); + debug!( + log, + "Partial sync selection proof from VC"; + "Sync selection proof" => ?selection, + ); + async move { + let response = beacon_node + .post_validator_sync_committee_selections(&[selection]) + .await; + debug!( + log, + "Response from middleware for sync"; + "response" => ?response, + ); + + response + } + }) + .await + { + Ok(response) => response, + Err(e) => { + warn! { + log, + "Unable to sign selection proof in middleware level"; + "error" => %e, + "pubkey" => ?duty.pubkey, + "slot" => proof_slot, + }; + return None; + } + }; + SyncSelectionProof::from(response.data[0].selection_proof.clone()) + } else { + match duties_service_ref + .validator_store + .produce_sync_selection_proof(&duty.pubkey, proof_slot, *subnet_id) + .await + { + Ok(proof) => proof, + Err(ValidatorStoreError::UnknownPubkey(pubkey)) => { + // A pubkey can be missing when a validator was recently + // removed via the API. + debug!( + log, + "Missing pubkey for sync selection proof"; + "pubkey" => ?pubkey, + "pubkey" => ?duty.pubkey, + "slot" => proof_slot, + ); + return None; + } + Err(e) => { + warn!( + log, + "Unable to sign selection proof"; + "error" => ?e, + "pubkey" => ?duty.pubkey, + "slot" => proof_slot, + ); + return None; + } + } + }; + + match proof.is_aggregator::() { + Ok(true) => { + debug!( + log, + "Validator is sync aggregator"; + "validator_index" => duty.validator_index, + "slot" => proof_slot, + "subnet_id" => %subnet_id, + ); + Some(((proof_slot, *subnet_id), proof)) + } + Ok(false) => None, + Err(e) => { + warn!( + log, + "Error determining is_aggregator"; + "pubkey" => ?duty.pubkey, + "slot" => proof_slot, + "error" => ?e, + ); + None + } } } }); From f7f5bf1b866a1bedd38cf5a329f3e64b699f3e57 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Mon, 24 Mar 2025 21:37:27 +0800 Subject: [PATCH 24/75] Modify only current slot --- validator_client/validator_services/src/sync.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index 73e4c69d71..4e2d3b4571 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -520,8 +520,15 @@ pub async fn fill_in_aggregation_proofs( "pre_compute_slot" => pre_compute_slot ); + let slots_to_process = if duties_service.distributed { + vec![current_slot] + } else { + (current_slot.as_u64()..=pre_compute_slot.as_u64()) + .map(Slot::new) + .collect::>() + }; // Generate selection proofs for each validator at each slot, one slot at a time. - for slot in (current_slot.as_u64()..=pre_compute_slot.as_u64()).map(Slot::new) { + for slot in slots_to_process { let mut validator_proofs = vec![]; for (validator_start_slot, duty) in pre_compute_duties { // Proofs are already known at this slot for this validator. From d2bb70ee8cd328ef1b690059d9e30e18d3a7de55 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Tue, 25 Mar 2025 12:06:15 +0800 Subject: [PATCH 25/75] Remove current slot --- validator_client/validator_services/src/sync.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index 4e2d3b4571..73e4c69d71 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -520,15 +520,8 @@ pub async fn fill_in_aggregation_proofs( "pre_compute_slot" => pre_compute_slot ); - let slots_to_process = if duties_service.distributed { - vec![current_slot] - } else { - (current_slot.as_u64()..=pre_compute_slot.as_u64()) - .map(Slot::new) - .collect::>() - }; // Generate selection proofs for each validator at each slot, one slot at a time. - for slot in slots_to_process { + for slot in (current_slot.as_u64()..=pre_compute_slot.as_u64()).map(Slot::new) { let mut validator_proofs = vec![]; for (validator_start_slot, duty) in pre_compute_duties { // Proofs are already known at this slot for this validator. From 3b883c3d057c7110d292ffb7e85d05038fcabfa5 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Wed, 26 Mar 2025 13:57:55 +0800 Subject: [PATCH 26/75] Testing distributed mode --- .../validator_services/src/sync.rs | 395 +++++++++++------- 1 file changed, 252 insertions(+), 143 deletions(-) diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index 73e4c69d71..8f42183aa3 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -522,125 +522,67 @@ pub async fn fill_in_aggregation_proofs( // Generate selection proofs for each validator at each slot, one slot at a time. for slot in (current_slot.as_u64()..=pre_compute_slot.as_u64()).map(Slot::new) { - let mut validator_proofs = vec![]; - for (validator_start_slot, duty) in pre_compute_duties { - // Proofs are already known at this slot for this validator. - if slot < *validator_start_slot { - continue; - } + // For distributed mode + if duties_service.distributed { + let mut partial_proofs = Vec::new(); - let subnet_ids = match duty.subnet_ids::() { - Ok(subnet_ids) => subnet_ids, - Err(e) => { - crit!( - log, - "Arithmetic error computing subnet IDs"; - "error" => ?e, - ); + for (validator_start_slot, duty) in pre_compute_duties { + // Proofs are already known at this slot for this validator. + if slot < *validator_start_slot { continue; } - }; - // Create futures to produce proofs. - let duties_service_ref = &duties_service; - let futures = subnet_ids.iter().map(|subnet_id| { - let duties_service = duties_service.clone(); - async move { - // Construct proof for prior slot. - let proof_slot = slot - 1; + let subnet_ids = match duty.subnet_ids::() { + Ok(subnet_ids) => subnet_ids, + Err(e) => { + crit!( + log, + "Arithmetic error computing subnet IDs"; + "error" => ?e, + ); + continue; + } + }; - let proof = if duties_service.distributed { - let sync_selection_proof = SyncCommitteeSelection { - validator_index: duty.validator_index, - slot: proof_slot, - subcommittee_index: **subnet_id, - selection_proof: match duties_service_ref - .validator_store - .produce_sync_selection_proof(&duty.pubkey, proof_slot, *subnet_id) - .await - { - Ok(proof) => proof.into(), - Err(e) => { - return match e { - ValidatorStoreError::UnknownPubkey(pubkey) => { - debug!( - log, - "Missing pubkey for sync selection proof"; - "pubkey" => ?pubkey, - "slot" => proof_slot, - ); - None - } - _ => { - warn!( - log, - "Unable to sign selection proof"; - "error" => ?e, - "pubkey" => ?duty.pubkey, - "slot" => proof_slot, - ); - None - } - }; - } - }, - }; + // Construct proof for prior slot. + let proof_slot = slot - 1; - let response = match duties_service - .beacon_nodes - .first_success(|beacon_node| { - let selection = sync_selection_proof.clone(); - debug!( - log, - "Partial sync selection proof from VC"; - "Sync selection proof" => ?selection, - ); - async move { - let response = beacon_node - .post_validator_sync_committee_selections(&[selection]) - .await; - debug!( - log, - "Response from middleware for sync"; - "response" => ?response, - ); + // Create futures for all subnet IDs for this validator + for subnet_id in subnet_ids { + let duties_service = duties_service.clone(); + let duty = duty.clone(); + let subnet_id = *subnet_id; - response - } - }) - .await - { - Ok(response) => response, - Err(e) => { - warn! { - log, - "Unable to sign selection proof in middleware level"; - "error" => %e, - "pubkey" => ?duty.pubkey, - "slot" => proof_slot, - }; - return None; - } - }; - SyncSelectionProof::from(response.data[0].selection_proof.clone()) - } else { - match duties_service_ref + // Store all partial sync selection proofs for this slot in partial_proofs so that it can be sent together + partial_proofs.push(async move { + // Produce partial selection proof + let sync_selection_proof = duties_service .validator_store - .produce_sync_selection_proof(&duty.pubkey, proof_slot, *subnet_id) - .await - { - Ok(proof) => proof, + .produce_sync_selection_proof( + &duty.pubkey, + proof_slot, + subnet_id.into(), + ) + .await; + + match sync_selection_proof { + Ok(proof) => { + let sync_committee_selection = SyncCommitteeSelection { + validator_index: duty.validator_index, + slot: proof_slot, + subcommittee_index: subnet_id, + selection_proof: proof.into(), + }; + Some(sync_committee_selection) + } Err(ValidatorStoreError::UnknownPubkey(pubkey)) => { - // A pubkey can be missing when a validator was recently - // removed via the API. debug!( log, "Missing pubkey for sync selection proof"; "pubkey" => ?pubkey, - "pubkey" => ?duty.pubkey, "slot" => proof_slot, ); - return None; + None } Err(e) => { warn!( @@ -650,9 +592,176 @@ pub async fn fill_in_aggregation_proofs( "pubkey" => ?duty.pubkey, "slot" => proof_slot, ); - return None; + None } } + }); + } + } + + // Execute partial_proofs in parallel + let partial_proof_results = join_all(partial_proofs).await; + + // Filter out None values and extract the selection proofs + let valid_results: Vec<_> = partial_proof_results.into_iter().flatten().collect(); + let selection_proofs: Vec<_> = valid_results.clone(); + + // If we have any valid proofs, send them to the middleware + if !selection_proofs.is_empty() { + debug!( + log, + "Sending batch of partial sync selection proofs"; + "count" => selection_proofs.len(), + ); + + let response = duties_service + .beacon_nodes + .first_success(|beacon_node| { + let proofs = selection_proofs.clone(); + async move { + beacon_node + .post_validator_sync_committee_selections(&proofs) + .await + } + }) + .await; + + match response { + Ok(response) => { + debug!( + log, + "Received batch response from middleware for sync"; + "count" => response.data.len(), + ); + + // Get the sync map to update duties + let sync_map = duties_service.sync_duties.committees.read(); + let Some(committee_duties) = sync_map.get(&sync_committee_period) else { + debug!( + log, + "Missing sync duties"; + "period" => sync_committee_period, + ); + continue; + }; + + let validators = committee_duties.validators.read(); + + // Process each response + for (i, selection_response) in response.data.iter().enumerate() { + if i >= valid_results.len() { + break; + } + + let selection = &valid_results[i]; + let subnet_id = SyncSubnetId::new(selection.subcommittee_index); + + // Convert the response to a SyncSelectionProof + let proof = SyncSelectionProof::from( + selection_response.selection_proof.clone(), + ); + + // Check if the validator is an aggregator + match proof.is_aggregator::() { + Ok(true) => { + if let Some(Some(duty)) = + validators.get(&selection.validator_index) + { + debug!( + log, + "Validator is sync aggregator"; + "validator_index" => selection.validator_index, + "slot" => selection.slot, + "subnet_id" => %subnet_id, + ); + + // Store the proof + duty.aggregation_duties + .proofs + .write() + .insert((selection.slot, subnet_id), proof); + } + } + Ok(false) => { + // Not an aggregator, nothing to do + } + Err(e) => { + warn!( + log, + "Error determining is_aggregator"; + "validator_index" => selection.validator_index, + "slot" => selection.slot, + "error" => ?e, + ); + } + } + } + } + Err(e) => { + warn!( + log, + "Failed to get sync selection proofs from middleware"; + "error" => %e, + "slot" => slot, + ); + } + } + } + } else { + // For non-distributed mode + let mut validator_proofs = vec![]; + for (validator_start_slot, duty) in pre_compute_duties { + // Proofs are already known at this slot for this validator. + if slot < *validator_start_slot { + continue; + } + + let subnet_ids = match duty.subnet_ids::() { + Ok(subnet_ids) => subnet_ids, + Err(e) => { + crit!( + log, + "Arithmetic error computing subnet IDs"; + "error" => ?e, + ); + continue; + } + }; + + // Create futures to produce proofs. + let duties_service_ref = &duties_service; + let futures = subnet_ids.iter().map(|subnet_id| async move { + // Construct proof for prior slot. + let proof_slot = slot - 1; + + let proof = match duties_service_ref + .validator_store + .produce_sync_selection_proof(&duty.pubkey, proof_slot, *subnet_id) + .await + { + Ok(proof) => proof, + Err(ValidatorStoreError::UnknownPubkey(pubkey)) => { + // A pubkey can be missing when a validator was recently + // removed via the API. + debug!( + log, + "Missing pubkey for sync selection proof"; + "pubkey" => ?pubkey, + "pubkey" => ?duty.pubkey, + "slot" => proof_slot, + ); + return None; + } + Err(e) => { + warn!( + log, + "Unable to sign selection proof"; + "error" => ?e, + "pubkey" => ?duty.pubkey, + "slot" => proof_slot, + ); + return None; + } }; match proof.is_aggregator::() { @@ -678,52 +787,52 @@ pub async fn fill_in_aggregation_proofs( None } } - } - }); + }); - // Execute all the futures in parallel, collecting any successful results. - let proofs = join_all(futures) - .await - .into_iter() - .flatten() - .collect::>(); + // Execute all the futures in parallel, collecting any successful results. + let proofs = join_all(futures) + .await + .into_iter() + .flatten() + .collect::>(); - validator_proofs.push((duty.validator_index, proofs)); - } + validator_proofs.push((duty.validator_index, proofs)); + } - // Add to global storage (we add regularly so the proofs can be used ASAP). - let sync_map = duties_service.sync_duties.committees.read(); - let Some(committee_duties) = sync_map.get(&sync_committee_period) else { - debug!( - log, - "Missing sync duties"; - "period" => sync_committee_period, - ); - continue; - }; - let validators = committee_duties.validators.read(); - let num_validators_updated = validator_proofs.len(); - - for (validator_index, proofs) in validator_proofs { - if let Some(Some(duty)) = validators.get(&validator_index) { - duty.aggregation_duties.proofs.write().extend(proofs); - } else { + // Add to global storage (we add regularly so the proofs can be used ASAP). + let sync_map = duties_service.sync_duties.committees.read(); + let Some(committee_duties) = sync_map.get(&sync_committee_period) else { debug!( log, - "Missing sync duty to update"; - "validator_index" => validator_index, + "Missing sync duties"; "period" => sync_committee_period, ); + continue; + }; + let validators = committee_duties.validators.read(); + let num_validators_updated = validator_proofs.len(); + + for (validator_index, proofs) in validator_proofs { + if let Some(Some(duty)) = validators.get(&validator_index) { + duty.aggregation_duties.proofs.write().extend(proofs); + } else { + debug!( + log, + "Missing sync duty to update"; + "validator_index" => validator_index, + "period" => sync_committee_period, + ); + } + } + + if num_validators_updated > 0 { + debug!( + log, + "Finished computing sync selection proofs"; + "slot" => slot, + "updated_validators" => num_validators_updated, + ); } } - - if num_validators_updated > 0 { - debug!( - log, - "Finished computing sync selection proofs"; - "slot" => slot, - "updated_validators" => num_validators_updated, - ); - } } } From 57e0bb60e2ab75ca5eaa9604523336dce843e958 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Wed, 26 Mar 2025 14:03:35 +0800 Subject: [PATCH 27/75] remove proof_slot --- validator_client/validator_services/src/sync.rs | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index 8f42183aa3..7cd8902d49 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -544,9 +544,6 @@ pub async fn fill_in_aggregation_proofs( } }; - // Construct proof for prior slot. - let proof_slot = slot - 1; - // Create futures for all subnet IDs for this validator for subnet_id in subnet_ids { let duties_service = duties_service.clone(); @@ -558,18 +555,14 @@ pub async fn fill_in_aggregation_proofs( // Produce partial selection proof let sync_selection_proof = duties_service .validator_store - .produce_sync_selection_proof( - &duty.pubkey, - proof_slot, - subnet_id.into(), - ) + .produce_sync_selection_proof(&duty.pubkey, slot, subnet_id.into()) .await; match sync_selection_proof { Ok(proof) => { let sync_committee_selection = SyncCommitteeSelection { validator_index: duty.validator_index, - slot: proof_slot, + slot, subcommittee_index: subnet_id, selection_proof: proof.into(), }; @@ -580,7 +573,7 @@ pub async fn fill_in_aggregation_proofs( log, "Missing pubkey for sync selection proof"; "pubkey" => ?pubkey, - "slot" => proof_slot, + "slot" => slot, ); None } @@ -590,7 +583,7 @@ pub async fn fill_in_aggregation_proofs( "Unable to sign selection proof"; "error" => ?e, "pubkey" => ?duty.pubkey, - "slot" => proof_slot, + "slot" => slot, ); None } @@ -612,6 +605,7 @@ pub async fn fill_in_aggregation_proofs( log, "Sending batch of partial sync selection proofs"; "count" => selection_proofs.len(), + "slot" => slot, ); let response = duties_service @@ -632,6 +626,7 @@ pub async fn fill_in_aggregation_proofs( log, "Received batch response from middleware for sync"; "count" => response.data.len(), + "slot" => slot, ); // Get the sync map to update duties From 1535f40f53049d82a30efd86a5c8e51c3e801b5e Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Thu, 27 Mar 2025 10:44:22 +0800 Subject: [PATCH 28/75] Add back proof slot --- validator_client/validator_services/src/sync.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index 7cd8902d49..2e1d311022 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -544,6 +544,9 @@ pub async fn fill_in_aggregation_proofs( } }; + // Construct proof for prior slot. + let proof_slot = slot - 1; + // Create futures for all subnet IDs for this validator for subnet_id in subnet_ids { let duties_service = duties_service.clone(); @@ -562,7 +565,7 @@ pub async fn fill_in_aggregation_proofs( Ok(proof) => { let sync_committee_selection = SyncCommitteeSelection { validator_index: duty.validator_index, - slot, + slot: proof_slot, subcommittee_index: subnet_id, selection_proof: proof.into(), }; @@ -573,7 +576,7 @@ pub async fn fill_in_aggregation_proofs( log, "Missing pubkey for sync selection proof"; "pubkey" => ?pubkey, - "slot" => slot, + "slot" => proof_slot, ); None } @@ -583,7 +586,7 @@ pub async fn fill_in_aggregation_proofs( "Unable to sign selection proof"; "error" => ?e, "pubkey" => ?duty.pubkey, - "slot" => slot, + "slot" => proof_slot, ); None } From b21451fc114810e4e36777ec6b3c3d5072e15de9 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Thu, 27 Mar 2025 11:17:09 +0800 Subject: [PATCH 29/75] Fix --- validator_client/validator_services/src/sync.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index 2e1d311022..5a5585c672 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -526,11 +526,8 @@ pub async fn fill_in_aggregation_proofs( if duties_service.distributed { let mut partial_proofs = Vec::new(); - for (validator_start_slot, duty) in pre_compute_duties { + for (_validator_start_slot, duty) in pre_compute_duties { // Proofs are already known at this slot for this validator. - if slot < *validator_start_slot { - continue; - } let subnet_ids = match duty.subnet_ids::() { Ok(subnet_ids) => subnet_ids, @@ -558,7 +555,11 @@ pub async fn fill_in_aggregation_proofs( // Produce partial selection proof let sync_selection_proof = duties_service .validator_store - .produce_sync_selection_proof(&duty.pubkey, slot, subnet_id.into()) + .produce_sync_selection_proof( + &duty.pubkey, + proof_slot, + subnet_id.into(), + ) .await; match sync_selection_proof { From 7ac6867c5bca9b9f2f8c6bc145b361c17c4891eb Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Thu, 27 Mar 2025 14:35:12 +0800 Subject: [PATCH 30/75] Add debug logging --- consensus/types/src/sync_selection_proof.rs | 1 + .../validator_services/src/sync.rs | 40 +++++++++++-------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/consensus/types/src/sync_selection_proof.rs b/consensus/types/src/sync_selection_proof.rs index e4f6ce43cd..61302a9dfb 100644 --- a/consensus/types/src/sync_selection_proof.rs +++ b/consensus/types/src/sync_selection_proof.rs @@ -13,6 +13,7 @@ use ssz_types::typenum::Unsigned; use std::cmp; #[derive(arbitrary::Arbitrary, PartialEq, Debug, Clone, Serialize, Deserialize)] +#[serde(transparent)] pub struct SyncSelectionProof(Signature); impl SyncSelectionProof { diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index 5a5585c672..1487999a39 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -568,8 +568,17 @@ pub async fn fill_in_aggregation_proofs( validator_index: duty.validator_index, slot: proof_slot, subcommittee_index: subnet_id, - selection_proof: proof.into(), + selection_proof: proof.clone().into(), }; + // Add log for debugging + debug!( + log, + "Partial sync selection proof"; + "validator_index" => duty.validator_index, + "slot" => proof_slot, + "subcommittee_index" => subnet_id, + "partial selection proof" => ?proof, + ); Some(sync_committee_selection) } Err(ValidatorStoreError::UnknownPubkey(pubkey)) => { @@ -647,13 +656,10 @@ pub async fn fill_in_aggregation_proofs( let validators = committee_duties.validators.read(); // Process each response - for (i, selection_response) in response.data.iter().enumerate() { - if i >= valid_results.len() { - break; - } - - let selection = &valid_results[i]; - let subnet_id = SyncSubnetId::new(selection.subcommittee_index); + for selection_response in response.data.iter() { + let validator_index = selection_response.validator_index; + let slot = selection_response.slot; + let subcommittee_index = selection_response.subcommittee_index; // Convert the response to a SyncSelectionProof let proof = SyncSelectionProof::from( @@ -663,22 +669,22 @@ pub async fn fill_in_aggregation_proofs( // Check if the validator is an aggregator match proof.is_aggregator::() { Ok(true) => { - if let Some(Some(duty)) = - validators.get(&selection.validator_index) - { + if let Some(Some(duty)) = validators.get(&validator_index) { debug!( log, "Validator is sync aggregator"; - "validator_index" => selection.validator_index, - "slot" => selection.slot, - "subnet_id" => %subnet_id, + "validator_index" => validator_index, + "slot" => slot, + "subnet_id" => subcommittee_index, + // log full selection proof for debugging + "full selection proof" => ?proof, ); // Store the proof duty.aggregation_duties .proofs .write() - .insert((selection.slot, subnet_id), proof); + .insert((slot, subcommittee_index.into()), proof); } } Ok(false) => { @@ -688,8 +694,8 @@ pub async fn fill_in_aggregation_proofs( warn!( log, "Error determining is_aggregator"; - "validator_index" => selection.validator_index, - "slot" => selection.slot, + "validator_index" => validator_index, + "slot" => slot, "error" => ?e, ); } From ef90462edbe9f1eafb39f9feb029d8db00a74097 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Thu, 27 Mar 2025 20:44:44 +0800 Subject: [PATCH 31/75] Remove transparent --- consensus/types/src/sync_selection_proof.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/consensus/types/src/sync_selection_proof.rs b/consensus/types/src/sync_selection_proof.rs index 61302a9dfb..e4f6ce43cd 100644 --- a/consensus/types/src/sync_selection_proof.rs +++ b/consensus/types/src/sync_selection_proof.rs @@ -13,7 +13,6 @@ use ssz_types::typenum::Unsigned; use std::cmp; #[derive(arbitrary::Arbitrary, PartialEq, Debug, Clone, Serialize, Deserialize)] -#[serde(transparent)] pub struct SyncSelectionProof(Signature); impl SyncSelectionProof { From 1118d622f57256a1aecbe1d0ce53e5c583695c22 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Mon, 31 Mar 2025 18:33:56 +0800 Subject: [PATCH 32/75] Update logging --- .../validator_services/src/duties_service.rs | 16 +- .../validator_services/src/sync.rs | 264 +++++++++--------- 2 files changed, 128 insertions(+), 152 deletions(-) diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index fde5e18f6a..7ae34a22d1 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -132,7 +132,6 @@ async fn make_selection_proof( spec: &ChainSpec, distributed: bool, beacon_nodes: &Arc>, - duties_service: &DutiesService, ) -> Result, Error> { let selection_proof = if distributed { // Submit a partial selection proof in the data field of the POST HTTP endpoint @@ -147,14 +146,12 @@ async fn make_selection_proof( }; // Call the endpoint /eth/v1/validator/beacon_committee_selections // The middleware should return a full selection proof here - let log = duties_service.context.log(); let response = beacon_nodes .first_success(|beacon_node| { let selections = selection.clone(); debug!( - log, - "Partial selection proof from VC"; - "Selection proof" => ?selections, + "Selection proof" = ?selections, + "Partial selection proof from VC" ); // println!("Selection proof: {:?}", selections); async move { @@ -162,12 +159,7 @@ async fn make_selection_proof( .post_validator_beacon_committee_selections(&[selections]) .await; - debug!( - log, - "Response from middleware"; - "response" => ?response, - - ); + debug!(?response, "Response from middleware"); // println!("Response from middleware {:?}", response); response @@ -1147,7 +1139,6 @@ async fn fill_in_selection_proofs( &duties_service.spec, duties_service.distributed, &duties_service.beacon_nodes, - &duties_service, ) .await?; Ok((duty, opt_selection_proof)) @@ -1163,7 +1154,6 @@ async fn fill_in_selection_proofs( &duties_service.spec, duties_service.distributed, &duties_service.beacon_nodes, - &duties_service, ) .await?; Ok((duty, opt_selection_proof)) diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index 7f27dd4263..a0a733d7a4 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -524,9 +524,8 @@ pub async fn fill_in_aggregation_proofs( Ok(subnet_ids) => subnet_ids, Err(e) => { crit!( - log, - "Arithmetic error computing subnet IDs"; - "error" => ?e, + "error" = ?e, + "Arithmetic error computing subnet IDs" ); continue; } @@ -563,31 +562,28 @@ pub async fn fill_in_aggregation_proofs( }; // Add log for debugging debug!( - log, - "Partial sync selection proof"; - "validator_index" => duty.validator_index, - "slot" => proof_slot, - "subcommittee_index" => subnet_id, - "partial selection proof" => ?proof, + "validator_index" = duty.validator_index, + "slot" = %proof_slot, + "subcommittee_index" = subnet_id, + "partial selection proof" = ?proof, + "Partial sync selection proof" ); Some(sync_committee_selection) } Err(ValidatorStoreError::UnknownPubkey(pubkey)) => { debug!( - log, - "Missing pubkey for sync selection proof"; - "pubkey" => ?pubkey, - "slot" => proof_slot, + ?pubkey, + "slot" = %proof_slot, + "Missing pubkey for sync selection proof" ); None } Err(e) => { warn!( - log, - "Unable to sign selection proof"; - "error" => ?e, - "pubkey" => ?duty.pubkey, - "slot" => proof_slot, + "error" = ?e, + "pubkey" = ?duty.pubkey, + "slot" = %proof_slot, + "Unable to sign selection proof" ); None } @@ -606,10 +602,8 @@ pub async fn fill_in_aggregation_proofs( // If we have any valid proofs, send them to the middleware if !selection_proofs.is_empty() { debug!( - log, - "Sending batch of partial sync selection proofs"; - "count" => selection_proofs.len(), - "slot" => slot, + "count" = selection_proofs.len(), + %slot, "Sending batch of partial sync selection proofs" ); let response = duties_service @@ -627,20 +621,14 @@ pub async fn fill_in_aggregation_proofs( match response { Ok(response) => { debug!( - log, - "Received batch response from middleware for sync"; - "count" => response.data.len(), - "slot" => slot, + "count" = response.data.len(), + %slot, "Received batch response from middleware for sync" ); // Get the sync map to update duties let sync_map = duties_service.sync_duties.committees.read(); let Some(committee_duties) = sync_map.get(&sync_committee_period) else { - debug!( - log, - "Missing sync duties"; - "period" => sync_committee_period, - ); + debug!("period" = sync_committee_period, "Missing sync duties"); continue; }; @@ -662,13 +650,12 @@ pub async fn fill_in_aggregation_proofs( Ok(true) => { if let Some(Some(duty)) = validators.get(&validator_index) { debug!( - log, - "Validator is sync aggregator"; - "validator_index" => validator_index, - "slot" => slot, - "subnet_id" => subcommittee_index, + validator_index, + %slot, + "subnet_id" = subcommittee_index, // log full selection proof for debugging - "full selection proof" => ?proof, + "full selection proof" = ?proof, + "Validator is sync aggregator" ); // Store the proof @@ -683,11 +670,10 @@ pub async fn fill_in_aggregation_proofs( } Err(e) => { warn!( - log, - "Error determining is_aggregator"; - "validator_index" => validator_index, - "slot" => slot, - "error" => ?e, + validator_index, + %slot, + "error" = ?e, + "Error determining is_aggregator" ); } } @@ -695,10 +681,9 @@ pub async fn fill_in_aggregation_proofs( } Err(e) => { warn!( - log, - "Failed to get sync selection proofs from middleware"; - "error" => %e, - "slot" => slot, + "error" = %e, + %slot, + "Failed to get sync selection proofs from middleware" ); } } @@ -711,112 +696,113 @@ pub async fn fill_in_aggregation_proofs( if slot < *validator_start_slot { continue; } - + let subnet_ids = match duty.subnet_ids::() { - Ok(subnet_ids) => subnet_ids, - Err(e) => { - crit!( - error = ?e, - "Arithmetic error computing subnet IDs" - ); - continue; - } - }; - - // Create futures to produce proofs. - let duties_service_ref = &duties_service; - let futures = subnet_ids.iter().map(|subnet_id| async move { - // Construct proof for prior slot. - let proof_slot = slot - 1; - - let proof = match duties_service_ref - .validator_store - .produce_sync_selection_proof(&duty.pubkey, proof_slot, *subnet_id) - .await - { - Ok(proof) => proof, - Err(ValidatorStoreError::UnknownPubkey(pubkey)) => { - // A pubkey can be missing when a validator was recently - // removed via the API. - debug!( - ?pubkey, - pubkey = ?duty.pubkey, - slot = %proof_slot, - "Missing pubkey for sync selection proof" - ); - return None; - } + Ok(subnet_ids) => subnet_ids, Err(e) => { - warn!( + crit!( error = ?e, - pubkey = ?duty.pubkey, - slot = %proof_slot, - "Unable to sign selection proof" + "Arithmetic error computing subnet IDs" ); - return None; + continue; } }; - match proof.is_aggregator::() { - Ok(true) => { - debug!( - validator_index = duty.validator_index, - slot = %proof_slot, - %subnet_id, - "Validator is sync aggregator" - ); - Some(((proof_slot, *subnet_id), proof)) - } - Ok(false) => None, - Err(e) => { - warn!( - pubkey = ?duty.pubkey, - slot = %proof_slot, - error = ?e, - "Error determining is_aggregator" - ); - None + // Create futures to produce proofs. + let duties_service_ref = &duties_service; + let futures = subnet_ids.iter().map(|subnet_id| async move { + // Construct proof for prior slot. + let proof_slot = slot - 1; + + let proof = match duties_service_ref + .validator_store + .produce_sync_selection_proof(&duty.pubkey, proof_slot, *subnet_id) + .await + { + Ok(proof) => proof, + Err(ValidatorStoreError::UnknownPubkey(pubkey)) => { + // A pubkey can be missing when a validator was recently + // removed via the API. + debug!( + ?pubkey, + pubkey = ?duty.pubkey, + slot = %proof_slot, + "Missing pubkey for sync selection proof" + ); + return None; + } + Err(e) => { + warn!( + error = ?e, + pubkey = ?duty.pubkey, + slot = %proof_slot, + "Unable to sign selection proof" + ); + return None; + } + }; + + match proof.is_aggregator::() { + Ok(true) => { + debug!( + validator_index = duty.validator_index, + slot = %proof_slot, + %subnet_id, + "Validator is sync aggregator" + ); + Some(((proof_slot, *subnet_id), proof)) + } + Ok(false) => None, + Err(e) => { + warn!( + pubkey = ?duty.pubkey, + slot = %proof_slot, + error = ?e, + "Error determining is_aggregator" + ); + None + } } + }); + + // Execute all the futures in parallel, collecting any successful results. + let proofs = join_all(futures) + .await + .into_iter() + .flatten() + .collect::>(); + + validator_proofs.push((duty.validator_index, proofs)); + } + + // Add to global storage (we add regularly so the proofs can be used ASAP). + let sync_map = duties_service.sync_duties.committees.read(); + let Some(committee_duties) = sync_map.get(&sync_committee_period) else { + debug!(period = sync_committee_period, "Missing sync duties"); + continue; + }; + let validators = committee_duties.validators.read(); + let num_validators_updated = validator_proofs.len(); + + for (validator_index, proofs) in validator_proofs { + if let Some(Some(duty)) = validators.get(&validator_index) { + duty.aggregation_duties.proofs.write().extend(proofs); + } else { + debug!( + validator_index, + period = sync_committee_period, + "Missing sync duty to update" + ); } - }); + } - // Execute all the futures in parallel, collecting any successful results. - let proofs = join_all(futures) - .await - .into_iter() - .flatten() - .collect::>(); - - validator_proofs.push((duty.validator_index, proofs)); - } - - // Add to global storage (we add regularly so the proofs can be used ASAP). - let sync_map = duties_service.sync_duties.committees.read(); - let Some(committee_duties) = sync_map.get(&sync_committee_period) else { - debug!(period = sync_committee_period, "Missing sync duties"); - continue; - }; - let validators = committee_duties.validators.read(); - let num_validators_updated = validator_proofs.len(); - - for (validator_index, proofs) in validator_proofs { - if let Some(Some(duty)) = validators.get(&validator_index) { - duty.aggregation_duties.proofs.write().extend(proofs); - } else { + if num_validators_updated > 0 { debug!( - validator_index, - period = sync_committee_period, - "Missing sync duty to update" + %slot, + updated_validators = num_validators_updated, + "Finished computing sync selection proofs" ); } } - - if num_validators_updated > 0 { - debug!( - %slot, - updated_validators = num_validators_updated, - "Finished computing sync selection proofs" - ); - } } } From 9459c362dde98fd8a993594e6218500e678f5b5c Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Tue, 1 Apr 2025 10:02:09 +0800 Subject: [PATCH 33/75] Simplify beacon committee selection logging and code --- validator_client/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 1198586840..13ecfe42ca 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -60,13 +60,13 @@ const WAITING_FOR_GENESIS_POLL_TIME: Duration = Duration::from_secs(12); const HTTP_ATTESTATION_TIMEOUT_QUOTIENT: u32 = 4; const HTTP_ATTESTER_DUTIES_TIMEOUT_QUOTIENT: u32 = 4; const HTTP_ATTESTATION_SUBSCRIPTIONS_TIMEOUT_QUOTIENT: u32 = 24; -const HTTP_ATTESTATION_AGGREGATOR_TIMEOUT_QUOTIENT: u32 = 24; +const HTTP_ATTESTATION_AGGREGATOR_TIMEOUT_QUOTIENT: u32 = 24; // For distributed mode only const HTTP_LIVENESS_TIMEOUT_QUOTIENT: u32 = 4; const HTTP_PROPOSAL_TIMEOUT_QUOTIENT: u32 = 2; const HTTP_PROPOSER_DUTIES_TIMEOUT_QUOTIENT: u32 = 4; const HTTP_SYNC_COMMITTEE_CONTRIBUTION_TIMEOUT_QUOTIENT: u32 = 4; const HTTP_SYNC_DUTIES_TIMEOUT_QUOTIENT: u32 = 4; -const HTTP_SYNC_AGGREGATOR_TIMEOUT_QUOTIENT: u32 = 24; +const HTTP_SYNC_AGGREGATOR_TIMEOUT_QUOTIENT: u32 = 24; // For distributed mode only const HTTP_GET_BEACON_BLOCK_SSZ_TIMEOUT_QUOTIENT: u32 = 4; const HTTP_GET_DEBUG_BEACON_STATE_QUOTIENT: u32 = 4; const HTTP_GET_DEPOSIT_SNAPSHOT_QUOTIENT: u32 = 4; From 101f24c4409193c0cf563f45849e4bb895f32585 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Tue, 1 Apr 2025 10:02:32 +0800 Subject: [PATCH 34/75] logging --- .../validator_services/src/duties_service.rs | 53 +++++++++---------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index 7ae34a22d1..9f9363215d 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -134,8 +134,7 @@ async fn make_selection_proof( beacon_nodes: &Arc>, ) -> Result, Error> { let selection_proof = if distributed { - // Submit a partial selection proof in the data field of the POST HTTP endpoint - let selection = BeaconCommitteeSelection { + let beacon_committee_selection = BeaconCommitteeSelection { validator_index: duty.validator_index, slot: duty.slot, selection_proof: validator_store @@ -145,38 +144,38 @@ async fn make_selection_proof( .into(), }; // Call the endpoint /eth/v1/validator/beacon_committee_selections - // The middleware should return a full selection proof here - let response = beacon_nodes + // by sending the BeaconCommitteeSelection that contains partial selection proof + // The middleware should return BeaconCommitteeSelection that contains full selection proof + let middleware_response = beacon_nodes .first_success(|beacon_node| { - let selections = selection.clone(); + let selection_data = beacon_committee_selection.clone(); debug!( - "Selection proof" = ?selections, - "Partial selection proof from VC" + "validator_index" = duty.validator_index, + "slot" = %duty.slot, + "partial selection proof" = ?beacon_committee_selection.selection_proof, + "Sending beacon committee selection to middleware" ); - // println!("Selection proof: {:?}", selections); async move { - let response = beacon_node - .post_validator_beacon_committee_selections(&[selections]) - .await; - - debug!(?response, "Response from middleware"); - // println!("Response from middleware {:?}", response); - - response + beacon_node + .post_validator_beacon_committee_selections(&[selection_data]) + .await } }) .await; - SelectionProof::from( - response - .map_err(|e| { - Error::FailedToProduceSelectionProof(ValidatorStoreError::Middleware( - e.to_string(), - )) - })? - .data[0] - .selection_proof - .clone(), - ) + + let response_data = &middleware_response + .map_err(|e| { + Error::FailedToProduceSelectionProof(ValidatorStoreError::Middleware(e.to_string())) + })? + .data[0]; + + debug!( + "validator_index" = response_data.validator_index, + "slot" = %response_data.slot, + "full selection proof" = ?response_data.selection_proof, + "Received beacon committee selection from middleware" + ); + SelectionProof::from(response_data.selection_proof.clone()) } else { validator_store .produce_selection_proof(duty.pubkey, duty.slot) From d222ce8a5c9182bd231ad1ce34ae5ca524078308 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Tue, 1 Apr 2025 16:06:14 +0800 Subject: [PATCH 35/75] tidy up sync.rs --- validator_client/validator_services/src/duties_service.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index 9f9363215d..50b01ce46e 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -137,6 +137,7 @@ async fn make_selection_proof( let beacon_committee_selection = BeaconCommitteeSelection { validator_index: duty.validator_index, slot: duty.slot, + // In distributed mode, this is partial selection proof selection_proof: validator_store .produce_selection_proof(duty.pubkey, duty.slot) .await @@ -153,7 +154,7 @@ async fn make_selection_proof( "validator_index" = duty.validator_index, "slot" = %duty.slot, "partial selection proof" = ?beacon_committee_selection.selection_proof, - "Sending beacon committee selection to middleware" + "Sending selection to middleware" ); async move { beacon_node @@ -173,7 +174,7 @@ async fn make_selection_proof( "validator_index" = response_data.validator_index, "slot" = %response_data.slot, "full selection proof" = ?response_data.selection_proof, - "Received beacon committee selection from middleware" + "Received selection from middleware" ); SelectionProof::from(response_data.selection_proof.clone()) } else { @@ -1105,6 +1106,7 @@ async fn fill_in_selection_proofs( let lookahead_slot = current_slot + selection_lookahead; let relevant_duties = if duties_service.distributed { + // Remove old slot duties and only keep current duties duties_by_slot .remove(&lookahead_slot) .map(|duties| BTreeMap::from([(lookahead_slot, duties)])) @@ -1127,7 +1129,7 @@ async fn fill_in_selection_proofs( ); // In distributed case, we want to send all partial selection proofs to the middleware to determine aggregation duties, - // as the middleware will need to have a threshold of partial selection proof to be able to return the full selection proof + // as the middleware will need to have a threshold of partial selection proofs to be able to return the full selection proof // Thus, sign selection proofs in parallel in distributed case; Otherwise, sign them serially in non-distributed (normal) case let duty_and_proof_results = if duties_service.distributed { futures::future::join_all(relevant_duties.into_values().flatten().map( From 6f2ea519395c54ae410586e612882b0be029c1d0 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Tue, 1 Apr 2025 16:07:32 +0800 Subject: [PATCH 36/75] sync.rs --- .../validator_services/src/sync.rs | 179 +++++++++--------- 1 file changed, 86 insertions(+), 93 deletions(-) diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index a0a733d7a4..b3c0d7d9bc 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -504,18 +504,11 @@ pub async fn fill_in_aggregation_proofs( current_slot: Slot, pre_compute_slot: Slot, ) { - debug!( - period = sync_committee_period, - %current_slot, - %pre_compute_slot, - "Calculating sync selection proofs" - ); - // Generate selection proofs for each validator at each slot, one slot at a time. for slot in (current_slot.as_u64()..=pre_compute_slot.as_u64()).map(Slot::new) { // For distributed mode if duties_service.distributed { - let mut partial_proofs = Vec::new(); + let mut sync_committee_selection = Vec::new(); for (_validator_start_slot, duty) in pre_compute_duties { // Proofs are already known at this slot for this validator. @@ -540,10 +533,10 @@ pub async fn fill_in_aggregation_proofs( let duty = duty.clone(); let subnet_id = *subnet_id; - // Store all partial sync selection proofs for this slot in partial_proofs so that it can be sent together - partial_proofs.push(async move { + // Store all partial sync selection proofs in partial_proofs so that it can be sent together later + sync_committee_selection.push(async move { // Produce partial selection proof - let sync_selection_proof = duties_service + let partial_sync_selection_proof = duties_service .validator_store .produce_sync_selection_proof( &duty.pubkey, @@ -552,7 +545,7 @@ pub async fn fill_in_aggregation_proofs( ) .await; - match sync_selection_proof { + match partial_sync_selection_proof { Ok(proof) => { let sync_committee_selection = SyncCommitteeSelection { validator_index: duty.validator_index, @@ -560,13 +553,12 @@ pub async fn fill_in_aggregation_proofs( subcommittee_index: subnet_id, selection_proof: proof.clone().into(), }; - // Add log for debugging debug!( "validator_index" = duty.validator_index, "slot" = %proof_slot, "subcommittee_index" = subnet_id, - "partial selection proof" = ?proof, - "Partial sync selection proof" + "partial sync selection proof" = ?proof, + "Sending sync selection to middleware" ); Some(sync_committee_selection) } @@ -592,104 +584,105 @@ pub async fn fill_in_aggregation_proofs( } } - // Execute partial_proofs in parallel - let partial_proof_results = join_all(partial_proofs).await; + let sync_committee_selection_data = join_all(sync_committee_selection).await; // Filter out None values and extract the selection proofs - let valid_results: Vec<_> = partial_proof_results.into_iter().flatten().collect(); - let selection_proofs: Vec<_> = valid_results.clone(); + let sync_selection_data: Vec<_> = sync_committee_selection_data + .into_iter() + .flatten() + .collect(); - // If we have any valid proofs, send them to the middleware - if !selection_proofs.is_empty() { - debug!( - "count" = selection_proofs.len(), - %slot, "Sending batch of partial sync selection proofs" - ); + let middleware_response = duties_service + .beacon_nodes + .first_success(|beacon_node| { + let selection_data = sync_selection_data.clone(); + async move { + beacon_node + .post_validator_sync_committee_selections(&selection_data) + .await + } + }) + .await; - let response = duties_service - .beacon_nodes - .first_success(|beacon_node| { - let proofs = selection_proofs.clone(); - async move { - beacon_node - .post_validator_sync_committee_selections(&proofs) - .await - } - }) - .await; + match middleware_response { + Ok(response) => { + // The selection proof from middleware response will be a full selection proof + debug!( + "validator_index" = response.data[0].validator_index, + "slot" = %response.data[0].slot, + "subcommittee_index" = response.data[0].subcommittee_index, + "full sync selection proof" = ?response.data[0].selection_proof, + "Received sync selection from middleware" + ); - match response { - Ok(response) => { - debug!( - "count" = response.data.len(), - %slot, "Received batch response from middleware for sync" - ); + // Get the sync map to update duties + let sync_map = duties_service.sync_duties.committees.read(); + let Some(committee_duties) = sync_map.get(&sync_committee_period) else { + debug!("period" = sync_committee_period, "Missing sync duties"); + continue; + }; - // Get the sync map to update duties - let sync_map = duties_service.sync_duties.committees.read(); - let Some(committee_duties) = sync_map.get(&sync_committee_period) else { - debug!("period" = sync_committee_period, "Missing sync duties"); - continue; - }; + let validators = committee_duties.validators.read(); - let validators = committee_duties.validators.read(); + // Process each response + for response_data in response.data.iter() { + let validator_index = response_data.validator_index; + let slot = response_data.slot; + let subcommittee_index = response_data.subcommittee_index; - // Process each response - for selection_response in response.data.iter() { - let validator_index = selection_response.validator_index; - let slot = selection_response.slot; - let subcommittee_index = selection_response.subcommittee_index; + // Convert the response to a SyncSelectionProof so we can call the is_aggregator method + let full_selection_proof = + SyncSelectionProof::from(response_data.selection_proof.clone()); - // Convert the response to a SyncSelectionProof - let proof = SyncSelectionProof::from( - selection_response.selection_proof.clone(), - ); - - // Check if the validator is an aggregator - match proof.is_aggregator::() { - Ok(true) => { - if let Some(Some(duty)) = validators.get(&validator_index) { - debug!( - validator_index, - %slot, - "subnet_id" = subcommittee_index, - // log full selection proof for debugging - "full selection proof" = ?proof, - "Validator is sync aggregator" - ); - - // Store the proof - duty.aggregation_duties - .proofs - .write() - .insert((slot, subcommittee_index.into()), proof); - } - } - Ok(false) => { - // Not an aggregator, nothing to do - } - Err(e) => { - warn!( + // Check if the validator is an aggregator + match full_selection_proof.is_aggregator::() { + Ok(true) => { + if let Some(Some(duty)) = validators.get(&validator_index) { + debug!( validator_index, %slot, - "error" = ?e, - "Error determining is_aggregator" + "subcommittee_index" = subcommittee_index, + // log full selection proof for debugging + "full selection proof" = ?response_data.selection_proof, + "Validator is sync aggregator" + ); + + // Store the proof + duty.aggregation_duties.proofs.write().insert( + (slot, subcommittee_index.into()), + full_selection_proof, ); } } + Ok(false) => {} // Not an aggregator + Err(e) => { + warn!( + validator_index, + %slot, + "error" = ?e, + "Error determining is_aggregator" + ); + } } } - Err(e) => { - warn!( - "error" = %e, - %slot, - "Failed to get sync selection proofs from middleware" - ); - } + } + Err(e) => { + warn!( + "error" = %e, + %slot, + "Failed to get sync selection proofs from middleware" + ); } } } else { // For non-distributed mode + debug!( + period = sync_committee_period, + %current_slot, + %pre_compute_slot, + "Calculating sync selection proofs" + ); + let mut validator_proofs = vec![]; for (validator_start_slot, duty) in pre_compute_duties { // Proofs are already known at this slot for this validator. From e8ca60ea23780259f0a0fecc94d1116d9572fe36 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Tue, 1 Apr 2025 17:33:26 +0800 Subject: [PATCH 37/75] Fix logging from middleware --- .../validator_services/src/sync.rs | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index b3c0d7d9bc..52683e08c6 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -557,7 +557,7 @@ pub async fn fill_in_aggregation_proofs( "validator_index" = duty.validator_index, "slot" = %proof_slot, "subcommittee_index" = subnet_id, - "partial sync selection proof" = ?proof, + "partial selection proof" = ?proof, "Sending sync selection to middleware" ); Some(sync_committee_selection) @@ -606,15 +606,6 @@ pub async fn fill_in_aggregation_proofs( match middleware_response { Ok(response) => { - // The selection proof from middleware response will be a full selection proof - debug!( - "validator_index" = response.data[0].validator_index, - "slot" = %response.data[0].slot, - "subcommittee_index" = response.data[0].subcommittee_index, - "full sync selection proof" = ?response.data[0].selection_proof, - "Received sync selection from middleware" - ); - // Get the sync map to update duties let sync_map = duties_service.sync_duties.committees.read(); let Some(committee_duties) = sync_map.get(&sync_committee_period) else { @@ -624,8 +615,16 @@ pub async fn fill_in_aggregation_proofs( let validators = committee_duties.validators.read(); - // Process each response + // Process each middleware response for response_data in response.data.iter() { + // The selection proof from middleware response will be a full selection proof + debug!( + "validator_index" = response_data.validator_index, + "slot" = %response_data.slot, + "subcommittee_index" = response_data.subcommittee_index, + "full selection proof" = ?response_data.selection_proof, + "Received sync selection from middleware" + ); let validator_index = response_data.validator_index; let slot = response_data.slot; let subcommittee_index = response_data.subcommittee_index; From e3406becb541834d9d8f8429d810d9402077f895 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Tue, 1 Apr 2025 19:58:53 +0800 Subject: [PATCH 38/75] remove subnet for loop --- .../validator_services/src/sync.rs | 40 ++++++++----------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index 52683e08c6..ede80b7e79 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -1,6 +1,6 @@ use crate::duties_service::{DutiesService, Error}; use doppelganger_service::DoppelgangerStatus; -use eth2::types::SyncCommitteeSelection; +use eth2::types::{Signature, SyncCommitteeSelection}; use futures::future::join_all; use logging::crit; use parking_lot::{MappedRwLockReadGuard, RwLock, RwLockReadGuard, RwLockWriteGuard}; @@ -527,39 +527,33 @@ pub async fn fill_in_aggregation_proofs( // Construct proof for prior slot. let proof_slot = slot - 1; - // Create futures for all subnet IDs for this validator - for subnet_id in subnet_ids { + // Store all partial sync selection proofs in partial_proofs so that it can be sent together later + sync_committee_selection.extend(subnet_ids.iter().map(|&subnet_id| { let duties_service = duties_service.clone(); let duty = duty.clone(); - let subnet_id = *subnet_id; - - // Store all partial sync selection proofs in partial_proofs so that it can be sent together later - sync_committee_selection.push(async move { + async move { // Produce partial selection proof let partial_sync_selection_proof = duties_service .validator_store - .produce_sync_selection_proof( - &duty.pubkey, - proof_slot, - subnet_id.into(), - ) + .produce_sync_selection_proof(&duty.pubkey, proof_slot, subnet_id) .await; match partial_sync_selection_proof { Ok(proof) => { - let sync_committee_selection = SyncCommitteeSelection { - validator_index: duty.validator_index, - slot: proof_slot, - subcommittee_index: subnet_id, - selection_proof: proof.clone().into(), - }; debug!( "validator_index" = duty.validator_index, "slot" = %proof_slot, - "subcommittee_index" = subnet_id, - "partial selection proof" = ?proof, + "subcommittee_index" = *subnet_id, + "partial selection proof" = ?Signature::from(proof.clone()), "Sending sync selection to middleware" ); + + let sync_committee_selection = SyncCommitteeSelection { + validator_index: duty.validator_index, + slot: proof_slot, + subcommittee_index: *subnet_id, + selection_proof: proof.clone().into(), + }; Some(sync_committee_selection) } Err(ValidatorStoreError::UnknownPubkey(pubkey)) => { @@ -580,13 +574,13 @@ pub async fn fill_in_aggregation_proofs( None } } - }); - } + } + })); } let sync_committee_selection_data = join_all(sync_committee_selection).await; - // Filter out None values and extract the selection proofs + // Collect the SyncCommitteeSelection data let sync_selection_data: Vec<_> = sync_committee_selection_data .into_iter() .flatten() From 1877f09a82b1d2a272fa2367db173a1128befdfe Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Tue, 1 Apr 2025 20:59:51 +0800 Subject: [PATCH 39/75] Add comment --- validator_client/validator_services/src/sync.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index ede80b7e79..a387bd65b0 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -527,7 +527,7 @@ pub async fn fill_in_aggregation_proofs( // Construct proof for prior slot. let proof_slot = slot - 1; - // Store all partial sync selection proofs in partial_proofs so that it can be sent together later + // Store all partial sync selection proofs so that it can be sent together later sync_committee_selection.extend(subnet_ids.iter().map(|&subnet_id| { let duties_service = duties_service.clone(); let duty = duty.clone(); @@ -586,6 +586,9 @@ pub async fn fill_in_aggregation_proofs( .flatten() .collect(); + // Call the endpoint /eth/v1/validator/sync_committee_selections + // by sending the SyncCommitteeSelection that contains partial sync selection proof + // The middleware should return SyncCommitteeSelection that contains full sync selection proof let middleware_response = duties_service .beacon_nodes .first_success(|beacon_node| { From 41422b1326415a0f5ef342feb84134289699d4fa Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Tue, 1 Apr 2025 21:11:17 +0800 Subject: [PATCH 40/75] reduce diff --- .../validator_services/src/sync.rs | 247 +++++++++--------- 1 file changed, 123 insertions(+), 124 deletions(-) diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index a387bd65b0..d24a5ad191 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -506,8 +506,129 @@ pub async fn fill_in_aggregation_proofs( ) { // Generate selection proofs for each validator at each slot, one slot at a time. for slot in (current_slot.as_u64()..=pre_compute_slot.as_u64()).map(Slot::new) { - // For distributed mode - if duties_service.distributed { + debug!( + period = sync_committee_period, + %current_slot, + %pre_compute_slot, + "Calculating sync selection proofs" + ); + if !duties_service.distributed { + // For non-distributed mode + let mut validator_proofs = vec![]; + for (validator_start_slot, duty) in pre_compute_duties { + // Proofs are already known at this slot for this validator. + if slot < *validator_start_slot { + continue; + } + + let subnet_ids = match duty.subnet_ids::() { + Ok(subnet_ids) => subnet_ids, + Err(e) => { + crit!( + error = ?e, + "Arithmetic error computing subnet IDs" + ); + continue; + } + }; + + // Create futures to produce proofs. + let duties_service_ref = &duties_service; + let futures = subnet_ids.iter().map(|subnet_id| async move { + // Construct proof for prior slot. + let proof_slot = slot - 1; + + let proof = match duties_service_ref + .validator_store + .produce_sync_selection_proof(&duty.pubkey, proof_slot, *subnet_id) + .await + { + Ok(proof) => proof, + Err(ValidatorStoreError::UnknownPubkey(pubkey)) => { + // A pubkey can be missing when a validator was recently + // removed via the API. + debug!( + ?pubkey, + pubkey = ?duty.pubkey, + slot = %proof_slot, + "Missing pubkey for sync selection proof" + ); + return None; + } + Err(e) => { + warn!( + error = ?e, + pubkey = ?duty.pubkey, + slot = %proof_slot, + "Unable to sign selection proof" + ); + return None; + } + }; + + match proof.is_aggregator::() { + Ok(true) => { + debug!( + validator_index = duty.validator_index, + slot = %proof_slot, + %subnet_id, + "Validator is sync aggregator" + ); + Some(((proof_slot, *subnet_id), proof)) + } + Ok(false) => None, + Err(e) => { + warn!( + pubkey = ?duty.pubkey, + slot = %proof_slot, + error = ?e, + "Error determining is_aggregator" + ); + None + } + } + }); + + // Execute all the futures in parallel, collecting any successful results. + let proofs = join_all(futures) + .await + .into_iter() + .flatten() + .collect::>(); + + validator_proofs.push((duty.validator_index, proofs)); + } + + // Add to global storage (we add regularly so the proofs can be used ASAP). + let sync_map = duties_service.sync_duties.committees.read(); + let Some(committee_duties) = sync_map.get(&sync_committee_period) else { + debug!(period = sync_committee_period, "Missing sync duties"); + continue; + }; + let validators = committee_duties.validators.read(); + let num_validators_updated = validator_proofs.len(); + + for (validator_index, proofs) in validator_proofs { + if let Some(Some(duty)) = validators.get(&validator_index) { + duty.aggregation_duties.proofs.write().extend(proofs); + } else { + debug!( + validator_index, + period = sync_committee_period, + "Missing sync duty to update" + ); + } + } + + if num_validators_updated > 0 { + debug!( + %slot, + updated_validators = num_validators_updated, + "Finished computing sync selection proofs" + ); + } + } else { + // For distributed mode let mut sync_committee_selection = Vec::new(); for (_validator_start_slot, duty) in pre_compute_duties { @@ -670,128 +791,6 @@ pub async fn fill_in_aggregation_proofs( ); } } - } else { - // For non-distributed mode - debug!( - period = sync_committee_period, - %current_slot, - %pre_compute_slot, - "Calculating sync selection proofs" - ); - - let mut validator_proofs = vec![]; - for (validator_start_slot, duty) in pre_compute_duties { - // Proofs are already known at this slot for this validator. - if slot < *validator_start_slot { - continue; - } - - let subnet_ids = match duty.subnet_ids::() { - Ok(subnet_ids) => subnet_ids, - Err(e) => { - crit!( - error = ?e, - "Arithmetic error computing subnet IDs" - ); - continue; - } - }; - - // Create futures to produce proofs. - let duties_service_ref = &duties_service; - let futures = subnet_ids.iter().map(|subnet_id| async move { - // Construct proof for prior slot. - let proof_slot = slot - 1; - - let proof = match duties_service_ref - .validator_store - .produce_sync_selection_proof(&duty.pubkey, proof_slot, *subnet_id) - .await - { - Ok(proof) => proof, - Err(ValidatorStoreError::UnknownPubkey(pubkey)) => { - // A pubkey can be missing when a validator was recently - // removed via the API. - debug!( - ?pubkey, - pubkey = ?duty.pubkey, - slot = %proof_slot, - "Missing pubkey for sync selection proof" - ); - return None; - } - Err(e) => { - warn!( - error = ?e, - pubkey = ?duty.pubkey, - slot = %proof_slot, - "Unable to sign selection proof" - ); - return None; - } - }; - - match proof.is_aggregator::() { - Ok(true) => { - debug!( - validator_index = duty.validator_index, - slot = %proof_slot, - %subnet_id, - "Validator is sync aggregator" - ); - Some(((proof_slot, *subnet_id), proof)) - } - Ok(false) => None, - Err(e) => { - warn!( - pubkey = ?duty.pubkey, - slot = %proof_slot, - error = ?e, - "Error determining is_aggregator" - ); - None - } - } - }); - - // Execute all the futures in parallel, collecting any successful results. - let proofs = join_all(futures) - .await - .into_iter() - .flatten() - .collect::>(); - - validator_proofs.push((duty.validator_index, proofs)); - } - - // Add to global storage (we add regularly so the proofs can be used ASAP). - let sync_map = duties_service.sync_duties.committees.read(); - let Some(committee_duties) = sync_map.get(&sync_committee_period) else { - debug!(period = sync_committee_period, "Missing sync duties"); - continue; - }; - let validators = committee_duties.validators.read(); - let num_validators_updated = validator_proofs.len(); - - for (validator_index, proofs) in validator_proofs { - if let Some(Some(duty)) = validators.get(&validator_index) { - duty.aggregation_duties.proofs.write().extend(proofs); - } else { - debug!( - validator_index, - period = sync_committee_period, - "Missing sync duty to update" - ); - } - } - - if num_validators_updated > 0 { - debug!( - %slot, - updated_validators = num_validators_updated, - "Finished computing sync selection proofs" - ); - } } } } From 8fd345c8777a58605d2eb910c3e03bb128bc7623 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Tue, 1 Apr 2025 21:12:33 +0800 Subject: [PATCH 41/75] move log --- validator_client/validator_services/src/sync.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index d24a5ad191..4b178e9633 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -504,14 +504,14 @@ pub async fn fill_in_aggregation_proofs( current_slot: Slot, pre_compute_slot: Slot, ) { + debug!( + period = sync_committee_period, + %current_slot, + %pre_compute_slot, + "Calculating sync selection proofs" + ); // Generate selection proofs for each validator at each slot, one slot at a time. for slot in (current_slot.as_u64()..=pre_compute_slot.as_u64()).map(Slot::new) { - debug!( - period = sync_committee_period, - %current_slot, - %pre_compute_slot, - "Calculating sync selection proofs" - ); if !duties_service.distributed { // For non-distributed mode let mut validator_proofs = vec![]; From bf75ee47e93c2902adab786afd51511bace03551 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Tue, 1 Apr 2025 21:14:46 +0800 Subject: [PATCH 42/75] revert --- .../validator_services/src/sync.rs | 247 +++++++++--------- 1 file changed, 124 insertions(+), 123 deletions(-) diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index 4b178e9633..a387bd65b0 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -504,131 +504,10 @@ pub async fn fill_in_aggregation_proofs( current_slot: Slot, pre_compute_slot: Slot, ) { - debug!( - period = sync_committee_period, - %current_slot, - %pre_compute_slot, - "Calculating sync selection proofs" - ); // Generate selection proofs for each validator at each slot, one slot at a time. for slot in (current_slot.as_u64()..=pre_compute_slot.as_u64()).map(Slot::new) { - if !duties_service.distributed { - // For non-distributed mode - let mut validator_proofs = vec![]; - for (validator_start_slot, duty) in pre_compute_duties { - // Proofs are already known at this slot for this validator. - if slot < *validator_start_slot { - continue; - } - - let subnet_ids = match duty.subnet_ids::() { - Ok(subnet_ids) => subnet_ids, - Err(e) => { - crit!( - error = ?e, - "Arithmetic error computing subnet IDs" - ); - continue; - } - }; - - // Create futures to produce proofs. - let duties_service_ref = &duties_service; - let futures = subnet_ids.iter().map(|subnet_id| async move { - // Construct proof for prior slot. - let proof_slot = slot - 1; - - let proof = match duties_service_ref - .validator_store - .produce_sync_selection_proof(&duty.pubkey, proof_slot, *subnet_id) - .await - { - Ok(proof) => proof, - Err(ValidatorStoreError::UnknownPubkey(pubkey)) => { - // A pubkey can be missing when a validator was recently - // removed via the API. - debug!( - ?pubkey, - pubkey = ?duty.pubkey, - slot = %proof_slot, - "Missing pubkey for sync selection proof" - ); - return None; - } - Err(e) => { - warn!( - error = ?e, - pubkey = ?duty.pubkey, - slot = %proof_slot, - "Unable to sign selection proof" - ); - return None; - } - }; - - match proof.is_aggregator::() { - Ok(true) => { - debug!( - validator_index = duty.validator_index, - slot = %proof_slot, - %subnet_id, - "Validator is sync aggregator" - ); - Some(((proof_slot, *subnet_id), proof)) - } - Ok(false) => None, - Err(e) => { - warn!( - pubkey = ?duty.pubkey, - slot = %proof_slot, - error = ?e, - "Error determining is_aggregator" - ); - None - } - } - }); - - // Execute all the futures in parallel, collecting any successful results. - let proofs = join_all(futures) - .await - .into_iter() - .flatten() - .collect::>(); - - validator_proofs.push((duty.validator_index, proofs)); - } - - // Add to global storage (we add regularly so the proofs can be used ASAP). - let sync_map = duties_service.sync_duties.committees.read(); - let Some(committee_duties) = sync_map.get(&sync_committee_period) else { - debug!(period = sync_committee_period, "Missing sync duties"); - continue; - }; - let validators = committee_duties.validators.read(); - let num_validators_updated = validator_proofs.len(); - - for (validator_index, proofs) in validator_proofs { - if let Some(Some(duty)) = validators.get(&validator_index) { - duty.aggregation_duties.proofs.write().extend(proofs); - } else { - debug!( - validator_index, - period = sync_committee_period, - "Missing sync duty to update" - ); - } - } - - if num_validators_updated > 0 { - debug!( - %slot, - updated_validators = num_validators_updated, - "Finished computing sync selection proofs" - ); - } - } else { - // For distributed mode + // For distributed mode + if duties_service.distributed { let mut sync_committee_selection = Vec::new(); for (_validator_start_slot, duty) in pre_compute_duties { @@ -791,6 +670,128 @@ pub async fn fill_in_aggregation_proofs( ); } } + } else { + // For non-distributed mode + debug!( + period = sync_committee_period, + %current_slot, + %pre_compute_slot, + "Calculating sync selection proofs" + ); + + let mut validator_proofs = vec![]; + for (validator_start_slot, duty) in pre_compute_duties { + // Proofs are already known at this slot for this validator. + if slot < *validator_start_slot { + continue; + } + + let subnet_ids = match duty.subnet_ids::() { + Ok(subnet_ids) => subnet_ids, + Err(e) => { + crit!( + error = ?e, + "Arithmetic error computing subnet IDs" + ); + continue; + } + }; + + // Create futures to produce proofs. + let duties_service_ref = &duties_service; + let futures = subnet_ids.iter().map(|subnet_id| async move { + // Construct proof for prior slot. + let proof_slot = slot - 1; + + let proof = match duties_service_ref + .validator_store + .produce_sync_selection_proof(&duty.pubkey, proof_slot, *subnet_id) + .await + { + Ok(proof) => proof, + Err(ValidatorStoreError::UnknownPubkey(pubkey)) => { + // A pubkey can be missing when a validator was recently + // removed via the API. + debug!( + ?pubkey, + pubkey = ?duty.pubkey, + slot = %proof_slot, + "Missing pubkey for sync selection proof" + ); + return None; + } + Err(e) => { + warn!( + error = ?e, + pubkey = ?duty.pubkey, + slot = %proof_slot, + "Unable to sign selection proof" + ); + return None; + } + }; + + match proof.is_aggregator::() { + Ok(true) => { + debug!( + validator_index = duty.validator_index, + slot = %proof_slot, + %subnet_id, + "Validator is sync aggregator" + ); + Some(((proof_slot, *subnet_id), proof)) + } + Ok(false) => None, + Err(e) => { + warn!( + pubkey = ?duty.pubkey, + slot = %proof_slot, + error = ?e, + "Error determining is_aggregator" + ); + None + } + } + }); + + // Execute all the futures in parallel, collecting any successful results. + let proofs = join_all(futures) + .await + .into_iter() + .flatten() + .collect::>(); + + validator_proofs.push((duty.validator_index, proofs)); + } + + // Add to global storage (we add regularly so the proofs can be used ASAP). + let sync_map = duties_service.sync_duties.committees.read(); + let Some(committee_duties) = sync_map.get(&sync_committee_period) else { + debug!(period = sync_committee_period, "Missing sync duties"); + continue; + }; + let validators = committee_duties.validators.read(); + let num_validators_updated = validator_proofs.len(); + + for (validator_index, proofs) in validator_proofs { + if let Some(Some(duty)) = validators.get(&validator_index) { + duty.aggregation_duties.proofs.write().extend(proofs); + } else { + debug!( + validator_index, + period = sync_committee_period, + "Missing sync duty to update" + ); + } + } + + if num_validators_updated > 0 { + debug!( + %slot, + updated_validators = num_validators_updated, + "Finished computing sync selection proofs" + ); + } } } } From 120fa54d92ee5ad8071d9c354cf6f183416f62d2 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Tue, 1 Apr 2025 22:06:17 +0800 Subject: [PATCH 43/75] simplify --- validator_client/validator_services/src/sync.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index a387bd65b0..844d751a25 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -510,9 +510,7 @@ pub async fn fill_in_aggregation_proofs( if duties_service.distributed { let mut sync_committee_selection = Vec::new(); - for (_validator_start_slot, duty) in pre_compute_duties { - // Proofs are already known at this slot for this validator. - + for (_, duty) in pre_compute_duties { let subnet_ids = match duty.subnet_ids::() { Ok(subnet_ids) => subnet_ids, Err(e) => { From d3dbe870efb536f3fa8af47b8380f46e30c688b8 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Wed, 9 Apr 2025 11:32:10 +0800 Subject: [PATCH 44/75] Add SelectionProofConfig --- .../validator_services/src/duties_service.rs | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index 50b01ce46e..0e08340b8d 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -123,6 +123,12 @@ pub struct SubscriptionSlots { duty_slot: Slot, } +pub struct SelectionProofConfig { + look_ahead: u64, + selections_endpoint: bool, + parallel_sign: bool, +} + /// Create a selection proof for `duty`. /// /// Return `Ok(None)` if the attesting validator is not an aggregator. @@ -979,9 +985,20 @@ async fn poll_beacon_attesters_for_epoch( // Spawn the background task to compute selection proofs. let subservice = duties_service.clone(); + + let config = SelectionProofConfig { + look_ahead: if duties_service.distributed { + SELECTION_PROOF_SLOT_LOOKAHEAD_DVT + } else { + SELECTION_PROOF_SLOT_LOOKAHEAD + }, + selections_endpoint: duties_service.distributed, + parallel_sign: duties_service.distributed, + }; + duties_service.context.executor.spawn( async move { - fill_in_selection_proofs(subservice, new_duties, dependent_root).await; + fill_in_selection_proofs(subservice, new_duties, dependent_root, config).await; }, "duties_service_selection_proofs_background", ); @@ -1076,6 +1093,7 @@ async fn fill_in_selection_proofs( duties_service: Arc>, duties: Vec, dependent_root: Hash256, + config: SelectionProofConfig, ) { // Sort duties by slot in a BTreeMap. let mut duties_by_slot: BTreeMap> = BTreeMap::new(); @@ -1097,11 +1115,7 @@ async fn fill_in_selection_proofs( continue; }; - let selection_lookahead = if duties_service.distributed { - SELECTION_PROOF_SLOT_LOOKAHEAD_DVT - } else { - SELECTION_PROOF_SLOT_LOOKAHEAD - }; + let selection_lookahead = config.look_ahead; let lookahead_slot = current_slot + selection_lookahead; @@ -1131,14 +1145,14 @@ async fn fill_in_selection_proofs( // In distributed case, we want to send all partial selection proofs to the middleware to determine aggregation duties, // as the middleware will need to have a threshold of partial selection proofs to be able to return the full selection proof // Thus, sign selection proofs in parallel in distributed case; Otherwise, sign them serially in non-distributed (normal) case - let duty_and_proof_results = if duties_service.distributed { + let duty_and_proof_results = if config.parallel_sign { futures::future::join_all(relevant_duties.into_values().flatten().map( |duty| async { let opt_selection_proof = make_selection_proof( &duty, &duties_service.validator_store, &duties_service.spec, - duties_service.distributed, + config.selections_endpoint, &duties_service.beacon_nodes, ) .await?; @@ -1153,7 +1167,7 @@ async fn fill_in_selection_proofs( &duty, &duties_service.validator_store, &duties_service.spec, - duties_service.distributed, + !config.selections_endpoint, // non-distributed case &duties_service.beacon_nodes, ) .await?; From a11cee21d3438ece6323b8299ae5b42dddaa6be2 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Wed, 9 Apr 2025 22:19:13 +0800 Subject: [PATCH 45/75] Use FuturesUnordered --- .../validator_services/src/duties_service.rs | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index 0e08340b8d..a8708eb749 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -16,7 +16,10 @@ use eth2::types::{ AttesterData, BeaconCommitteeSelection, BeaconCommitteeSubscription, DutiesResponse, ProposerData, StateId, ValidatorId, }; -use futures::{stream, StreamExt}; +use futures::{ + stream::{self, FuturesUnordered}, + StreamExt, +}; use parking_lot::RwLock; use safe_arith::{ArithError, SafeArith}; use slot_clock::SlotClock; @@ -123,7 +126,7 @@ pub struct SubscriptionSlots { duty_slot: Slot, } -pub struct SelectionProofConfig { +struct SelectionProofConfig { look_ahead: u64, selections_endpoint: bool, parallel_sign: bool, @@ -1146,8 +1149,10 @@ async fn fill_in_selection_proofs( // as the middleware will need to have a threshold of partial selection proofs to be able to return the full selection proof // Thus, sign selection proofs in parallel in distributed case; Otherwise, sign them serially in non-distributed (normal) case let duty_and_proof_results = if config.parallel_sign { - futures::future::join_all(relevant_duties.into_values().flatten().map( - |duty| async { + let futures = relevant_duties + .into_values() + .flatten() + .map(|duty| async { let opt_selection_proof = make_selection_proof( &duty, &duties_service.validator_store, @@ -1157,9 +1162,10 @@ async fn fill_in_selection_proofs( ) .await?; Ok((duty, opt_selection_proof)) - }, - )) - .await + }) + .collect::>(); + + futures.collect::>().await } else { stream::iter(relevant_duties.into_values().flatten()) .then(|duty| async { From af676d43bee4b51c46ebee088cc1226e57041a56 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Thu, 10 Apr 2025 19:45:53 +0800 Subject: [PATCH 46/75] Remove negation --- validator_client/validator_services/src/duties_service.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index a8708eb749..b0a8c7a95f 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -1173,7 +1173,7 @@ async fn fill_in_selection_proofs( &duty, &duties_service.validator_store, &duties_service.spec, - !config.selections_endpoint, // non-distributed case + config.selections_endpoint, // non-distributed case &duties_service.beacon_nodes, ) .await?; From 76125fa0fac11225288c1b147343454c18bb19f5 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Fri, 11 Apr 2025 11:39:45 +0800 Subject: [PATCH 47/75] Process each result --- .../validator_services/src/duties_service.rs | 78 +++++++++++++++++-- 1 file changed, 72 insertions(+), 6 deletions(-) diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index b0a8c7a95f..6e0c1e05a2 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -139,10 +139,10 @@ async fn make_selection_proof( duty: &AttesterData, validator_store: &ValidatorStore, spec: &ChainSpec, - distributed: bool, + config: &SelectionProofConfig, beacon_nodes: &Arc>, ) -> Result, Error> { - let selection_proof = if distributed { + let selection_proof = if config.selections_endpoint { let beacon_committee_selection = BeaconCommitteeSelection { validator_index: duty.validator_index, slot: duty.slot, @@ -1149,7 +1149,7 @@ async fn fill_in_selection_proofs( // as the middleware will need to have a threshold of partial selection proofs to be able to return the full selection proof // Thus, sign selection proofs in parallel in distributed case; Otherwise, sign them serially in non-distributed (normal) case let duty_and_proof_results = if config.parallel_sign { - let futures = relevant_duties + let mut futures = relevant_duties .into_values() .flatten() .map(|duty| async { @@ -1157,7 +1157,7 @@ async fn fill_in_selection_proofs( &duty, &duties_service.validator_store, &duties_service.spec, - config.selections_endpoint, + &config, &duties_service.beacon_nodes, ) .await?; @@ -1165,7 +1165,73 @@ async fn fill_in_selection_proofs( }) .collect::>(); - futures.collect::>().await + while let Some(result) = futures.next().await { + let mut attesters = duties_service.attesters.write(); + let (duty, selection_proof) = match result { + Ok(duty_and_proof) => duty_and_proof, + Err(Error::FailedToProduceSelectionProof( + ValidatorStoreError::UnknownPubkey(pubkey), + )) => { + // A pubkey can be missing when a validator was recently + // removed via the API. + warn!( + info = "a validator may have recently been removed from this VC", + ?pubkey, + "Missing pubkey for duty and proof" + ); + // Do not abort the entire batch for a single failure. + continue; + } + Err(e) => { + error!( + error = ?e, + msg = "may impair attestation duties", + "Failed to produce duty and proof" + ); + // Do not abort the entire batch for a single failure. + continue; + } + }; + + let attester_map = attesters.entry(duty.pubkey).or_default(); + let epoch = duty.slot.epoch(E::slots_per_epoch()); + match attester_map.entry(epoch) { + hash_map::Entry::Occupied(mut entry) => { + // No need to update duties for which no proof was computed. + let Some(selection_proof) = selection_proof else { + continue; + }; + + let (existing_dependent_root, existing_duty) = entry.get_mut(); + + if *existing_dependent_root == dependent_root { + // Replace existing proof. + existing_duty.selection_proof = Some(selection_proof); + } else { + // Our selection proofs are no longer relevant due to a reorg, abandon + // this entire background process. + debug!( + reason = "re-org", + "Stopping selection proof background task" + ); + return; + } + } + hash_map::Entry::Vacant(entry) => { + // This probably shouldn't happen, but we have enough info to fill in the + // entry so we may as well. + let subscription_slots = + SubscriptionSlots::new(duty.slot, current_slot); + let duty_and_proof = DutyAndProof { + duty, + selection_proof, + subscription_slots, + }; + entry.insert((dependent_root, duty_and_proof)); + } + } + } + Vec::new() } else { stream::iter(relevant_duties.into_values().flatten()) .then(|duty| async { @@ -1173,7 +1239,7 @@ async fn fill_in_selection_proofs( &duty, &duties_service.validator_store, &duties_service.spec, - config.selections_endpoint, // non-distributed case + &config, &duties_service.beacon_nodes, ) .await?; From ea13f74c065c566a0868976ac5520b4002a25d5c Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Fri, 11 Apr 2025 12:03:54 +0800 Subject: [PATCH 48/75] Add duration of computation offset --- .../validator_services/src/duties_service.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index 6e0c1e05a2..b9f8966905 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -127,7 +127,8 @@ pub struct SubscriptionSlots { } struct SelectionProofConfig { - look_ahead: u64, + lookahead_slot: u64, + computation_offset: Duration, // The seconds to compute the selection proof before a slot selections_endpoint: bool, parallel_sign: bool, } @@ -990,11 +991,13 @@ async fn poll_beacon_attesters_for_epoch( let subservice = duties_service.clone(); let config = SelectionProofConfig { - look_ahead: if duties_service.distributed { + lookahead_slot: if duties_service.distributed { SELECTION_PROOF_SLOT_LOOKAHEAD_DVT } else { SELECTION_PROOF_SLOT_LOOKAHEAD }, + computation_offset: duties_service.slot_clock.slot_duration() + / SELECTION_PROOF_SCHEDULE_DENOM, selections_endpoint: duties_service.distributed, parallel_sign: duties_service.distributed, }; @@ -1108,17 +1111,16 @@ async fn fill_in_selection_proofs( // At halfway through each slot when nothing else is likely to be getting signed, sign a batch // of selection proofs and insert them into the duties service `attesters` map. let slot_clock = &duties_service.slot_clock; - let slot_offset = duties_service.slot_clock.slot_duration() / SELECTION_PROOF_SCHEDULE_DENOM; while !duties_by_slot.is_empty() { if let Some(duration) = slot_clock.duration_to_next_slot() { - sleep(duration.saturating_sub(slot_offset)).await; + sleep(duration.saturating_sub(config.computation_offset)).await; let Some(current_slot) = slot_clock.now() else { continue; }; - let selection_lookahead = config.look_ahead; + let selection_lookahead = config.lookahead_slot; let lookahead_slot = current_slot + selection_lookahead; From 44bd5f13f6c10f30d5b1a5d953daaf1c7507bc4c Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Fri, 11 Apr 2025 19:37:46 +0800 Subject: [PATCH 49/75] rearrange --- .../validator_services/src/duties_service.rs | 140 +++++++++--------- 1 file changed, 70 insertions(+), 70 deletions(-) diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index b9f8966905..55acca5fc2 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -1150,8 +1150,8 @@ async fn fill_in_selection_proofs( // In distributed case, we want to send all partial selection proofs to the middleware to determine aggregation duties, // as the middleware will need to have a threshold of partial selection proofs to be able to return the full selection proof // Thus, sign selection proofs in parallel in distributed case; Otherwise, sign them serially in non-distributed (normal) case - let duty_and_proof_results = if config.parallel_sign { - let mut futures = relevant_duties + if config.parallel_sign { + let mut duty_and_proof_results = relevant_duties .into_values() .flatten() .map(|duty| async { @@ -1167,7 +1167,7 @@ async fn fill_in_selection_proofs( }) .collect::>(); - while let Some(result) = futures.next().await { + while let Some(result) = duty_and_proof_results.next().await { let mut attesters = duties_service.attesters.write(); let (duty, selection_proof) = match result { Ok(duty_and_proof) => duty_and_proof, @@ -1233,9 +1233,8 @@ async fn fill_in_selection_proofs( } } } - Vec::new() } else { - stream::iter(relevant_duties.into_values().flatten()) + let duty_and_proof_results = stream::iter(relevant_duties.into_values().flatten()) .then(|duty| async { let opt_selection_proof = make_selection_proof( &duty, @@ -1248,76 +1247,77 @@ async fn fill_in_selection_proofs( Ok((duty, opt_selection_proof)) }) .collect::>() - .await - }; + .await; - // Add to attesters store. - let mut attesters = duties_service.attesters.write(); - for result in duty_and_proof_results { - let (duty, selection_proof) = match result { - Ok(duty_and_proof) => duty_and_proof, - Err(Error::FailedToProduceSelectionProof( - ValidatorStoreError::UnknownPubkey(pubkey), - )) => { - // A pubkey can be missing when a validator was recently - // removed via the API. - warn!( - info = "a validator may have recently been removed from this VC", - ?pubkey, - "Missing pubkey for duty and proof" - ); - // Do not abort the entire batch for a single failure. - continue; - } - Err(e) => { - error!( - error = ?e, - msg = "may impair attestation duties", - "Failed to produce duty and proof" - ); - // Do not abort the entire batch for a single failure. - continue; - } - }; - - let attester_map = attesters.entry(duty.pubkey).or_default(); - let epoch = duty.slot.epoch(E::slots_per_epoch()); - match attester_map.entry(epoch) { - hash_map::Entry::Occupied(mut entry) => { - // No need to update duties for which no proof was computed. - let Some(selection_proof) = selection_proof else { - continue; - }; - - let (existing_dependent_root, existing_duty) = entry.get_mut(); - - if *existing_dependent_root == dependent_root { - // Replace existing proof. - existing_duty.selection_proof = Some(selection_proof); - } else { - // Our selection proofs are no longer relevant due to a reorg, abandon - // this entire background process. - debug!( - reason = "re-org", - "Stopping selection proof background task" + // Add to attesters store. + let mut attesters = duties_service.attesters.write(); + for result in duty_and_proof_results { + let (duty, selection_proof) = match result { + Ok(duty_and_proof) => duty_and_proof, + Err(Error::FailedToProduceSelectionProof( + ValidatorStoreError::UnknownPubkey(pubkey), + )) => { + // A pubkey can be missing when a validator was recently + // removed via the API. + warn!( + info = "a validator may have recently been removed from this VC", + ?pubkey, + "Missing pubkey for duty and proof" ); - return; + // Do not abort the entire batch for a single failure. + continue; + } + Err(e) => { + error!( + error = ?e, + msg = "may impair attestation duties", + "Failed to produce duty and proof" + ); + // Do not abort the entire batch for a single failure. + continue; + } + }; + + let attester_map = attesters.entry(duty.pubkey).or_default(); + let epoch = duty.slot.epoch(E::slots_per_epoch()); + match attester_map.entry(epoch) { + hash_map::Entry::Occupied(mut entry) => { + // No need to update duties for which no proof was computed. + let Some(selection_proof) = selection_proof else { + continue; + }; + + let (existing_dependent_root, existing_duty) = entry.get_mut(); + + if *existing_dependent_root == dependent_root { + // Replace existing proof. + existing_duty.selection_proof = Some(selection_proof); + } else { + // Our selection proofs are no longer relevant due to a reorg, abandon + // this entire background process. + debug!( + reason = "re-org", + "Stopping selection proof background task" + ); + return; + } + } + hash_map::Entry::Vacant(entry) => { + // This probably shouldn't happen, but we have enough info to fill in the + // entry so we may as well. + let subscription_slots = + SubscriptionSlots::new(duty.slot, current_slot); + let duty_and_proof = DutyAndProof { + duty, + selection_proof, + subscription_slots, + }; + entry.insert((dependent_root, duty_and_proof)); } } - hash_map::Entry::Vacant(entry) => { - // This probably shouldn't happen, but we have enough info to fill in the - // entry so we may as well. - let subscription_slots = SubscriptionSlots::new(duty.slot, current_slot); - let duty_and_proof = DutyAndProof { - duty, - selection_proof, - subscription_slots, - }; - entry.insert((dependent_root, duty_and_proof)); - } } - } - drop(attesters); + drop(attesters); + }; let time_taken_ms = Duration::from_secs_f64(timer.map_or(0.0, |t| t.stop_and_record())).as_millis(); From b506fa5369d25337972941d9b98e6bde7d41ff7c Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Tue, 15 Apr 2025 10:40:48 +0800 Subject: [PATCH 50/75] Remove duplication with a function --- .../validator_services/src/duties_service.rs | 206 +++++++----------- 1 file changed, 81 insertions(+), 125 deletions(-) diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index 55acca5fc2..3df04c84f3 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -20,7 +20,7 @@ use futures::{ stream::{self, FuturesUnordered}, StreamExt, }; -use parking_lot::RwLock; +use parking_lot::{RwLock, RwLockWriteGuard}; use safe_arith::{ArithError, SafeArith}; use slot_clock::SlotClock; use std::cmp::min; @@ -1091,6 +1091,72 @@ async fn post_validator_duties_attester( .map_err(|e| Error::FailedToDownloadAttesters(e.to_string())) } +fn process_duty_and_proof( + attesters: &mut RwLockWriteGuard, + result: Result<(AttesterData, Option), Error>, + dependent_root: Hash256, + current_slot: Slot, +) -> bool { + let (duty, selection_proof) = match result { + Ok(duty_and_proof) => duty_and_proof, + Err(Error::FailedToProduceSelectionProof(ValidatorStoreError::UnknownPubkey(pubkey))) => { + // A pubkey can be missing when a validator was recently removed via the API. + warn!( + info = "A validator may have recently been removed from this VC", + ?pubkey, + "Missing pubkey for duty and proof" + ); + // Do not abort the entire batch for a single failure. + return true; + } + Err(e) => { + error!( + error = ?e, + msg = "may impair attestation duties", + "Failed to produce duty and proof" + ); + return true; + } + }; + + let attester_map = attesters.entry(duty.pubkey).or_default(); + let epoch = duty.slot.epoch(E::slots_per_epoch()); + match attester_map.entry(epoch) { + hash_map::Entry::Occupied(mut entry) => { + // No need to update duties for which no proof was computed. + let Some(selection_proof) = selection_proof else { + return true; + }; + + let (existing_dependent_root, existing_duty) = entry.get_mut(); + + if *existing_dependent_root == dependent_root { + // Replace existing proof. + existing_duty.selection_proof = Some(selection_proof); + true + } else { + // Our selection proofs are no longer relevant due to a reorg, abandon this entire background process. + debug!( + reason = "re-org", + "Stopping selection proof background task" + ); + false + } + } + + hash_map::Entry::Vacant(entry) => { + // This probably shouldn't happen, but we have enough info to fill in the entry so we may as well. + let subscription_slots = SubscriptionSlots::new(duty.slot, current_slot); + let duty_and_proof = DutyAndProof { + duty, + selection_proof, + subscription_slots, + }; + entry.insert((dependent_root, duty_and_proof)); + true + } + } +} /// Compute the attestation selection proofs for the `duties` and add them to the `attesters` map. /// /// Duties are computed in batches each slot. If a re-org is detected then the process will @@ -1169,68 +1235,13 @@ async fn fill_in_selection_proofs( while let Some(result) = duty_and_proof_results.next().await { let mut attesters = duties_service.attesters.write(); - let (duty, selection_proof) = match result { - Ok(duty_and_proof) => duty_and_proof, - Err(Error::FailedToProduceSelectionProof( - ValidatorStoreError::UnknownPubkey(pubkey), - )) => { - // A pubkey can be missing when a validator was recently - // removed via the API. - warn!( - info = "a validator may have recently been removed from this VC", - ?pubkey, - "Missing pubkey for duty and proof" - ); - // Do not abort the entire batch for a single failure. - continue; - } - Err(e) => { - error!( - error = ?e, - msg = "may impair attestation duties", - "Failed to produce duty and proof" - ); - // Do not abort the entire batch for a single failure. - continue; - } - }; - - let attester_map = attesters.entry(duty.pubkey).or_default(); - let epoch = duty.slot.epoch(E::slots_per_epoch()); - match attester_map.entry(epoch) { - hash_map::Entry::Occupied(mut entry) => { - // No need to update duties for which no proof was computed. - let Some(selection_proof) = selection_proof else { - continue; - }; - - let (existing_dependent_root, existing_duty) = entry.get_mut(); - - if *existing_dependent_root == dependent_root { - // Replace existing proof. - existing_duty.selection_proof = Some(selection_proof); - } else { - // Our selection proofs are no longer relevant due to a reorg, abandon - // this entire background process. - debug!( - reason = "re-org", - "Stopping selection proof background task" - ); - return; - } - } - hash_map::Entry::Vacant(entry) => { - // This probably shouldn't happen, but we have enough info to fill in the - // entry so we may as well. - let subscription_slots = - SubscriptionSlots::new(duty.slot, current_slot); - let duty_and_proof = DutyAndProof { - duty, - selection_proof, - subscription_slots, - }; - entry.insert((dependent_root, duty_and_proof)); - } + if !process_duty_and_proof::( + &mut attesters, + result, + dependent_root, + current_slot, + ) { + return; } } } else { @@ -1252,68 +1263,13 @@ async fn fill_in_selection_proofs( // Add to attesters store. let mut attesters = duties_service.attesters.write(); for result in duty_and_proof_results { - let (duty, selection_proof) = match result { - Ok(duty_and_proof) => duty_and_proof, - Err(Error::FailedToProduceSelectionProof( - ValidatorStoreError::UnknownPubkey(pubkey), - )) => { - // A pubkey can be missing when a validator was recently - // removed via the API. - warn!( - info = "a validator may have recently been removed from this VC", - ?pubkey, - "Missing pubkey for duty and proof" - ); - // Do not abort the entire batch for a single failure. - continue; - } - Err(e) => { - error!( - error = ?e, - msg = "may impair attestation duties", - "Failed to produce duty and proof" - ); - // Do not abort the entire batch for a single failure. - continue; - } - }; - - let attester_map = attesters.entry(duty.pubkey).or_default(); - let epoch = duty.slot.epoch(E::slots_per_epoch()); - match attester_map.entry(epoch) { - hash_map::Entry::Occupied(mut entry) => { - // No need to update duties for which no proof was computed. - let Some(selection_proof) = selection_proof else { - continue; - }; - - let (existing_dependent_root, existing_duty) = entry.get_mut(); - - if *existing_dependent_root == dependent_root { - // Replace existing proof. - existing_duty.selection_proof = Some(selection_proof); - } else { - // Our selection proofs are no longer relevant due to a reorg, abandon - // this entire background process. - debug!( - reason = "re-org", - "Stopping selection proof background task" - ); - return; - } - } - hash_map::Entry::Vacant(entry) => { - // This probably shouldn't happen, but we have enough info to fill in the - // entry so we may as well. - let subscription_slots = - SubscriptionSlots::new(duty.slot, current_slot); - let duty_and_proof = DutyAndProof { - duty, - selection_proof, - subscription_slots, - }; - entry.insert((dependent_root, duty_and_proof)); - } + if !process_duty_and_proof::( + &mut attesters, + result, + dependent_root, + current_slot, + ) { + return; } } drop(attesters); From ab1d2c06c6a25c826c4e16e4135f4ff454edcc19 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Tue, 15 Apr 2025 21:39:55 +0800 Subject: [PATCH 51/75] Modify to FuturesUnordered for Sync --- .../validator_services/src/duties_service.rs | 10 ++--- .../validator_services/src/sync.rs | 43 +++++++++++++++---- 2 files changed, 40 insertions(+), 13 deletions(-) diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index 3df04c84f3..0e2d5537b1 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -126,11 +126,11 @@ pub struct SubscriptionSlots { duty_slot: Slot, } -struct SelectionProofConfig { - lookahead_slot: u64, - computation_offset: Duration, // The seconds to compute the selection proof before a slot - selections_endpoint: bool, - parallel_sign: bool, +pub struct SelectionProofConfig { + pub lookahead_slot: u64, + pub computation_offset: Duration, // The seconds to compute the selection proof before a slot + pub selections_endpoint: bool, + pub parallel_sign: bool, } /// Create a selection proof for `duty`. diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index 844d751a25..12060d8a2d 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -1,13 +1,15 @@ -use crate::duties_service::{DutiesService, Error}; +use crate::duties_service::{DutiesService, Error, SelectionProofConfig}; use doppelganger_service::DoppelgangerStatus; use eth2::types::{Signature, SyncCommitteeSelection}; use futures::future::join_all; +use futures::stream::{FuturesUnordered, StreamExt}; use logging::crit; use parking_lot::{MappedRwLockReadGuard, RwLock, RwLockReadGuard, RwLockWriteGuard}; use slot_clock::SlotClock; use std::collections::{HashMap, HashSet}; use std::marker::PhantomData; use std::sync::Arc; +use std::time::Duration; use tracing::{debug, info, warn}; use types::{ChainSpec, EthSpec, PublicKeyBytes, Slot, SyncDuty, SyncSelectionProof, SyncSubnetId}; use validator_store::Error as ValidatorStoreError; @@ -349,12 +351,22 @@ pub async fn poll_sync_committee_duties( let sub_duties_service = duties_service.clone(); duties_service.context.executor.spawn( async move { + let config = SelectionProofConfig { + lookahead_slot: sub_duties_service + .sync_duties + .aggregation_pre_compute_slots(), + computation_offset: Duration::from_secs(12), + selections_endpoint: sub_duties_service.distributed, + parallel_sign: sub_duties_service.distributed, + }; + fill_in_aggregation_proofs( sub_duties_service, &new_pre_compute_duties, current_sync_committee_period, current_slot, current_pre_compute_slot, + config, ) .await }, @@ -393,12 +405,22 @@ pub async fn poll_sync_committee_duties( let sub_duties_service = duties_service.clone(); duties_service.context.executor.spawn( async move { + let config = SelectionProofConfig { + lookahead_slot: sub_duties_service + .sync_duties + .aggregation_pre_compute_slots(), + computation_offset: Duration::from_secs(12), + selections_endpoint: sub_duties_service.distributed, + parallel_sign: sub_duties_service.distributed, + }; + fill_in_aggregation_proofs( sub_duties_service, &new_pre_compute_duties, next_sync_committee_period, current_slot, pre_compute_slot, + config, ) .await }, @@ -503,11 +525,12 @@ pub async fn fill_in_aggregation_proofs( sync_committee_period: u64, current_slot: Slot, pre_compute_slot: Slot, + config: SelectionProofConfig, ) { // Generate selection proofs for each validator at each slot, one slot at a time. for slot in (current_slot.as_u64()..=pre_compute_slot.as_u64()).map(Slot::new) { // For distributed mode - if duties_service.distributed { + if config.parallel_sign { let mut sync_committee_selection = Vec::new(); for (_, duty) in pre_compute_duties { @@ -576,14 +599,18 @@ pub async fn fill_in_aggregation_proofs( })); } - let sync_committee_selection_data = join_all(sync_committee_selection).await; + let mut futures_unordered = FuturesUnordered::new(); - // Collect the SyncCommitteeSelection data - let sync_selection_data: Vec<_> = sync_committee_selection_data - .into_iter() - .flatten() - .collect(); + for future in sync_committee_selection { + futures_unordered.push(future); + } + let mut sync_selection_data = Vec::new(); + while let Some(result) = futures_unordered.next().await { + if let Some(selection) = result { + sync_selection_data.push(selection); + } + } // Call the endpoint /eth/v1/validator/sync_committee_selections // by sending the SyncCommitteeSelection that contains partial sync selection proof // The middleware should return SyncCommitteeSelection that contains full sync selection proof From 2612a6b5d8f3b3c31e5e695c58c62355ea41775c Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Wed, 16 Apr 2025 13:20:34 +0800 Subject: [PATCH 52/75] create helper function in sync --- .../validator_services/src/sync.rs | 102 +++++++++++++++++- 1 file changed, 101 insertions(+), 1 deletion(-) diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index 12060d8a2d..e377ea1d2f 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -1,4 +1,5 @@ use crate::duties_service::{DutiesService, Error, SelectionProofConfig}; +use beacon_node_fallback::BeaconNodeFallback; use doppelganger_service::DoppelgangerStatus; use eth2::types::{Signature, SyncCommitteeSelection}; use futures::future::join_all; @@ -10,7 +11,7 @@ use std::collections::{HashMap, HashSet}; use std::marker::PhantomData; use std::sync::Arc; use std::time::Duration; -use tracing::{debug, info, warn}; +use tracing::{debug, error, info, warn}; use types::{ChainSpec, EthSpec, PublicKeyBytes, Slot, SyncDuty, SyncSelectionProof, SyncSubnetId}; use validator_store::Error as ValidatorStoreError; @@ -519,6 +520,105 @@ pub async fn poll_sync_committee_duties_for_period( + duties_service: &Arc>, + duty: &SyncDuty, + proof_slot: Slot, + subnet_id: SyncSubnetId, + config: &SelectionProofConfig, + _beacon_nodes: &Arc>, +) -> Option { + let sync_selection_proof = duties_service + .validator_store + .produce_sync_selection_proof(&duty.pubkey, proof_slot, subnet_id) + .await; + + let selection_proof = match sync_selection_proof { + Ok(proof) => proof, + Err(ValidatorStoreError::UnknownPubkey(pubkey)) => { + // A pubkey can be missing when a validator was recently removed via the API + debug!( + ?pubkey, + "slot" = %proof_slot, + "Missing pubkey for sync selection proof"); + return None; + } + Err(e) => { + warn!( + "error" = ?e, + "pubkey" = ?duty.pubkey, + "slot" = %proof_slot, + "Unable to sign selection proof" + ); + return None; + } + }; + + // In --distributed mode when we want to call the selections endpoint + if config.selections_endpoint { + debug!( + "validator_index" = duty.validator_index, + "slot" = %proof_slot, + "subcommittee_index" = *subnet_id, + "partial selection proof" = ?Signature::from(selection_proof.clone()), + "Created sync selection proof" + ); + + let sync_committee_selection = SyncCommitteeSelection { + validator_index: duty.validator_index, + slot: proof_slot, + subcommittee_index: *subnet_id, + selection_proof: selection_proof.clone().into(), + }; + + // Call the endpoint /eth/v1/validator/sync_committee_selections + // by sending the SyncCommitteeSelection that contains partial sync selection proof + // The middleware should return SyncCommitteeSelection that contains full sync selection proof + let middleware_response = duties_service + .beacon_nodes + .first_success(|beacon_node| { + let selection_data = sync_committee_selection.clone(); + async move { + beacon_node + .post_validator_sync_committee_selections(&[selection_data]) + .await + } + }) + .await; + + match middleware_response { + Ok(response) => { + // Process each middleware response + let response_data = &response.data[0]; + // The selection proof from middleware response will be a full selection proof + debug!( + "validator_index" = response_data.validator_index, + "slot" = %response_data.slot, + "subcommittee_index" = response_data.subcommittee_index, + "full selection proof" = ?response_data.selection_proof, + "Received sync selection from middleware" + ); + + // Convert the response to a SyncSelectionProof so we can call the is_aggregator method + let selection_proof = + SyncSelectionProof::from(response_data.selection_proof.clone()); + Some(selection_proof) + } + Err(e) => { + error!( + "error" = %e, + %proof_slot, + "Failed to get sync selection proofs from middleware" + ); + None + } + } + } else { + // When calling the selections endpoint is not required, return the selection_proof + Some(selection_proof) + } +} + pub async fn fill_in_aggregation_proofs( duties_service: Arc>, pre_compute_duties: &[(Slot, SyncDuty)], From efa14b08e454c3ea28141ae03fc3ba4c1dbe7279 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Thu, 17 Apr 2025 11:52:09 +0800 Subject: [PATCH 53/75] refactor distributed sync part --- .../validator_services/src/sync.rs | 170 +++++------------- 1 file changed, 44 insertions(+), 126 deletions(-) diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index e377ea1d2f..65bdb15308 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -631,7 +631,8 @@ pub async fn fill_in_aggregation_proofs( for slot in (current_slot.as_u64()..=pre_compute_slot.as_u64()).map(Slot::new) { // For distributed mode if config.parallel_sign { - let mut sync_committee_selection = Vec::new(); + let mut futures_unordered = FuturesUnordered::new(); + let config_ref = &config; for (_, duty) in pre_compute_duties { let subnet_ids = match duty.subnet_ids::() { @@ -649,86 +650,28 @@ pub async fn fill_in_aggregation_proofs( let proof_slot = slot - 1; // Store all partial sync selection proofs so that it can be sent together later - sync_committee_selection.extend(subnet_ids.iter().map(|&subnet_id| { + for &subnet_id in &subnet_ids { let duties_service = duties_service.clone(); let duty = duty.clone(); - async move { - // Produce partial selection proof - let partial_sync_selection_proof = duties_service - .validator_store - .produce_sync_selection_proof(&duty.pubkey, proof_slot, subnet_id) - .await; + futures_unordered.push(async move { + let result = make_sync_selection_proof( + &duties_service, + &duty, + proof_slot, + subnet_id, + config_ref, + &duties_service.beacon_nodes, + ) + .await; - match partial_sync_selection_proof { - Ok(proof) => { - debug!( - "validator_index" = duty.validator_index, - "slot" = %proof_slot, - "subcommittee_index" = *subnet_id, - "partial selection proof" = ?Signature::from(proof.clone()), - "Sending sync selection to middleware" - ); - - let sync_committee_selection = SyncCommitteeSelection { - validator_index: duty.validator_index, - slot: proof_slot, - subcommittee_index: *subnet_id, - selection_proof: proof.clone().into(), - }; - Some(sync_committee_selection) - } - Err(ValidatorStoreError::UnknownPubkey(pubkey)) => { - debug!( - ?pubkey, - "slot" = %proof_slot, - "Missing pubkey for sync selection proof" - ); - None - } - Err(e) => { - warn!( - "error" = ?e, - "pubkey" = ?duty.pubkey, - "slot" = %proof_slot, - "Unable to sign selection proof" - ); - None - } - } - } - })); - } - - let mut futures_unordered = FuturesUnordered::new(); - - for future in sync_committee_selection { - futures_unordered.push(future); - } - - let mut sync_selection_data = Vec::new(); - while let Some(result) = futures_unordered.next().await { - if let Some(selection) = result { - sync_selection_data.push(selection); + result.map(|proof| (duty.validator_index, proof_slot, subnet_id, proof)) + }); } } - // Call the endpoint /eth/v1/validator/sync_committee_selections - // by sending the SyncCommitteeSelection that contains partial sync selection proof - // The middleware should return SyncCommitteeSelection that contains full sync selection proof - let middleware_response = duties_service - .beacon_nodes - .first_success(|beacon_node| { - let selection_data = sync_selection_data.clone(); - async move { - beacon_node - .post_validator_sync_committee_selections(&selection_data) - .await - } - }) - .await; - match middleware_response { - Ok(response) => { - // Get the sync map to update duties + let mut successful_results = Vec::new(); + while let Some(result) = futures_unordered.next().await { + if let Some((validator_index, proof_slot, subnet_id, proof)) = result { let sync_map = duties_service.sync_duties.committees.read(); let Some(committee_duties) = sync_map.get(&sync_committee_period) else { debug!("period" = sync_committee_period, "Missing sync duties"); @@ -737,63 +680,38 @@ pub async fn fill_in_aggregation_proofs( let validators = committee_duties.validators.read(); - // Process each middleware response - for response_data in response.data.iter() { - // The selection proof from middleware response will be a full selection proof - debug!( - "validator_index" = response_data.validator_index, - "slot" = %response_data.slot, - "subcommittee_index" = response_data.subcommittee_index, - "full selection proof" = ?response_data.selection_proof, - "Received sync selection from middleware" - ); - let validator_index = response_data.validator_index; - let slot = response_data.slot; - let subcommittee_index = response_data.subcommittee_index; - - // Convert the response to a SyncSelectionProof so we can call the is_aggregator method - let full_selection_proof = - SyncSelectionProof::from(response_data.selection_proof.clone()); - - // Check if the validator is an aggregator - match full_selection_proof.is_aggregator::() { - Ok(true) => { - if let Some(Some(duty)) = validators.get(&validator_index) { - debug!( - validator_index, - %slot, - "subcommittee_index" = subcommittee_index, - // log full selection proof for debugging - "full selection proof" = ?response_data.selection_proof, - "Validator is sync aggregator" - ); - - // Store the proof - duty.aggregation_duties.proofs.write().insert( - (slot, subcommittee_index.into()), - full_selection_proof, - ); - } - } - Ok(false) => {} // Not an aggregator - Err(e) => { - warn!( + // Check if the validator is an aggregator + match proof.is_aggregator::() { + Ok(true) => { + if let Some(Some(duty)) = validators.get(&validator_index) { + debug!( validator_index, - %slot, - "error" = ?e, - "Error determining is_aggregator" + "slot" = %proof_slot, + "subcommittee_index" = *subnet_id, + // log full selection proof for debugging + "full selection proof" = ?proof, + "Validator is sync aggregator" ); + + // Store the proof + duty.aggregation_duties + .proofs + .write() + .insert((slot, subnet_id), proof); + successful_results.push(validator_index); } } + Ok(false) => {} // Not an aggregator + Err(e) => { + warn!( + validator_index, + %slot, + "error" = ?e, + "Error determining is_aggregator" + ); + } } } - Err(e) => { - warn!( - "error" = %e, - %slot, - "Failed to get sync selection proofs from middleware" - ); - } } } else { // For non-distributed mode From 24b125f8515f42a4cc8ab7fc5217f87782515d15 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Fri, 18 Apr 2025 10:43:44 +0800 Subject: [PATCH 54/75] Try to fix --- validator_client/validator_services/src/sync.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index 65bdb15308..d148fe7a75 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -561,7 +561,7 @@ pub async fn make_sync_selection_proof( "slot" = %proof_slot, "subcommittee_index" = *subnet_id, "partial selection proof" = ?Signature::from(selection_proof.clone()), - "Created sync selection proof" + "Sending sync selection to middleware" ); let sync_committee_selection = SyncCommitteeSelection { @@ -588,7 +588,6 @@ pub async fn make_sync_selection_proof( match middleware_response { Ok(response) => { - // Process each middleware response let response_data = &response.data[0]; // The selection proof from middleware response will be a full selection proof debug!( @@ -664,14 +663,14 @@ pub async fn fill_in_aggregation_proofs( ) .await; - result.map(|proof| (duty.validator_index, proof_slot, subnet_id, proof)) + result.map(|proof| (duty.validator_index, slot, subnet_id, proof)) }); } } - let mut successful_results = Vec::new(); + // let mut successful_results = Vec::new(); while let Some(result) = futures_unordered.next().await { - if let Some((validator_index, proof_slot, subnet_id, proof)) = result { + if let Some((validator_index, slot, subnet_id, proof)) = result { let sync_map = duties_service.sync_duties.committees.read(); let Some(committee_duties) = sync_map.get(&sync_committee_period) else { debug!("period" = sync_committee_period, "Missing sync duties"); @@ -686,7 +685,7 @@ pub async fn fill_in_aggregation_proofs( if let Some(Some(duty)) = validators.get(&validator_index) { debug!( validator_index, - "slot" = %proof_slot, + "slot" = %slot, "subcommittee_index" = *subnet_id, // log full selection proof for debugging "full selection proof" = ?proof, @@ -698,7 +697,7 @@ pub async fn fill_in_aggregation_proofs( .proofs .write() .insert((slot, subnet_id), proof); - successful_results.push(validator_index); + //successful_results.push(validator_index); } } Ok(false) => {} // Not an aggregator From f2725ce0ccbd6bbaba3c80d57e8450a2c49bdb98 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Fri, 18 Apr 2025 12:12:16 +0800 Subject: [PATCH 55/75] Add log to debug --- validator_client/validator_services/src/sync.rs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index d148fe7a75..b7d774d394 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -599,9 +599,9 @@ pub async fn make_sync_selection_proof( ); // Convert the response to a SyncSelectionProof so we can call the is_aggregator method - let selection_proof = + let full_selection_proof = SyncSelectionProof::from(response_data.selection_proof.clone()); - Some(selection_proof) + Some(full_selection_proof) } Err(e) => { error!( @@ -663,12 +663,20 @@ pub async fn fill_in_aggregation_proofs( ) .await; + if let Some(proof) = &result { + debug!( + validator_index = duty.validator_index, + "slot" = %proof_slot, + "subcommittee_index" = *subnet_id, + "full selection proof" = ?proof, + "Selection proof in result variable" + ); + } result.map(|proof| (duty.validator_index, slot, subnet_id, proof)) }); } } - // let mut successful_results = Vec::new(); while let Some(result) = futures_unordered.next().await { if let Some((validator_index, slot, subnet_id, proof)) = result { let sync_map = duties_service.sync_duties.committees.read(); @@ -697,7 +705,6 @@ pub async fn fill_in_aggregation_proofs( .proofs .write() .insert((slot, subnet_id), proof); - //successful_results.push(validator_index); } } Ok(false) => {} // Not an aggregator From 23fefa338a30048296e115a8cbf654ed740e5959 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Fri, 18 Apr 2025 14:14:06 +0800 Subject: [PATCH 56/75] Add more logging --- validator_client/validator_services/src/sync.rs | 6 +++--- .../src/sync_committee_service.rs | 13 +++++++++++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index b7d774d394..388c0365f8 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -672,13 +672,13 @@ pub async fn fill_in_aggregation_proofs( "Selection proof in result variable" ); } - result.map(|proof| (duty.validator_index, slot, subnet_id, proof)) + result.map(|proof| (duty.validator_index, proof_slot, subnet_id, proof)) }); } } while let Some(result) = futures_unordered.next().await { - if let Some((validator_index, slot, subnet_id, proof)) = result { + if let Some((validator_index, proof_slot, subnet_id, proof)) = result { let sync_map = duties_service.sync_duties.committees.read(); let Some(committee_duties) = sync_map.get(&sync_committee_period) else { debug!("period" = sync_committee_period, "Missing sync duties"); @@ -693,7 +693,7 @@ pub async fn fill_in_aggregation_proofs( if let Some(Some(duty)) = validators.get(&validator_index) { debug!( validator_index, - "slot" = %slot, + "slot" = %proof_slot, "subcommittee_index" = *subnet_id, // log full selection proof for debugging "full selection proof" = ?proof, diff --git a/validator_client/validator_services/src/sync_committee_service.rs b/validator_client/validator_services/src/sync_committee_service.rs index d99c0d3107..233100b570 100644 --- a/validator_client/validator_services/src/sync_committee_service.rs +++ b/validator_client/validator_services/src/sync_committee_service.rs @@ -380,11 +380,20 @@ impl SyncCommitteeService { aggregator_index, aggregator_pk, contribution.clone(), - selection_proof, + selection_proof.clone(), ) .await { - Ok(signed_contribution) => Some(signed_contribution), + Ok(signed_contribution) => { + debug!( + validator_index = aggregator_index, + %slot, + subcommittee_index = %subnet_id, + full_selection_proof = ?selection_proof, + "Produced sync contribution and proof" + ); + Some(signed_contribution) + } Err(ValidatorStoreError::UnknownPubkey(pubkey)) => { // A pubkey can be missing when a validator was recently // removed via the API. From 67d2f9b4f84a21b827bd79716d2e043bf07377ae Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Mon, 21 Apr 2025 13:54:08 +0800 Subject: [PATCH 57/75] change computation offset --- validator_client/validator_services/src/sync.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index 388c0365f8..afa1fc6a3b 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -356,7 +356,7 @@ pub async fn poll_sync_committee_duties( lookahead_slot: sub_duties_service .sync_duties .aggregation_pre_compute_slots(), - computation_offset: Duration::from_secs(12), + computation_offset: Duration::from_secs(0), selections_endpoint: sub_duties_service.distributed, parallel_sign: sub_duties_service.distributed, }; From 247f133b794c8dd5de206c8bd28e449218989442 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Mon, 21 Apr 2025 15:28:35 +0800 Subject: [PATCH 58/75] change to slot --- validator_client/validator_services/src/sync.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index afa1fc6a3b..bf9c70b6ce 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -672,13 +672,13 @@ pub async fn fill_in_aggregation_proofs( "Selection proof in result variable" ); } - result.map(|proof| (duty.validator_index, proof_slot, subnet_id, proof)) + result.map(|proof| (duty.validator_index, slot, subnet_id, proof)) }); } } while let Some(result) = futures_unordered.next().await { - if let Some((validator_index, proof_slot, subnet_id, proof)) = result { + if let Some((validator_index, slot, subnet_id, proof)) = result { let sync_map = duties_service.sync_duties.committees.read(); let Some(committee_duties) = sync_map.get(&sync_committee_period) else { debug!("period" = sync_committee_period, "Missing sync duties"); @@ -693,7 +693,7 @@ pub async fn fill_in_aggregation_proofs( if let Some(Some(duty)) = validators.get(&validator_index) { debug!( validator_index, - "slot" = %proof_slot, + "slot" = %slot, "subcommittee_index" = *subnet_id, // log full selection proof for debugging "full selection proof" = ?proof, From 239c1839c385c36cf55a842f7dac12809d333295 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Mon, 21 Apr 2025 17:11:06 +0800 Subject: [PATCH 59/75] proof_slot --- validator_client/validator_services/src/sync.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index bf9c70b6ce..f626c08eef 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -672,13 +672,13 @@ pub async fn fill_in_aggregation_proofs( "Selection proof in result variable" ); } - result.map(|proof| (duty.validator_index, slot, subnet_id, proof)) + result.map(|proof| (duty.validator_index, proof_slot, subnet_id, proof)) }); } } while let Some(result) = futures_unordered.next().await { - if let Some((validator_index, slot, subnet_id, proof)) = result { + if let Some((validator_index, proof_slot, subnet_id, proof)) = result { let sync_map = duties_service.sync_duties.committees.read(); let Some(committee_duties) = sync_map.get(&sync_committee_period) else { debug!("period" = sync_committee_period, "Missing sync duties"); @@ -693,7 +693,7 @@ pub async fn fill_in_aggregation_proofs( if let Some(Some(duty)) = validators.get(&validator_index) { debug!( validator_index, - "slot" = %slot, + "slot" = %proof_slot, "subcommittee_index" = *subnet_id, // log full selection proof for debugging "full selection proof" = ?proof, @@ -704,7 +704,7 @@ pub async fn fill_in_aggregation_proofs( duty.aggregation_duties .proofs .write() - .insert((slot, subnet_id), proof); + .insert((proof_slot, subnet_id), proof); } } Ok(false) => {} // Not an aggregator From 67bfd112b5cc97eb38fd925c13db4339d53d127f Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Wed, 23 Apr 2025 10:24:59 +0800 Subject: [PATCH 60/75] refactor for normal mode --- .../validator_services/src/sync.rs | 84 ++++++++----------- 1 file changed, 35 insertions(+), 49 deletions(-) diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index f626c08eef..5e149bf272 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -651,11 +651,10 @@ pub async fn fill_in_aggregation_proofs( // Store all partial sync selection proofs so that it can be sent together later for &subnet_id in &subnet_ids { let duties_service = duties_service.clone(); - let duty = duty.clone(); futures_unordered.push(async move { let result = make_sync_selection_proof( &duties_service, - &duty, + duty, proof_slot, subnet_id, config_ref, @@ -748,58 +747,45 @@ pub async fn fill_in_aggregation_proofs( // Create futures to produce proofs. let duties_service_ref = &duties_service; + let config_ref = &config; let futures = subnet_ids.iter().map(|subnet_id| async move { // Construct proof for prior slot. let proof_slot = slot - 1; - let proof = match duties_service_ref - .validator_store - .produce_sync_selection_proof(&duty.pubkey, proof_slot, *subnet_id) - .await - { - Ok(proof) => proof, - Err(ValidatorStoreError::UnknownPubkey(pubkey)) => { - // A pubkey can be missing when a validator was recently - // removed via the API. - debug!( - ?pubkey, - pubkey = ?duty.pubkey, - slot = %proof_slot, - "Missing pubkey for sync selection proof" - ); - return None; - } - Err(e) => { - warn!( - error = ?e, - pubkey = ?duty.pubkey, - slot = %proof_slot, - "Unable to sign selection proof" - ); - return None; - } - }; + let proof = make_sync_selection_proof( + duties_service_ref, + duty, + proof_slot, + *subnet_id, + config_ref, + &duties_service_ref.beacon_nodes, + ) + .await; - match proof.is_aggregator::() { - Ok(true) => { - debug!( - validator_index = duty.validator_index, - slot = %proof_slot, - %subnet_id, - "Validator is sync aggregator" - ); - Some(((proof_slot, *subnet_id), proof)) - } - Ok(false) => None, - Err(e) => { - warn!( - pubkey = ?duty.pubkey, - slot = %proof_slot, - error = ?e, - "Error determining is_aggregator" - ); - None - } + match proof { + Some(proof) => match proof.is_aggregator::() { + Ok(true) => { + debug!( + validator_index = duty.validator_index, + slot = %proof_slot, + %subnet_id, + "Validator is sync aggregator" + ); + Some(((proof_slot, *subnet_id), proof)) + } + Ok(false) => None, + Err(e) => { + warn!( + pubkey = ?duty.pubkey, + slot = %proof_slot, + error = ?e, + "Error determining is_aggregator" + ); + None + } + }, + + None => None, } }); From 980e3997beadd4998fd6ce6de173337e5b1ba074 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Wed, 23 Apr 2025 11:40:27 +0800 Subject: [PATCH 61/75] Revise to SyncDutiesMap --- validator_client/validator_services/src/sync.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index 5e149bf272..247a32e0c8 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -357,8 +357,8 @@ pub async fn poll_sync_committee_duties( .sync_duties .aggregation_pre_compute_slots(), computation_offset: Duration::from_secs(0), - selections_endpoint: sub_duties_service.distributed, - parallel_sign: sub_duties_service.distributed, + selections_endpoint: sub_duties_service.sync_duties.distributed, + parallel_sign: sub_duties_service.sync_duties.distributed, }; fill_in_aggregation_proofs( From c71de7185d364d9c9df18bcd68b3187bcd048a1f Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Wed, 23 Apr 2025 11:44:59 +0800 Subject: [PATCH 62/75] change to 0 --- validator_client/validator_services/src/sync.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index 247a32e0c8..45a254b02f 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -410,7 +410,7 @@ pub async fn poll_sync_committee_duties( lookahead_slot: sub_duties_service .sync_duties .aggregation_pre_compute_slots(), - computation_offset: Duration::from_secs(12), + computation_offset: Duration::from_secs(0), selections_endpoint: sub_duties_service.distributed, parallel_sign: sub_duties_service.distributed, }; From 2bcb984f80f68a3af28404173f2d4ea5b58ef274 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Wed, 23 Apr 2025 13:20:10 +0800 Subject: [PATCH 63/75] Revised to sync_duties --- validator_client/validator_services/src/sync.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index 45a254b02f..c5c6fbfcfb 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -411,8 +411,8 @@ pub async fn poll_sync_committee_duties( .sync_duties .aggregation_pre_compute_slots(), computation_offset: Duration::from_secs(0), - selections_endpoint: sub_duties_service.distributed, - parallel_sign: sub_duties_service.distributed, + selections_endpoint: sub_duties_service.sync_duties.distributed, + parallel_sign: sub_duties_service.sync_duties.distributed, }; fill_in_aggregation_proofs( From 23c901345a5ca7b765d8ddf7e764a594e92144f0 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Wed, 23 Apr 2025 13:24:50 +0800 Subject: [PATCH 64/75] remove logging --- common/eth2/src/types.rs | 1 - validator_client/validator_services/src/sync.rs | 9 --------- .../src/sync_committee_service.rs | 13 ++----------- 3 files changed, 2 insertions(+), 21 deletions(-) diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 428cc17415..06c983b1a3 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -939,7 +939,6 @@ pub struct BeaconCommitteeSelection { } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] - pub struct SyncCommitteeSelection { #[serde(with = "serde_utils::quoted_u64")] pub validator_index: u64, diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index c5c6fbfcfb..aa7703e2cd 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -662,15 +662,6 @@ pub async fn fill_in_aggregation_proofs( ) .await; - if let Some(proof) = &result { - debug!( - validator_index = duty.validator_index, - "slot" = %proof_slot, - "subcommittee_index" = *subnet_id, - "full selection proof" = ?proof, - "Selection proof in result variable" - ); - } result.map(|proof| (duty.validator_index, proof_slot, subnet_id, proof)) }); } diff --git a/validator_client/validator_services/src/sync_committee_service.rs b/validator_client/validator_services/src/sync_committee_service.rs index 233100b570..d99c0d3107 100644 --- a/validator_client/validator_services/src/sync_committee_service.rs +++ b/validator_client/validator_services/src/sync_committee_service.rs @@ -380,20 +380,11 @@ impl SyncCommitteeService { aggregator_index, aggregator_pk, contribution.clone(), - selection_proof.clone(), + selection_proof, ) .await { - Ok(signed_contribution) => { - debug!( - validator_index = aggregator_index, - %slot, - subcommittee_index = %subnet_id, - full_selection_proof = ?selection_proof, - "Produced sync contribution and proof" - ); - Some(signed_contribution) - } + Ok(signed_contribution) => Some(signed_contribution), Err(ValidatorStoreError::UnknownPubkey(pubkey)) => { // A pubkey can be missing when a validator was recently // removed via the API. From 43b209f5292a711ba3bb9ce9c6163e64a2c2693f Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Wed, 23 Apr 2025 14:53:25 +0800 Subject: [PATCH 65/75] add transparent --- consensus/types/src/sync_selection_proof.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/consensus/types/src/sync_selection_proof.rs b/consensus/types/src/sync_selection_proof.rs index e4f6ce43cd..61302a9dfb 100644 --- a/consensus/types/src/sync_selection_proof.rs +++ b/consensus/types/src/sync_selection_proof.rs @@ -13,6 +13,7 @@ use ssz_types::typenum::Unsigned; use std::cmp; #[derive(arbitrary::Arbitrary, PartialEq, Debug, Clone, Serialize, Deserialize)] +#[serde(transparent)] pub struct SyncSelectionProof(Signature); impl SyncSelectionProof { From f20f7491d70f9ef47d65f7e3a24d47fd31ce1153 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Wed, 23 Apr 2025 15:17:29 +0800 Subject: [PATCH 66/75] Add comments --- .../validator_services/src/duties_service.rs | 12 +++++++++-- .../validator_services/src/sync.rs | 21 ++++++++++--------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index 0e2d5537b1..a5d582381e 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -990,6 +990,9 @@ async fn poll_beacon_attesters_for_epoch( // Spawn the background task to compute selection proofs. let subservice = duties_service.clone(); + // Define a config to be pass to fill_in_selection_proofs. + // The defined config here defaults to using selections_endpoint and parallel_sign (i.e., distributed mode) + // Other DVT applications, e.g., Anchor can pass in different configs to suit different needs. let config = SelectionProofConfig { lookahead_slot: if duties_service.distributed { SELECTION_PROOF_SLOT_LOOKAHEAD_DVT @@ -1091,6 +1094,7 @@ async fn post_validator_duties_attester( .map_err(|e| Error::FailedToDownloadAttesters(e.to_string())) } +// Create a helper function here to reduce code duplication for normal and distributed mode fn process_duty_and_proof( attesters: &mut RwLockWriteGuard, result: Result<(AttesterData, Option), Error>, @@ -1107,6 +1111,7 @@ fn process_duty_and_proof( "Missing pubkey for duty and proof" ); // Do not abort the entire batch for a single failure. + // return true means continue processing duties. return true; } Err(e) => { @@ -1157,6 +1162,7 @@ fn process_duty_and_proof( } } } + /// Compute the attestation selection proofs for the `duties` and add them to the `attesters` map. /// /// Duties are computed in batches each slot. If a re-org is detected then the process will @@ -1190,8 +1196,8 @@ async fn fill_in_selection_proofs( let lookahead_slot = current_slot + selection_lookahead; - let relevant_duties = if duties_service.distributed { - // Remove old slot duties and only keep current duties + let relevant_duties = if config.selections_endpoint { + // Remove old slot duties and only keep current duties in distributed mode duties_by_slot .remove(&lookahead_slot) .map(|duties| BTreeMap::from([(lookahead_slot, duties)])) @@ -1235,6 +1241,7 @@ async fn fill_in_selection_proofs( while let Some(result) = duty_and_proof_results.next().await { let mut attesters = duties_service.attesters.write(); + // if process_duty_and_proof returns false, exit the loop if !process_duty_and_proof::( &mut attesters, result, @@ -1245,6 +1252,7 @@ async fn fill_in_selection_proofs( } } } else { + // In normal (non-distributed case), sign selection proofs serially let duty_and_proof_results = stream::iter(relevant_duties.into_values().flatten()) .then(|duty| async { let opt_selection_proof = make_selection_proof( diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index aa7703e2cd..6e0ba9d83f 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -1,5 +1,5 @@ use crate::duties_service::{DutiesService, Error, SelectionProofConfig}; -use beacon_node_fallback::BeaconNodeFallback; +// use beacon_node_fallback::BeaconNodeFallback; use doppelganger_service::DoppelgangerStatus; use eth2::types::{Signature, SyncCommitteeSelection}; use futures::future::join_all; @@ -352,10 +352,12 @@ pub async fn poll_sync_committee_duties( let sub_duties_service = duties_service.clone(); duties_service.context.executor.spawn( async move { + // The defined config here defaults to using selections_endpoint and parallel_sign (i.e., distributed mode) + // Other DVT applications, e.g., Anchor can pass in different configs to suit different needs. let config = SelectionProofConfig { lookahead_slot: sub_duties_service .sync_duties - .aggregation_pre_compute_slots(), + .aggregation_pre_compute_slots(), // Use the current behaviour defined in the method computation_offset: Duration::from_secs(0), selections_endpoint: sub_duties_service.sync_duties.distributed, parallel_sign: sub_duties_service.sync_duties.distributed, @@ -520,13 +522,13 @@ pub async fn poll_sync_committee_duties_for_period( duties_service: &Arc>, duty: &SyncDuty, proof_slot: Slot, subnet_id: SyncSubnetId, config: &SelectionProofConfig, - _beacon_nodes: &Arc>, ) -> Option { let sync_selection_proof = duties_service .validator_store @@ -554,12 +556,13 @@ pub async fn make_sync_selection_proof( } }; - // In --distributed mode when we want to call the selections endpoint + // In distributed mode when we want to call the selections endpoint if config.selections_endpoint { debug!( "validator_index" = duty.validator_index, "slot" = %proof_slot, "subcommittee_index" = *subnet_id, + // In distributed mode, this is partial selection proof "partial selection proof" = ?Signature::from(selection_proof.clone()), "Sending sync selection to middleware" ); @@ -589,16 +592,16 @@ pub async fn make_sync_selection_proof( match middleware_response { Ok(response) => { let response_data = &response.data[0]; - // The selection proof from middleware response will be a full selection proof debug!( "validator_index" = response_data.validator_index, "slot" = %response_data.slot, "subcommittee_index" = response_data.subcommittee_index, + // The selection proof from middleware response will be a full selection proof "full selection proof" = ?response_data.selection_proof, "Received sync selection from middleware" ); - // Convert the response to a SyncSelectionProof so we can call the is_aggregator method + // Convert the response to a SyncSelectionProof let full_selection_proof = SyncSelectionProof::from(response_data.selection_proof.clone()); Some(full_selection_proof) @@ -613,7 +616,7 @@ pub async fn make_sync_selection_proof( } } } else { - // When calling the selections endpoint is not required, return the selection_proof + // When calling the selections endpoint is not required, the selection_proof is already a full selection proof Some(selection_proof) } } @@ -648,7 +651,7 @@ pub async fn fill_in_aggregation_proofs( // Construct proof for prior slot. let proof_slot = slot - 1; - // Store all partial sync selection proofs so that it can be sent together later + // Calling the make_sync_selection_proof will return a full selection proof for &subnet_id in &subnet_ids { let duties_service = duties_service.clone(); futures_unordered.push(async move { @@ -658,7 +661,6 @@ pub async fn fill_in_aggregation_proofs( proof_slot, subnet_id, config_ref, - &duties_service.beacon_nodes, ) .await; @@ -749,7 +751,6 @@ pub async fn fill_in_aggregation_proofs( proof_slot, *subnet_id, config_ref, - &duties_service_ref.beacon_nodes, ) .await; From 05df47bed8714ed3a445611bf483cdcd7630ea0f Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Wed, 23 Apr 2025 15:39:52 +0800 Subject: [PATCH 67/75] Comment --- validator_client/validator_services/src/sync.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index 6e0ba9d83f..d85212c3e6 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -616,7 +616,7 @@ pub async fn make_sync_selection_proof( } } } else { - // When calling the selections endpoint is not required, the selection_proof is already a full selection proof + // In non-distributed mode, the selection_proof is already a full selection proof Some(selection_proof) } } From b71e40d4a6617914a67ab48834a5b29bc82c9ab8 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Tue, 29 Apr 2025 15:12:58 +0800 Subject: [PATCH 68/75] Revise config move to lib.rs --- validator_client/src/lib.rs | 35 ++++++++- .../validator_services/src/duties_service.rs | 71 +++++++++---------- .../validator_services/src/sync.rs | 66 ++++++----------- 3 files changed, 85 insertions(+), 87 deletions(-) diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 13ecfe42ca..0367b8da82 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -4,6 +4,7 @@ mod latency; mod notifier; use crate::cli::ValidatorClient; +use crate::duties_service::SelectionProofConfig; pub use config::Config; use initialized_validators::InitializedValidators; use metrics::set_gauge; @@ -74,6 +75,17 @@ const HTTP_GET_VALIDATOR_BLOCK_TIMEOUT_QUOTIENT: u32 = 4; const DOPPELGANGER_SERVICE_NAME: &str = "doppelganger"; +/// Compute attestation selection proofs this many slots before they are required. +/// +/// At start-up selection proofs will be computed with less lookahead out of necessity. +const SELECTION_PROOF_SLOT_LOOKAHEAD: u64 = 8; + +/// The attestation selection proof lookahead for those running with the --distributed flag. +const SELECTION_PROOF_SLOT_LOOKAHEAD_DVT: u64 = 1; + +/// Fraction of a slot at which selection proof signing should happen (2 means half way). +const SELECTION_PROOF_SCHEDULE_DENOM: u32 = 2; + #[derive(Clone)] pub struct ProductionValidatorClient { context: RuntimeContext, @@ -444,11 +456,30 @@ impl ProductionValidatorClient { validator_store.prune_slashing_protection_db(slot.epoch(E::slots_per_epoch()), true); } + // Define a config to be pass to fill_in_selection_proofs. + // The defined config here defaults to using selections_endpoint and parallel_sign (i.e., distributed mode) + // Other DVT applications, e.g., Anchor can pass in different configs to suit different needs. + let selection_proof_config = if config.distributed { + SelectionProofConfig { + lookahead_slot: SELECTION_PROOF_SLOT_LOOKAHEAD_DVT, + computation_offset: slot_clock.slot_duration() / SELECTION_PROOF_SCHEDULE_DENOM, + selections_endpoint: true, + parallel_sign: true, + } + } else { + SelectionProofConfig { + lookahead_slot: SELECTION_PROOF_SLOT_LOOKAHEAD, + computation_offset: Duration::from_secs(6), + selections_endpoint: false, + parallel_sign: false, + } + }; + let duties_context = context.service_context("duties".into()); let duties_service = Arc::new(DutiesService { attesters: <_>::default(), proposers: <_>::default(), - sync_duties: SyncDutiesMap::new(config.distributed), + sync_duties: SyncDutiesMap::new(selection_proof_config.clone()), slot_clock: slot_clock.clone(), beacon_nodes: beacon_nodes.clone(), validator_store: validator_store.clone(), @@ -456,7 +487,7 @@ impl ProductionValidatorClient { spec: context.eth2_config.spec.clone(), context: duties_context, enable_high_validator_count_metrics: config.enable_high_validator_count_metrics, - distributed: config.distributed, + selection_proof_config, disable_attesting: config.disable_attesting, }); diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index a5d582381e..24999ba0c3 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -37,17 +37,6 @@ use validator_store::{Error as ValidatorStoreError, ValidatorStore}; /// Only retain `HISTORICAL_DUTIES_EPOCHS` duties prior to the current epoch. const HISTORICAL_DUTIES_EPOCHS: u64 = 2; -/// Compute attestation selection proofs this many slots before they are required. -/// -/// At start-up selection proofs will be computed with less lookahead out of necessity. -const SELECTION_PROOF_SLOT_LOOKAHEAD: u64 = 8; - -/// The attestation selection proof lookahead for those running with the --distributed flag. -const SELECTION_PROOF_SLOT_LOOKAHEAD_DVT: u64 = 1; - -/// Fraction of a slot at which selection proof signing should happen (2 means half way). -const SELECTION_PROOF_SCHEDULE_DENOM: u32 = 2; - /// Minimum number of validators for which we auto-enable per-validator metrics. /// For validators greater than this value, we need to manually set the `enable-per-validator-metrics` /// flag in the cli to enable collection of per validator metrics. @@ -126,6 +115,7 @@ pub struct SubscriptionSlots { duty_slot: Slot, } +#[derive(Clone)] pub struct SelectionProofConfig { pub lookahead_slot: u64, pub computation_offset: Duration, // The seconds to compute the selection proof before a slot @@ -140,10 +130,10 @@ async fn make_selection_proof( duty: &AttesterData, validator_store: &ValidatorStore, spec: &ChainSpec, - config: &SelectionProofConfig, + duties_service: &DutiesService, beacon_nodes: &Arc>, ) -> Result, Error> { - let selection_proof = if config.selections_endpoint { + let selection_proof = if duties_service.selection_proof_config.selections_endpoint { let beacon_committee_selection = BeaconCommitteeSelection { validator_index: duty.validator_index, slot: duty.slot, @@ -285,10 +275,10 @@ pub struct DutiesService { pub context: RuntimeContext, /// The current chain spec. pub spec: Arc, - //// Whether we permit large validator counts in the metrics. + /// Whether we permit large validator counts in the metrics. pub enable_high_validator_count_metrics: bool, - /// If this validator is running in distributed mode. - pub distributed: bool, + /// Pass the config for distributed or non-distributed mode. + pub selection_proof_config: SelectionProofConfig, pub disable_attesting: bool, } @@ -990,24 +980,9 @@ async fn poll_beacon_attesters_for_epoch( // Spawn the background task to compute selection proofs. let subservice = duties_service.clone(); - // Define a config to be pass to fill_in_selection_proofs. - // The defined config here defaults to using selections_endpoint and parallel_sign (i.e., distributed mode) - // Other DVT applications, e.g., Anchor can pass in different configs to suit different needs. - let config = SelectionProofConfig { - lookahead_slot: if duties_service.distributed { - SELECTION_PROOF_SLOT_LOOKAHEAD_DVT - } else { - SELECTION_PROOF_SLOT_LOOKAHEAD - }, - computation_offset: duties_service.slot_clock.slot_duration() - / SELECTION_PROOF_SCHEDULE_DENOM, - selections_endpoint: duties_service.distributed, - parallel_sign: duties_service.distributed, - }; - duties_service.context.executor.spawn( async move { - fill_in_selection_proofs(subservice, new_duties, dependent_root, config).await; + fill_in_selection_proofs(subservice, new_duties, dependent_root).await; }, "duties_service_selection_proofs_background", ); @@ -1171,7 +1146,6 @@ async fn fill_in_selection_proofs( duties_service: Arc>, duties: Vec, dependent_root: Hash256, - config: SelectionProofConfig, ) { // Sort duties by slot in a BTreeMap. let mut duties_by_slot: BTreeMap> = BTreeMap::new(); @@ -1186,17 +1160,32 @@ async fn fill_in_selection_proofs( while !duties_by_slot.is_empty() { if let Some(duration) = slot_clock.duration_to_next_slot() { - sleep(duration.saturating_sub(config.computation_offset)).await; + sleep( + duration.saturating_sub( + duties_service + .sync_duties + .selection_proof_config + .computation_offset, + ), + ) + .await; let Some(current_slot) = slot_clock.now() else { continue; }; - let selection_lookahead = config.lookahead_slot; + let selection_lookahead = duties_service + .sync_duties + .selection_proof_config + .lookahead_slot; let lookahead_slot = current_slot + selection_lookahead; - let relevant_duties = if config.selections_endpoint { + let relevant_duties = if duties_service + .sync_duties + .selection_proof_config + .parallel_sign + { // Remove old slot duties and only keep current duties in distributed mode duties_by_slot .remove(&lookahead_slot) @@ -1222,7 +1211,11 @@ async fn fill_in_selection_proofs( // In distributed case, we want to send all partial selection proofs to the middleware to determine aggregation duties, // as the middleware will need to have a threshold of partial selection proofs to be able to return the full selection proof // Thus, sign selection proofs in parallel in distributed case; Otherwise, sign them serially in non-distributed (normal) case - if config.parallel_sign { + if duties_service + .sync_duties + .selection_proof_config + .parallel_sign + { let mut duty_and_proof_results = relevant_duties .into_values() .flatten() @@ -1231,7 +1224,7 @@ async fn fill_in_selection_proofs( &duty, &duties_service.validator_store, &duties_service.spec, - &config, + &duties_service, &duties_service.beacon_nodes, ) .await?; @@ -1259,7 +1252,7 @@ async fn fill_in_selection_proofs( &duty, &duties_service.validator_store, &duties_service.spec, - &config, + &duties_service, &duties_service.beacon_nodes, ) .await?; diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index d85212c3e6..a2ad36e0a6 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -10,7 +10,6 @@ use slot_clock::SlotClock; use std::collections::{HashMap, HashSet}; use std::marker::PhantomData; use std::sync::Arc; -use std::time::Duration; use tracing::{debug, error, info, warn}; use types::{ChainSpec, EthSpec, PublicKeyBytes, Slot, SyncDuty, SyncSelectionProof, SyncSubnetId}; use validator_store::Error as ValidatorStoreError; @@ -36,7 +35,7 @@ pub struct SyncDutiesMap { /// Map from sync committee period to duties for members of that sync committee. committees: RwLock>, /// Whether we are in `distributed` mode and using reduced lookahead for aggregate pre-compute. - distributed: bool, + pub selection_proof_config: SelectionProofConfig, _phantom: PhantomData, } @@ -86,10 +85,10 @@ pub struct SlotDuties { } impl SyncDutiesMap { - pub fn new(distributed: bool) -> Self { + pub fn new(selection_proof_config: SelectionProofConfig) -> Self { Self { committees: RwLock::new(HashMap::new()), - distributed, + selection_proof_config, _phantom: PhantomData, } } @@ -109,7 +108,7 @@ impl SyncDutiesMap { /// Number of slots in advance to compute selection proofs fn aggregation_pre_compute_slots(&self) -> u64 { - if self.distributed { + if self.selection_proof_config.parallel_sign { AGGREGATION_PRE_COMPUTE_SLOTS_DISTRIBUTED } else { E::slots_per_epoch() * AGGREGATION_PRE_COMPUTE_EPOCHS @@ -354,14 +353,6 @@ pub async fn poll_sync_committee_duties( async move { // The defined config here defaults to using selections_endpoint and parallel_sign (i.e., distributed mode) // Other DVT applications, e.g., Anchor can pass in different configs to suit different needs. - let config = SelectionProofConfig { - lookahead_slot: sub_duties_service - .sync_duties - .aggregation_pre_compute_slots(), // Use the current behaviour defined in the method - computation_offset: Duration::from_secs(0), - selections_endpoint: sub_duties_service.sync_duties.distributed, - parallel_sign: sub_duties_service.sync_duties.distributed, - }; fill_in_aggregation_proofs( sub_duties_service, @@ -369,7 +360,6 @@ pub async fn poll_sync_committee_duties( current_sync_committee_period, current_slot, current_pre_compute_slot, - config, ) .await }, @@ -408,22 +398,12 @@ pub async fn poll_sync_committee_duties( let sub_duties_service = duties_service.clone(); duties_service.context.executor.spawn( async move { - let config = SelectionProofConfig { - lookahead_slot: sub_duties_service - .sync_duties - .aggregation_pre_compute_slots(), - computation_offset: Duration::from_secs(0), - selections_endpoint: sub_duties_service.sync_duties.distributed, - parallel_sign: sub_duties_service.sync_duties.distributed, - }; - fill_in_aggregation_proofs( sub_duties_service, &new_pre_compute_duties, next_sync_committee_period, current_slot, pre_compute_slot, - config, ) .await }, @@ -528,7 +508,6 @@ pub async fn make_sync_selection_proof( duty: &SyncDuty, proof_slot: Slot, subnet_id: SyncSubnetId, - config: &SelectionProofConfig, ) -> Option { let sync_selection_proof = duties_service .validator_store @@ -557,7 +536,11 @@ pub async fn make_sync_selection_proof( }; // In distributed mode when we want to call the selections endpoint - if config.selections_endpoint { + if duties_service + .sync_duties + .selection_proof_config + .selections_endpoint + { debug!( "validator_index" = duty.validator_index, "slot" = %proof_slot, @@ -627,14 +610,16 @@ pub async fn fill_in_aggregation_proofs( sync_committee_period: u64, current_slot: Slot, pre_compute_slot: Slot, - config: SelectionProofConfig, ) { // Generate selection proofs for each validator at each slot, one slot at a time. for slot in (current_slot.as_u64()..=pre_compute_slot.as_u64()).map(Slot::new) { // For distributed mode - if config.parallel_sign { + if duties_service + .sync_duties + .selection_proof_config + .parallel_sign + { let mut futures_unordered = FuturesUnordered::new(); - let config_ref = &config; for (_, duty) in pre_compute_duties { let subnet_ids = match duty.subnet_ids::() { @@ -655,14 +640,9 @@ pub async fn fill_in_aggregation_proofs( for &subnet_id in &subnet_ids { let duties_service = duties_service.clone(); futures_unordered.push(async move { - let result = make_sync_selection_proof( - &duties_service, - duty, - proof_slot, - subnet_id, - config_ref, - ) - .await; + let result = + make_sync_selection_proof(&duties_service, duty, proof_slot, subnet_id) + .await; result.map(|proof| (duty.validator_index, proof_slot, subnet_id, proof)) }); @@ -740,19 +720,13 @@ pub async fn fill_in_aggregation_proofs( // Create futures to produce proofs. let duties_service_ref = &duties_service; - let config_ref = &config; let futures = subnet_ids.iter().map(|subnet_id| async move { // Construct proof for prior slot. let proof_slot = slot - 1; - let proof = make_sync_selection_proof( - duties_service_ref, - duty, - proof_slot, - *subnet_id, - config_ref, - ) - .await; + let proof = + make_sync_selection_proof(duties_service_ref, duty, proof_slot, *subnet_id) + .await; match proof { Some(proof) => match proof.is_aggregator::() { From 9d6d1cb654ded8d2bacd38dc2f5b956fcf758952 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Tue, 29 Apr 2025 15:33:18 +0800 Subject: [PATCH 69/75] Revise a bit --- validator_client/src/lib.rs | 2 +- validator_client/validator_services/src/sync.rs | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 0367b8da82..c1a3217753 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -469,7 +469,7 @@ impl ProductionValidatorClient { } else { SelectionProofConfig { lookahead_slot: SELECTION_PROOF_SLOT_LOOKAHEAD, - computation_offset: Duration::from_secs(6), + computation_offset: slot_clock.slot_duration() / SELECTION_PROOF_SCHEDULE_DENOM, selections_endpoint: false, parallel_sign: false, } diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index a2ad36e0a6..15252f4d92 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -351,9 +351,6 @@ pub async fn poll_sync_committee_duties( let sub_duties_service = duties_service.clone(); duties_service.context.executor.spawn( async move { - // The defined config here defaults to using selections_endpoint and parallel_sign (i.e., distributed mode) - // Other DVT applications, e.g., Anchor can pass in different configs to suit different needs. - fill_in_aggregation_proofs( sub_duties_service, &new_pre_compute_duties, From d34a91ac07fcef1f41e09563b68d033a92296bab Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Tue, 29 Apr 2025 16:28:42 +0800 Subject: [PATCH 70/75] address comments --- validator_client/src/lib.rs | 27 +++++++++++++-- .../validator_services/src/duties_service.rs | 33 +++++-------------- .../validator_services/src/sync.rs | 18 ++-------- 3 files changed, 34 insertions(+), 44 deletions(-) diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index c1a3217753..5cb1c362ba 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -86,6 +86,11 @@ const SELECTION_PROOF_SLOT_LOOKAHEAD_DVT: u64 = 1; /// Fraction of a slot at which selection proof signing should happen (2 means half way). const SELECTION_PROOF_SCHEDULE_DENOM: u32 = 2; +/// Number of epochs in advance to compute selection proofs when not in `distributed` mode. +pub const AGGREGATION_PRE_COMPUTE_EPOCHS: u64 = 2; +/// Number of slots in advance to compute selection proofs when in `distributed` mode. +pub const AGGREGATION_PRE_COMPUTE_SLOTS_DISTRIBUTED: u64 = 1; + #[derive(Clone)] pub struct ProductionValidatorClient { context: RuntimeContext, @@ -459,7 +464,7 @@ impl ProductionValidatorClient { // Define a config to be pass to fill_in_selection_proofs. // The defined config here defaults to using selections_endpoint and parallel_sign (i.e., distributed mode) // Other DVT applications, e.g., Anchor can pass in different configs to suit different needs. - let selection_proof_config = if config.distributed { + let attestation_selection_proof_config = if config.distributed { SelectionProofConfig { lookahead_slot: SELECTION_PROOF_SLOT_LOOKAHEAD_DVT, computation_offset: slot_clock.slot_duration() / SELECTION_PROOF_SCHEDULE_DENOM, @@ -475,11 +480,27 @@ impl ProductionValidatorClient { } }; + let sync_selection_proof_config = if config.distributed { + SelectionProofConfig { + lookahead_slot: AGGREGATION_PRE_COMPUTE_SLOTS_DISTRIBUTED, + computation_offset: Duration::default(), + selections_endpoint: true, + parallel_sign: true, + } + } else { + SelectionProofConfig { + lookahead_slot: E::slots_per_epoch() * AGGREGATION_PRE_COMPUTE_EPOCHS, + computation_offset: Duration::default(), + selections_endpoint: false, + parallel_sign: false, + } + }; + let duties_context = context.service_context("duties".into()); let duties_service = Arc::new(DutiesService { attesters: <_>::default(), proposers: <_>::default(), - sync_duties: SyncDutiesMap::new(selection_proof_config.clone()), + sync_duties: SyncDutiesMap::new(sync_selection_proof_config), slot_clock: slot_clock.clone(), beacon_nodes: beacon_nodes.clone(), validator_store: validator_store.clone(), @@ -487,7 +508,7 @@ impl ProductionValidatorClient { spec: context.eth2_config.spec.clone(), context: duties_context, enable_high_validator_count_metrics: config.enable_high_validator_count_metrics, - selection_proof_config, + selection_proof_config: attestation_selection_proof_config, disable_attesting: config.disable_attesting, }); diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index 24999ba0c3..27f0817b0d 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -115,7 +115,6 @@ pub struct SubscriptionSlots { duty_slot: Slot, } -#[derive(Clone)] pub struct SelectionProofConfig { pub lookahead_slot: u64, pub computation_offset: Duration, // The seconds to compute the selection proof before a slot @@ -130,10 +129,10 @@ async fn make_selection_proof( duty: &AttesterData, validator_store: &ValidatorStore, spec: &ChainSpec, - duties_service: &DutiesService, beacon_nodes: &Arc>, + config: &SelectionProofConfig, ) -> Result, Error> { - let selection_proof = if duties_service.selection_proof_config.selections_endpoint { + let selection_proof = if config.selections_endpoint { let beacon_committee_selection = BeaconCommitteeSelection { validator_index: duty.validator_index, slot: duty.slot, @@ -1161,12 +1160,7 @@ async fn fill_in_selection_proofs( while !duties_by_slot.is_empty() { if let Some(duration) = slot_clock.duration_to_next_slot() { sleep( - duration.saturating_sub( - duties_service - .sync_duties - .selection_proof_config - .computation_offset, - ), + duration.saturating_sub(duties_service.selection_proof_config.computation_offset), ) .await; @@ -1174,18 +1168,11 @@ async fn fill_in_selection_proofs( continue; }; - let selection_lookahead = duties_service - .sync_duties - .selection_proof_config - .lookahead_slot; + let selection_lookahead = duties_service.selection_proof_config.lookahead_slot; let lookahead_slot = current_slot + selection_lookahead; - let relevant_duties = if duties_service - .sync_duties - .selection_proof_config - .parallel_sign - { + let relevant_duties = if duties_service.selection_proof_config.parallel_sign { // Remove old slot duties and only keep current duties in distributed mode duties_by_slot .remove(&lookahead_slot) @@ -1211,11 +1198,7 @@ async fn fill_in_selection_proofs( // In distributed case, we want to send all partial selection proofs to the middleware to determine aggregation duties, // as the middleware will need to have a threshold of partial selection proofs to be able to return the full selection proof // Thus, sign selection proofs in parallel in distributed case; Otherwise, sign them serially in non-distributed (normal) case - if duties_service - .sync_duties - .selection_proof_config - .parallel_sign - { + if duties_service.selection_proof_config.parallel_sign { let mut duty_and_proof_results = relevant_duties .into_values() .flatten() @@ -1224,8 +1207,8 @@ async fn fill_in_selection_proofs( &duty, &duties_service.validator_store, &duties_service.spec, - &duties_service, &duties_service.beacon_nodes, + &duties_service.selection_proof_config, ) .await?; Ok((duty, opt_selection_proof)) @@ -1252,8 +1235,8 @@ async fn fill_in_selection_proofs( &duty, &duties_service.validator_store, &duties_service.spec, - &duties_service, &duties_service.beacon_nodes, + &duties_service.selection_proof_config, ) .await?; Ok((duty, opt_selection_proof)) diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index 15252f4d92..0df51adb32 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -14,11 +14,6 @@ use tracing::{debug, error, info, warn}; use types::{ChainSpec, EthSpec, PublicKeyBytes, Slot, SyncDuty, SyncSelectionProof, SyncSubnetId}; use validator_store::Error as ValidatorStoreError; -/// Number of epochs in advance to compute selection proofs when not in `distributed` mode. -pub const AGGREGATION_PRE_COMPUTE_EPOCHS: u64 = 2; -/// Number of slots in advance to compute selection proofs when in `distributed` mode. -pub const AGGREGATION_PRE_COMPUTE_SLOTS_DISTRIBUTED: u64 = 1; - /// Top-level data-structure containing sync duty information. /// /// This data is structured as a series of nested `HashMap`s wrapped in `RwLock`s. Fine-grained @@ -106,15 +101,6 @@ impl SyncDutiesMap { }) } - /// Number of slots in advance to compute selection proofs - fn aggregation_pre_compute_slots(&self) -> u64 { - if self.selection_proof_config.parallel_sign { - AGGREGATION_PRE_COMPUTE_SLOTS_DISTRIBUTED - } else { - E::slots_per_epoch() * AGGREGATION_PRE_COMPUTE_EPOCHS - } - } - /// Prepare for pre-computation of selection proofs for `committee_period`. /// /// Return the slot up to which proofs should be pre-computed, as well as a vec of @@ -130,7 +116,7 @@ impl SyncDutiesMap { current_slot, first_slot_of_period::(committee_period, spec), ); - let pre_compute_lookahead_slots = self.aggregation_pre_compute_slots(); + let pre_compute_lookahead_slots = self.selection_proof_config.lookahead_slot; let pre_compute_slot = std::cmp::min( current_slot + pre_compute_lookahead_slots, last_slot_of_period::(committee_period, spec), @@ -382,7 +368,7 @@ pub async fn poll_sync_committee_duties( } // Pre-compute aggregator selection proofs for the next period. - let aggregate_pre_compute_lookahead_slots = sync_duties.aggregation_pre_compute_slots(); + let aggregate_pre_compute_lookahead_slots = sync_duties.selection_proof_config.lookahead_slot; if (current_slot + aggregate_pre_compute_lookahead_slots) .epoch(E::slots_per_epoch()) .sync_committee_period(spec)? From 9d8a9e7e701158ca8b70a8fcfdd55df3dfe4a39b Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Tue, 6 May 2025 15:12:21 +0800 Subject: [PATCH 71/75] revise comment --- validator_client/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 5cb1c362ba..17ce92cce0 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -461,7 +461,7 @@ impl ProductionValidatorClient { validator_store.prune_slashing_protection_db(slot.epoch(E::slots_per_epoch()), true); } - // Define a config to be pass to fill_in_selection_proofs. + // Define a config to be pass to duties_service. // The defined config here defaults to using selections_endpoint and parallel_sign (i.e., distributed mode) // Other DVT applications, e.g., Anchor can pass in different configs to suit different needs. let attestation_selection_proof_config = if config.distributed { From 46c29b09b7d044bedcccf0921a1e87cf9e658a5a Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Fri, 9 May 2025 14:15:46 +0800 Subject: [PATCH 72/75] fix --- validator_client/src/lib.rs | 31 +++++++------ .../validator_services/src/duties_service.rs | 44 ++++++++++++------- .../validator_services/src/sync.rs | 12 +++-- validator_client/validator_store/src/lib.rs | 1 - 4 files changed, 48 insertions(+), 40 deletions(-) diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index daf80895c3..0eadffacfb 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -483,7 +483,7 @@ impl ProductionValidatorClient { } }; - let sync_selection_proof_config = if config.distributed { + let _sync_selection_proof_config = if config.distributed { SelectionProofConfig { lookahead_slot: AGGREGATION_PRE_COMPUTE_SLOTS_DISTRIBUTED, computation_offset: Duration::default(), @@ -499,21 +499,20 @@ impl ProductionValidatorClient { } }; - let duties_context = context.service_context("duties".into()); - let duties_service = Arc::new(DutiesService { - attesters: <_>::default(), - proposers: <_>::default(), - sync_duties: SyncDutiesMap::new(sync_selection_proof_config), - slot_clock: slot_clock.clone(), - beacon_nodes: beacon_nodes.clone(), - validator_store: validator_store.clone(), - unknown_validator_next_poll_slots: <_>::default(), - spec: context.eth2_config.spec.clone(), - context: duties_context, - enable_high_validator_count_metrics: config.enable_high_validator_count_metrics, - selection_proof_config: attestation_selection_proof_config, - disable_attesting: config.disable_attesting, - }); + let duties_service: Arc< + DutiesService, SystemTimeSlotClock>, + > = Arc::new( + DutiesServiceBuilder::new() + .slot_clock(slot_clock.clone()) + .beacon_nodes(beacon_nodes.clone()) + .validator_store(validator_store.clone()) + .spec(context.eth2_config.spec.clone()) + .executor(context.executor.clone()) + .enable_high_validator_count_metrics(config.enable_high_validator_count_metrics) + .selection_proof_config(attestation_selection_proof_config) + .disable_attesting(config.disable_attesting) + .build()?, + ); // Update the metrics server. if let Some(ctx) = &validator_metrics_ctx { diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index b1c418de88..8902684cb1 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -114,6 +114,7 @@ pub struct SubscriptionSlots { duty_slot: Slot, } +#[derive(Copy, Clone)] pub struct SelectionProofConfig { pub lookahead_slot: u64, pub computation_offset: Duration, // The seconds to compute the selection proof before a slot @@ -121,16 +122,27 @@ pub struct SelectionProofConfig { pub parallel_sign: bool, } +impl SelectionProofConfig { + fn default() -> Self { + Self { + lookahead_slot: 0, + computation_offset: Duration::default(), + selections_endpoint: false, + parallel_sign: false, + } + } +} + /// Create a selection proof for `duty`. /// /// Return `Ok(None)` if the attesting validator is not an aggregator. -async fn make_selection_proof( +async fn make_selection_proof( duty: &AttesterData, validator_store: &S, spec: &ChainSpec, - beacon_nodes: &Arc>, + beacon_nodes: &Arc>, config: &SelectionProofConfig, -) -> Result, Error> { +) -> Result, Error> { let selection_proof = if config.selections_endpoint { let beacon_committee_selection = BeaconCommitteeSelection { validator_index: duty.validator_index, @@ -266,7 +278,7 @@ pub struct DutiesServiceBuilder { //// Whether we permit large validator counts in the metrics. enable_high_validator_count_metrics: bool, /// If this validator is running in distributed mode. - distributed: bool, + selection_proof_config: SelectionProofConfig, disable_attesting: bool, } @@ -285,7 +297,7 @@ impl DutiesServiceBuilder { executor: None, spec: None, enable_high_validator_count_metrics: false, - distributed: false, + selection_proof_config: SelectionProofConfig::default(), disable_attesting: false, } } @@ -323,8 +335,8 @@ impl DutiesServiceBuilder { self } - pub fn distributed(mut self, distributed: bool) -> Self { - self.distributed = distributed; + pub fn selection_proof_config(mut self, selection_proof_config: SelectionProofConfig) -> Self { + self.selection_proof_config = selection_proof_config; self } @@ -337,7 +349,7 @@ impl DutiesServiceBuilder { Ok(DutiesService { attesters: Default::default(), proposers: Default::default(), - sync_duties: SyncDutiesMap::new(self.distributed), + sync_duties: SyncDutiesMap::new(self.selection_proof_config), validator_store: self .validator_store .ok_or("Cannot build DutiesService without validator_store")?, @@ -353,7 +365,7 @@ impl DutiesServiceBuilder { .ok_or("Cannot build DutiesService without executor")?, spec: self.spec.ok_or("Cannot build DutiesService without spec")?, enable_high_validator_count_metrics: self.enable_high_validator_count_metrics, - distributed: self.distributed, + selection_proof_config: self.selection_proof_config, disable_attesting: self.disable_attesting, }) } @@ -1168,9 +1180,9 @@ async fn post_validator_duties_attester( +fn process_duty_and_proof( attesters: &mut RwLockWriteGuard, - result: Result<(AttesterData, Option), Error>, + result: Result<(AttesterData, Option), Error>, dependent_root: Hash256, current_slot: Slot, ) -> bool { @@ -1198,7 +1210,7 @@ fn process_duty_and_proof( }; let attester_map = attesters.entry(duty.pubkey).or_default(); - let epoch = duty.slot.epoch(E::slots_per_epoch()); + let epoch = duty.slot.epoch(S::E::slots_per_epoch()); match attester_map.entry(epoch) { hash_map::Entry::Occupied(mut entry) => { // No need to update duties for which no proof was computed. @@ -1304,7 +1316,7 @@ async fn fill_in_selection_proofs( + if !process_duty_and_proof::( &mut attesters, result, dependent_root, @@ -1332,7 +1344,7 @@ async fn fill_in_selection_proofs( + if !process_duty_and_proof::( &mut attesters, result, dependent_root, diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index da8aab6401..d2a1aabc5b 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -1,6 +1,4 @@ use crate::duties_service::{DutiesService, Error, SelectionProofConfig}; -// use beacon_node_fallback::BeaconNodeFallback; -use doppelganger_service::DoppelgangerStatus; use eth2::types::{Signature, SyncCommitteeSelection}; use futures::future::join_all; use futures::stream::{FuturesUnordered, StreamExt}; @@ -489,8 +487,8 @@ pub async fn poll_sync_committee_duties_for_period( - duties_service: &Arc>, +pub async fn make_sync_selection_proof( + duties_service: &Arc>, duty: &SyncDuty, proof_slot: Slot, subnet_id: SyncSubnetId, @@ -608,7 +606,7 @@ pub async fn fill_in_aggregation_proofs() { + let subnet_ids = match duty.subnet_ids::() { Ok(subnet_ids) => subnet_ids, Err(e) => { crit!( @@ -646,7 +644,7 @@ pub async fn fill_in_aggregation_proofs() { + match proof.is_aggregator::() { Ok(true) => { if let Some(Some(duty)) = validators.get(&validator_index) { debug!( @@ -703,7 +701,7 @@ pub async fn fill_in_aggregation_proofs { GreaterThanCurrentEpoch { epoch: Epoch, current_epoch: Epoch }, UnableToSignAttestation(AttestationError), SpecificError(T), - UnableToSign(SigningError), Middleware(String), } From c790d9dd451c6fb58703cf783ebf8beba83ae10c Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Fri, 9 May 2025 16:00:56 +0800 Subject: [PATCH 73/75] Fix --- validator_client/src/lib.rs | 5 ++-- .../validator_services/src/duties_service.rs | 25 ++++++++++++++----- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 0eadffacfb..ec7a084f55 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -483,7 +483,7 @@ impl ProductionValidatorClient { } }; - let _sync_selection_proof_config = if config.distributed { + let sync_selection_proof_config = if config.distributed { SelectionProofConfig { lookahead_slot: AGGREGATION_PRE_COMPUTE_SLOTS_DISTRIBUTED, computation_offset: Duration::default(), @@ -509,7 +509,8 @@ impl ProductionValidatorClient { .spec(context.eth2_config.spec.clone()) .executor(context.executor.clone()) .enable_high_validator_count_metrics(config.enable_high_validator_count_metrics) - .selection_proof_config(attestation_selection_proof_config) + .attestation_selection_proof_config(attestation_selection_proof_config) + .sync_selection_proof_config(sync_selection_proof_config) .disable_attesting(config.disable_attesting) .build()?, ); diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index 8902684cb1..48aaf351a1 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -278,7 +278,8 @@ pub struct DutiesServiceBuilder { //// Whether we permit large validator counts in the metrics. enable_high_validator_count_metrics: bool, /// If this validator is running in distributed mode. - selection_proof_config: SelectionProofConfig, + attestation_selection_proof_config: SelectionProofConfig, + sync_selection_proof_config: SelectionProofConfig, disable_attesting: bool, } @@ -297,7 +298,8 @@ impl DutiesServiceBuilder { executor: None, spec: None, enable_high_validator_count_metrics: false, - selection_proof_config: SelectionProofConfig::default(), + attestation_selection_proof_config: SelectionProofConfig::default(), + sync_selection_proof_config: SelectionProofConfig::default(), disable_attesting: false, } } @@ -335,8 +337,19 @@ impl DutiesServiceBuilder { self } - pub fn selection_proof_config(mut self, selection_proof_config: SelectionProofConfig) -> Self { - self.selection_proof_config = selection_proof_config; + pub fn attestation_selection_proof_config( + mut self, + attestation_selection_proof_config: SelectionProofConfig, + ) -> Self { + self.attestation_selection_proof_config = attestation_selection_proof_config; + self + } + + pub fn sync_selection_proof_config( + mut self, + sync_selection_proof_config: SelectionProofConfig, + ) -> Self { + self.sync_selection_proof_config = sync_selection_proof_config; self } @@ -349,7 +362,7 @@ impl DutiesServiceBuilder { Ok(DutiesService { attesters: Default::default(), proposers: Default::default(), - sync_duties: SyncDutiesMap::new(self.selection_proof_config), + sync_duties: SyncDutiesMap::new(self.sync_selection_proof_config), validator_store: self .validator_store .ok_or("Cannot build DutiesService without validator_store")?, @@ -365,7 +378,7 @@ impl DutiesServiceBuilder { .ok_or("Cannot build DutiesService without executor")?, spec: self.spec.ok_or("Cannot build DutiesService without spec")?, enable_high_validator_count_metrics: self.enable_high_validator_count_metrics, - selection_proof_config: self.selection_proof_config, + selection_proof_config: self.attestation_selection_proof_config, disable_attesting: self.disable_attesting, }) } From 4e77e39b661bf35f9359d29c306b3831bc2a64ae Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Fri, 9 May 2025 16:07:40 +0800 Subject: [PATCH 74/75] Add debug --- validator_client/validator_services/src/duties_service.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index 48aaf351a1..cff3426cb7 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -114,7 +114,7 @@ pub struct SubscriptionSlots { duty_slot: Slot, } -#[derive(Copy, Clone)] +#[derive(Copy, Clone, Debug)] pub struct SelectionProofConfig { pub lookahead_slot: u64, pub computation_offset: Duration, // The seconds to compute the selection proof before a slot From 69e7e60f93e38e888e88f8e88e148c9f1b919500 Mon Sep 17 00:00:00 2001 From: Tan Chee Keong Date: Fri, 9 May 2025 19:59:48 +0800 Subject: [PATCH 75/75] Add comment for clarity --- validator_client/src/lib.rs | 14 ++++++-------- .../validator_services/src/duties_service.rs | 13 ++++++++----- validator_client/validator_services/src/sync.rs | 6 +++--- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index ec7a084f55..5dccda20f0 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -60,13 +60,13 @@ const WAITING_FOR_GENESIS_POLL_TIME: Duration = Duration::from_secs(12); const HTTP_ATTESTATION_TIMEOUT_QUOTIENT: u32 = 4; const HTTP_ATTESTER_DUTIES_TIMEOUT_QUOTIENT: u32 = 4; const HTTP_ATTESTATION_SUBSCRIPTIONS_TIMEOUT_QUOTIENT: u32 = 24; -const HTTP_ATTESTATION_AGGREGATOR_TIMEOUT_QUOTIENT: u32 = 24; // For distributed mode only +const HTTP_ATTESTATION_AGGREGATOR_TIMEOUT_QUOTIENT: u32 = 24; // For DVT involving middleware only const HTTP_LIVENESS_TIMEOUT_QUOTIENT: u32 = 4; const HTTP_PROPOSAL_TIMEOUT_QUOTIENT: u32 = 2; const HTTP_PROPOSER_DUTIES_TIMEOUT_QUOTIENT: u32 = 4; const HTTP_SYNC_COMMITTEE_CONTRIBUTION_TIMEOUT_QUOTIENT: u32 = 4; const HTTP_SYNC_DUTIES_TIMEOUT_QUOTIENT: u32 = 4; -const HTTP_SYNC_AGGREGATOR_TIMEOUT_QUOTIENT: u32 = 24; // For distributed mode only +const HTTP_SYNC_AGGREGATOR_TIMEOUT_QUOTIENT: u32 = 24; // For DVT involving middleware only const HTTP_GET_BEACON_BLOCK_SSZ_TIMEOUT_QUOTIENT: u32 = 4; const HTTP_GET_DEBUG_BEACON_STATE_QUOTIENT: u32 = 4; const HTTP_GET_DEPOSIT_SNAPSHOT_QUOTIENT: u32 = 4; @@ -83,12 +83,12 @@ const SELECTION_PROOF_SLOT_LOOKAHEAD: u64 = 8; /// The attestation selection proof lookahead for those running with the --distributed flag. const SELECTION_PROOF_SLOT_LOOKAHEAD_DVT: u64 = 1; -/// Fraction of a slot at which selection proof signing should happen (2 means half way). +/// Fraction of a slot at which attestation selection proof signing should happen (2 means half way). const SELECTION_PROOF_SCHEDULE_DENOM: u32 = 2; -/// Number of epochs in advance to compute selection proofs when not in `distributed` mode. +/// Number of epochs in advance to compute sync selection proofs when not in `distributed` mode. pub const AGGREGATION_PRE_COMPUTE_EPOCHS: u64 = 2; -/// Number of slots in advance to compute selection proofs when in `distributed` mode. +/// Number of slots in advance to compute sync selection proofs when in `distributed` mode. pub const AGGREGATION_PRE_COMPUTE_SLOTS_DISTRIBUTED: u64 = 1; type ValidatorStore = LighthouseValidatorStore; @@ -499,9 +499,7 @@ impl ProductionValidatorClient { } }; - let duties_service: Arc< - DutiesService, SystemTimeSlotClock>, - > = Arc::new( + let duties_service = Arc::new( DutiesServiceBuilder::new() .slot_clock(slot_clock.clone()) .beacon_nodes(beacon_nodes.clone()) diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index cff3426cb7..34607eaa13 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -118,11 +118,12 @@ pub struct SubscriptionSlots { pub struct SelectionProofConfig { pub lookahead_slot: u64, pub computation_offset: Duration, // The seconds to compute the selection proof before a slot - pub selections_endpoint: bool, - pub parallel_sign: bool, + pub selections_endpoint: bool, // whether to call the selections endpoint, true for DVT with middleware + pub parallel_sign: bool, // whether to sign the selection proof in parallel, true in distributed mode } impl SelectionProofConfig { + // Create a default associated function to be passed in DutiesServiceBuilder::new() fn default() -> Self { Self { lookahead_slot: 0, @@ -136,7 +137,7 @@ impl SelectionProofConfig { /// Create a selection proof for `duty`. /// /// Return `Ok(None)` if the attesting validator is not an aggregator. -async fn make_selection_proof( +async fn make_selection_proof( duty: &AttesterData, validator_store: &S, spec: &ChainSpec, @@ -147,7 +148,7 @@ async fn make_selection_proof { spec: Option>, //// Whether we permit large validator counts in the metrics. enable_high_validator_count_metrics: bool, - /// If this validator is running in distributed mode. + /// Create attestation selection proof config attestation_selection_proof_config: SelectionProofConfig, + /// Create sync selection proof config sync_selection_proof_config: SelectionProofConfig, disable_attesting: bool, } diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index d2a1aabc5b..cdda9a6690 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -487,7 +487,7 @@ pub async fn poll_sync_committee_duties_for_period( +pub async fn make_sync_selection_proof( duties_service: &Arc>, duty: &SyncDuty, proof_slot: Slot, @@ -519,7 +519,7 @@ pub async fn make_sync_selection_proof