From e34a9a0c65d5f8287c4e2e8805ee4ea7b36ac3a9 Mon Sep 17 00:00:00 2001 From: Mac L Date: Mon, 23 Jun 2025 13:59:34 +1000 Subject: [PATCH] Allow the `--beacon-nodes` list to be updated at runtime (#6551) Adds a new `/lighthouse` API call to the VC which allows the list of beacon nodes to be updated dynamically at runtime. An entirely new beacon node list is provided to the VC so it effectively adds, removes or reorders nodes to match the new list. This can then be used in Siren, which will enable a "drag to reorder" system along with adding and removing beacon nodes while the VC is on. This will make it unnecessary to reboot the VC when users want to simply add or remove a BN from the list. --- Cargo.lock | 1 + book/src/api_vc_endpoints.md | 55 ++++++++++++++++ common/eth2/src/lib.rs | 36 +++++++++++ common/eth2/src/lighthouse_vc/types.rs | 10 +++ .../beacon_node_fallback/Cargo.toml | 1 + .../beacon_node_fallback/src/lib.rs | 36 ++++++++++- validator_client/http_api/src/lib.rs | 64 ++++++++++++++++++- validator_client/http_api/src/test_utils.rs | 1 + validator_client/http_api/src/tests.rs | 1 + validator_client/src/lib.rs | 35 +--------- 10 files changed, 203 insertions(+), 37 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d3394fce29..7d77ce4044 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -942,6 +942,7 @@ dependencies = [ "eth2", "futures", "itertools 0.10.5", + "sensitive_url", "serde", "slot_clock", "strum", diff --git a/book/src/api_vc_endpoints.md b/book/src/api_vc_endpoints.md index 87c9a517a5..14f4933e17 100644 --- a/book/src/api_vc_endpoints.md +++ b/book/src/api_vc_endpoints.md @@ -19,6 +19,7 @@ | [`POST /lighthouse/validators/web3signer`](#post-lighthousevalidatorsweb3signer) | Add web3signer validators. | | [`GET /lighthouse/logs`](#get-lighthouselogs) | Get logs | | [`GET /lighthouse/beacon/health`](#get-lighthousebeaconhealth) | Get health information for each connected beacon node. | +| [`POST /lighthouse/beacon/update`](#post-lighthousebeaconupdate) | Update the `--beacon-nodes` list. | The query to Lighthouse API endpoints requires authorization, see [Authorization Header](./api_vc_auth_header.md). @@ -926,3 +927,57 @@ curl -X GET http://localhost:5062/lighthouse/beacon/health \ } } ``` + +## `POST /lighthouse/beacon/update` + +Updates the list of beacon nodes originally specified by the `--beacon-nodes` CLI flag. +Use this endpoint when you don't want to restart the VC to add, remove or reorder beacon nodes. + +### HTTP Specification + +| Property | Specification | +|-------------------|--------------------------------------------| +| Path | `/lighthouse/beacon/update` | +| Method | POST | +| Required Headers | [`Authorization`](./api_vc_auth_header.md) | +| Typical Responses | 200, 400 | + +### Example Request Body + +```json +{ + "beacon_nodes": [ + "http://beacon-node1:5052", + "http://beacon-node2:5052", + "http://beacon-node3:5052" + ] +} +``` + +Command: + +```bash +DATADIR=/var/lib/lighthouse +curl -X POST http://localhost:5062/lighthouse/beacon/update \ + -H "Authorization: Bearer $(cat ${DATADIR}/validators/api-token.txt)" \ + -H "Content-Type: application/json" \ + -d "{\"beacon_nodes\":[\"http://beacon-node1:5052\",\"http://beacon-node2:5052\",\"http://beacon-node3:5052\"]}" +``` + +### Example Response Body + +```json +{ + "data": { + "new_beacon_nodes_list": [ + "http://beacon-node1:5052", + "http://beacon-node2:5052", + "http://beacon-node3:5052" + ] + } +} +``` + +If successful, the response will be a copy of the new list included in the request. +If unsuccessful, an error will be shown and the beacon nodes list will not be updated. +You can verify the results of the endpoint by using the `/lighthouse/beacon/health` endpoint. diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index b3f8f0becd..70fa52e60a 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -50,6 +50,22 @@ pub const CONTENT_TYPE_HEADER: &str = "Content-Type"; pub const SSZ_CONTENT_TYPE_HEADER: &str = "application/octet-stream"; pub const JSON_CONTENT_TYPE_HEADER: &str = "application/json"; +/// Specific optimized timeout constants for HTTP requests involved in different validator duties. +/// This can help ensure that proper endpoint fallback occurs. +const HTTP_ATTESTATION_TIMEOUT_QUOTIENT: u32 = 4; +const HTTP_ATTESTER_DUTIES_TIMEOUT_QUOTIENT: u32 = 4; +const HTTP_ATTESTATION_SUBSCRIPTIONS_TIMEOUT_QUOTIENT: u32 = 24; +const HTTP_LIVENESS_TIMEOUT_QUOTIENT: u32 = 4; +const HTTP_PROPOSAL_TIMEOUT_QUOTIENT: u32 = 2; +const HTTP_PROPOSER_DUTIES_TIMEOUT_QUOTIENT: u32 = 4; +const HTTP_SYNC_COMMITTEE_CONTRIBUTION_TIMEOUT_QUOTIENT: u32 = 4; +const HTTP_SYNC_DUTIES_TIMEOUT_QUOTIENT: u32 = 4; +const HTTP_GET_BEACON_BLOCK_SSZ_TIMEOUT_QUOTIENT: u32 = 4; +const HTTP_GET_DEBUG_BEACON_STATE_QUOTIENT: u32 = 4; +const HTTP_GET_DEPOSIT_SNAPSHOT_QUOTIENT: u32 = 4; +const HTTP_GET_VALIDATOR_BLOCK_TIMEOUT_QUOTIENT: u32 = 4; +const HTTP_DEFAULT_TIMEOUT_QUOTIENT: u32 = 4; + #[derive(Debug)] pub enum Error { /// The `reqwest` client raised an error. @@ -164,6 +180,26 @@ impl Timeouts { default: timeout, } } + + pub fn use_optimized_timeouts(base_timeout: Duration) -> Self { + Timeouts { + attestation: base_timeout / HTTP_ATTESTATION_TIMEOUT_QUOTIENT, + attester_duties: base_timeout / HTTP_ATTESTER_DUTIES_TIMEOUT_QUOTIENT, + attestation_subscriptions: base_timeout + / HTTP_ATTESTATION_SUBSCRIPTIONS_TIMEOUT_QUOTIENT, + liveness: base_timeout / HTTP_LIVENESS_TIMEOUT_QUOTIENT, + proposal: base_timeout / HTTP_PROPOSAL_TIMEOUT_QUOTIENT, + proposer_duties: base_timeout / HTTP_PROPOSER_DUTIES_TIMEOUT_QUOTIENT, + sync_committee_contribution: base_timeout + / HTTP_SYNC_COMMITTEE_CONTRIBUTION_TIMEOUT_QUOTIENT, + sync_duties: base_timeout / HTTP_SYNC_DUTIES_TIMEOUT_QUOTIENT, + get_beacon_blocks_ssz: base_timeout / HTTP_GET_BEACON_BLOCK_SSZ_TIMEOUT_QUOTIENT, + get_debug_beacon_states: base_timeout / HTTP_GET_DEBUG_BEACON_STATE_QUOTIENT, + get_deposit_snapshot: base_timeout / HTTP_GET_DEPOSIT_SNAPSHOT_QUOTIENT, + get_validator_block: base_timeout / HTTP_GET_VALIDATOR_BLOCK_TIMEOUT_QUOTIENT, + default: base_timeout / HTTP_DEFAULT_TIMEOUT_QUOTIENT, + } + } } /// A wrapper around `reqwest::Client` which provides convenience methods for interfacing with a diff --git a/common/eth2/src/lighthouse_vc/types.rs b/common/eth2/src/lighthouse_vc/types.rs index d7d5a00df5..4407e30e43 100644 --- a/common/eth2/src/lighthouse_vc/types.rs +++ b/common/eth2/src/lighthouse_vc/types.rs @@ -197,3 +197,13 @@ pub struct SingleExportKeystoresResponse { pub struct SetGraffitiRequest { pub graffiti: GraffitiString, } + +#[derive(Serialize, Deserialize, Debug)] +pub struct UpdateCandidatesRequest { + pub beacon_nodes: Vec, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct UpdateCandidatesResponse { + pub new_beacon_nodes_list: Vec, +} diff --git a/validator_client/beacon_node_fallback/Cargo.toml b/validator_client/beacon_node_fallback/Cargo.toml index 3bcb0d7034..5fe2af4cb0 100644 --- a/validator_client/beacon_node_fallback/Cargo.toml +++ b/validator_client/beacon_node_fallback/Cargo.toml @@ -13,6 +13,7 @@ clap = { workspace = true } eth2 = { workspace = true } futures = { workspace = true } itertools = { workspace = true } +sensitive_url = { workspace = true } serde = { workspace = true } slot_clock = { workspace = true } strum = { workspace = true } diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index e11cc97e79..b3158cd380 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -8,8 +8,9 @@ use beacon_node_health::{ IsOptimistic, SyncDistanceTier, }; use clap::ValueEnum; -use eth2::BeaconNodeHttpClient; +use eth2::{BeaconNodeHttpClient, Timeouts}; use futures::future; +use sensitive_url::SensitiveUrl; use serde::{ser::SerializeStruct, Deserialize, Serialize, Serializer}; use slot_clock::SlotClock; use std::cmp::Ordering; @@ -455,6 +456,39 @@ impl BeaconNodeFallback { (candidate_info, num_available, num_synced) } + /// Update the list of candidates with a new list. + /// Returns `Ok(new_list)` if the update was successful. + /// Returns `Err(some_err)` if the list is empty. + pub async fn update_candidates_list( + &self, + new_list: Vec, + use_long_timeouts: bool, + ) -> Result, String> { + if new_list.is_empty() { + return Err("list cannot be empty".to_string()); + } + + let timeouts: Timeouts = if new_list.len() == 1 || use_long_timeouts { + Timeouts::set_all(Duration::from_secs(self.spec.seconds_per_slot)) + } else { + Timeouts::use_optimized_timeouts(Duration::from_secs(self.spec.seconds_per_slot)) + }; + + let new_candidates: Vec = new_list + .clone() + .into_iter() + .enumerate() + .map(|(index, url)| { + CandidateBeaconNode::new(BeaconNodeHttpClient::new(url, timeouts.clone()), index) + }) + .collect(); + + let mut candidates = self.candidates.write().await; + *candidates = new_candidates; + + Ok(new_list) + } + /// Loop through ALL candidates in `self.candidates` and update their sync status. /// /// It is possible for a node to return an unsynced status while continuing to serve diff --git a/validator_client/http_api/src/lib.rs b/validator_client/http_api/src/lib.rs index aebe179567..d5de24229c 100644 --- a/validator_client/http_api/src/lib.rs +++ b/validator_client/http_api/src/lib.rs @@ -22,6 +22,7 @@ use account_utils::{ }; pub use api_secret::ApiSecret; use beacon_node_fallback::CandidateInfo; +use core::convert::Infallible; use create_validator::{ create_validators_mnemonic, create_validators_web3signer, get_voting_password_storage, }; @@ -30,7 +31,7 @@ use eth2::lighthouse_vc::{ std_types::{AuthResponse, GetFeeRecipientResponse, GetGasLimitResponse}, types::{ self as api_types, GenericResponse, GetGraffitiResponse, Graffiti, PublicKey, - PublicKeyBytes, SetGraffitiRequest, + PublicKeyBytes, SetGraffitiRequest, UpdateCandidatesRequest, UpdateCandidatesResponse, }, }; use health_metrics::observe::Observe; @@ -38,6 +39,7 @@ use lighthouse_version::version_with_platform; use logging::crit; use logging::SSELoggingComponents; use parking_lot::RwLock; +use sensitive_url::SensitiveUrl; use serde::{Deserialize, Serialize}; use slot_clock::SlotClock; use std::collections::HashMap; @@ -53,7 +55,8 @@ use tracing::{info, warn}; use types::{ChainSpec, ConfigAndPreset, EthSpec}; use validator_dir::Builder as ValidatorDirBuilder; use validator_services::block_service::BlockService; -use warp::{sse::Event, Filter}; +use warp::{reply::Response, sse::Event, Filter}; +use warp_utils::reject::convert_rejection; use warp_utils::task::blocking_json_task; #[derive(Debug)] @@ -102,6 +105,7 @@ pub struct Config { pub allow_keystore_export: bool, pub store_passwords_in_secrets_dir: bool, pub http_token_path: PathBuf, + pub bn_long_timeouts: bool, } impl Default for Config { @@ -121,6 +125,7 @@ impl Default for Config { allow_keystore_export: false, store_passwords_in_secrets_dir: false, http_token_path, + bn_long_timeouts: false, } } } @@ -147,6 +152,7 @@ pub fn serve( let config = &ctx.config; let allow_keystore_export = config.allow_keystore_export; let store_passwords_in_secrets_dir = config.store_passwords_in_secrets_dir; + let use_long_timeouts = config.bn_long_timeouts; // Configure CORS. let cors_builder = { @@ -839,6 +845,59 @@ pub fn serve( }) }); + // POST /lighthouse/beacon/update + let post_lighthouse_beacon_update = warp::path("lighthouse") + .and(warp::path("beacon")) + .and(warp::path("update")) + .and(warp::path::end()) + .and(warp::body::json()) + .and(block_service_filter.clone()) + .then( + move |request: UpdateCandidatesRequest, + block_service: BlockService, T>| async move { + async fn parse_urls(urls: &[String]) -> Result, Response> { + match urls + .iter() + .map(|url| SensitiveUrl::parse(url).map_err(|e| e.to_string())) + .collect() + { + Ok(sensitive_urls) => Ok(sensitive_urls), + Err(_) => Err(convert_rejection::(Err( + warp_utils::reject::custom_bad_request( + "one or more urls could not be parsed".to_string(), + ), + )) + .await), + } + } + + let beacons: Vec = match parse_urls(&request.beacon_nodes).await { + Ok(new_beacons) => { + match block_service + .beacon_nodes + .update_candidates_list(new_beacons, use_long_timeouts) + .await + { + Ok(beacons) => beacons, + Err(e) => { + return convert_rejection::(Err( + warp_utils::reject::custom_bad_request(e.to_string()), + )) + .await + } + } + } + Err(e) => return e, + }; + + let response: UpdateCandidatesResponse = UpdateCandidatesResponse { + new_beacon_nodes_list: beacons.iter().map(|surl| surl.to_string()).collect(), + }; + + blocking_json_task(move || Ok(api_types::GenericResponse::from(response))).await + }, + ); + // Standard key-manager endpoints. let eth_v1 = warp::path("eth").and(warp::path("v1")); let std_keystores = eth_v1.and(warp::path("keystores")).and(warp::path::end()); @@ -1316,6 +1375,7 @@ pub fn serve( .or(post_std_keystores) .or(post_std_remotekeys) .or(post_graffiti) + .or(post_lighthouse_beacon_update) .recover(warp_utils::reject::handle_rejection), )) .or(warp::patch() diff --git a/validator_client/http_api/src/test_utils.rs b/validator_client/http_api/src/test_utils.rs index 08447a82ce..8c23f79fd3 100644 --- a/validator_client/http_api/src/test_utils.rs +++ b/validator_client/http_api/src/test_utils.rs @@ -173,6 +173,7 @@ impl ApiTester { allow_keystore_export: true, store_passwords_in_secrets_dir: false, http_token_path: tempdir().unwrap().path().join(PK_FILENAME), + bn_long_timeouts: false, } } diff --git a/validator_client/http_api/src/tests.rs b/validator_client/http_api/src/tests.rs index 4b1a3c0059..7d421cd7d5 100644 --- a/validator_client/http_api/src/tests.rs +++ b/validator_client/http_api/src/tests.rs @@ -126,6 +126,7 @@ impl ApiTester { allow_keystore_export: true, store_passwords_in_secrets_dir: false, http_token_path: token_path, + bn_long_timeouts: false, }, sse_logging_components: None, slot_clock: slot_clock.clone(), diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index a96780b335..73dcb793dc 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -53,22 +53,6 @@ const RETRY_DELAY: Duration = Duration::from_secs(2); /// The time between polls when waiting for genesis. const WAITING_FOR_GENESIS_POLL_TIME: Duration = Duration::from_secs(12); -/// Specific timeout constants for HTTP requests involved in different validator duties. -/// This can help ensure that proper endpoint fallback occurs. -const HTTP_ATTESTATION_TIMEOUT_QUOTIENT: u32 = 4; -const HTTP_ATTESTER_DUTIES_TIMEOUT_QUOTIENT: u32 = 4; -const HTTP_ATTESTATION_SUBSCRIPTIONS_TIMEOUT_QUOTIENT: u32 = 24; -const HTTP_LIVENESS_TIMEOUT_QUOTIENT: u32 = 4; -const HTTP_PROPOSAL_TIMEOUT_QUOTIENT: u32 = 2; -const HTTP_PROPOSER_DUTIES_TIMEOUT_QUOTIENT: u32 = 4; -const HTTP_SYNC_COMMITTEE_CONTRIBUTION_TIMEOUT_QUOTIENT: u32 = 4; -const HTTP_SYNC_DUTIES_TIMEOUT_QUOTIENT: u32 = 4; -const HTTP_GET_BEACON_BLOCK_SSZ_TIMEOUT_QUOTIENT: u32 = 4; -const HTTP_GET_DEBUG_BEACON_STATE_QUOTIENT: u32 = 4; -const HTTP_GET_DEPOSIT_SNAPSHOT_QUOTIENT: u32 = 4; -const HTTP_GET_VALIDATOR_BLOCK_TIMEOUT_QUOTIENT: u32 = 4; -const HTTP_DEFAULT_TIMEOUT_QUOTIENT: u32 = 4; - const DOPPELGANGER_SERVICE_NAME: &str = "doppelganger"; type ValidatorStore = LighthouseValidatorStore; @@ -290,24 +274,7 @@ impl ProductionValidatorClient { // Use quicker timeouts if a fallback beacon node exists. let timeouts = if i < last_beacon_node_index && !config.use_long_timeouts { info!("Fallback endpoints are available, using optimized timeouts."); - Timeouts { - attestation: slot_duration / HTTP_ATTESTATION_TIMEOUT_QUOTIENT, - attester_duties: slot_duration / HTTP_ATTESTER_DUTIES_TIMEOUT_QUOTIENT, - attestation_subscriptions: slot_duration - / HTTP_ATTESTATION_SUBSCRIPTIONS_TIMEOUT_QUOTIENT, - liveness: slot_duration / HTTP_LIVENESS_TIMEOUT_QUOTIENT, - proposal: slot_duration / HTTP_PROPOSAL_TIMEOUT_QUOTIENT, - proposer_duties: slot_duration / HTTP_PROPOSER_DUTIES_TIMEOUT_QUOTIENT, - sync_committee_contribution: slot_duration - / HTTP_SYNC_COMMITTEE_CONTRIBUTION_TIMEOUT_QUOTIENT, - sync_duties: slot_duration / HTTP_SYNC_DUTIES_TIMEOUT_QUOTIENT, - get_beacon_blocks_ssz: slot_duration - / HTTP_GET_BEACON_BLOCK_SSZ_TIMEOUT_QUOTIENT, - get_debug_beacon_states: slot_duration / HTTP_GET_DEBUG_BEACON_STATE_QUOTIENT, - get_deposit_snapshot: slot_duration / HTTP_GET_DEPOSIT_SNAPSHOT_QUOTIENT, - get_validator_block: slot_duration / HTTP_GET_VALIDATOR_BLOCK_TIMEOUT_QUOTIENT, - default: slot_duration / HTTP_DEFAULT_TIMEOUT_QUOTIENT, - } + Timeouts::use_optimized_timeouts(slot_duration) } else { Timeouts::set_all(slot_duration.saturating_mul(config.long_timeouts_multiplier)) };