Allow the --beacon-nodes list to be updated at runtime (#6551)

Adds a new `/lighthouse` API call to the VC which allows the list of beacon nodes to be updated dynamically at runtime. An entirely new beacon node list is provided to the VC so it effectively adds, removes or reorders nodes to match the new list. This can then be used in Siren, which will enable a "drag to reorder" system along with adding and removing beacon nodes while the VC is on. This will make it unnecessary to reboot the VC when users want to simply add or remove a BN from the list.
2026-03-04 17:21:44 +00:00 · 2025-06-23 13:59:34 +10:00
parent 11bcccb353
commit e34a9a0c65
10 changed files with 203 additions and 37 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -942,6 +942,7 @@ dependencies = [
 "eth2",
 "futures",
 "itertools 0.10.5",
+ "sensitive_url",
 "serde",
 "slot_clock",
 "strum",
--- a/book/src/api_vc_endpoints.md
+++ b/book/src/api_vc_endpoints.md
@@ -19,6 +19,7 @@
 | [`POST /lighthouse/validators/web3signer`](#post-lighthousevalidatorsweb3signer) | Add web3signer validators. |
 | [`GET /lighthouse/logs`](#get-lighthouselogs) | Get logs |
 | [`GET /lighthouse/beacon/health`](#get-lighthousebeaconhealth) | Get health information for each connected beacon node. |
+| [`POST /lighthouse/beacon/update`](#post-lighthousebeaconupdate) | Update the `--beacon-nodes` list. |

 The query to Lighthouse API endpoints requires authorization, see [Authorization Header](./api_vc_auth_header.md).

@@ -926,3 +927,57 @@ curl -X GET http://localhost:5062/lighthouse/beacon/health \
    }
 }
 ```
+
+## `POST /lighthouse/beacon/update`
+
+Updates the list of beacon nodes originally specified by the `--beacon-nodes` CLI flag.
+Use this endpoint when you don't want to restart the VC to add, remove or reorder beacon nodes.
+
+### HTTP Specification
+
+| Property          | Specification                              |
+|-------------------|--------------------------------------------|
+| Path              | `/lighthouse/beacon/update`                |
+| Method            | POST                                       |
+| Required Headers  | [`Authorization`](./api_vc_auth_header.md) |
+| Typical Responses | 200, 400                                   |
+
+### Example Request Body
+
+```json
+{
+    "beacon_nodes": [
+        "http://beacon-node1:5052",
+        "http://beacon-node2:5052",
+        "http://beacon-node3:5052"
+    ]
+}
+```
+
+Command:
+
+```bash
+DATADIR=/var/lib/lighthouse
+curl -X POST http://localhost:5062/lighthouse/beacon/update \
+ -H "Authorization: Bearer $(cat ${DATADIR}/validators/api-token.txt)" \
+ -H "Content-Type: application/json" \
+ -d "{\"beacon_nodes\":[\"http://beacon-node1:5052\",\"http://beacon-node2:5052\",\"http://beacon-node3:5052\"]}"
+```
+
+### Example Response Body
+
+```json
+{
+    "data": {
+        "new_beacon_nodes_list": [
+            "http://beacon-node1:5052",
+            "http://beacon-node2:5052",
+            "http://beacon-node3:5052"
+        ]
+    }
+}
+```
+
+If successful, the response will be a copy of the new list included in the request.
+If unsuccessful, an error will be shown and the beacon nodes list will not be updated.
+You can verify the results of the endpoint by using the `/lighthouse/beacon/health` endpoint.
--- a/common/eth2/src/lib.rs
+++ b/common/eth2/src/lib.rs
@@ -50,6 +50,22 @@ pub const CONTENT_TYPE_HEADER: &str = "Content-Type";
 pub const SSZ_CONTENT_TYPE_HEADER: &str = "application/octet-stream";
 pub const JSON_CONTENT_TYPE_HEADER: &str = "application/json";

+/// Specific optimized timeout constants for HTTP requests involved in different validator duties.
+/// This can help ensure that proper endpoint fallback occurs.
+const HTTP_ATTESTATION_TIMEOUT_QUOTIENT: u32 = 4;
+const HTTP_ATTESTER_DUTIES_TIMEOUT_QUOTIENT: u32 = 4;
+const HTTP_ATTESTATION_SUBSCRIPTIONS_TIMEOUT_QUOTIENT: u32 = 24;
+const HTTP_LIVENESS_TIMEOUT_QUOTIENT: u32 = 4;
+const HTTP_PROPOSAL_TIMEOUT_QUOTIENT: u32 = 2;
+const HTTP_PROPOSER_DUTIES_TIMEOUT_QUOTIENT: u32 = 4;
+const HTTP_SYNC_COMMITTEE_CONTRIBUTION_TIMEOUT_QUOTIENT: u32 = 4;
+const HTTP_SYNC_DUTIES_TIMEOUT_QUOTIENT: u32 = 4;
+const HTTP_GET_BEACON_BLOCK_SSZ_TIMEOUT_QUOTIENT: u32 = 4;
+const HTTP_GET_DEBUG_BEACON_STATE_QUOTIENT: u32 = 4;
+const HTTP_GET_DEPOSIT_SNAPSHOT_QUOTIENT: u32 = 4;
+const HTTP_GET_VALIDATOR_BLOCK_TIMEOUT_QUOTIENT: u32 = 4;
+const HTTP_DEFAULT_TIMEOUT_QUOTIENT: u32 = 4;
+
 #[derive(Debug)]
 pub enum Error {
    /// The `reqwest` client raised an error.
@@ -164,6 +180,26 @@ impl Timeouts {
            default: timeout,
        }
    }
+
+    pub fn use_optimized_timeouts(base_timeout: Duration) -> Self {
+        Timeouts {
+            attestation: base_timeout / HTTP_ATTESTATION_TIMEOUT_QUOTIENT,
+            attester_duties: base_timeout / HTTP_ATTESTER_DUTIES_TIMEOUT_QUOTIENT,
+            attestation_subscriptions: base_timeout
+                / HTTP_ATTESTATION_SUBSCRIPTIONS_TIMEOUT_QUOTIENT,
+            liveness: base_timeout / HTTP_LIVENESS_TIMEOUT_QUOTIENT,
+            proposal: base_timeout / HTTP_PROPOSAL_TIMEOUT_QUOTIENT,
+            proposer_duties: base_timeout / HTTP_PROPOSER_DUTIES_TIMEOUT_QUOTIENT,
+            sync_committee_contribution: base_timeout
+                / HTTP_SYNC_COMMITTEE_CONTRIBUTION_TIMEOUT_QUOTIENT,
+            sync_duties: base_timeout / HTTP_SYNC_DUTIES_TIMEOUT_QUOTIENT,
+            get_beacon_blocks_ssz: base_timeout / HTTP_GET_BEACON_BLOCK_SSZ_TIMEOUT_QUOTIENT,
+            get_debug_beacon_states: base_timeout / HTTP_GET_DEBUG_BEACON_STATE_QUOTIENT,
+            get_deposit_snapshot: base_timeout / HTTP_GET_DEPOSIT_SNAPSHOT_QUOTIENT,
+            get_validator_block: base_timeout / HTTP_GET_VALIDATOR_BLOCK_TIMEOUT_QUOTIENT,
+            default: base_timeout / HTTP_DEFAULT_TIMEOUT_QUOTIENT,
+        }
+    }
 }

 /// A wrapper around `reqwest::Client` which provides convenience methods for interfacing with a
--- a/common/eth2/src/lighthouse_vc/types.rs
+++ b/common/eth2/src/lighthouse_vc/types.rs
@@ -197,3 +197,13 @@ pub struct SingleExportKeystoresResponse {
 pub struct SetGraffitiRequest {
    pub graffiti: GraffitiString,
 }
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct UpdateCandidatesRequest {
+    pub beacon_nodes: Vec<String>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct UpdateCandidatesResponse {
+    pub new_beacon_nodes_list: Vec<String>,
+}
--- a/validator_client/beacon_node_fallback/Cargo.toml
+++ b/validator_client/beacon_node_fallback/Cargo.toml
@@ -13,6 +13,7 @@ clap = { workspace = true }
 eth2 = { workspace = true }
 futures = { workspace = true }
 itertools = { workspace = true }
+sensitive_url = { workspace = true }
 serde = { workspace = true }
 slot_clock = { workspace = true }
 strum = { workspace = true }
--- a/validator_client/beacon_node_fallback/src/lib.rs
+++ b/validator_client/beacon_node_fallback/src/lib.rs
@@ -8,8 +8,9 @@ use beacon_node_health::{
    IsOptimistic, SyncDistanceTier,
 };
 use clap::ValueEnum;
-use eth2::BeaconNodeHttpClient;
+use eth2::{BeaconNodeHttpClient, Timeouts};
 use futures::future;
+use sensitive_url::SensitiveUrl;
 use serde::{ser::SerializeStruct, Deserialize, Serialize, Serializer};
 use slot_clock::SlotClock;
 use std::cmp::Ordering;
@@ -455,6 +456,39 @@ impl<T: SlotClock> BeaconNodeFallback<T> {
        (candidate_info, num_available, num_synced)
    }

+    /// Update the list of candidates with a new list.
+    /// Returns `Ok(new_list)` if the update was successful.
+    /// Returns `Err(some_err)` if the list is empty.
+    pub async fn update_candidates_list(
+        &self,
+        new_list: Vec<SensitiveUrl>,
+        use_long_timeouts: bool,
+    ) -> Result<Vec<SensitiveUrl>, String> {
+        if new_list.is_empty() {
+            return Err("list cannot be empty".to_string());
+        }
+
+        let timeouts: Timeouts = if new_list.len() == 1 || use_long_timeouts {
+            Timeouts::set_all(Duration::from_secs(self.spec.seconds_per_slot))
+        } else {
+            Timeouts::use_optimized_timeouts(Duration::from_secs(self.spec.seconds_per_slot))
+        };
+
+        let new_candidates: Vec<CandidateBeaconNode> = new_list
+            .clone()
+            .into_iter()
+            .enumerate()
+            .map(|(index, url)| {
+                CandidateBeaconNode::new(BeaconNodeHttpClient::new(url, timeouts.clone()), index)
+            })
+            .collect();
+
+        let mut candidates = self.candidates.write().await;
+        *candidates = new_candidates;
+
+        Ok(new_list)
+    }
+
    /// Loop through ALL candidates in `self.candidates` and update their sync status.
    ///
    /// It is possible for a node to return an unsynced status while continuing to serve
--- a/validator_client/http_api/src/lib.rs
+++ b/validator_client/http_api/src/lib.rs
@@ -22,6 +22,7 @@ use account_utils::{
 };
 pub use api_secret::ApiSecret;
 use beacon_node_fallback::CandidateInfo;
+use core::convert::Infallible;
 use create_validator::{
    create_validators_mnemonic, create_validators_web3signer, get_voting_password_storage,
 };
@@ -30,7 +31,7 @@ use eth2::lighthouse_vc::{
    std_types::{AuthResponse, GetFeeRecipientResponse, GetGasLimitResponse},
    types::{
        self as api_types, GenericResponse, GetGraffitiResponse, Graffiti, PublicKey,
-        PublicKeyBytes, SetGraffitiRequest,
+        PublicKeyBytes, SetGraffitiRequest, UpdateCandidatesRequest, UpdateCandidatesResponse,
    },
 };
 use health_metrics::observe::Observe;
@@ -38,6 +39,7 @@ use lighthouse_version::version_with_platform;
 use logging::crit;
 use logging::SSELoggingComponents;
 use parking_lot::RwLock;
+use sensitive_url::SensitiveUrl;
 use serde::{Deserialize, Serialize};
 use slot_clock::SlotClock;
 use std::collections::HashMap;
@@ -53,7 +55,8 @@ use tracing::{info, warn};
 use types::{ChainSpec, ConfigAndPreset, EthSpec};
 use validator_dir::Builder as ValidatorDirBuilder;
 use validator_services::block_service::BlockService;
-use warp::{sse::Event, Filter};
+use warp::{reply::Response, sse::Event, Filter};
+use warp_utils::reject::convert_rejection;
 use warp_utils::task::blocking_json_task;

 #[derive(Debug)]
@@ -102,6 +105,7 @@ pub struct Config {
    pub allow_keystore_export: bool,
    pub store_passwords_in_secrets_dir: bool,
    pub http_token_path: PathBuf,
+    pub bn_long_timeouts: bool,
 }

 impl Default for Config {
@@ -121,6 +125,7 @@ impl Default for Config {
            allow_keystore_export: false,
            store_passwords_in_secrets_dir: false,
            http_token_path,
+            bn_long_timeouts: false,
        }
    }
 }
@@ -147,6 +152,7 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
    let config = &ctx.config;
    let allow_keystore_export = config.allow_keystore_export;
    let store_passwords_in_secrets_dir = config.store_passwords_in_secrets_dir;
+    let use_long_timeouts = config.bn_long_timeouts;

    // Configure CORS.
    let cors_builder = {
@@ -839,6 +845,59 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
            })
        });

+    // POST /lighthouse/beacon/update
+    let post_lighthouse_beacon_update = warp::path("lighthouse")
+        .and(warp::path("beacon"))
+        .and(warp::path("update"))
+        .and(warp::path::end())
+        .and(warp::body::json())
+        .and(block_service_filter.clone())
+        .then(
+            move |request: UpdateCandidatesRequest,
+                  block_service: BlockService<LighthouseValidatorStore<T, E>, T>| async move {
+                async fn parse_urls(urls: &[String]) -> Result<Vec<SensitiveUrl>, Response> {
+                    match urls
+                        .iter()
+                        .map(|url| SensitiveUrl::parse(url).map_err(|e| e.to_string()))
+                        .collect()
+                    {
+                        Ok(sensitive_urls) => Ok(sensitive_urls),
+                        Err(_) => Err(convert_rejection::<Infallible>(Err(
+                            warp_utils::reject::custom_bad_request(
+                                "one or more urls could not be parsed".to_string(),
+                            ),
+                        ))
+                        .await),
+                    }
+                }
+
+                let beacons: Vec<SensitiveUrl> = match parse_urls(&request.beacon_nodes).await {
+                    Ok(new_beacons) => {
+                        match block_service
+                            .beacon_nodes
+                            .update_candidates_list(new_beacons, use_long_timeouts)
+                            .await
+                        {
+                            Ok(beacons) => beacons,
+                            Err(e) => {
+                                return convert_rejection::<Infallible>(Err(
+                                    warp_utils::reject::custom_bad_request(e.to_string()),
+                                ))
+                                .await
+                            }
+                        }
+                    }
+                    Err(e) => return e,
+                };
+
+                let response: UpdateCandidatesResponse = UpdateCandidatesResponse {
+                    new_beacon_nodes_list: beacons.iter().map(|surl| surl.to_string()).collect(),
+                };
+
+                blocking_json_task(move || Ok(api_types::GenericResponse::from(response))).await
+            },
+        );
+
    // Standard key-manager endpoints.
    let eth_v1 = warp::path("eth").and(warp::path("v1"));
    let std_keystores = eth_v1.and(warp::path("keystores")).and(warp::path::end());
@@ -1316,6 +1375,7 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
                        .or(post_std_keystores)
                        .or(post_std_remotekeys)
                        .or(post_graffiti)
+                        .or(post_lighthouse_beacon_update)
                        .recover(warp_utils::reject::handle_rejection),
                ))
                .or(warp::patch()
--- a/validator_client/http_api/src/test_utils.rs
+++ b/validator_client/http_api/src/test_utils.rs
@@ -173,6 +173,7 @@ impl ApiTester {
            allow_keystore_export: true,
            store_passwords_in_secrets_dir: false,
            http_token_path: tempdir().unwrap().path().join(PK_FILENAME),
+            bn_long_timeouts: false,
        }
    }

--- a/validator_client/http_api/src/tests.rs
+++ b/validator_client/http_api/src/tests.rs
@@ -126,6 +126,7 @@ impl ApiTester {
                allow_keystore_export: true,
                store_passwords_in_secrets_dir: false,
                http_token_path: token_path,
+                bn_long_timeouts: false,
            },
            sse_logging_components: None,
            slot_clock: slot_clock.clone(),
--- a/validator_client/src/lib.rs
+++ b/validator_client/src/lib.rs
@@ -53,22 +53,6 @@ const RETRY_DELAY: Duration = Duration::from_secs(2);
 /// The time between polls when waiting for genesis.
 const WAITING_FOR_GENESIS_POLL_TIME: Duration = Duration::from_secs(12);

-/// Specific timeout constants for HTTP requests involved in different validator duties.
-/// This can help ensure that proper endpoint fallback occurs.
-const HTTP_ATTESTATION_TIMEOUT_QUOTIENT: u32 = 4;
-const HTTP_ATTESTER_DUTIES_TIMEOUT_QUOTIENT: u32 = 4;
-const HTTP_ATTESTATION_SUBSCRIPTIONS_TIMEOUT_QUOTIENT: u32 = 24;
-const HTTP_LIVENESS_TIMEOUT_QUOTIENT: u32 = 4;
-const HTTP_PROPOSAL_TIMEOUT_QUOTIENT: u32 = 2;
-const HTTP_PROPOSER_DUTIES_TIMEOUT_QUOTIENT: u32 = 4;
-const HTTP_SYNC_COMMITTEE_CONTRIBUTION_TIMEOUT_QUOTIENT: u32 = 4;
-const HTTP_SYNC_DUTIES_TIMEOUT_QUOTIENT: u32 = 4;
-const HTTP_GET_BEACON_BLOCK_SSZ_TIMEOUT_QUOTIENT: u32 = 4;
-const HTTP_GET_DEBUG_BEACON_STATE_QUOTIENT: u32 = 4;
-const HTTP_GET_DEPOSIT_SNAPSHOT_QUOTIENT: u32 = 4;
-const HTTP_GET_VALIDATOR_BLOCK_TIMEOUT_QUOTIENT: u32 = 4;
-const HTTP_DEFAULT_TIMEOUT_QUOTIENT: u32 = 4;
-
 const DOPPELGANGER_SERVICE_NAME: &str = "doppelganger";

 type ValidatorStore<E> = LighthouseValidatorStore<SystemTimeSlotClock, E>;
@@ -290,24 +274,7 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
            // Use quicker timeouts if a fallback beacon node exists.
            let timeouts = if i < last_beacon_node_index && !config.use_long_timeouts {
                info!("Fallback endpoints are available, using optimized timeouts.");
-                Timeouts {
-                    attestation: slot_duration / HTTP_ATTESTATION_TIMEOUT_QUOTIENT,
-                    attester_duties: slot_duration / HTTP_ATTESTER_DUTIES_TIMEOUT_QUOTIENT,
-                    attestation_subscriptions: slot_duration
-                        / HTTP_ATTESTATION_SUBSCRIPTIONS_TIMEOUT_QUOTIENT,
-                    liveness: slot_duration / HTTP_LIVENESS_TIMEOUT_QUOTIENT,
-                    proposal: slot_duration / HTTP_PROPOSAL_TIMEOUT_QUOTIENT,
-                    proposer_duties: slot_duration / HTTP_PROPOSER_DUTIES_TIMEOUT_QUOTIENT,
-                    sync_committee_contribution: slot_duration
-                        / HTTP_SYNC_COMMITTEE_CONTRIBUTION_TIMEOUT_QUOTIENT,
-                    sync_duties: slot_duration / HTTP_SYNC_DUTIES_TIMEOUT_QUOTIENT,
-                    get_beacon_blocks_ssz: slot_duration
-                        / HTTP_GET_BEACON_BLOCK_SSZ_TIMEOUT_QUOTIENT,
-                    get_debug_beacon_states: slot_duration / HTTP_GET_DEBUG_BEACON_STATE_QUOTIENT,
-                    get_deposit_snapshot: slot_duration / HTTP_GET_DEPOSIT_SNAPSHOT_QUOTIENT,
-                    get_validator_block: slot_duration / HTTP_GET_VALIDATOR_BLOCK_TIMEOUT_QUOTIENT,
-                    default: slot_duration / HTTP_DEFAULT_TIMEOUT_QUOTIENT,
-                }
+                Timeouts::use_optimized_timeouts(slot_duration)
            } else {
                Timeouts::set_all(slot_duration.saturating_mul(config.long_timeouts_multiplier))
            };