Add CLI flag for HTTP API token path (VC) (#6577)

* Add cli flag for HTTP API token path (VC)

* Add http_token_path_flag test

* Add pre-check for directory case & Fix test utils

* Update docs

* Apply review: move http_token_path into validator_http_api config

* Lint

* Make diff lesser to replace PK_FILENAME

* Merge branch 'unstable' into feature/cli-token-path

* Applt review: help_vc.md

Co-authored-by: chonghe <44791194+chong-he@users.noreply.github.com>

* Fix help for cli

* Fix issues on ci

* Merge branch 'unstable' into feature/cli-token-path

* Merge branch 'unstable' into feature/cli-token-path

* Merge branch 'unstable' into feature/cli-token-path

* Merge branch 'unstable' into feature/cli-token-path

Cargo.lock

@@ -9552,6 +9552,8 @@ dependencies = [
  "beacon_node_fallback",
  "bls",
  "deposit_contract",
+ "directory",
+ "dirs",
  "doppelganger_service",
  "eth2",
  "eth2_keystore",

book/src/api-vc-auth-header.md

@@ -18,7 +18,8 @@ Authorization: Bearer hGut6B8uEujufDXSmZsT0thnxvdvKFBvh
 ## Obtaining the API token
 
 The API token is stored as a file in the `validators` directory. For most users
-this is `~/.lighthouse/{network}/validators/api-token.txt`. Here's an
+this is `~/.lighthouse/{network}/validators/api-token.txt`, unless overridden using the
+`--http-token-path` CLI parameter. Here's an
 example using the `cat` command to print the token to the terminal, but any
 text editor will suffice:
 

book/src/api-vc-endpoints.md

@@ -53,7 +53,7 @@ Example Response Body:
 }
 ```
 
-> Note: The command provided in this documentation links to the API token file. In this documentation, it is assumed that the API token file is located in `/var/lib/lighthouse/validators/api-token.txt`. If your database is saved in another directory, modify the `DATADIR` accordingly. If you are having permission issue with accessing the API token file, you can modify the header to become `-H "Authorization: Bearer $(sudo cat ${DATADIR}/validators/api-token.txt)"`.
+> Note: The command provided in this documentation links to the API token file. In this documentation, it is assumed that the API token file is located in `/var/lib/lighthouse/validators/api-token.txt`. If your database is saved in another directory, modify the `DATADIR` accordingly. If you've specified a custom token path using `--http-token-path`, use that path instead. If you are having permission issue with accessing the API token file, you can modify the header to become `-H "Authorization: Bearer $(sudo cat ${DATADIR}/validators/api-token.txt)"`.
 
 > As an alternative, you can also provide the API token directly, for example, `-H "Authorization: Bearer hGut6B8uEujufDXSmZsT0thnxvdvKFBvh`. In this case, you obtain the token from the file `api-token.txt` and the command becomes:
 

book/src/help_vc.md

@@ -69,6 +69,10 @@ Options:
           this server (e.g., http://localhost:5062).
       --http-port <PORT>
           Set the listen TCP port for the RESTful HTTP API server.
+      --http-token-path <HTTP_TOKEN_PATH>
+          Path to file containing the HTTP API token for validator client
+          authentication. If not specified, defaults to
+          {validators-dir}/api-token.txt.
       --log-format <FORMAT>
           Specifies the log format used when emitting logs to the terminal.
           [possible values: JSON]

lighthouse/tests/validator_client.rs

@@ -344,6 +344,21 @@ fn http_store_keystore_passwords_in_secrets_dir_present() {
         .with_config(|config| assert!(config.http_api.store_passwords_in_secrets_dir));
 }
 
+#[test]
+fn http_token_path_flag() {
+    let dir = TempDir::new().expect("Unable to create temporary directory");
+    CommandLineTest::new()
+        .flag("http", None)
+        .flag("http-token-path", dir.path().join("api-token.txt").to_str())
+        .run()
+        .with_config(|config| {
+            assert_eq!(
+                config.http_api.http_token_path,
+                dir.path().join("api-token.txt")
+            );
+        });
+}
+
 // Tests for Metrics flags.
 #[test]
 fn metrics_flag() {

validator_client/http_api/Cargo.toml

@@ -13,7 +13,9 @@ account_utils = { workspace = true }
 bls = { workspace = true }
 beacon_node_fallback = { workspace = true }
 deposit_contract = { workspace = true }
+directory = { workspace = true }
 doppelganger_service = { workspace = true }
+dirs = { workspace = true }
 graffiti_file = { workspace = true }
 eth2  = { workspace = true }
 eth2_keystore  = { workspace = true }

validator_client/http_api/src/api_secret.rs

@@ -5,7 +5,7 @@ use std::fs;
 use std::path::{Path, PathBuf};
 use warp::Filter;
 
-/// The name of the file which stores the API token.
+/// The default name of the file which stores the API token.
 pub const PK_FILENAME: &str = "api-token.txt";
 
 pub const PK_LEN: usize = 33;
@@ -31,14 +31,32 @@ pub struct ApiSecret {
 impl ApiSecret {
     /// If the public key is already on-disk, use it.
     ///
-    /// The provided `dir` is a directory containing `PK_FILENAME`.
+    /// The provided `pk_path` is a path containing API token.
     ///
     /// If the public key file is missing on disk, create a new key and
     /// write it to disk (over-writing any existing files).
-    pub fn create_or_open<P: AsRef<Path>>(dir: P) -> Result<Self, String> {
-        let pk_path = dir.as_ref().join(PK_FILENAME);
+    pub fn create_or_open<P: AsRef<Path>>(pk_path: P) -> Result<Self, String> {
+        let pk_path = pk_path.as_ref();
+
+        // Check if the path is a directory
+        if pk_path.is_dir() {
+            return Err(format!(
+                "API token path {:?} is a directory, not a file",
+                pk_path
+            ));
+        }
 
         if !pk_path.exists() {
+            // Create parent directories if they don't exist
+            if let Some(parent) = pk_path.parent() {
+                std::fs::create_dir_all(parent).map_err(|e| {
+                    format!(
+                        "Unable to create parent directories for {:?}: {:?}",
+                        pk_path, e
+                    )
+                })?;
+            }
+
             let length = PK_LEN;
             let pk: String = thread_rng()
                 .sample_iter(&Alphanumeric)
@@ -47,7 +65,7 @@ impl ApiSecret {
                 .collect();
 
             // Create and write the public key to file with appropriate permissions
-            create_with_600_perms(&pk_path, pk.to_string().as_bytes()).map_err(|e| {
+            create_with_600_perms(pk_path, pk.to_string().as_bytes()).map_err(|e| {
                 format!(
                     "Unable to create file with permissions for {:?}: {:?}",
                     pk_path, e
@@ -55,13 +73,16 @@ impl ApiSecret {
             })?;
         }
 
-        let pk = fs::read(&pk_path)
-            .map_err(|e| format!("cannot read {}: {}", PK_FILENAME, e))?
+        let pk = fs::read(pk_path)
+            .map_err(|e| format!("cannot read {}: {}", pk_path.display(), e))?
             .iter()
             .map(|&c| char::from(c))
             .collect();
 
-        Ok(Self { pk, pk_path })
+        Ok(Self {
+            pk,
+            pk_path: pk_path.to_path_buf(),
+        })
     }
 
     /// Returns the API token.

validator_client/http_api/src/lib.rs

@@ -7,6 +7,7 @@ mod remotekeys;
 mod tests;
 
 pub mod test_utils;
+pub use api_secret::PK_FILENAME;
 
 use graffiti::{delete_graffiti, get_graffiti, set_graffiti};
 
@@ -23,6 +24,7 @@ use beacon_node_fallback::CandidateInfo;
 use create_validator::{
     create_validators_mnemonic, create_validators_web3signer, get_voting_password_storage,
 };
+use directory::{DEFAULT_HARDCODED_NETWORK, DEFAULT_ROOT_DIR, DEFAULT_VALIDATOR_DIR};
 use eth2::lighthouse_vc::{
     std_types::{AuthResponse, GetFeeRecipientResponse, GetGasLimitResponse},
     types::{
@@ -99,10 +101,17 @@ pub struct Config {
     pub allow_origin: Option<String>,
     pub allow_keystore_export: bool,
     pub store_passwords_in_secrets_dir: bool,
+    pub http_token_path: PathBuf,
 }
 
 impl Default for Config {
     fn default() -> Self {
+        let http_token_path = dirs::home_dir()
+            .unwrap_or_else(|| PathBuf::from("."))
+            .join(DEFAULT_ROOT_DIR)
+            .join(DEFAULT_HARDCODED_NETWORK)
+            .join(DEFAULT_VALIDATOR_DIR)
+            .join(PK_FILENAME);
         Self {
             enabled: false,
             listen_addr: IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)),
@@ -110,6 +119,7 @@ impl Default for Config {
             allow_origin: None,
             allow_keystore_export: false,
             store_passwords_in_secrets_dir: false,
+            http_token_path,
         }
     }
 }

validator_client/http_api/src/test_utils.rs

@@ -1,3 +1,4 @@
+use crate::api_secret::PK_FILENAME;
 use crate::{ApiSecret, Config as HttpConfig, Context};
 use account_utils::validator_definitions::ValidatorDefinitions;
 use account_utils::{
@@ -73,6 +74,7 @@ impl ApiTester {
 
         let validator_dir = tempdir().unwrap();
         let secrets_dir = tempdir().unwrap();
+        let token_path = tempdir().unwrap().path().join(PK_FILENAME);
 
         let validator_defs = ValidatorDefinitions::open_or_create(validator_dir.path()).unwrap();
 
@@ -85,7 +87,7 @@ impl ApiTester {
         .await
         .unwrap();
 
-        let api_secret = ApiSecret::create_or_open(validator_dir.path()).unwrap();
+        let api_secret = ApiSecret::create_or_open(token_path).unwrap();
         let api_pubkey = api_secret.api_token();
 
         let config = ValidatorStoreConfig {
@@ -177,6 +179,7 @@ impl ApiTester {
             allow_origin: None,
             allow_keystore_export: true,
             store_passwords_in_secrets_dir: false,
+            http_token_path: tempdir().unwrap().path().join(PK_FILENAME),
         }
     }
 
@@ -199,8 +202,8 @@ impl ApiTester {
     }
 
     pub fn invalid_token_client(&self) -> ValidatorClientHttpClient {
-        let tmp = tempdir().unwrap();
-        let api_secret = ApiSecret::create_or_open(tmp.path()).unwrap();
+        let tmp = tempdir().unwrap().path().join("invalid-token.txt");
+        let api_secret = ApiSecret::create_or_open(tmp).unwrap();
         let invalid_pubkey = api_secret.api_token();
         ValidatorClientHttpClient::new(self.url.clone(), invalid_pubkey).unwrap()
     }

validator_client/http_api/src/tests.rs

@@ -63,6 +63,7 @@ impl ApiTester {
 
         let validator_dir = tempdir().unwrap();
         let secrets_dir = tempdir().unwrap();
+        let token_path = tempdir().unwrap().path().join("api-token.txt");
 
         let validator_defs = ValidatorDefinitions::open_or_create(validator_dir.path()).unwrap();
 
@@ -75,7 +76,7 @@ impl ApiTester {
         .await
         .unwrap();
 
-        let api_secret = ApiSecret::create_or_open(validator_dir.path()).unwrap();
+        let api_secret = ApiSecret::create_or_open(&token_path).unwrap();
         let api_pubkey = api_secret.api_token();
 
         let spec = Arc::new(E::default_spec());
@@ -127,6 +128,7 @@ impl ApiTester {
                 allow_origin: None,
                 allow_keystore_export: true,
                 store_passwords_in_secrets_dir: false,
+                http_token_path: token_path,
             },
             sse_logging_components: None,
             log,
@@ -161,8 +163,8 @@ impl ApiTester {
     }
 
     pub fn invalid_token_client(&self) -> ValidatorClientHttpClient {
-        let tmp = tempdir().unwrap();
-        let api_secret = ApiSecret::create_or_open(tmp.path()).unwrap();
+        let tmp = tempdir().unwrap().path().join("invalid-token.txt");
+        let api_secret = ApiSecret::create_or_open(tmp).unwrap();
         let invalid_pubkey = api_secret.api_token();
         ValidatorClientHttpClient::new(self.url.clone(), invalid_pubkey.clone()).unwrap()
     }

validator_client/src/cli.rs

@@ -247,6 +247,18 @@ pub fn cli_app() -> Command {
                 .help_heading(FLAG_HEADER)
                 .display_order(0)
         )
+        .arg(
+            Arg::new("http-token-path")
+                .long("http-token-path")
+                .requires("http")
+                .value_name("HTTP_TOKEN_PATH")
+                .help(
+                    "Path to file containing the HTTP API token for validator client authentication. \
+                    If not specified, defaults to {validators-dir}/api-token.txt."
+                )
+                .action(ArgAction::Set)
+                .display_order(0)
+        )
         /* Prometheus metrics HTTP server related arguments */
         .arg(
             Arg::new("metrics")

validator_client/src/config.rs

@@ -17,7 +17,7 @@ use std::path::PathBuf;
 use std::str::FromStr;
 use std::time::Duration;
 use types::{Address, GRAFFITI_BYTES_LEN};
-use validator_http_api;
+use validator_http_api::{self, PK_FILENAME};
 use validator_http_metrics;
 use validator_store::Config as ValidatorStoreConfig;
 
@@ -314,6 +314,12 @@ impl Config {
             config.http_api.store_passwords_in_secrets_dir = true;
         }
 
+        if cli_args.get_one::<String>("http-token-path").is_some() {
+            config.http_api.http_token_path = parse_required(cli_args, "http-token-path")
+                // For backward compatibility, default to the path under the validator dir if not provided.
+                .unwrap_or_else(|_| config.validator_dir.join(PK_FILENAME));
+        }
+
         /*
          * Prometheus metrics HTTP server
          */

validator_client/src/lib.rs

@@ -551,7 +551,7 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
         let (block_service_tx, block_service_rx) = mpsc::channel(channel_capacity);
         let log = self.context.log();
 
-        let api_secret = ApiSecret::create_or_open(&self.config.validator_dir)?;
+        let api_secret = ApiSecret::create_or_open(&self.config.http_api.http_token_path)?;
 
         self.http_api_listen_addr = if self.config.http_api.enabled {
             let ctx = Arc::new(validator_http_api::Context {