CI fix: move download web3signer binary out of build script (#4163)

## Issue Addressed

Attempt to fix #3812 

## Proposed Changes

Move web3signer binary download script out of build script to avoid downloading unless necessary. If this works, it should also reduce the build time for all jobs that runs compilation.

testing/web3signer_tests/src/get_web3signer.rs

@@ -0,0 +1,102 @@
+//! This build script downloads the latest Web3Signer release and places it in the `OUT_DIR` so it
+//! can be used for integration testing.
+
+use reqwest::{
+    header::{self, HeaderValue},
+    Client,
+};
+use serde_json::Value;
+use std::env;
+use std::fs;
+use std::path::PathBuf;
+use zip::ZipArchive;
+
+/// Use `None` to download the latest Github release.
+/// Use `Some("21.8.1")` to download a specific version.
+const FIXED_VERSION_STRING: Option<&str> = None;
+
+pub async fn download_binary(dest_dir: PathBuf, github_token: &str) {
+    let version_file = dest_dir.join("version");
+
+    let client = Client::builder()
+        // Github gives a 403 without a user agent.
+        .user_agent("web3signer_tests")
+        .build()
+        .unwrap();
+
+    let version = if let Some(version) = FIXED_VERSION_STRING {
+        version.to_string()
+    } else if let Ok(env_version) = env::var("LIGHTHOUSE_WEB3SIGNER_VERSION") {
+        env_version
+    } else {
+        // Get the latest release of the web3 signer repo.
+        let mut token_header_value = HeaderValue::from_str(github_token).unwrap();
+        token_header_value.set_sensitive(true);
+        let latest_response: Value = client
+            .get("https://api.github.com/repos/ConsenSys/web3signer/releases/latest")
+            .header(header::AUTHORIZATION, token_header_value)
+            .send()
+            .await
+            .unwrap()
+            .error_for_status()
+            .unwrap()
+            .json()
+            .await
+            .unwrap();
+        latest_response
+            .get("tag_name")
+            .unwrap()
+            .as_str()
+            .unwrap()
+            .to_string()
+    };
+
+    if version_file.exists() && fs::read(&version_file).unwrap() == version.as_bytes() {
+        // The latest version is already downloaded, do nothing.
+        return;
+    } else {
+        // Ignore the result since we don't care if the version file already exists.
+        let _ = fs::remove_file(&version_file);
+    }
+
+    // Download the latest release zip.
+    let zip_url = format!("https://artifacts.consensys.net/public/web3signer/raw/names/web3signer.zip/versions/{}/web3signer-{}.zip", version, version);
+    let zip_response = client
+        .get(zip_url)
+        .send()
+        .await
+        .unwrap()
+        .error_for_status()
+        .unwrap()
+        .bytes()
+        .await
+        .unwrap();
+
+    // Write the zip to a file.
+    let zip_path = dest_dir.join(format!("{}.zip", version));
+    fs::write(&zip_path, zip_response).unwrap();
+    // Unzip the zip.
+    let mut zip_file = fs::File::open(&zip_path).unwrap();
+    ZipArchive::new(&mut zip_file)
+        .unwrap()
+        .extract(&dest_dir)
+        .unwrap();
+
+    // Rename the web3signer directory so it doesn't include the version string. This ensures the
+    // path to the binary is predictable.
+    let web3signer_dir = dest_dir.join("web3signer");
+    if web3signer_dir.exists() {
+        fs::remove_dir_all(&web3signer_dir).unwrap();
+    }
+    fs::rename(
+        dest_dir.join(format!("web3signer-{}", version)),
+        web3signer_dir,
+    )
+    .unwrap();
+
+    // Delete zip and unzipped dir.
+    fs::remove_file(&zip_path).unwrap();
+
+    // Update the version file to avoid duplicate downloads.
+    fs::write(&version_file, version).unwrap();
+}

testing/web3signer_tests/src/lib.rs

@@ -9,16 +9,21 @@
 //! - Lighthouse can issue valid requests to Web3Signer.
 //! - The signatures generated by Web3Signer are identical to those which Lighthouse generates.
 //!
-//! There is a build script in this crate which obtains the latest version of Web3Signer and makes
-//! it available via the `OUT_DIR`.
+//! There is a `download_binary` function in the `get_web3signer` module which obtains the latest version of Web3Signer and makes
+//! it available via the `TEMP_DIR`.
+#![cfg(all(test, unix, not(debug_assertions)))]
+
+mod get_web3signer;
 
-#[cfg(all(test, unix, not(debug_assertions)))]
 mod tests {
+    use crate::get_web3signer::download_binary;
     use account_utils::validator_definitions::{
         SigningDefinition, ValidatorDefinition, ValidatorDefinitions, Web3SignerDefinition,
     };
     use eth2_keystore::KeystoreBuilder;
     use eth2_network_config::Eth2NetworkConfig;
+    use lazy_static::lazy_static;
+    use parking_lot::Mutex;
     use reqwest::Client;
     use serde::Serialize;
     use slot_clock::{SlotClock, TestingSlotClock};
@@ -31,7 +36,8 @@ mod tests {
     use std::sync::Arc;
     use std::time::{Duration, Instant};
     use task_executor::TaskExecutor;
-    use tempfile::TempDir;
+    use tempfile::{tempdir, TempDir};
+    use tokio::sync::OnceCell;
     use tokio::time::sleep;
     use types::*;
     use url::Url;
@@ -51,6 +57,13 @@ mod tests {
     /// debugging.
     const SUPPRESS_WEB3SIGNER_LOGS: bool = true;
 
+    lazy_static! {
+        static ref TEMP_DIR: Arc<Mutex<TempDir>> = Arc::new(Mutex::new(
+            tempdir().expect("Failed to create temporary directory")
+        ));
+        static ref GET_WEB3SIGNER_BIN: OnceCell<()> = OnceCell::new();
+    }
+
     type E = MainnetEthSpec;
 
     /// This marker trait is implemented for objects that we wish to compare to ensure Web3Signer
@@ -99,7 +112,10 @@ mod tests {
 
     /// The location of the Web3Signer binary generated by the build script.
     fn web3signer_binary() -> PathBuf {
-        PathBuf::from(env::var("OUT_DIR").unwrap())
+        TEMP_DIR
+            .lock()
+            .path()
+            .to_path_buf()
             .join("web3signer")
             .join("bin")
             .join("web3signer")
@@ -143,6 +159,19 @@ mod tests {
 
     impl Web3SignerRig {
         pub async fn new(network: &str, listen_address: &str, listen_port: u16) -> Self {
+            GET_WEB3SIGNER_BIN
+                .get_or_init(|| async {
+                    // Read a Github API token from the environment. This is intended to prevent rate-limits on CI.
+                    // We use a name that is unlikely to accidentally collide with anything the user has configured.
+                    let github_token = env::var("LIGHTHOUSE_GITHUB_TOKEN");
+                    download_binary(
+                        TEMP_DIR.lock().path().to_path_buf(),
+                        github_token.as_deref().unwrap_or(""),
+                    )
+                    .await;
+                })
+                .await;
+
             let keystore_dir = TempDir::new().unwrap();
             let keypair = testing_keypair();
             let keystore =