Fix execution engine integration tests with latest geth version (#6996)

https://github.com/sigp/lighthouse/issues/6937


  - Use `ethers-rs` [`Signer`](https://www.gakonst.com/ethers-rs/middleware/signer.html) middleware for local signing and sending raw txs to geth
- ~~Set `totalDifficulty = 0` through `serde` default if the block does not contain a `totalDifficulty` field~~

testing/execution_engine_integration/Cargo.toml

@@ -7,7 +7,9 @@ edition = { workspace = true }
 async-channel = { workspace = true }
 deposit_contract = { workspace = true }
 ethers-core = { workspace = true }
+ethers-middleware = { workspace = true }
 ethers-providers = { workspace = true }
+ethers-signers = { workspace = true }
 execution_layer = { workspace = true }
 fork_choice = { workspace = true }
 futures = { workspace = true }

testing/execution_engine_integration/src/geth.rs

@@ -7,10 +7,7 @@ use std::{env, fs};
 use tempfile::TempDir;
 use unused_port::unused_tcp4_port;
 
-// This is not currently used due to the following breaking changes in geth that requires updating our tests:
-// 1. removal of `personal` namespace in v1.14.12: See #30704
-// 2. removal of `totalDifficulty` field from RPC in v1.14.11. See #30386.
-// const GETH_BRANCH: &str = "master";
+const GETH_BRANCH: &str = "master";
 const GETH_REPO_URL: &str = "https://github.com/ethereum/go-ethereum";
 
 pub fn build_result(repo_dir: &Path) -> Output {
@@ -30,14 +27,12 @@ pub fn build(execution_clients_dir: &Path) {
     }
 
     // Get the latest tag on the branch
-    // let last_release = build_utils::get_latest_release(&repo_dir, GETH_BRANCH).unwrap();
-    // Using an older release due to breaking changes in recent releases. See comment on `GETH_BRANCH` const.
-    let release_tag = "v1.14.10";
-    build_utils::checkout(&repo_dir, dbg!(release_tag)).unwrap();
+    let last_release = build_utils::get_latest_release(&repo_dir, GETH_BRANCH).unwrap();
+    build_utils::checkout(&repo_dir, dbg!(&last_release)).unwrap();
 
     // Build geth
     build_utils::check_command_output(build_result(&repo_dir), || {
-        format!("geth make failed using release {release_tag}")
+        format!("geth make failed using release {last_release}")
     });
 }
 
@@ -102,7 +97,7 @@ impl GenericExecutionEngine for GethEngine {
             .arg(datadir.path().to_str().unwrap())
             .arg("--http")
             .arg("--http.api")
-            .arg("engine,eth,personal")
+            .arg("engine,eth")
             .arg("--http.port")
             .arg(http_port.to_string())
             .arg("--authrpc.port")

testing/execution_engine_integration/src/main.rs

@@ -32,12 +32,12 @@ fn main() {
 fn test_geth() {
     let test_dir = build_utils::prepare_dir();
     geth::build(&test_dir);
-    TestRig::new(GethEngine).perform_tests_blocking();
+    TestRig::new(GethEngine, true).perform_tests_blocking();
     geth::clean(&test_dir);
 }
 
 fn test_nethermind() {
     let test_dir = build_utils::prepare_dir();
     nethermind::build(&test_dir);
-    TestRig::new(NethermindEngine).perform_tests_blocking();
+    TestRig::new(NethermindEngine, false).perform_tests_blocking();
 }

testing/execution_engine_integration/src/test_rig.rs

@@ -2,7 +2,9 @@ use crate::execution_engine::{
     ExecutionEngine, GenericExecutionEngine, ACCOUNT1, ACCOUNT2, KEYSTORE_PASSWORD, PRIVATE_KEYS,
 };
 use crate::transactions::transactions;
+use ethers_middleware::SignerMiddleware;
 use ethers_providers::Middleware;
+use ethers_signers::LocalWallet;
 use execution_layer::test_utils::DEFAULT_GAS_LIMIT;
 use execution_layer::{
     BlockProposalContentsType, BuilderParams, ChainHealth, ExecutionLayer, PayloadAttributes,
@@ -44,6 +46,7 @@ pub struct TestRig<Engine, E: EthSpec = MainnetEthSpec> {
     ee_b: ExecutionPair<Engine, E>,
     spec: ChainSpec,
     _runtime_shutdown: async_channel::Sender<()>,
+    use_local_signing: bool,
 }
 
 /// Import a private key into the execution engine and unlock it so that we can
@@ -104,7 +107,7 @@ async fn import_and_unlock(http_url: SensitiveUrl, priv_keys: &[&str], password:
 }
 
 impl<Engine: GenericExecutionEngine> TestRig<Engine> {
-    pub fn new(generic_engine: Engine) -> Self {
+    pub fn new(generic_engine: Engine, use_local_signing: bool) -> Self {
         let runtime = Arc::new(
             tokio::runtime::Builder::new_multi_thread()
                 .enable_all()
@@ -166,6 +169,7 @@ impl<Engine: GenericExecutionEngine> TestRig<Engine> {
             ee_b,
             spec,
             _runtime_shutdown: runtime_shutdown,
+            use_local_signing,
         }
     }
 
@@ -197,15 +201,9 @@ impl<Engine: GenericExecutionEngine> TestRig<Engine> {
     pub async fn perform_tests(&self) {
         self.wait_until_synced().await;
 
-        // Import and unlock all private keys to sign transactions
-        let _ = futures::future::join_all([&self.ee_a, &self.ee_b].iter().map(|ee| {
-            import_and_unlock(
-                ee.execution_engine.http_url(),
-                &PRIVATE_KEYS,
-                KEYSTORE_PASSWORD,
-            )
-        }))
-        .await;
+        // Create a local signer in case we need to sign transactions locally
+        let wallet1: LocalWallet = PRIVATE_KEYS[0].parse().expect("Invalid private key");
+        let signer = SignerMiddleware::new(&self.ee_a.execution_engine.provider, wallet1);
 
         // We hardcode the accounts here since some EEs start with a default unlocked account
         let account1 = ethers_core::types::Address::from_slice(&hex::decode(ACCOUNT1).unwrap());
@@ -236,15 +234,38 @@ impl<Engine: GenericExecutionEngine> TestRig<Engine> {
         // Submit transactions before getting payload
         let txs = transactions::<MainnetEthSpec>(account1, account2);
         let mut pending_txs = Vec::new();
-        for tx in txs.clone().into_iter() {
-            let pending_tx = self
-                .ee_a
-                .execution_engine
-                .provider
-                .send_transaction(tx, None)
-                .await
-                .unwrap();
-            pending_txs.push(pending_tx);
+
+        if self.use_local_signing {
+            // Sign locally with the Signer middleware
+            for (i, tx) in txs.clone().into_iter().enumerate() {
+                // The local signer uses eth_sendRawTransaction, so we need to manually set the nonce
+                let mut tx = tx.clone();
+                tx.set_nonce(i as u64);
+                let pending_tx = signer.send_transaction(tx, None).await.unwrap();
+                pending_txs.push(pending_tx);
+            }
+        } else {
+            // Sign on the EE
+            // Import and unlock all private keys to sign transactions on the EE
+            let _ = futures::future::join_all([&self.ee_a, &self.ee_b].iter().map(|ee| {
+                import_and_unlock(
+                    ee.execution_engine.http_url(),
+                    &PRIVATE_KEYS,
+                    KEYSTORE_PASSWORD,
+                )
+            }))
+            .await;
+
+            for tx in txs.clone().into_iter() {
+                let pending_tx = self
+                    .ee_a
+                    .execution_engine
+                    .provider
+                    .send_transaction(tx, None)
+                    .await
+                    .unwrap();
+                pending_txs.push(pending_tx);
+            }
         }
 
         /*