[Merge] Implement execution_layer (#2635)

* Checkout serde_utils from rayonism

* Make eth1::http functions pub

* Add bones of execution_layer

* Modify decoding

* Expose Transaction, cargo fmt

* Add executePayload

* Add all minimal spec endpoints

* Start adding json rpc wrapper

* Finish custom JSON response handler

* Switch to new rpc sending method

* Add first test

* Fix camelCase

* Finish adding tests

* Begin threading execution layer into BeaconChain

* Fix clippy lints

* Fix clippy lints

* Thread execution layer into ClientBuilder

* Add CLI flags

* Add block processing methods to ExecutionLayer

* Add block_on to execution_layer

* Integrate execute_payload

* Add extra_data field

* Begin implementing payload handle

* Send consensus valid/invalid messages

* Fix minor type in task_executor

* Call forkchoiceUpdated

* Add search for TTD block

* Thread TTD into execution layer

* Allow producing block with execution payload

* Add LRU cache for execution blocks

* Remove duplicate 0x on ssz_types serialization

* Add tests for block getter methods

* Add basic block generator impl

* Add is_valid_terminal_block to EL

* Verify merge block in block_verification

* Partially implement --terminal-block-hash-override

* Add terminal_block_hash to ChainSpec

* Remove Option from terminal_block_hash in EL

* Revert merge changes to consensus/fork_choice

* Remove commented-out code

* Add bones for handling RPC methods on test server

* Add first ExecutionLayer tests

* Add testing for finding terminal block

* Prevent infinite loops

* Add insert_merge_block to block gen

* Add block gen test for pos blocks

* Start adding payloads to block gen

* Fix clippy lints

* Add execution payload to block gen

* Add execute_payload to block_gen

* Refactor block gen

* Add all routes to mock server

* Use Uint256 for base_fee_per_gas

* Add working execution chain build

* Remove unused var

* Revert "Use Uint256 for base_fee_per_gas"

This reverts commit 6c88f19ac4.

* Fix base_fee_for_gas Uint256

* Update execute payload handle

* Improve testing, fix bugs

* Fix default fee-recipient

* Fix fee-recipient address (again)

* Add check for terminal block, add comments, tidy

* Apply suggestions from code review

Co-authored-by: realbigsean <seananderson33@GMAIL.com>

* Fix is_none on handle Drop

* Remove commented-out tests

Co-authored-by: realbigsean <seananderson33@GMAIL.com>

consensus/fork_choice/Cargo.toml

@@ -8,7 +8,6 @@ edition = "2018"
 
 [dependencies]
 types = { path = "../types" }
-state_processing = { path = "../state_processing" }
 proto_array = { path = "../proto_array" }
 eth2_ssz = "0.4.0"
 eth2_ssz_derive = "0.3.0"

consensus/fork_choice/src/fork_choice.rs

@@ -2,11 +2,9 @@ use std::marker::PhantomData;
 
 use proto_array::{Block as ProtoBlock, ProtoArrayForkChoice};
 use ssz_derive::{Decode, Encode};
-use state_processing::per_block_processing::is_merge_block;
 use types::{
-    AttestationShufflingId, BeaconBlock, BeaconState, BeaconStateError, ChainSpec, Checkpoint,
-    Epoch, EthSpec, Hash256, IndexedAttestation, PowBlock, RelativeEpoch, SignedBeaconBlock, Slot,
-    Uint256,
+    AttestationShufflingId, BeaconBlock, BeaconState, BeaconStateError, Checkpoint, Epoch, EthSpec,
+    Hash256, IndexedAttestation, RelativeEpoch, SignedBeaconBlock, Slot,
 };
 
 use crate::ForkChoiceStore;
@@ -63,10 +61,6 @@ pub enum InvalidBlock {
         finalized_root: Hash256,
         block_ancestor: Option<Hash256>,
     },
-    InvalidTerminalPowBlock {
-        block_total_difficulty: Uint256,
-        parent_total_difficulty: Uint256,
-    },
 }
 
 #[derive(Debug)]
@@ -238,14 +232,6 @@ where
     }
 }
 
-/// https://github.com/ethereum/consensus-specs/blob/dev/specs/merge/fork-choice.md#is_valid_terminal_pow_block
-fn is_valid_terminal_pow_block(block: &PowBlock, parent: &PowBlock, spec: &ChainSpec) -> bool {
-    let is_total_difficulty_reached = block.total_difficulty >= spec.terminal_total_difficulty;
-    let is_parent_total_difficulty_valid = parent.total_difficulty < spec.terminal_total_difficulty;
-
-    is_total_difficulty_reached && is_parent_total_difficulty_valid
-}
-
 impl<T, E> ForkChoice<T, E>
 where
     T: ForkChoiceStore<E>,
@@ -460,7 +446,6 @@ where
         block: &BeaconBlock<E>,
         block_root: Hash256,
         state: &BeaconState<E>,
-        spec: &ChainSpec,
     ) -> Result<(), Error<T::Error>> {
         let current_slot = self.update_time(current_slot)?;
 
@@ -511,19 +496,6 @@ where
             }));
         }
 
-        // https://github.com/ethereum/consensus-specs/blob/dev/specs/merge/fork-choice.md#on_block
-        if is_merge_block(state, block.body()) {
-            // TODO: get POW blocks from eth1 chain here as indicated in the merge spec link ^
-            let pow_block = PowBlock::default();
-            let pow_parent = PowBlock::default();
-            if !is_valid_terminal_pow_block(&pow_block, &pow_parent, spec) {
-                return Err(Error::InvalidBlock(InvalidBlock::InvalidTerminalPowBlock {
-                    block_total_difficulty: pow_block.total_difficulty,
-                    parent_total_difficulty: pow_parent.total_difficulty,
-                }));
-            }
-        }
-
         // Update justified checkpoint.
         if state.current_justified_checkpoint().epoch > self.fc_store.justified_checkpoint().epoch {
             if state.current_justified_checkpoint().epoch

consensus/fork_choice/tests/tests.rs

@@ -268,13 +268,7 @@ impl ForkChoiceTest {
             .chain
             .fork_choice
             .write()
-            .on_block(
-                current_slot,
-                &block,
-                block.canonical_root(),
-                &state,
-                &self.harness.chain.spec,
-            )
+            .on_block(current_slot, &block, block.canonical_root(), &state)
             .unwrap();
         self
     }
@@ -309,13 +303,7 @@ impl ForkChoiceTest {
             .chain
             .fork_choice
             .write()
-            .on_block(
-                current_slot,
-                &block,
-                block.canonical_root(),
-                &state,
-                &self.harness.chain.spec,
-            )
+            .on_block(current_slot, &block, block.canonical_root(), &state)
             .err()
             .expect("on_block did not return an error");
         comparison_func(err);

consensus/serde_utils/src/hex.rs

@@ -6,6 +6,7 @@ use std::fmt;
 /// Encode `data` as a 0x-prefixed hex string.
 pub fn encode<T: AsRef<[u8]>>(data: T) -> String {
     let hex = hex::encode(data);
+
     let mut s = "0x".to_string();
     s.push_str(hex.as_str());
     s
@@ -33,12 +34,7 @@ impl<'de> Visitor<'de> for PrefixedHexVisitor {
     where
         E: de::Error,
     {
-        if let Some(stripped) = value.strip_prefix("0x") {
-            Ok(hex::decode(stripped)
-                .map_err(|e| de::Error::custom(format!("invalid hex ({:?})", e)))?)
-        } else {
-            Err(de::Error::custom("missing 0x prefix"))
-        }
+        decode(value).map_err(de::Error::custom)
     }
 }
 

consensus/serde_utils/src/hex_vec.rs

@@ -0,0 +1,23 @@
+//! Formats `Vec<u8>` as a 0x-prefixed hex string.
+//!
+//! E.g., `vec![0, 1, 2, 3]` serializes as `"0x00010203"`.
+
+use crate::hex::PrefixedHexVisitor;
+use serde::{Deserializer, Serializer};
+
+pub fn serialize<S>(bytes: &[u8], serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    let mut hex_string: String = "0x".to_string();
+    hex_string.push_str(&hex::encode(&bytes));
+
+    serializer.serialize_str(&hex_string)
+}
+
+pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    deserializer.deserialize_str(PrefixedHexVisitor)
+}

consensus/serde_utils/src/lib.rs

@@ -2,8 +2,11 @@ mod quoted_int;
 
 pub mod bytes_4_hex;
 pub mod hex;
+pub mod hex_vec;
+pub mod list_of_bytes_lists;
 pub mod quoted_u64_vec;
 pub mod u32_hex;
+pub mod u64_hex_be;
 pub mod u8_hex;
 
 pub use quoted_int::{quoted_u32, quoted_u64, quoted_u8};

consensus/serde_utils/src/list_of_bytes_lists.rs

@@ -0,0 +1,49 @@
+//! Formats `Vec<u64>` using quotes.
+//!
+//! E.g., `vec![0, 1, 2]` serializes as `["0", "1", "2"]`.
+//!
+//! Quotes can be optional during decoding.
+
+use crate::hex;
+use serde::ser::SerializeSeq;
+use serde::{de, Deserializer, Serializer};
+
+pub struct ListOfBytesListVisitor;
+impl<'a> serde::de::Visitor<'a> for ListOfBytesListVisitor {
+    type Value = Vec<Vec<u8>>;
+
+    fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(formatter, "a list of 0x-prefixed byte lists")
+    }
+
+    fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
+    where
+        A: serde::de::SeqAccess<'a>,
+    {
+        let mut vec = vec![];
+
+        while let Some(val) = seq.next_element::<String>()? {
+            vec.push(hex::decode(&val).map_err(de::Error::custom)?);
+        }
+
+        Ok(vec)
+    }
+}
+
+pub fn serialize<S>(value: &[Vec<u8>], serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    let mut seq = serializer.serialize_seq(Some(value.len()))?;
+    for val in value {
+        seq.serialize_element(&hex::encode(val))?;
+    }
+    seq.end()
+}
+
+pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<Vec<u8>>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    deserializer.deserialize_any(ListOfBytesListVisitor)
+}

consensus/serde_utils/src/quoted_int.rs

@@ -70,17 +70,6 @@ macro_rules! define_mod {
             pub value: T,
         }
 
-        /// Compositional wrapper type that allows quotes or no quotes.
-        #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize, Serialize)]
-        #[serde(transparent)]
-        pub struct MaybeQuoted<T>
-        where
-            T: From<$int> + Into<$int> + Copy + TryFrom<u64>,
-        {
-            #[serde(with = "self")]
-            pub value: T,
-        }
-
         /// Serialize with quotes.
         pub fn serialize<S, T>(value: &T, serializer: S) -> Result<S::Ok, S::Error>
         where

consensus/serde_utils/src/u64_hex_be.rs

@@ -0,0 +1,134 @@
+//! Formats `u64` as a 0x-prefixed, big-endian hex string.
+//!
+//! E.g., `0` serializes as `"0x0000000000000000"`.
+
+use serde::de::{self, Error, Visitor};
+use serde::{Deserializer, Serializer};
+use std::fmt;
+
+const BYTES_LEN: usize = 8;
+
+pub struct QuantityVisitor;
+impl<'de> Visitor<'de> for QuantityVisitor {
+    type Value = Vec<u8>;
+
+    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+        formatter.write_str("a hex string")
+    }
+
+    fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        if !value.starts_with("0x") {
+            return Err(de::Error::custom("must start with 0x"));
+        }
+
+        let stripped = value.trim_start_matches("0x");
+
+        if stripped.is_empty() {
+            Err(de::Error::custom(format!(
+                "quantity cannot be {}",
+                stripped
+            )))
+        } else if stripped == "0" {
+            Ok(vec![0])
+        } else if stripped.starts_with('0') {
+            Err(de::Error::custom("cannot have leading zero"))
+        } else if stripped.len() % 2 != 0 {
+            hex::decode(&format!("0{}", stripped))
+                .map_err(|e| de::Error::custom(format!("invalid hex ({:?})", e)))
+        } else {
+            hex::decode(&stripped).map_err(|e| de::Error::custom(format!("invalid hex ({:?})", e)))
+        }
+    }
+}
+
+pub fn serialize<S>(num: &u64, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    let raw = hex::encode(num.to_be_bytes());
+    let trimmed = raw.trim_start_matches('0');
+
+    let hex = if trimmed.is_empty() { "0" } else { &trimmed };
+
+    serializer.serialize_str(&format!("0x{}", &hex))
+}
+
+pub fn deserialize<'de, D>(deserializer: D) -> Result<u64, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let decoded = deserializer.deserialize_str(QuantityVisitor)?;
+
+    // TODO: this is not strict about byte length like other methods.
+    if decoded.len() > BYTES_LEN {
+        return Err(D::Error::custom(format!(
+            "expected max {} bytes for array, got {}",
+            BYTES_LEN,
+            decoded.len()
+        )));
+    }
+
+    let mut array = [0; BYTES_LEN];
+    array[BYTES_LEN - decoded.len()..].copy_from_slice(&decoded);
+    Ok(u64::from_be_bytes(array))
+}
+
+#[cfg(test)]
+mod test {
+    use serde::{Deserialize, Serialize};
+    use serde_json;
+
+    #[derive(Debug, PartialEq, Serialize, Deserialize)]
+    #[serde(transparent)]
+    struct Wrapper {
+        #[serde(with = "super")]
+        val: u64,
+    }
+
+    #[test]
+    fn encoding() {
+        assert_eq!(
+            &serde_json::to_string(&Wrapper { val: 0 }).unwrap(),
+            "\"0x0\""
+        );
+        assert_eq!(
+            &serde_json::to_string(&Wrapper { val: 1 }).unwrap(),
+            "\"0x1\""
+        );
+        assert_eq!(
+            &serde_json::to_string(&Wrapper { val: 256 }).unwrap(),
+            "\"0x100\""
+        );
+        assert_eq!(
+            &serde_json::to_string(&Wrapper { val: 65 }).unwrap(),
+            "\"0x41\""
+        );
+        assert_eq!(
+            &serde_json::to_string(&Wrapper { val: 1024 }).unwrap(),
+            "\"0x400\""
+        );
+    }
+
+    #[test]
+    fn decoding() {
+        assert_eq!(
+            serde_json::from_str::<Wrapper>("\"0x0\"").unwrap(),
+            Wrapper { val: 0 },
+        );
+        assert_eq!(
+            serde_json::from_str::<Wrapper>("\"0x41\"").unwrap(),
+            Wrapper { val: 65 },
+        );
+        assert_eq!(
+            serde_json::from_str::<Wrapper>("\"0x400\"").unwrap(),
+            Wrapper { val: 1024 },
+        );
+        serde_json::from_str::<Wrapper>("\"0x\"").unwrap_err();
+        serde_json::from_str::<Wrapper>("\"0x0400\"").unwrap_err();
+        serde_json::from_str::<Wrapper>("\"400\"").unwrap_err();
+        serde_json::from_str::<Wrapper>("\"ff\"").unwrap_err();
+    }
+}

consensus/ssz_types/src/serde_utils/hex_fixed_vec.rs

@@ -8,10 +8,7 @@ where
     S: Serializer,
     U: Unsigned,
 {
-    let mut hex_string: String = "0x".to_string();
-    hex_string.push_str(&hex::encode(&bytes[..]));
-
-    serializer.serialize_str(&hex_string)
+    serializer.serialize_str(&hex::encode(&bytes[..]))
 }
 
 pub fn deserialize<'de, D, U>(deserializer: D) -> Result<FixedVector<u8, U>, D::Error>

consensus/ssz_types/src/serde_utils/hex_var_list.rs

@@ -9,10 +9,7 @@ where
     S: Serializer,
     N: Unsigned,
 {
-    let mut hex_string: String = "0x".to_string();
-    hex_string.push_str(&hex::encode(&**bytes));
-
-    serializer.serialize_str(&hex_string)
+    serializer.serialize_str(&hex::encode(&**bytes))
 }
 
 pub fn deserialize<'de, D, N>(deserializer: D) -> Result<VariableList<u8, N>, D::Error>

consensus/types/src/chain_spec.rs

@@ -131,6 +131,7 @@ pub struct ChainSpec {
     /// The Merge fork epoch is optional, with `None` representing "Merge never happens".
     pub merge_fork_epoch: Option<Epoch>,
     pub terminal_total_difficulty: Uint256,
+    pub terminal_block_hash: Hash256,
 
     /*
      * Networking
@@ -483,6 +484,7 @@ impl ChainSpec {
             terminal_total_difficulty: Uint256::MAX
                 .checked_sub(Uint256::from(2u64.pow(10)))
                 .expect("calculation does not overflow"),
+            terminal_block_hash: Hash256::zero(),
 
             /*
              * Network specific

consensus/types/src/lib.rs

@@ -113,7 +113,7 @@ pub use crate::deposit_message::DepositMessage;
 pub use crate::enr_fork_id::EnrForkId;
 pub use crate::eth1_data::Eth1Data;
 pub use crate::eth_spec::EthSpecId;
-pub use crate::execution_payload::ExecutionPayload;
+pub use crate::execution_payload::{ExecutionPayload, Transaction};
 pub use crate::execution_payload_header::ExecutionPayloadHeader;
 pub use crate::fork::Fork;
 pub use crate::fork_context::ForkContext;