[Merge] Implement execution_layer (#2635)

* Checkout serde_utils from rayonism

* Make eth1::http functions pub

* Add bones of execution_layer

* Modify decoding

* Expose Transaction, cargo fmt

* Add executePayload

* Add all minimal spec endpoints

* Start adding json rpc wrapper

* Finish custom JSON response handler

* Switch to new rpc sending method

* Add first test

* Fix camelCase

* Finish adding tests

* Begin threading execution layer into BeaconChain

* Fix clippy lints

* Fix clippy lints

* Thread execution layer into ClientBuilder

* Add CLI flags

* Add block processing methods to ExecutionLayer

* Add block_on to execution_layer

* Integrate execute_payload

* Add extra_data field

* Begin implementing payload handle

* Send consensus valid/invalid messages

* Fix minor type in task_executor

* Call forkchoiceUpdated

* Add search for TTD block

* Thread TTD into execution layer

* Allow producing block with execution payload

* Add LRU cache for execution blocks

* Remove duplicate 0x on ssz_types serialization

* Add tests for block getter methods

* Add basic block generator impl

* Add is_valid_terminal_block to EL

* Verify merge block in block_verification

* Partially implement --terminal-block-hash-override

* Add terminal_block_hash to ChainSpec

* Remove Option from terminal_block_hash in EL

* Revert merge changes to consensus/fork_choice

* Remove commented-out code

* Add bones for handling RPC methods on test server

* Add first ExecutionLayer tests

* Add testing for finding terminal block

* Prevent infinite loops

* Add insert_merge_block to block gen

* Add block gen test for pos blocks

* Start adding payloads to block gen

* Fix clippy lints

* Add execution payload to block gen

* Add execute_payload to block_gen

* Refactor block gen

* Add all routes to mock server

* Use Uint256 for base_fee_per_gas

* Add working execution chain build

* Remove unused var

* Revert "Use Uint256 for base_fee_per_gas"

This reverts commit 6c88f19ac4.

* Fix base_fee_for_gas Uint256

* Update execute payload handle

* Improve testing, fix bugs

* Fix default fee-recipient

* Fix fee-recipient address (again)

* Add check for terminal block, add comments, tidy

* Apply suggestions from code review

Co-authored-by: realbigsean <seananderson33@GMAIL.com>

* Fix is_none on handle Drop

* Remove commented-out tests

Co-authored-by: realbigsean <seananderson33@GMAIL.com>

consensus/serde_utils/src/hex.rs

@@ -6,6 +6,7 @@ use std::fmt;
 /// Encode `data` as a 0x-prefixed hex string.
 pub fn encode<T: AsRef<[u8]>>(data: T) -> String {
     let hex = hex::encode(data);
+
     let mut s = "0x".to_string();
     s.push_str(hex.as_str());
     s
@@ -33,12 +34,7 @@ impl<'de> Visitor<'de> for PrefixedHexVisitor {
     where
         E: de::Error,
     {
-        if let Some(stripped) = value.strip_prefix("0x") {
-            Ok(hex::decode(stripped)
-                .map_err(|e| de::Error::custom(format!("invalid hex ({:?})", e)))?)
-        } else {
-            Err(de::Error::custom("missing 0x prefix"))
-        }
+        decode(value).map_err(de::Error::custom)
     }
 }
 

consensus/serde_utils/src/hex_vec.rs

@@ -0,0 +1,23 @@
+//! Formats `Vec<u8>` as a 0x-prefixed hex string.
+//!
+//! E.g., `vec![0, 1, 2, 3]` serializes as `"0x00010203"`.
+
+use crate::hex::PrefixedHexVisitor;
+use serde::{Deserializer, Serializer};
+
+pub fn serialize<S>(bytes: &[u8], serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    let mut hex_string: String = "0x".to_string();
+    hex_string.push_str(&hex::encode(&bytes));
+
+    serializer.serialize_str(&hex_string)
+}
+
+pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    deserializer.deserialize_str(PrefixedHexVisitor)
+}

consensus/serde_utils/src/lib.rs

@@ -2,8 +2,11 @@ mod quoted_int;
 
 pub mod bytes_4_hex;
 pub mod hex;
+pub mod hex_vec;
+pub mod list_of_bytes_lists;
 pub mod quoted_u64_vec;
 pub mod u32_hex;
+pub mod u64_hex_be;
 pub mod u8_hex;
 
 pub use quoted_int::{quoted_u32, quoted_u64, quoted_u8};

consensus/serde_utils/src/list_of_bytes_lists.rs

@@ -0,0 +1,49 @@
+//! Formats `Vec<u64>` using quotes.
+//!
+//! E.g., `vec![0, 1, 2]` serializes as `["0", "1", "2"]`.
+//!
+//! Quotes can be optional during decoding.
+
+use crate::hex;
+use serde::ser::SerializeSeq;
+use serde::{de, Deserializer, Serializer};
+
+pub struct ListOfBytesListVisitor;
+impl<'a> serde::de::Visitor<'a> for ListOfBytesListVisitor {
+    type Value = Vec<Vec<u8>>;
+
+    fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(formatter, "a list of 0x-prefixed byte lists")
+    }
+
+    fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
+    where
+        A: serde::de::SeqAccess<'a>,
+    {
+        let mut vec = vec![];
+
+        while let Some(val) = seq.next_element::<String>()? {
+            vec.push(hex::decode(&val).map_err(de::Error::custom)?);
+        }
+
+        Ok(vec)
+    }
+}
+
+pub fn serialize<S>(value: &[Vec<u8>], serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    let mut seq = serializer.serialize_seq(Some(value.len()))?;
+    for val in value {
+        seq.serialize_element(&hex::encode(val))?;
+    }
+    seq.end()
+}
+
+pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<Vec<u8>>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    deserializer.deserialize_any(ListOfBytesListVisitor)
+}

consensus/serde_utils/src/quoted_int.rs

@@ -70,17 +70,6 @@ macro_rules! define_mod {
             pub value: T,
         }
 
-        /// Compositional wrapper type that allows quotes or no quotes.
-        #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize, Serialize)]
-        #[serde(transparent)]
-        pub struct MaybeQuoted<T>
-        where
-            T: From<$int> + Into<$int> + Copy + TryFrom<u64>,
-        {
-            #[serde(with = "self")]
-            pub value: T,
-        }
-
         /// Serialize with quotes.
         pub fn serialize<S, T>(value: &T, serializer: S) -> Result<S::Ok, S::Error>
         where

consensus/serde_utils/src/u64_hex_be.rs

@@ -0,0 +1,134 @@
+//! Formats `u64` as a 0x-prefixed, big-endian hex string.
+//!
+//! E.g., `0` serializes as `"0x0000000000000000"`.
+
+use serde::de::{self, Error, Visitor};
+use serde::{Deserializer, Serializer};
+use std::fmt;
+
+const BYTES_LEN: usize = 8;
+
+pub struct QuantityVisitor;
+impl<'de> Visitor<'de> for QuantityVisitor {
+    type Value = Vec<u8>;
+
+    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+        formatter.write_str("a hex string")
+    }
+
+    fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        if !value.starts_with("0x") {
+            return Err(de::Error::custom("must start with 0x"));
+        }
+
+        let stripped = value.trim_start_matches("0x");
+
+        if stripped.is_empty() {
+            Err(de::Error::custom(format!(
+                "quantity cannot be {}",
+                stripped
+            )))
+        } else if stripped == "0" {
+            Ok(vec![0])
+        } else if stripped.starts_with('0') {
+            Err(de::Error::custom("cannot have leading zero"))
+        } else if stripped.len() % 2 != 0 {
+            hex::decode(&format!("0{}", stripped))
+                .map_err(|e| de::Error::custom(format!("invalid hex ({:?})", e)))
+        } else {
+            hex::decode(&stripped).map_err(|e| de::Error::custom(format!("invalid hex ({:?})", e)))
+        }
+    }
+}
+
+pub fn serialize<S>(num: &u64, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    let raw = hex::encode(num.to_be_bytes());
+    let trimmed = raw.trim_start_matches('0');
+
+    let hex = if trimmed.is_empty() { "0" } else { &trimmed };
+
+    serializer.serialize_str(&format!("0x{}", &hex))
+}
+
+pub fn deserialize<'de, D>(deserializer: D) -> Result<u64, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let decoded = deserializer.deserialize_str(QuantityVisitor)?;
+
+    // TODO: this is not strict about byte length like other methods.
+    if decoded.len() > BYTES_LEN {
+        return Err(D::Error::custom(format!(
+            "expected max {} bytes for array, got {}",
+            BYTES_LEN,
+            decoded.len()
+        )));
+    }
+
+    let mut array = [0; BYTES_LEN];
+    array[BYTES_LEN - decoded.len()..].copy_from_slice(&decoded);
+    Ok(u64::from_be_bytes(array))
+}
+
+#[cfg(test)]
+mod test {
+    use serde::{Deserialize, Serialize};
+    use serde_json;
+
+    #[derive(Debug, PartialEq, Serialize, Deserialize)]
+    #[serde(transparent)]
+    struct Wrapper {
+        #[serde(with = "super")]
+        val: u64,
+    }
+
+    #[test]
+    fn encoding() {
+        assert_eq!(
+            &serde_json::to_string(&Wrapper { val: 0 }).unwrap(),
+            "\"0x0\""
+        );
+        assert_eq!(
+            &serde_json::to_string(&Wrapper { val: 1 }).unwrap(),
+            "\"0x1\""
+        );
+        assert_eq!(
+            &serde_json::to_string(&Wrapper { val: 256 }).unwrap(),
+            "\"0x100\""
+        );
+        assert_eq!(
+            &serde_json::to_string(&Wrapper { val: 65 }).unwrap(),
+            "\"0x41\""
+        );
+        assert_eq!(
+            &serde_json::to_string(&Wrapper { val: 1024 }).unwrap(),
+            "\"0x400\""
+        );
+    }
+
+    #[test]
+    fn decoding() {
+        assert_eq!(
+            serde_json::from_str::<Wrapper>("\"0x0\"").unwrap(),
+            Wrapper { val: 0 },
+        );
+        assert_eq!(
+            serde_json::from_str::<Wrapper>("\"0x41\"").unwrap(),
+            Wrapper { val: 65 },
+        );
+        assert_eq!(
+            serde_json::from_str::<Wrapper>("\"0x400\"").unwrap(),
+            Wrapper { val: 1024 },
+        );
+        serde_json::from_str::<Wrapper>("\"0x\"").unwrap_err();
+        serde_json::from_str::<Wrapper>("\"0x0400\"").unwrap_err();
+        serde_json::from_str::<Wrapper>("\"400\"").unwrap_err();
+        serde_json::from_str::<Wrapper>("\"ff\"").unwrap_err();
+    }
+}