Enable proposer boost re-orging (#2860)

## Proposed Changes

With proposer boosting implemented (#2822) we have an opportunity to re-org out late blocks.

This PR adds three flags to the BN to control this behaviour:

* `--disable-proposer-reorgs`: turn aggressive re-orging off (it's on by default).
* `--proposer-reorg-threshold N`: attempt to orphan blocks with less than N% of the committee vote. If this parameter isn't set then N defaults to 20% when the feature is enabled.
* `--proposer-reorg-epochs-since-finalization N`: only attempt to re-org late blocks when the number of epochs since finalization is less than or equal to N. The default is 2 epochs, meaning re-orgs will only be attempted when the chain is finalizing optimally.

For safety Lighthouse will only attempt a re-org under very specific conditions:

1. The block being proposed is 1 slot after the canonical head, and the canonical head is 1 slot after its parent. i.e. at slot `n + 1` rather than building on the block from slot `n` we build on the block from slot `n - 1`.
2. The current canonical head received less than N% of the committee vote. N should be set depending on the proposer boost fraction itself, the fraction of the network that is believed to be applying it, and the size of the largest entity that could be hoarding votes.
3. The current canonical head arrived after the attestation deadline from our perspective. This condition was only added to support suppression of forkchoiceUpdated messages, but makes intuitive sense.
4. The block is being proposed in the first 2 seconds of the slot. This gives it time to propagate and receive the proposer boost.


## Additional Info

For the initial idea and background, see: https://github.com/ethereum/consensus-specs/pull/2353#issuecomment-950238004

There is also a specification for this feature here: https://github.com/ethereum/consensus-specs/pull/3034

Co-authored-by: Michael Sproul <micsproul@gmail.com>
Co-authored-by: pawan <pawandhananjay@gmail.com>

beacon_node/execution_layer/src/test_utils/execution_block_generator.rs

@@ -105,13 +105,15 @@ pub struct PoWBlock {
     pub timestamp: u64,
 }
 
-#[derive(Clone)]
+#[derive(Debug, Clone)]
 pub struct ExecutionBlockGenerator<T: EthSpec> {
     /*
      * Common database
      */
+    head_block: Option<Block<T>>,
+    finalized_block_hash: Option<ExecutionBlockHash>,
     blocks: HashMap<ExecutionBlockHash, Block<T>>,
-    block_hashes: HashMap<u64, ExecutionBlockHash>,
+    block_hashes: HashMap<u64, Vec<ExecutionBlockHash>>,
     /*
      * PoW block parameters
      */
@@ -133,6 +135,8 @@ impl<T: EthSpec> ExecutionBlockGenerator<T> {
         terminal_block_hash: ExecutionBlockHash,
     ) -> Self {
         let mut gen = Self {
+            head_block: <_>::default(),
+            finalized_block_hash: <_>::default(),
             blocks: <_>::default(),
             block_hashes: <_>::default(),
             terminal_total_difficulty,
@@ -149,13 +153,7 @@ impl<T: EthSpec> ExecutionBlockGenerator<T> {
     }
 
     pub fn latest_block(&self) -> Option<Block<T>> {
-        let hash = *self
-            .block_hashes
-            .iter()
-            .max_by_key(|(number, _)| *number)
-            .map(|(_, hash)| hash)?;
-
-        self.block_by_hash(hash)
+        self.head_block.clone()
     }
 
     pub fn latest_execution_block(&self) -> Option<ExecutionBlock> {
@@ -164,8 +162,18 @@ impl<T: EthSpec> ExecutionBlockGenerator<T> {
     }
 
     pub fn block_by_number(&self, number: u64) -> Option<Block<T>> {
-        let hash = *self.block_hashes.get(&number)?;
-        self.block_by_hash(hash)
+        // Get the latest canonical head block
+        let mut latest_block = self.latest_block()?;
+        loop {
+            let block_number = latest_block.block_number();
+            if block_number < number {
+                return None;
+            }
+            if block_number == number {
+                return Some(latest_block);
+            }
+            latest_block = self.block_by_hash(latest_block.parent_hash())?;
+        }
     }
 
     pub fn execution_block_by_number(&self, number: u64) -> Option<ExecutionBlock> {
@@ -226,10 +234,16 @@ impl<T: EthSpec> ExecutionBlockGenerator<T> {
     }
 
     pub fn insert_pow_block(&mut self, block_number: u64) -> Result<(), String> {
+        if let Some(finalized_block_hash) = self.finalized_block_hash {
+            return Err(format!(
+                "terminal block {} has been finalized. PoW chain has stopped building",
+                finalized_block_hash
+            ));
+        }
         let parent_hash = if block_number == 0 {
             ExecutionBlockHash::zero()
-        } else if let Some(hash) = self.block_hashes.get(&(block_number - 1)) {
-            *hash
+        } else if let Some(block) = self.block_by_number(block_number - 1) {
+            block.block_hash()
         } else {
             return Err(format!(
                 "parent with block number {} not found",
@@ -244,49 +258,118 @@ impl<T: EthSpec> ExecutionBlockGenerator<T> {
             parent_hash,
         )?;
 
-        self.insert_block(Block::PoW(block))
-    }
+        // Insert block into block tree
+        self.insert_block(Block::PoW(block))?;
 
-    pub fn insert_block(&mut self, block: Block<T>) -> Result<(), String> {
-        if self.blocks.contains_key(&block.block_hash()) {
-            return Err(format!("{:?} is already known", block.block_hash()));
-        } else if self.block_hashes.contains_key(&block.block_number()) {
-            return Err(format!(
-                "block {} is already known, forking is not supported",
-                block.block_number()
-            ));
-        } else if block.block_number() != 0 && !self.blocks.contains_key(&block.parent_hash()) {
-            return Err(format!("parent block {:?} is unknown", block.parent_hash()));
+        // Set head
+        if let Some(head_total_difficulty) =
+            self.head_block.as_ref().and_then(|b| b.total_difficulty())
+        {
+            if block.total_difficulty >= head_total_difficulty {
+                self.head_block = Some(Block::PoW(block));
+            }
+        } else {
+            self.head_block = Some(Block::PoW(block));
         }
-
-        self.insert_block_without_checks(block)
-    }
-
-    pub fn insert_block_without_checks(&mut self, block: Block<T>) -> Result<(), String> {
-        self.block_hashes
-            .insert(block.block_number(), block.block_hash());
-        self.blocks.insert(block.block_hash(), block);
-
         Ok(())
     }
 
+    /// Insert a PoW block given the parent hash.
+    ///
+    /// Returns `Ok(hash)` of the inserted block.
+    /// Returns an error if the `parent_hash` does not exist in the block tree or
+    /// if the parent block is the terminal block.
+    pub fn insert_pow_block_by_hash(
+        &mut self,
+        parent_hash: ExecutionBlockHash,
+        unique_id: u64,
+    ) -> Result<ExecutionBlockHash, String> {
+        let parent_block = self.block_by_hash(parent_hash).ok_or_else(|| {
+            format!(
+                "Block corresponding to parent hash does not exist: {}",
+                parent_hash
+            )
+        })?;
+
+        let mut block = generate_pow_block(
+            self.terminal_total_difficulty,
+            self.terminal_block_number,
+            parent_block.block_number() + 1,
+            parent_hash,
+        )?;
+
+        // Hack the block hash to make this block distinct from any other block with a different
+        // `unique_id` (the default is 0).
+        block.block_hash = ExecutionBlockHash::from_root(Hash256::from_low_u64_be(unique_id));
+        block.block_hash = ExecutionBlockHash::from_root(block.tree_hash_root());
+
+        let hash = self.insert_block(Block::PoW(block))?;
+
+        // Set head
+        if let Some(head_total_difficulty) =
+            self.head_block.as_ref().and_then(|b| b.total_difficulty())
+        {
+            if block.total_difficulty >= head_total_difficulty {
+                self.head_block = Some(Block::PoW(block));
+            }
+        } else {
+            self.head_block = Some(Block::PoW(block));
+        }
+        Ok(hash)
+    }
+
+    pub fn insert_block(&mut self, block: Block<T>) -> Result<ExecutionBlockHash, String> {
+        if self.blocks.contains_key(&block.block_hash()) {
+            return Err(format!("{:?} is already known", block.block_hash()));
+        } else if block.parent_hash() != ExecutionBlockHash::zero()
+            && !self.blocks.contains_key(&block.parent_hash())
+        {
+            return Err(format!("parent block {:?} is unknown", block.parent_hash()));
+        }
+
+        Ok(self.insert_block_without_checks(block))
+    }
+
+    pub fn insert_block_without_checks(&mut self, block: Block<T>) -> ExecutionBlockHash {
+        let block_hash = block.block_hash();
+        self.block_hashes
+            .entry(block.block_number())
+            .or_insert_with(Vec::new)
+            .push(block_hash);
+        self.blocks.insert(block_hash, block);
+
+        block_hash
+    }
+
     pub fn modify_last_block(&mut self, block_modifier: impl FnOnce(&mut Block<T>)) {
-        if let Some((last_block_hash, block_number)) =
-            self.block_hashes.keys().max().and_then(|block_number| {
-                self.block_hashes
-                    .get(block_number)
-                    .map(|block| (block, *block_number))
+        if let Some(last_block_hash) = self
+            .block_hashes
+            .iter_mut()
+            .max_by_key(|(block_number, _)| *block_number)
+            .and_then(|(_, block_hashes)| {
+                // Remove block hash, we will re-insert with the new block hash after modifying it.
+                block_hashes.pop()
             })
         {
-            let mut block = self.blocks.remove(last_block_hash).unwrap();
+            let mut block = self.blocks.remove(&last_block_hash).unwrap();
             block_modifier(&mut block);
+
             // Update the block hash after modifying the block
             match &mut block {
                 Block::PoW(b) => b.block_hash = ExecutionBlockHash::from_root(b.tree_hash_root()),
                 Block::PoS(b) => b.block_hash = ExecutionBlockHash::from_root(b.tree_hash_root()),
             }
-            self.block_hashes.insert(block_number, block.block_hash());
-            self.blocks.insert(block.block_hash(), block);
+
+            // Update head.
+            if self
+                .head_block
+                .as_ref()
+                .map_or(true, |head| head.block_hash() == last_block_hash)
+            {
+                self.head_block = Some(block.clone());
+            }
+
+            self.insert_block_without_checks(block);
         }
     }
 
@@ -405,6 +488,17 @@ impl<T: EthSpec> ExecutionBlockGenerator<T> {
             }
         };
 
+        self.head_block = Some(
+            self.blocks
+                .get(&forkchoice_state.head_block_hash)
+                .unwrap()
+                .clone(),
+        );
+
+        if forkchoice_state.finalized_block_hash != ExecutionBlockHash::zero() {
+            self.finalized_block_hash = Some(forkchoice_state.finalized_block_hash);
+        }
+
         Ok(JsonForkchoiceUpdatedV1Response {
             payload_status: JsonPayloadStatusV1 {
                 status: JsonPayloadStatusV1Status::Valid,

beacon_node/execution_layer/src/test_utils/handle_rpc.rs

@@ -123,6 +123,14 @@ pub async fn handle_rpc<T: EthSpec>(
             let forkchoice_state: JsonForkChoiceStateV1 = get_param(params, 0)?;
             let payload_attributes: Option<JsonPayloadAttributesV1> = get_param(params, 1)?;
 
+            if let Some(hook_response) = ctx
+                .hook
+                .lock()
+                .on_forkchoice_updated(forkchoice_state.clone(), payload_attributes.clone())
+            {
+                return Ok(serde_json::to_value(hook_response).unwrap());
+            }
+
             let head_block_hash = forkchoice_state.head_block_hash;
 
             // Canned responses set by block hash take priority.

beacon_node/execution_layer/src/test_utils/hook.rs

@@ -0,0 +1,27 @@
+use crate::json_structures::*;
+
+type ForkChoiceUpdatedHook = dyn Fn(
+        JsonForkChoiceStateV1,
+        Option<JsonPayloadAttributesV1>,
+    ) -> Option<JsonForkchoiceUpdatedV1Response>
+    + Send
+    + Sync;
+
+#[derive(Default)]
+pub struct Hook {
+    forkchoice_updated: Option<Box<ForkChoiceUpdatedHook>>,
+}
+
+impl Hook {
+    pub fn on_forkchoice_updated(
+        &self,
+        state: JsonForkChoiceStateV1,
+        payload_attributes: Option<JsonPayloadAttributesV1>,
+    ) -> Option<JsonForkchoiceUpdatedV1Response> {
+        (self.forkchoice_updated.as_ref()?)(state, payload_attributes)
+    }
+
+    pub fn set_forkchoice_updated_hook(&mut self, f: Box<ForkChoiceUpdatedHook>) {
+        self.forkchoice_updated = Some(f);
+    }
+}

beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs

@@ -234,6 +234,21 @@ impl<T: EthSpec> MockExecutionLayer<T> {
         self
     }
 
+    pub fn produce_forked_pow_block(self) -> (Self, ExecutionBlockHash) {
+        let head_block = self
+            .server
+            .execution_block_generator()
+            .latest_block()
+            .unwrap();
+
+        let block_hash = self
+            .server
+            .execution_block_generator()
+            .insert_pow_block_by_hash(head_block.parent_hash(), 1)
+            .unwrap();
+        (self, block_hash)
+    }
+
     pub async fn with_terminal_block<'a, U, V>(self, func: U) -> Self
     where
         U: Fn(ChainSpec, ExecutionLayer<T>, Option<ExecutionBlock>) -> V,

beacon_node/execution_layer/src/test_utils/mod.rs

@@ -23,6 +23,7 @@ use types::{EthSpec, ExecutionBlockHash, Uint256};
 use warp::{http::StatusCode, Filter, Rejection};
 
 pub use execution_block_generator::{generate_pow_block, Block, ExecutionBlockGenerator};
+pub use hook::Hook;
 pub use mock_builder::{Context as MockBuilderContext, MockBuilder, Operation, TestingBuilder};
 pub use mock_execution_layer::MockExecutionLayer;
 
@@ -33,6 +34,7 @@ pub const DEFAULT_BUILDER_THRESHOLD_WEI: u128 = 1_000_000_000_000_000_000;
 
 mod execution_block_generator;
 mod handle_rpc;
+mod hook;
 mod mock_builder;
 mod mock_execution_layer;
 
@@ -99,6 +101,7 @@ impl<T: EthSpec> MockServer<T> {
             static_new_payload_response: <_>::default(),
             static_forkchoice_updated_response: <_>::default(),
             static_get_block_by_hash_response: <_>::default(),
+            hook: <_>::default(),
             new_payload_statuses: <_>::default(),
             fcu_payload_statuses: <_>::default(),
             _phantom: PhantomData,
@@ -359,8 +362,7 @@ impl<T: EthSpec> MockServer<T> {
             .write()
             // The EF tests supply blocks out of order, so we must import them "without checks" and
             // trust they form valid chains.
-            .insert_block_without_checks(block)
-            .unwrap()
+            .insert_block_without_checks(block);
     }
 
     pub fn get_block(&self, block_hash: ExecutionBlockHash) -> Option<Block<T>> {
@@ -441,6 +443,7 @@ pub struct Context<T: EthSpec> {
     pub static_new_payload_response: Arc<Mutex<Option<StaticNewPayloadResponse>>>,
     pub static_forkchoice_updated_response: Arc<Mutex<Option<PayloadStatusV1>>>,
     pub static_get_block_by_hash_response: Arc<Mutex<Option<Option<ExecutionBlock>>>>,
+    pub hook: Arc<Mutex<Hook>>,
 
     // Canned responses by block hash.
     //