//! This module provides all functionality for finding the canonical head, updating all necessary
//! components (e.g. caches) and maintaining a cached head block and state.
//!
//! For practically all applications, the "canonical head" can be read using
//! `beacon_chain.canonical_head.cached_head()`.
//!
//! The canonical head can be updated using `beacon_chain.recompute_head()`.
//!
//! ## Deadlock safety
//!
//! This module contains three locks:
//!
//! 1. `RwLock<BeaconForkChoice>`: Contains `proto_array` fork choice.
//! 2. `RwLock<CachedHead>`: Contains a cached block/state from the last run of `proto_array`.
//! 3. `Mutex<()>`: Is used to prevent concurrent execution of `BeaconChain::recompute_head`.
//!
//! This module has to take great efforts to avoid causing a deadlock with these three methods. Any
//! developers working in this module should tread carefully and seek a detailed review.
//!
//! To encourage safe use of this module, it should **only ever return a read or write lock for the
//! fork choice lock (lock 1)**. Whilst public functions might indirectly utilise locks (2) and (3),
//! the fundamental `RwLockWriteGuard` or `RwLockReadGuard` should never be exposed. This prevents
//! external functions from acquiring these locks in conflicting orders and causing a deadlock.
//!
//! ## Design Considerations
//!
//! We separate the `BeaconForkChoice` and `CachedHead` into two `RwLocks` because we want to ensure
//! fast access to the `CachedHead`. If we were to put them both under the same lock, we would need
//! to take an exclusive write-lock on it in order to run `ForkChoice::get_head`. This can take tens
//! of milliseconds and would block all downstream functions that want to know simple things like
//! the head block root. This is unacceptable for fast-responding functions like the networking
//! stack.

use crate::persisted_fork_choice::PersistedForkChoice;
use crate::shuffling_cache::BlockShufflingIds;
use crate::{
    BeaconChain, BeaconChainError as Error, BeaconChainTypes, BeaconSnapshot,
    beacon_chain::{BeaconForkChoice, BeaconStore, FORK_CHOICE_DB_KEY, OverrideForkchoiceUpdate},
    block_times_cache::BlockTimesCache,
    events::ServerSentEventHandler,
    metrics,
    validator_monitor::get_slot_delay_ms,
};
use eth2::types::{EventKind, SseChainReorg, SseFinalizedCheckpoint, SseLateHead};
use fork_choice::{
    ExecutionStatus, ForkChoiceStore, ForkChoiceView, ForkchoiceUpdateParameters, PayloadStatus,
    ProtoBlock, ResetPayloadStatuses,
};
use itertools::process_results;

use logging::crit;
use parking_lot::{Mutex, RwLock, RwLockReadGuard, RwLockUpgradableReadGuard, RwLockWriteGuard};
use slot_clock::SlotClock;
use state_processing::AllCaches;
use std::sync::Arc;
use std::time::Duration;
use store::{
    Error as StoreError, KeyValueStore, KeyValueStoreOp, StoreConfig, iter::StateRootsIterator,
};
use task_executor::{JoinHandle, ShutdownReason};
use tracing::{debug, error, info, instrument, warn};
use types::*;

/// Simple wrapper around `RwLock` that uses private visibility to prevent any other modules from
/// accessing the contained lock without it being explicitly noted in this module.
pub struct CanonicalHeadRwLock<T>(RwLock<T>);

impl<T> From<RwLock<T>> for CanonicalHeadRwLock<T> {
    fn from(rw_lock: RwLock<T>) -> Self {
        Self(rw_lock)
    }
}

impl<T> CanonicalHeadRwLock<T> {
    fn new(item: T) -> Self {
        Self::from(RwLock::new(item))
    }

    fn read(&self) -> RwLockReadGuard<'_, T> {
        self.0.read()
    }

    fn upgradable_read(&self) -> RwLockUpgradableReadGuard<'_, T> {
        self.0.upgradable_read()
    }

    fn write(&self) -> RwLockWriteGuard<'_, T> {
        self.0.write()
    }
}

/// Provides a series of cached values from the last time `BeaconChain::recompute_head` was run.
///
/// This struct is designed to be cheap-to-clone, any large fields should be wrapped in an `Arc` (or
/// similar).
#[derive(Clone)]
pub struct CachedHead<E: EthSpec> {
    /// Provides the head block and state from the last time the head was updated.
    pub snapshot: Arc<BeaconSnapshot<E>>,
    /// The justified checkpoint as per `self.fork_choice`.
    ///
    /// This value may be distinct to the `self.snapshot.beacon_state.justified_checkpoint`.
    /// This value should be used over the beacon state value in practically all circumstances.
    justified_checkpoint: Checkpoint,
    /// The finalized checkpoint as per `self.fork_choice`.
    ///
    /// This value may be distinct to the `self.snapshot.beacon_state.finalized_checkpoint`.
    /// This value should be used over the beacon state value in practically all circumstances.
    finalized_checkpoint: Checkpoint,
    /// The payload status of the head block, as determined by fork choice.
    head_payload_status: proto_array::PayloadStatus,
    /// The `execution_payload.block_hash` of the block at the head of the chain. Set to `None`
    /// before Bellatrix.
    head_hash: Option<ExecutionBlockHash>,
    /// The `execution_payload.block_hash` of the justified block. Set to `None` before Bellatrix.
    justified_hash: Option<ExecutionBlockHash>,
    /// The `execution_payload.block_hash` of the finalized block. Set to `None` before Bellatrix.
    finalized_hash: Option<ExecutionBlockHash>,
}

impl<E: EthSpec> CachedHead<E> {
    /// Returns root of the block at the head of the beacon chain.
    pub fn head_block_root(&self) -> Hash256 {
        self.snapshot.beacon_block_root
    }

    /// Returns the root of the parent of the head block.
    pub fn parent_block_root(&self) -> Hash256 {
        self.snapshot.beacon_block.parent_root()
    }

    /// Returns root of the `BeaconState` at the head of the beacon chain.
    ///
    /// ## Note
    ///
    /// This `BeaconState` has *not* been advanced to the current slot, it has the same slot as the
    /// head block.
    pub fn head_state_root(&self) -> Hash256 {
        self.snapshot.beacon_state_root()
    }

    /// Returns slot of the block at the head of the beacon chain.
    ///
    /// ## Notes
    ///
    /// This is *not* the current slot as per the system clock. Use `BeaconChain::slot` for the
    /// system clock (aka "wall clock") slot.
    pub fn head_slot(&self) -> Slot {
        self.snapshot.beacon_block.slot()
    }

    /// Returns the `Fork` from the `BeaconState` at the head of the chain.
    pub fn head_fork(&self) -> Fork {
        self.snapshot.beacon_state.fork()
    }

    /// Returns the randao mix for the block at the head of the chain.
    pub fn head_random(&self) -> Result<Hash256, BeaconStateError> {
        let state = &self.snapshot.beacon_state;
        let root = *state.get_randao_mix(state.current_epoch())?;
        Ok(root)
    }

    /// Returns the randao mix for the parent of the block at the head of the chain.
    ///
    /// This is useful for re-orging the current head. The parent's RANDAO value is read from
    /// the head's execution payload because it is unavailable in the beacon state's RANDAO mixes
    /// array after being overwritten by the head block's RANDAO mix.
    ///
    /// This will error if the head block is not execution-enabled (post Bellatrix).
    pub fn parent_random(&self) -> Result<Hash256, BeaconStateError> {
        self.snapshot
            .beacon_block
            .message()
            .execution_payload()
            .map(|payload| payload.prev_randao())
    }

    /// Returns the execution block number of the block at the head of the chain.
    ///
    /// Returns an error if the chain is prior to Bellatrix.
    pub fn head_block_number(&self) -> Result<u64, BeaconStateError> {
        self.snapshot
            .beacon_block
            .message()
            .execution_payload()
            .map(|payload| payload.block_number())
    }

    /// Returns the active validator count for the current epoch of the head state.
    ///
    /// Should only return `None` if the caches have not been built on the head state (this should
    /// never happen).
    pub fn active_validator_count(&self) -> Option<usize> {
        self.snapshot
            .beacon_state
            .get_cached_active_validator_indices(RelativeEpoch::Current)
            .map(|indices| indices.len())
            .ok()
    }

    /// Returns the finalized checkpoint, as determined by fork choice.
    ///
    /// ## Note
    ///
    /// This is *not* the finalized checkpoint of the `head_snapshot.beacon_state`, rather it is the
    /// best finalized checkpoint that has been observed by `self.fork_choice`. It is possible that
    /// the `head_snapshot.beacon_state` finalized value is earlier than the one returned here.
    pub fn finalized_checkpoint(&self) -> Checkpoint {
        self.finalized_checkpoint
    }

    /// Returns the justified checkpoint, as determined by fork choice.
    ///
    /// ## Note
    ///
    /// This is *not* the "current justified checkpoint" of the `head_snapshot.beacon_state`, rather
    /// it is the justified checkpoint in the view of `self.fork_choice`. It is possible that the
    /// `head_snapshot.beacon_state` justified value is different to, but not conflicting with, the
    /// one returned here.
    pub fn justified_checkpoint(&self) -> Checkpoint {
        self.justified_checkpoint
    }

    /// Returns the cached values of `ForkChoice::forkchoice_update_parameters`.
    ///
    /// Useful for supplying to the execution layer.
    pub fn forkchoice_update_parameters(&self) -> ForkchoiceUpdateParameters {
        ForkchoiceUpdateParameters {
            head_root: self.snapshot.beacon_block_root,
            head_hash: self.head_hash,
            justified_hash: self.justified_hash,
            finalized_hash: self.finalized_hash,
        }
    }

    pub fn head_payload_status(&self) -> proto_array::PayloadStatus {
        self.head_payload_status
    }
}

/// Represents the "canonical head" of the beacon chain.
///
/// The `cached_head` is elected by the `fork_choice` algorithm contained in this struct.
///
/// There is no guarantee that the state of the `fork_choice` struct will always represent the
/// `cached_head` (i.e. we may call `fork_choice` *without* updating the cached values), however
/// there is a guarantee that the `cached_head` represents some past state of `fork_choice` (i.e.
/// `fork_choice` never lags *behind* the `cached_head`).
pub struct CanonicalHead<T: BeaconChainTypes> {
    /// Provides an in-memory representation of the non-finalized block tree and is used to run the
    /// fork choice algorithm and determine the canonical head.
    fork_choice: CanonicalHeadRwLock<BeaconForkChoice<T>>,
    /// Provides values cached from a previous execution of `self.fork_choice.get_head`.
    ///
    /// Although `self.fork_choice` might be slightly more advanced that this value, it is safe to
    /// consider that these values represent the "canonical head" of the beacon chain.
    cached_head: CanonicalHeadRwLock<CachedHead<T::EthSpec>>,
    /// A lock used to prevent concurrent runs of `BeaconChain::recompute_head`.
    ///
    /// This lock **should not be made public**, it should only be used inside this module.
    recompute_head_lock: Mutex<()>,
}

impl<T: BeaconChainTypes> CanonicalHead<T> {
    /// Instantiate `Self`.
    pub fn new(
        fork_choice: BeaconForkChoice<T>,
        snapshot: Arc<BeaconSnapshot<T::EthSpec>>,
        head_payload_status: proto_array::PayloadStatus,
    ) -> Self {
        let fork_choice_view = fork_choice.cached_fork_choice_view();
        let forkchoice_update_params = fork_choice.get_forkchoice_update_parameters();
        let cached_head = CachedHead {
            snapshot,
            justified_checkpoint: fork_choice_view.justified_checkpoint,
            finalized_checkpoint: fork_choice_view.finalized_checkpoint,
            head_payload_status,
            head_hash: forkchoice_update_params.head_hash,
            justified_hash: forkchoice_update_params.justified_hash,
            finalized_hash: forkchoice_update_params.finalized_hash,
        };

        Self {
            fork_choice: CanonicalHeadRwLock::new(fork_choice),
            cached_head: CanonicalHeadRwLock::new(cached_head),
            recompute_head_lock: Mutex::new(()),
        }
    }

    /// Load a persisted version of `BeaconForkChoice` from the `store` and restore `self` to that
    /// state.
    ///
    /// This is useful if some database corruption is expected and we wish to go back to our last
    /// save-point.
    pub(crate) fn restore_from_store(
        &self,
        // We don't actually need this value, however it's always present when we call this function
        // and it needs to be dropped to prevent a dead-lock. Requiring it to be passed here is
        // defensive programming.
        mut fork_choice_write_lock: RwLockWriteGuard<BeaconForkChoice<T>>,
        reset_payload_statuses: ResetPayloadStatuses,
        store: &BeaconStore<T>,
        spec: &ChainSpec,
    ) -> Result<(), Error> {
        let mut fork_choice =
            <BeaconChain<T>>::load_fork_choice(store.clone(), reset_payload_statuses, spec)?
                .ok_or(Error::MissingPersistedForkChoice)?;
        let current_slot_for_head = fork_choice.fc_store().get_current_slot();
        let (_, head_payload_status) = fork_choice.get_head(current_slot_for_head, spec)?;
        let fork_choice_view = fork_choice.cached_fork_choice_view();
        let beacon_block_root = fork_choice_view.head_block_root;
        let beacon_block = store
            .get_full_block(&beacon_block_root)?
            .ok_or(Error::MissingBeaconBlock(beacon_block_root))?;
        let current_slot = fork_choice.fc_store().get_current_slot();

        let (_, beacon_state) = store
            .get_advanced_hot_state(beacon_block_root, current_slot, beacon_block.state_root())?
            .ok_or(Error::MissingBeaconState(beacon_block.state_root()))?;

        // Load the execution envelope from the store if the head has a Full payload.
        let execution_envelope = if head_payload_status == PayloadStatus::Full {
            store
                .get_payload_envelope(&beacon_block_root)?
                .map(Arc::new)
        } else {
            None
        };

        let snapshot = BeaconSnapshot {
            beacon_block_root,
            execution_envelope,
            beacon_block: Arc::new(beacon_block),
            beacon_state,
        };

        let forkchoice_update_params = fork_choice.get_forkchoice_update_parameters();
        let cached_head = CachedHead {
            snapshot: Arc::new(snapshot),
            justified_checkpoint: fork_choice_view.justified_checkpoint,
            finalized_checkpoint: fork_choice_view.finalized_checkpoint,
            head_payload_status,
            head_hash: forkchoice_update_params.head_hash,
            justified_hash: forkchoice_update_params.justified_hash,
            finalized_hash: forkchoice_update_params.finalized_hash,
        };

        *fork_choice_write_lock = fork_choice;
        // Avoid interleaving the fork choice and cached head locks.
        drop(fork_choice_write_lock);
        *self.cached_head.write() = cached_head;

        Ok(())
    }

    /// Returns the execution status of the block at the head of the beacon chain.
    ///
    /// This will only return `Err` in the scenario where `self.fork_choice` has advanced
    /// significantly past the cached `head_snapshot`. In such a scenario it is likely prudent to
    /// run `BeaconChain::recompute_head` to update the cached values.
    pub fn head_execution_status(&self) -> Result<ExecutionStatus, Error> {
        let head_block_root = self.cached_head().head_block_root();
        self.fork_choice_read_lock()
            .get_block_execution_status(&head_block_root)
            .ok_or(Error::HeadMissingFromForkChoice(head_block_root))
    }

    /// Returns a clone of the `CachedHead` and the execution status of the contained head block.
    ///
    /// This will only return `Err` in the scenario where `self.fork_choice` has advanced
    /// significantly past the cached `head_snapshot`. In such a scenario it is likely prudent to
    /// run `BeaconChain::recompute_head` to update the cached values.
    pub fn head_and_execution_status(
        &self,
    ) -> Result<(CachedHead<T::EthSpec>, ExecutionStatus), Error> {
        let head = self.cached_head();
        let head_block_root = head.head_block_root();
        let execution_status = self
            .fork_choice_read_lock()
            .get_block_execution_status(&head_block_root)
            .ok_or(Error::HeadMissingFromForkChoice(head_block_root))?;
        Ok((head, execution_status))
    }

    /// Returns `true` if the payload for this block is canonical (Full) according to fork choice.
    pub fn block_has_canonical_payload(
        &self,
        root: &Hash256,
        spec: &ChainSpec,
    ) -> Result<bool, Error> {
        let cached_head = self.cached_head();
        let head_root = cached_head.head_block_root();
        let head_payload_status = cached_head.head_payload_status();

        if *root == head_root {
            return Ok(head_payload_status == PayloadStatus::Full);
        }

        self.fork_choice_read_lock()
            .get_canonical_payload_status(root, spec)
            .map(|status| status == PayloadStatus::Full)
            .map_err(Error::ForkChoiceError)
    }

    /// Returns a clone of `self.cached_head`.
    ///
    /// Takes a read-lock on `self.cached_head` for a short time (just long enough to clone it).
    /// The `CachedHead` is designed to be fast-to-clone so this is preferred to passing back a
    /// `RwLockReadGuard`, which may cause deadlock issues (see module-level documentation).
    ///
    /// This function is safe to be public since it does not expose any locks.
    pub fn cached_head(&self) -> CachedHead<T::EthSpec> {
        self.cached_head_read_lock().clone()
    }

    /// Access a read-lock for the cached head.
    ///
    /// This function is **not safe** to be public. See the module-level documentation for more
    /// information about protecting from deadlocks.
    fn cached_head_read_lock(&self) -> RwLockReadGuard<'_, CachedHead<T::EthSpec>> {
        self.cached_head.read()
    }

    /// Access a write-lock for the cached head.
    ///
    /// This function is **not safe** to be public. See the module-level documentation for more
    /// information about protecting from deadlocks.
    #[instrument(skip_all)]
    fn cached_head_write_lock(&self) -> RwLockWriteGuard<'_, CachedHead<T::EthSpec>> {
        self.cached_head.write()
    }

    /// Access a read-lock for fork choice.
    pub fn fork_choice_read_lock(&self) -> RwLockReadGuard<'_, BeaconForkChoice<T>> {
        let _timer = metrics::start_timer(&metrics::FORK_CHOICE_READ_LOCK_AQUIRE_TIMES);
        self.fork_choice.read()
    }

    /// Access an upgradable read-lock for fork choice.
    pub fn fork_choice_upgradable_read_lock(
        &self,
    ) -> RwLockUpgradableReadGuard<'_, BeaconForkChoice<T>> {
        let _timer = metrics::start_timer(&metrics::FORK_CHOICE_UPGRADABLE_READ_LOCK_AQUIRE_TIMES);
        self.fork_choice.upgradable_read()
    }

    /// Access a write-lock for fork choice.
    #[instrument(skip_all)]
    pub fn fork_choice_write_lock(&self) -> RwLockWriteGuard<'_, BeaconForkChoice<T>> {
        let _timer = metrics::start_timer(&metrics::FORK_CHOICE_WRITE_LOCK_AQUIRE_TIMES);
        self.fork_choice.write()
    }
}

impl<T: BeaconChainTypes> BeaconChain<T> {
    /// Contains the "best block"; the head of the canonical `BeaconChain`.
    ///
    /// It is important to note that the `snapshot.beacon_state` returned may not match the present slot. It
    /// is the state as it was when the head block was received, which could be some slots prior to
    /// now.
    pub fn head(&self) -> CachedHead<T::EthSpec> {
        self.canonical_head.cached_head()
    }

    /// Apply a function to an `Arc`-clone of the canonical head snapshot.
    ///
    /// This method is a relic from an old implementation where the canonical head was not behind
    /// an `Arc` and the canonical head lock had to be held whenever it was read. This method is
    /// fine to be left here, it just seems a bit weird.
    pub fn with_head<U, E>(
        &self,
        f: impl FnOnce(&BeaconSnapshot<T::EthSpec>) -> Result<U, E>,
    ) -> Result<U, E>
    where
        E: From<Error>,
    {
        let head_snapshot = self.head_snapshot();
        f(&head_snapshot)
    }

    /// Returns the beacon block root at the head of the canonical chain.
    ///
    /// See `Self::head` for more information.
    pub fn head_beacon_block_root(&self) -> Hash256 {
        self.canonical_head
            .cached_head_read_lock()
            .snapshot
            .beacon_block_root
    }

    /// Returns the slot of the highest block in the canonical chain.
    pub fn best_slot(&self) -> Slot {
        self.canonical_head
            .cached_head_read_lock()
            .snapshot
            .beacon_block
            .slot()
    }

    /// Returns a `Arc` of the `BeaconSnapshot` at the head of the canonical chain.
    ///
    /// See `Self::head` for more information.
    pub fn head_snapshot(&self) -> Arc<BeaconSnapshot<T::EthSpec>> {
        self.canonical_head.cached_head_read_lock().snapshot.clone()
    }

    /// Returns the beacon block at the head of the canonical chain.
    ///
    /// See `Self::head` for more information.
    pub fn head_beacon_block(&self) -> Arc<SignedBeaconBlock<T::EthSpec>> {
        self.canonical_head
            .cached_head_read_lock()
            .snapshot
            .beacon_block
            .clone()
    }

    /// Returns a clone of the beacon state at the head of the canonical chain.
    ///
    /// Cloning the head state is expensive and should generally be avoided outside of tests.
    ///
    /// See `Self::head` for more information.
    pub fn head_beacon_state_cloned(&self) -> BeaconState<T::EthSpec> {
        // Don't clone whilst holding the read-lock, take an Arc-clone to reduce lock contention.
        let snapshot: Arc<_> = self.head_snapshot();
        snapshot.beacon_state.clone()
    }

    /// Execute the fork choice algorithm and enthrone the result as the canonical head.
    ///
    /// This method replaces the old `BeaconChain::fork_choice` method.
    #[instrument(skip_all, level = "debug")]
    pub async fn recompute_head_at_current_slot(self: &Arc<Self>) {
        match self.slot() {
            Ok(current_slot) => self.recompute_head_at_slot(current_slot).await,
            Err(e) => error!(
                error = ?e,
                "No slot when recomputing head"
            ),
        }
    }

    /// Execute the fork choice algorithm and enthrone the result as the canonical head.
    ///
    /// The `current_slot` is specified rather than relying on the wall-clock slot. Using a
    /// different slot to the wall-clock can be useful for pushing fork choice into the next slot
    /// *just* before the start of the slot. This ensures that block production can use the correct
    /// head value without being delayed.
    ///
    /// This function purposefully does *not* return a `Result`. It's possible for fork choice to
    /// fail to update if there is only one viable head and it has an invalid execution payload. In
    /// such a case it's critical that the `BeaconChain` keeps importing blocks so that the
    /// situation can be rectified. We avoid returning an error here so that calling functions
    /// can't abort block import because an error is returned here.
    #[instrument(name = "lh_recompute_head_at_slot", skip(self), level = "info", fields(slot = %current_slot))]
    pub async fn recompute_head_at_slot(self: &Arc<Self>, current_slot: Slot) {
        metrics::inc_counter(&metrics::FORK_CHOICE_REQUESTS);
        let _timer = metrics::start_timer(&metrics::FORK_CHOICE_TIMES);

        let chain = self.clone();
        match self
            .spawn_blocking_handle(
                move || chain.recompute_head_at_slot_internal(current_slot),
                "recompute_head_internal",
            )
            .await
        {
            // Fork choice returned successfully and did not need to update the EL.
            Ok(Ok(None)) => (),
            // Fork choice returned successfully and needed to update the EL. It has returned a
            // join-handle from when it spawned some async tasks. We should await those tasks.
            Ok(Ok(Some(join_handle))) => match join_handle.await {
                // The async task completed successfully.
                Ok(Some(())) => (),
                // The async task did not complete successfully since the runtime is shutting down.
                Ok(None) => {
                    debug!(info = "shutting down", "Did not update EL fork choice");
                }
                // The async task did not complete successfully, tokio returned an error.
                Err(e) => {
                    error!(
                        error = ?e,
                        "Did not update EL fork choice"
                    );
                }
            },
            // There was an error recomputing the head.
            Ok(Err(e)) => {
                metrics::inc_counter(&metrics::FORK_CHOICE_ERRORS);
                error!(
                    error = ?e,
                    "Error whist recomputing head"
                );
            }
            // There was an error spawning the task.
            Err(e) => {
                error!(
                    error = ?e,
                    "Failed to spawn recompute head task"
                );
            }
        }
    }

    /// A non-async (blocking) function which recomputes the canonical head and spawns async tasks.
    ///
    /// This function performs long-running, heavy-lifting tasks which should not be performed on
    /// the core `tokio` executor.
    fn recompute_head_at_slot_internal(
        self: &Arc<Self>,
        current_slot: Slot,
    ) -> Result<Option<JoinHandle<Option<()>>>, Error> {
        let recompute_head_lock = self.canonical_head.recompute_head_lock.lock();

        // Take a clone of the current ("old") head.
        let old_cached_head = self.canonical_head.cached_head();

        // Determine the current ("old") fork choice parameters.
        //
        // It is important to read the `fork_choice_view` from the cached head rather than from fork
        // choice, since the fork choice value might have changed between calls to this function. We
        // are interested in the changes since we last cached the head values, not since fork choice
        // was last run.
        let old_view = ForkChoiceView {
            head_block_root: old_cached_head.head_block_root(),
            justified_checkpoint: old_cached_head.justified_checkpoint(),
            finalized_checkpoint: old_cached_head.finalized_checkpoint(),
        };
        let old_payload_status = old_cached_head.head_payload_status();

        let mut fork_choice_write_lock = self.canonical_head.fork_choice_write_lock();

        // Recompute the current head via the fork choice algorithm.
        let (_, new_payload_status) = fork_choice_write_lock.get_head(current_slot, &self.spec)?;

        // Downgrade the fork choice write-lock to a read lock, without allowing access to any
        // other writers.
        let fork_choice_read_lock = RwLockWriteGuard::downgrade(fork_choice_write_lock);

        // Read the current head value from the fork choice algorithm.
        let new_view = fork_choice_read_lock.cached_fork_choice_view();

        // Check to ensure that the finalized block hasn't been marked as invalid. If it has,
        // shut down Lighthouse.
        let finalized_proto_block = fork_choice_read_lock.get_finalized_block()?;
        check_finalized_payload_validity(self, &finalized_proto_block)?;

        // Sanity check the finalized checkpoint.
        //
        // The new finalized checkpoint must be either equal to or better than the previous
        // finalized checkpoint.
        check_against_finality_reversion(&old_view, &new_view)?;

        let new_head_proto_block = fork_choice_read_lock
            .get_block(&new_view.head_block_root)
            .ok_or(Error::HeadBlockMissingFromForkChoice(
                new_view.head_block_root,
            ))?;

        // Do not allow an invalid block to become the head.
        //
        // This check avoids the following infinite loop:
        //
        // 1. A new block is set as the head.
        // 2. The EL is updated with the new head, and returns INVALID.
        // 3. We call `process_invalid_execution_payload` and it calls this function.
        // 4. This function elects an invalid block as the head.
        // 5. GOTO 2
        //
        // In theory, fork choice should never select an invalid head (i.e., step #3 is impossible).
        // However, this check is cheap.
        if new_head_proto_block.execution_status.is_invalid() {
            return Err(Error::HeadHasInvalidPayload {
                block_root: new_head_proto_block.root,
                execution_status: new_head_proto_block.execution_status,
            });
        }

        // Exit early if the head, checkpoints, and payload status have not changed.
        if new_view == old_view && new_payload_status == old_payload_status {
            debug!(
                head = ?new_view.head_block_root,
                "No change in canonical head"
            );
            return Ok(None);
        }

        // Get the parameters to update the execution layer since either the head or some finality
        // parameters have changed.
        let new_forkchoice_update_parameters =
            fork_choice_read_lock.get_forkchoice_update_parameters();

        perform_debug_logging::<T>(&old_view, &new_view, &fork_choice_read_lock);

        // Drop the read lock, it's no longer required and holding it any longer than necessary
        // will just cause lock contention.
        drop(fork_choice_read_lock);

        // If the head has changed, update `self.canonical_head`.
        let new_cached_head = if new_view.head_block_root != old_view.head_block_root
            || new_payload_status != old_payload_status
        {
            metrics::inc_counter(&metrics::FORK_CHOICE_CHANGED_HEAD);

            // TODO(gloas): could optimise this to reuse state and rest of snapshot if just the
            // payload status has changed.
            let mut new_snapshot = {
                let beacon_block = self
                    .store
                    .get_full_block(&new_view.head_block_root)?
                    .ok_or(Error::MissingBeaconBlock(new_view.head_block_root))?;

                // Load the execution envelope from the store if the head has a Full payload.
                let state_root = beacon_block.state_root();
                let execution_envelope = if new_payload_status == PayloadStatus::Full {
                    let envelope = self
                        .store
                        .get_payload_envelope(&new_view.head_block_root)?
                        .map(Arc::new)
                        .ok_or(Error::MissingExecutionPayloadEnvelope(
                            new_view.head_block_root,
                        ))?;

                    Some(envelope)
                } else {
                    None
                };
                let (_, beacon_state) = self
                    .store
                    .get_advanced_hot_state(new_view.head_block_root, current_slot, state_root)?
                    .ok_or(Error::MissingBeaconState(state_root))?;

                BeaconSnapshot {
                    beacon_block: Arc::new(beacon_block),
                    execution_envelope,
                    beacon_block_root: new_view.head_block_root,
                    beacon_state,
                }
            };

            // Regardless of where we got the state from, attempt to build all the
            // caches except the tree hash cache.
            new_snapshot.beacon_state.build_all_caches(&self.spec)?;

            let new_cached_head = CachedHead {
                snapshot: Arc::new(new_snapshot),
                justified_checkpoint: new_view.justified_checkpoint,
                finalized_checkpoint: new_view.finalized_checkpoint,
                head_payload_status: new_payload_status,
                head_hash: new_forkchoice_update_parameters.head_hash,
                justified_hash: new_forkchoice_update_parameters.justified_hash,
                finalized_hash: new_forkchoice_update_parameters.finalized_hash,
            };

            let new_head = {
                // Now the new snapshot has been obtained, take a write-lock on the cached head so
                // we can update it quickly.
                let mut cached_head_write_lock = self.canonical_head.cached_head_write_lock();
                // Enshrine the new head as the canonical cached head.
                *cached_head_write_lock = new_cached_head;
                // Take a clone of the cached head for later use. It is cloned whilst
                // holding the write-lock to ensure we get exactly the head we just enshrined.
                cached_head_write_lock.clone()
            };

            // Clear the early attester cache in case it conflicts with `self.canonical_head`.
            self.early_attester_cache.clear();

            new_head
        } else {
            let new_cached_head = CachedHead {
                // The head hasn't changed, take a relatively cheap `Arc`-clone of the existing
                // head.
                snapshot: old_cached_head.snapshot.clone(),
                justified_checkpoint: new_view.justified_checkpoint,
                finalized_checkpoint: new_view.finalized_checkpoint,
                head_payload_status: new_payload_status,
                head_hash: new_forkchoice_update_parameters.head_hash,
                justified_hash: new_forkchoice_update_parameters.justified_hash,
                finalized_hash: new_forkchoice_update_parameters.finalized_hash,
            };

            let mut cached_head_write_lock = self.canonical_head.cached_head_write_lock();

            // Enshrine the new head as the canonical cached head. Whilst the head block hasn't
            // changed, the FFG checkpoints must have changed.
            *cached_head_write_lock = new_cached_head;

            // Take a clone of the cached head for later use. It is cloned whilst
            // holding the write-lock to ensure we get exactly the head we just enshrined.
            cached_head_write_lock.clone()
        };

        // Alias for readability.
        let new_snapshot = &new_cached_head.snapshot;
        let old_snapshot = &old_cached_head.snapshot;

        // Only run on head *block* changes - payload status changes only need the
        // `cached_head` update above, not re-org detection or event emission.
        if new_snapshot.beacon_block_root != old_snapshot.beacon_block_root
            && let Err(e) =
                self.after_new_head(&old_cached_head, &new_cached_head, new_head_proto_block)
        {
            crit!(
                error = ?e,
                "Error updating canonical head"
            );
        }

        // Drop the old cache head nice and early to try and free the memory as soon as possible.
        drop(old_cached_head);

        // If the finalized checkpoint changed, perform some updates.
        //
        // The `after_finalization` function will take a write-lock on `fork_choice`, therefore it
        // is a dead-lock risk to hold any other lock on fork choice at this point.
        if new_view.finalized_checkpoint != old_view.finalized_checkpoint
            && let Err(e) =
                self.after_finalization(&new_cached_head, new_view, finalized_proto_block)
        {
            crit!(
                error = ?e,
                "Error updating finalization"
            );
        }

        // The execution layer updates might attempt to take a write-lock on fork choice, so it's
        // important to ensure the fork-choice lock isn't being held.
        let el_update_handle = spawn_execution_layer_updates(
            self.clone(),
            new_forkchoice_update_parameters,
            new_payload_status,
        )?;

        // We have completed recomputing the head and it's now valid for another process to do the
        // same.
        drop(recompute_head_lock);

        Ok(Some(el_update_handle))
    }

    /// Perform updates to caches and other components after the canonical head has been changed.
    #[instrument(skip_all)]
    fn after_new_head(
        self: &Arc<Self>,
        old_cached_head: &CachedHead<T::EthSpec>,
        new_cached_head: &CachedHead<T::EthSpec>,
        new_head_proto_block: ProtoBlock,
    ) -> Result<(), Error> {
        let _timer = metrics::start_timer(&metrics::FORK_CHOICE_AFTER_NEW_HEAD_TIMES);
        let old_snapshot = &old_cached_head.snapshot;
        let new_snapshot = &new_cached_head.snapshot;
        let new_head_is_optimistic = new_head_proto_block
            .execution_status
            .is_optimistic_or_invalid();

        // Update the state cache so it doesn't mistakenly prune the new head.
        self.store
            .state_cache
            .lock()
            .update_head_block_root(new_cached_head.head_block_root());

        // Detect and potentially report any re-orgs.
        let reorg_distance = detect_reorg(
            &old_snapshot.beacon_state,
            old_snapshot.beacon_block_root,
            &new_snapshot.beacon_state,
            new_snapshot.beacon_block_root,
            &self.spec,
        );

        // Determine if the new head is in a later epoch to the previous head.
        let is_epoch_transition = old_snapshot
            .beacon_block
            .slot()
            .epoch(T::EthSpec::slots_per_epoch())
            < new_snapshot
                .beacon_state
                .slot()
                .epoch(T::EthSpec::slots_per_epoch());

        // This field is used for server-sent events.
        let head_slot = new_snapshot.beacon_state.slot();

        match BlockShufflingIds::try_from_head(
            new_snapshot.beacon_block_root,
            &new_snapshot.beacon_state,
        ) {
            Ok(head_shuffling_ids) => self
                .shuffling_cache
                .write()
                .update_head_shuffling_ids(head_shuffling_ids),
            Err(e) => {
                error!(
                    error = ?e,
                    head_block_root = ?new_snapshot.beacon_block_root,
                    "Failed to get head shuffling ids"
                );
            }
        }

        observe_head_block_delays(
            &mut self.block_times_cache.write(),
            &new_head_proto_block,
            new_snapshot.beacon_block.message().proposer_index(),
            new_snapshot
                .beacon_block
                .message()
                .body()
                .graffiti()
                .as_utf8_lossy(),
            &self.slot_clock,
            self.event_handler.as_ref(),
            &self.spec,
        );

        if is_epoch_transition || reorg_distance.is_some() {
            self.persist_fork_choice()?;
            self.op_pool.prune_attestations(self.epoch()?);
        }

        // Register a server-sent-event for a reorg (if necessary).
        if let Some(depth) = reorg_distance
            && let Some(event_handler) = self
                .event_handler
                .as_ref()
                .filter(|handler| handler.has_reorg_subscribers())
        {
            event_handler.register(EventKind::ChainReorg(SseChainReorg {
                slot: head_slot,
                depth: depth.as_u64(),
                old_head_block: old_snapshot.beacon_block_root,
                old_head_state: old_snapshot.beacon_state_root(),
                new_head_block: new_snapshot.beacon_block_root,
                new_head_state: new_snapshot.beacon_state_root(),
                epoch: head_slot.epoch(T::EthSpec::slots_per_epoch()),
                execution_optimistic: new_head_is_optimistic,
            }));
        }

        Ok(())
    }

    /// Perform updates to caches and other components after the finalized checkpoint has been
    /// changed.
    ///
    /// This function will take a write-lock on `canonical_head.fork_choice`, therefore it would be
    /// unwise to hold any lock on fork choice while calling this function.
    #[instrument(skip_all)]
    fn after_finalization(
        self: &Arc<Self>,
        new_cached_head: &CachedHead<T::EthSpec>,
        new_view: ForkChoiceView,
        finalized_proto_block: ProtoBlock,
    ) -> Result<(), Error> {
        let _timer = metrics::start_timer(&metrics::FORK_CHOICE_AFTER_FINALIZATION_TIMES);
        let new_snapshot = &new_cached_head.snapshot;
        let finalized_block_is_optimistic = finalized_proto_block
            .execution_status
            .is_optimistic_or_invalid();

        self.observed_block_producers.write().prune(
            new_view
                .finalized_checkpoint
                .epoch
                .start_slot(T::EthSpec::slots_per_epoch()),
        );

        self.observed_column_sidecars.write().prune(
            new_view
                .finalized_checkpoint
                .epoch
                .start_slot(T::EthSpec::slots_per_epoch()),
        );

        self.observed_slashable.write().prune(
            new_view
                .finalized_checkpoint
                .epoch
                .start_slot(T::EthSpec::slots_per_epoch()),
        );

        // Prune the Gloas pending-payload cache. Anything older than the data-availability
        // boundary cannot still be in flight; finalised entries are also safe to drop.
        if self.spec.gloas_fork_epoch.is_some() {
            let finalized_epoch = new_view.finalized_checkpoint.epoch;
            let current_epoch = new_snapshot
                .beacon_state
                .slot()
                .epoch(T::EthSpec::slots_per_epoch());
            if let Some(min_epochs_for_blobs) = self
                .spec
                .min_epoch_data_availability_boundary(current_epoch)
            {
                let cutoff_epoch = std::cmp::max(finalized_epoch + 1, min_epochs_for_blobs);
                if let Err(e) = self.pending_payload_cache.do_maintenance(cutoff_epoch) {
                    error!(error = ?e, "Failed to prune pending payload cache on finalization");
                }
            }
        }

        if let Some(event_handler) = self.event_handler.as_ref()
            && event_handler.has_finalized_subscribers()
        {
            event_handler.register(EventKind::FinalizedCheckpoint(SseFinalizedCheckpoint {
                epoch: new_view.finalized_checkpoint.epoch,
                block: new_view.finalized_checkpoint.root,
                // Provide the state root of the latest finalized block, rather than the
                // specific state root at the first slot of the finalized epoch (which
                // might be a skip slot).
                state: finalized_proto_block.state_root,
                execution_optimistic: finalized_block_is_optimistic,
            }));
        }

        // The store migration task and op pool pruning require the *state at the first slot of the
        // finalized epoch*, rather than the state of the latest finalized block. These two values
        // will only differ when the first slot of the finalized epoch is a skip slot.
        let new_finalized_slot = new_view
            .finalized_checkpoint
            .epoch
            .start_slot(T::EthSpec::slots_per_epoch());
        let new_finalized_state_root = if new_finalized_slot == finalized_proto_block.slot {
            // Fast-path for the common case where the finalized state is not at a skipped slot.
            finalized_proto_block.state_root
        } else {
            // Use the `StateRootsIterator` directly rather than `BeaconChain::state_root_at_slot`
            // to ensure we use the same state that we just set as the head.
            process_results(
                StateRootsIterator::new(&self.store, &new_snapshot.beacon_state),
                |mut iter| {
                    iter.find_map(|(state_root, slot)| {
                        if slot == new_finalized_slot {
                            Some(state_root)
                        } else {
                            None
                        }
                    })
                },
            )?
            .ok_or(Error::MissingFinalizedStateRoot(new_finalized_slot))?
        };

        let update_cache = true;
        let new_finalized_state = self
            .store
            .get_hot_state(&new_finalized_state_root, update_cache)?
            .ok_or(Error::MissingBeaconState(new_finalized_state_root))?;

        self.op_pool.prune_all(
            &new_snapshot.beacon_block,
            &new_snapshot.beacon_state,
            &new_finalized_state,
            self.epoch()?,
            &self.spec,
        );

        // We just pass the state root to the finalization thread. It should be able to reload the
        // state from the state_cache near instantly anyway. We could experiment with sending the
        // state over a channel in future, but it's probably no quicker.
        self.store_migrator.process_finalization(
            new_finalized_state_root.into(),
            new_view.finalized_checkpoint,
        )?;

        // Prune blobs in the background.
        if let Some(data_availability_boundary) = self.data_availability_boundary() {
            self.store_migrator
                .process_prune_blobs(data_availability_boundary);
        }

        // Take a write-lock on the canonical head and signal for it to prune.
        self.canonical_head.fork_choice_write_lock().prune()?;

        Ok(())
    }

    /// Persist fork choice to disk, writing immediately.
    pub fn persist_fork_choice(&self) -> Result<(), Error> {
        let _fork_choice_timer = metrics::start_timer(&metrics::PERSIST_FORK_CHOICE);
        let batch = vec![self.persist_fork_choice_in_batch()?];
        self.store.hot_db.do_atomically(batch)?;
        Ok(())
    }

    /// Return a database operation for writing fork choice to disk.
    pub fn persist_fork_choice_in_batch(&self) -> Result<KeyValueStoreOp, Error> {
        Self::persist_fork_choice_in_batch_standalone(
            &self.canonical_head.fork_choice_read_lock(),
            self.store.get_config(),
        )
        .map_err(Into::into)
    }

    /// Return a database operation for writing fork choice to disk.
    pub fn persist_fork_choice_in_batch_standalone(
        fork_choice: &BeaconForkChoice<T>,
        store_config: &StoreConfig,
    ) -> Result<KeyValueStoreOp, StoreError> {
        let persisted_fork_choice = PersistedForkChoice {
            fork_choice: fork_choice.to_persisted(),
            fork_choice_store: fork_choice.fc_store().to_persisted(),
        };
        persisted_fork_choice.as_kv_store_op(FORK_CHOICE_DB_KEY, store_config)
    }
}

/// Check to see if the `finalized_proto_block` has an invalid execution payload. If so, shut down
/// Lighthouse.
///
/// ## Notes
///
/// This function is called whilst holding a write-lock on the `canonical_head`. To ensure dead-lock
/// safety, **do not take any other locks inside this function**.
#[instrument(skip_all)]
fn check_finalized_payload_validity<T: BeaconChainTypes>(
    chain: &BeaconChain<T>,
    finalized_proto_block: &ProtoBlock,
) -> Result<(), Error> {
    if let ExecutionStatus::Invalid(block_hash) = finalized_proto_block.execution_status {
        crit!(
            ?block_hash,
            msg = "You must use the `--purge-db` flag to clear the database and restart sync. \
            You may be on a hostile network.",
            "Finalized block has an invalid payload"
        );
        let mut shutdown_sender = chain.shutdown_sender();
        shutdown_sender
            .try_send(ShutdownReason::Failure(
                "Finalized block has an invalid execution payload.",
            ))
            .map_err(Error::InvalidFinalizedPayloadShutdownError)?;

        // Exit now, the node is in an invalid state.
        return Err(Error::InvalidFinalizedPayload {
            finalized_root: finalized_proto_block.root,
            execution_block_hash: block_hash,
        });
    }

    Ok(())
}

/// Check to ensure that the transition from `old_view` to `new_view` will not revert finality.
fn check_against_finality_reversion(
    old_view: &ForkChoiceView,
    new_view: &ForkChoiceView,
) -> Result<(), Error> {
    let finalization_equal = new_view.finalized_checkpoint == old_view.finalized_checkpoint;
    let finalization_advanced =
        new_view.finalized_checkpoint.epoch > old_view.finalized_checkpoint.epoch;

    if finalization_equal || finalization_advanced {
        Ok(())
    } else {
        Err(Error::RevertedFinalizedEpoch {
            old: old_view.finalized_checkpoint,
            new: new_view.finalized_checkpoint,
        })
    }
}

fn perform_debug_logging<T: BeaconChainTypes>(
    old_view: &ForkChoiceView,
    new_view: &ForkChoiceView,
    fork_choice: &BeaconForkChoice<T>,
) {
    if new_view.head_block_root != old_view.head_block_root {
        debug!(
            new_head_weight = ?fork_choice
            .get_block_weight(&new_view.head_block_root),
        new_head = ?new_view.head_block_root,
        old_head_weight = ?fork_choice
            .get_block_weight(&old_view.head_block_root),
        old_head = ?old_view.head_block_root,
            "Fork choice updated head"
        )
    }
    if new_view.justified_checkpoint != old_view.justified_checkpoint {
        debug!(
            new_root = ?new_view.justified_checkpoint.root,
            new_epoch = %new_view.justified_checkpoint.epoch,
            old_root = ?old_view.justified_checkpoint.root,
            old_epoch = %old_view.justified_checkpoint.epoch,
            "Fork choice justified"
        )
    }
    if new_view.finalized_checkpoint != old_view.finalized_checkpoint {
        debug!(
            new_root = ?new_view.finalized_checkpoint.root,
            new_epoch = %new_view.finalized_checkpoint.epoch,
            old_root = ?old_view.finalized_checkpoint.root,
            old_epoch = %old_view.finalized_checkpoint.epoch,
            "Fork choice finalized"
        )
    }
}

#[instrument(skip_all)]
fn spawn_execution_layer_updates<T: BeaconChainTypes>(
    chain: Arc<BeaconChain<T>>,
    forkchoice_update_params: ForkchoiceUpdateParameters,
    head_payload_status: PayloadStatus,
) -> Result<JoinHandle<Option<()>>, Error> {
    let current_slot = chain
        .slot_clock
        .now_or_genesis()
        .ok_or(Error::UnableToReadSlot)?;

    chain
        .task_executor
        .clone()
        .spawn_handle(
            async move {
                // Avoids raising an error before Bellatrix.
                //
                // See `Self::prepare_beacon_proposer` for more detail.
                if chain.slot_is_prior_to_bellatrix(current_slot + 1) {
                    return;
                }

                if let Err(e) = chain
                    .update_execution_engine_forkchoice(
                        current_slot,
                        forkchoice_update_params,
                        head_payload_status,
                        OverrideForkchoiceUpdate::Yes,
                    )
                    .await
                {
                    crit!(
                        error = ?e,
                        "Failed to update execution head"
                    );
                }

                // Update the mechanism for preparing for block production on the execution layer.
                //
                // Performing this call immediately after `update_execution_engine_forkchoice_blocking`
                // might result in two calls to fork choice updated, one *without* payload attributes and
                // then a second *with* payload attributes.
                //
                // This seems OK. It's not a significant waste of EL<>CL bandwidth or resources, as far as I
                // know.
                if let Err(e) = chain.prepare_beacon_proposer(current_slot).await {
                    crit!(
                        error = ?e,
                        "Failed to prepare proposers after fork choice"
                    );
                }
            },
            "update_el_forkchoice",
        )
        .ok_or(Error::RuntimeShutdown)
}

/// Attempt to detect if the new head is not on the same chain as the previous block
/// (i.e., a re-org).
///
/// Note: this will declare a re-org if we skip `SLOTS_PER_HISTORICAL_ROOT` blocks
/// between calls to fork choice without swapping between chains. This seems like an
/// extreme-enough scenario that a warning is fine.
fn detect_reorg<E: EthSpec>(
    old_state: &BeaconState<E>,
    old_block_root: Hash256,
    new_state: &BeaconState<E>,
    new_block_root: Hash256,
    spec: &ChainSpec,
) -> Option<Slot> {
    let is_reorg = new_state
        .get_block_root(old_state.slot())
        .map_or(true, |root| *root != old_block_root);

    if is_reorg {
        let reorg_distance =
            match find_reorg_slot(old_state, old_block_root, new_state, new_block_root, spec) {
                Ok(slot) => old_state.slot().saturating_sub(slot),
                Err(e) => {
                    warn!(error = ?e, "Could not find re-org depth");
                    return None;
                }
            };

        metrics::inc_counter(&metrics::FORK_CHOICE_REORG_COUNT);
        metrics::inc_counter(&metrics::FORK_CHOICE_REORG_COUNT_INTEROP);
        metrics::set_gauge(
            &metrics::FORK_CHOICE_REORG_DISTANCE,
            reorg_distance.as_u64() as i64,
        );
        info!(
            previous_head = ?old_block_root,
            previous_slot = %old_state.slot(),
            new_head = ?new_block_root,
            new_slot = %new_state.slot(),
            %reorg_distance,
            "Beacon chain re-org"
        );

        Some(reorg_distance)
    } else {
        None
    }
}

/// Iterate through the current chain to find the slot intersecting with the given beacon state.
/// The maximum depth this will search is `SLOTS_PER_HISTORICAL_ROOT`, and if that depth is reached
/// and no intersection is found, the finalized slot will be returned.
pub fn find_reorg_slot<E: EthSpec>(
    old_state: &BeaconState<E>,
    old_block_root: Hash256,
    new_state: &BeaconState<E>,
    new_block_root: Hash256,
    spec: &ChainSpec,
) -> Result<Slot, Error> {
    // The earliest slot for which the two chains may have a common history.
    let lowest_slot = std::cmp::min(new_state.slot(), old_state.slot());

    // Create an iterator across `$state`, assuming that the block at `$state.slot` has the
    // block root of `$block_root`.
    //
    // The iterator will be skipped until the next value returns `lowest_slot`.
    //
    // This is a macro instead of a function or closure due to the complex types invloved
    // in all the iterator wrapping.
    macro_rules! aligned_roots_iter {
        ($state: ident, $block_root: ident) => {
            std::iter::once(Ok(($state.slot(), $block_root)))
                .chain($state.rev_iter_block_roots(spec))
                .skip_while(|result| result.as_ref().is_ok_and(|(slot, _)| *slot > lowest_slot))
        };
    }

    // Create iterators across old/new roots where iterators both start at the same slot.
    let mut new_roots = aligned_roots_iter!(new_state, new_block_root);
    let mut old_roots = aligned_roots_iter!(old_state, old_block_root);

    // Whilst *both* of the iterators are still returning values, try and find a common
    // ancestor between them.
    while let (Some(old), Some(new)) = (old_roots.next(), new_roots.next()) {
        let (old_slot, old_root) = old?;
        let (new_slot, new_root) = new?;

        // Sanity check to detect programming errors.
        if old_slot != new_slot {
            return Err(Error::InvalidReorgSlotIter { new_slot, old_slot });
        }

        if old_root == new_root {
            // A common ancestor has been found.
            return Ok(old_slot);
        }
    }

    // If no common ancestor is found, declare that the re-org happened at the previous
    // finalized slot.
    //
    // Sometimes this will result in the return slot being *lower* than the actual reorg
    // slot. However, assuming we don't re-org through a finalized slot, it will never be
    // *higher*.
    //
    // We provide this potentially-inaccurate-but-safe information to avoid onerous
    // database reads during times of deep reorgs.
    Ok(old_state
        .finalized_checkpoint()
        .epoch
        .start_slot(E::slots_per_epoch()))
}

fn observe_head_block_delays<E: EthSpec, S: SlotClock>(
    block_times_cache: &mut BlockTimesCache,
    head_block: &ProtoBlock,
    head_block_proposer_index: u64,
    head_block_graffiti: String,
    slot_clock: &S,
    event_handler: Option<&ServerSentEventHandler<E>>,
    spec: &ChainSpec,
) {
    let Some(block_time_set_as_head) = slot_clock.now_duration() else {
        // Practically unreachable: the slot clock's time should not be before the UNIX epoch.
        return;
    };
    let head_block_root = head_block.root;
    let head_block_slot = head_block.slot;
    let head_block_is_optimistic = head_block.execution_status.is_optimistic_or_invalid();

    // Calculate the total delay between the start of the slot and when it was set as head.
    let block_delay_total = get_slot_delay_ms(block_time_set_as_head, head_block_slot, slot_clock);

    // Do not write to the cache for blocks older than 2 epochs, this helps reduce writes to
    // the cache during sync.
    if block_delay_total < slot_clock.slot_duration() * 64 {
        block_times_cache.set_time_set_as_head(
            head_block_root,
            head_block_slot,
            block_time_set_as_head,
        );
    }

    // If a block comes in from over 4 slots ago, it is most likely a block from sync.
    let block_from_sync = block_delay_total > slot_clock.slot_duration() * 4;

    // Do not store metrics if the block was > 4 slots old, this helps prevent noise during
    // sync.
    if !block_from_sync {
        // Observe the delay between when we imported the block and when we set the block as
        // head.
        let block_delays = block_times_cache.get_block_delays(
            head_block_root,
            slot_clock
                .start_of(head_block_slot)
                .unwrap_or_else(|| Duration::from_secs(0)),
        );

        // Update all the metrics

        // Convention here is to use "Time" to indicate the duration of the event and "Delay"
        // to indicate the time since the start of the slot.
        //
        // Observe the total block delay. This is the delay between the time the slot started
        // and when the block was set as head.
        metrics::set_gauge(
            &metrics::BEACON_BLOCK_DELAY_TOTAL,
            block_delay_total.as_millis() as i64,
        );

        // The time at which the beacon block was first observed to be processed
        metrics::set_gauge(
            &metrics::BEACON_BLOCK_DELAY_OBSERVED_SLOT_START,
            block_delays
                .observed
                .unwrap_or_else(|| Duration::from_secs(0))
                .as_millis() as i64,
        );

        // The time from the start of the slot when all blobs/data columns have been observed. Technically this
        // is the time we last saw a blob/data column related to this block/slot.
        metrics::set_gauge(
            &metrics::BEACON_BLOB_DELAY_ALL_OBSERVED_SLOT_START,
            block_delays
                .all_blobs_observed
                .unwrap_or_else(|| Duration::from_secs(0))
                .as_millis() as i64,
        );

        // The time it took to check the validity within Lighthouse
        metrics::set_gauge(
            &metrics::BEACON_BLOCK_DELAY_CONSENSUS_VERIFICATION_TIME,
            block_delays
                .consensus_verification_time
                .unwrap_or_else(|| Duration::from_secs(0))
                .as_millis() as i64,
        );

        // The time it took to check the validity with the EL
        metrics::set_gauge(
            &metrics::BEACON_BLOCK_DELAY_EXECUTION_TIME,
            block_delays
                .execution_time
                .unwrap_or_else(|| Duration::from_secs(0))
                .as_millis() as i64,
        );

        // The time the block became available after the start of the slot. Available here means
        // that all the blobs have arrived and the block has been verified by the execution layer.
        metrics::set_gauge(
            &metrics::BEACON_BLOCK_DELAY_AVAILABLE_SLOT_START,
            block_delays
                .available
                .unwrap_or_else(|| Duration::from_secs(0))
                .as_millis() as i64,
        );

        // The time the block became attestable after the start of the slot.
        metrics::set_gauge(
            &metrics::BEACON_BLOCK_DELAY_ATTESTABLE_SLOT_START,
            block_delays
                .attestable
                .unwrap_or_else(|| Duration::from_secs(0))
                .as_millis() as i64,
        );

        // The time the block was imported since becoming available.
        metrics::set_gauge(
            &metrics::BEACON_BLOCK_DELAY_IMPORTED_TIME,
            block_delays
                .imported
                .unwrap_or_else(|| Duration::from_secs(0))
                .as_millis() as i64,
        );

        // The time the block was imported and setting it as head
        metrics::set_gauge(
            &metrics::BEACON_BLOCK_DELAY_HEAD_IMPORTED_TIME,
            block_delays
                .set_as_head
                .unwrap_or_else(|| Duration::from_secs(0))
                .as_millis() as i64,
        );

        // Consider the block late if the time it became attestable is after the attestation
        // deadline. If the block was not made attestable, use the set-as-head time.
        let attestable_delay = block_delays.attestable.unwrap_or(block_delay_total);

        // Determine whether the block has been set as head too late for proper attestation
        // production.
        let late_head = attestable_delay >= spec.get_unaggregated_attestation_due();

        // If the block was enshrined as head too late for attestations to be created for it,
        // log a debug warning and increment a metric.
        let format_delay = |delay: &Option<Duration>| {
            delay.map_or("unknown".to_string(), |d| format!("{}", d.as_millis()))
        };
        if late_head {
            metrics::inc_counter(&metrics::BEACON_BLOCK_DELAY_HEAD_SLOT_START_EXCEEDED_TOTAL);
            debug!(
                block_root = ?head_block_root,
                proposer_index = head_block_proposer_index,
                slot = %head_block_slot,
                total_delay_ms = block_delay_total.as_millis(),
                observed_delay_ms = format_delay(&block_delays.observed),
                blob_delay_ms = format_delay(&block_delays.all_blobs_observed),
                consensus_time_ms = format_delay(&block_delays.consensus_verification_time),
                execution_time_ms = format_delay(&block_delays.execution_time),
                available_delay_ms = format_delay(&block_delays.available),
                attestable_delay_ms = format_delay(&block_delays.attestable),
                imported_time_ms = format_delay(&block_delays.imported),
                set_as_head_time_ms = format_delay(&block_delays.set_as_head),
                "Delayed head block"
            );
            if let Some(event_handler) = event_handler
                && event_handler.has_late_head_subscribers()
            {
                let peer_info = block_times_cache.get_peer_info(head_block_root);
                event_handler.register(EventKind::LateHead(SseLateHead {
                    slot: head_block_slot,
                    block: head_block_root,
                    peer_id: peer_info.id,
                    peer_client: peer_info.client,
                    proposer_index: head_block_proposer_index,
                    proposer_graffiti: head_block_graffiti,
                    block_delay: block_delay_total,
                    observed_delay: block_delays.observed,
                    imported_delay: block_delays.imported,
                    set_as_head_delay: block_delays.set_as_head,
                    execution_optimistic: head_block_is_optimistic,
                }));
            }
        } else {
            debug!(
                block_root = ?head_block_root,
                proposer_index = head_block_proposer_index,
                slot = %head_block_slot,
                total_delay_ms = block_delay_total.as_millis(),
                observed_delay_ms = format_delay(&block_delays.observed),
                blob_delay_ms = format_delay(&block_delays.all_blobs_observed),
                consensus_time_ms = format_delay(&block_delays.consensus_verification_time),
                execution_time_ms = format_delay(&block_delays.execution_time),
                available_delay_ms = format_delay(&block_delays.available),
                attestable_delay_ms = format_delay(&block_delays.attestable),
                imported_time_ms = format_delay(&block_delays.imported),
                set_as_head_time_ms = format_delay(&block_delays.set_as_head),
                "On-time head block"
            );
        }
    }
}
