Gloas cold DB (#8991)

Closes:

- https://github.com/sigp/lighthouse/issues/8958


  - Update the `HotColdStore` to handle storage of cold states.
- Update `BeaconSnapshot` to hold the execution envelope. This is required to make `chain_dump`-related checks sane, and will be generally useful (see: https://github.com/sigp/lighthouse/issues/8956).
- Bug fix in the `BlockReplayer` for the case where the starting state is already `Full` (we should not try to apply another payload). This happens on the cold DB path because we try to replay from the closest cached state (which is often full).
- Update `test_gloas_hot_state_hierarchy` to cover the cold DB migration.


Co-Authored-By: Michael Sproul <michael@sigmaprime.io>

Co-Authored-By: Michael Sproul <michaelsproul@users.noreply.github.com>

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -6689,6 +6689,9 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         let mut prev_block_root = None;
         let mut prev_beacon_state = None;
 
+        // Collect all blocks.
+        let mut blocks = vec![];
+
         for res in self.forwards_iter_block_roots(from_slot)? {
             let (beacon_block_root, _) = res?;
 
@@ -6704,16 +6707,42 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                 .ok_or_else(|| {
                     Error::DBInconsistent(format!("Missing block {}", beacon_block_root))
                 })?;
-            let beacon_state_root = beacon_block.state_root();
+            blocks.push((beacon_block_root, Arc::new(beacon_block)));
+        }
+
+        // Collect states, using the next blocks to determine if states are full (have Gloas
+        // payloads).
+        for (i, (block_root, block)) in blocks.iter().enumerate() {
+            let (opt_envelope, state_root) = if block.fork_name_unchecked().gloas_enabled() {
+                let opt_envelope = self.store.get_payload_envelope(block_root)?.map(Arc::new);
+
+                if let Some((_, next_block)) = blocks.get(i + 1) {
+                    let block_hash = block.payload_bid_block_hash()?;
+                    if next_block.is_parent_block_full(block_hash) {
+                        let envelope = opt_envelope.ok_or_else(|| {
+                            Error::DBInconsistent(format!("Missing envelope {block_root:?}"))
+                        })?;
+                        let state_root = envelope.message.state_root;
+                        (Some(envelope), state_root)
+                    } else {
+                        (None, block.state_root())
+                    }
+                } else {
+                    // TODO(gloas): should use fork choice/cached head for last block in sequence
+                    opt_envelope
+                        .as_ref()
+                        .map_or((None, block.state_root()), |envelope| {
+                            (Some(envelope.clone()), envelope.message.state_root)
+                        })
+                }
+            } else {
+                (None, block.state_root())
+            };
 
-            // This branch is reached from the HTTP API. We assume the user wants
-            // to cache states so that future calls are faster.
             let mut beacon_state = self
                 .store
-                .get_state(&beacon_state_root, Some(beacon_block.slot()), true)?
-                .ok_or_else(|| {
-                    Error::DBInconsistent(format!("Missing state {:?}", beacon_state_root))
-                })?;
+                .get_state(&state_root, Some(block.slot()), true)?
+                .ok_or_else(|| Error::DBInconsistent(format!("Missing state {:?}", state_root)))?;
 
             // This beacon state might come from the freezer DB, which means it could have pending
             // updates or lots of untethered memory. We rebase it on the previous state in order to
@@ -6726,12 +6755,14 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             prev_beacon_state = Some(beacon_state.clone());
 
             let snapshot = BeaconSnapshot {
-                beacon_block: Arc::new(beacon_block),
-                beacon_block_root,
+                beacon_block: block.clone(),
+                execution_envelope: opt_envelope,
+                beacon_block_root: *block_root,
                 beacon_state,
             };
             dump.push(snapshot);
         }
+
         Ok(dump)
     }
 

beacon_node/beacon_chain/src/beacon_snapshot.rs

@@ -2,7 +2,7 @@ use serde::Serialize;
 use std::sync::Arc;
 use types::{
     AbstractExecPayload, BeaconState, EthSpec, FullPayload, Hash256, SignedBeaconBlock,
-    SignedBlindedBeaconBlock,
+    SignedBlindedBeaconBlock, SignedExecutionPayloadEnvelope,
 };
 
 /// Represents some block and its associated state. Generally, this will be used for tracking the
@@ -10,6 +10,7 @@ use types::{
 #[derive(Clone, Serialize, PartialEq, Debug)]
 pub struct BeaconSnapshot<E: EthSpec, Payload: AbstractExecPayload<E> = FullPayload<E>> {
     pub beacon_block: Arc<SignedBeaconBlock<E, Payload>>,
+    pub execution_envelope: Option<Arc<SignedExecutionPayloadEnvelope<E>>>,
     pub beacon_block_root: Hash256,
     pub beacon_state: BeaconState<E>,
 }
@@ -31,33 +32,42 @@ impl<E: EthSpec, Payload: AbstractExecPayload<E>> BeaconSnapshot<E, Payload> {
     /// Create a new checkpoint.
     pub fn new(
         beacon_block: Arc<SignedBeaconBlock<E, Payload>>,
+        execution_envelope: Option<Arc<SignedExecutionPayloadEnvelope<E>>>,
         beacon_block_root: Hash256,
         beacon_state: BeaconState<E>,
     ) -> Self {
         Self {
             beacon_block,
+            execution_envelope,
             beacon_block_root,
             beacon_state,
         }
     }
 
-    /// Returns the state root from `self.beacon_block`.
+    /// Returns the state root from `self.beacon_block` or `self.execution_envelope` as
+    /// appropriate.
     ///
     /// ## Caution
     ///
     /// It is not strictly enforced that `root(self.beacon_state) == self.beacon_state_root()`.
     pub fn beacon_state_root(&self) -> Hash256 {
-        self.beacon_block.message().state_root()
+        if let Some(ref envelope) = self.execution_envelope {
+            envelope.message.state_root
+        } else {
+            self.beacon_block.message().state_root()
+        }
     }
 
     /// Update all fields of the checkpoint.
     pub fn update(
         &mut self,
         beacon_block: Arc<SignedBeaconBlock<E, Payload>>,
+        execution_envelope: Option<Arc<SignedExecutionPayloadEnvelope<E>>>,
         beacon_block_root: Hash256,
         beacon_state: BeaconState<E>,
     ) {
         self.beacon_block = beacon_block;
+        self.execution_envelope = execution_envelope;
         self.beacon_block_root = beacon_block_root;
         self.beacon_state = beacon_state;
     }

beacon_node/beacon_chain/src/builder.rs

@@ -358,6 +358,7 @@ where
         Ok((
             BeaconSnapshot {
                 beacon_block_root,
+                execution_envelope: None,
                 beacon_block: Arc::new(beacon_block),
                 beacon_state,
             },
@@ -616,8 +617,10 @@ where
                 .map_err(|e| format!("Failed to initialize data column info: {:?}", e))?,
         );
 
+        // TODO(gloas): add check that checkpoint state is Pending
         let snapshot = BeaconSnapshot {
             beacon_block_root: weak_subj_block_root,
+            execution_envelope: None,
             beacon_block: Arc::new(weak_subj_block),
             beacon_state: weak_subj_state,
         };
@@ -800,6 +803,7 @@ where
 
         let mut head_snapshot = BeaconSnapshot {
             beacon_block_root: head_block_root,
+            execution_envelope: None,
             beacon_block: Arc::new(head_block),
             beacon_state: head_state,
         };

beacon_node/beacon_chain/src/canonical_head.rs

@@ -319,6 +319,7 @@ impl<T: BeaconChainTypes> CanonicalHead<T> {
 
         let snapshot = BeaconSnapshot {
             beacon_block_root,
+            execution_envelope: None,
             beacon_block: Arc::new(beacon_block),
             beacon_state,
         };
@@ -695,6 +696,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
 
                 BeaconSnapshot {
                     beacon_block: Arc::new(beacon_block),
+                    execution_envelope: None,
                     beacon_block_root: new_view.head_block_root,
                     beacon_state,
                 }