Merge branch 'gloas-containers' into gloas_envelope_processing_merge_containers

beacon_node/http_api/src/custody.rs

@@ -0,0 +1,53 @@
+use beacon_chain::{BeaconChain, BeaconChainTypes};
+use eth2::lighthouse::CustodyInfo;
+use std::sync::Arc;
+use types::EthSpec;
+use warp_utils::reject::{custom_bad_request, custom_server_error};
+
+pub fn info<T: BeaconChainTypes>(
+    chain: Arc<BeaconChain<T>>,
+) -> Result<CustodyInfo, warp::Rejection> {
+    if !chain.spec.is_fulu_scheduled() {
+        return Err(custom_bad_request("Fulu is not scheduled".to_string()));
+    }
+
+    let opt_data_column_custody_info = chain
+        .store
+        .get_data_column_custody_info()
+        .map_err(|e| custom_server_error(format!("error reading DataColumnCustodyInfo: {e:?}")))?;
+
+    let column_data_availability_boundary = chain
+        .column_data_availability_boundary()
+        .ok_or_else(|| custom_server_error("unreachable: Fulu should be enabled".to_string()))?;
+
+    let earliest_custodied_data_column_slot = opt_data_column_custody_info
+        .and_then(|info| info.earliest_data_column_slot)
+        .unwrap_or_else(|| {
+            // If there's no data column custody info/earliest data column slot, it means *column*
+            // backfill is not running. Block backfill could still be running, so our earliest
+            // available column is either the oldest block slot or the DA boundary, whichever is
+            // more recent.
+            let oldest_block_slot = chain.store.get_anchor_info().oldest_block_slot;
+            column_data_availability_boundary
+                .start_slot(T::EthSpec::slots_per_epoch())
+                .max(oldest_block_slot)
+        });
+    let earliest_custodied_data_column_epoch =
+        earliest_custodied_data_column_slot.epoch(T::EthSpec::slots_per_epoch());
+
+    // Compute the custody columns and the CGC *at the earliest custodied slot*. The node might
+    // have some columns prior to this, but this value is the most up-to-date view of the data the
+    // node is custodying.
+    let custody_context = chain.data_availability_checker.custody_context();
+    let custody_columns = custody_context
+        .custody_columns_for_epoch(Some(earliest_custodied_data_column_epoch), &chain.spec)
+        .to_vec();
+    let custody_group_count = custody_context
+        .custody_group_count_at_epoch(earliest_custodied_data_column_epoch, &chain.spec);
+
+    Ok(CustodyInfo {
+        earliest_custodied_data_column_slot,
+        custody_group_count,
+        custody_columns,
+    })
+}

beacon_node/http_api/src/lib.rs

@@ -13,6 +13,7 @@ mod block_packing_efficiency;
 mod block_rewards;
 mod build_block_contents;
 mod builder_states;
+mod custody;
 mod database;
 mod light_client;
 mod metrics;
@@ -4590,6 +4591,50 @@ pub fn serve<T: BeaconChainTypes>(
             },
         );
 
+    // GET lighthouse/custody/info
+    let get_lighthouse_custody_info = warp::path("lighthouse")
+        .and(warp::path("custody"))
+        .and(warp::path("info"))
+        .and(warp::path::end())
+        .and(task_spawner_filter.clone())
+        .and(chain_filter.clone())
+        .then(
+            |task_spawner: TaskSpawner<T::EthSpec>, chain: Arc<BeaconChain<T>>| {
+                task_spawner.blocking_json_task(Priority::P1, move || custody::info(chain))
+            },
+        );
+
+    // POST lighthouse/custody/backfill
+    let post_lighthouse_custody_backfill = warp::path("lighthouse")
+        .and(warp::path("custody"))
+        .and(warp::path("backfill"))
+        .and(warp::path::end())
+        .and(task_spawner_filter.clone())
+        .and(chain_filter.clone())
+        .then(
+            |task_spawner: TaskSpawner<T::EthSpec>, chain: Arc<BeaconChain<T>>| {
+                task_spawner.blocking_json_task(Priority::P1, move || {
+                    // Calling this endpoint will trigger custody backfill once `effective_epoch``
+                    // is finalized.
+                    let effective_epoch = chain
+                        .canonical_head
+                        .cached_head()
+                        .head_slot()
+                        .epoch(T::EthSpec::slots_per_epoch())
+                        + 1;
+                    let custody_context = chain.data_availability_checker.custody_context();
+                    // Reset validator custody requirements to `effective_epoch` with the latest
+                    // cgc requiremnets.
+                    custody_context.reset_validator_custody_requirements(effective_epoch);
+                    // Update `DataColumnCustodyInfo` to reflect the custody change.
+                    chain.update_data_column_custody_info(Some(
+                        effective_epoch.start_slot(T::EthSpec::slots_per_epoch()),
+                    ));
+                    Ok(())
+                })
+            },
+        );
+
     // GET lighthouse/analysis/block_rewards
     let get_lighthouse_block_rewards = warp::path("lighthouse")
         .and(warp::path("analysis"))
@@ -4891,6 +4936,7 @@ pub fn serve<T: BeaconChainTypes>(
                 .uor(get_lighthouse_validator_inclusion)
                 .uor(get_lighthouse_staking)
                 .uor(get_lighthouse_database_info)
+                .uor(get_lighthouse_custody_info)
                 .uor(get_lighthouse_block_rewards)
                 .uor(get_lighthouse_attestation_performance)
                 .uor(get_beacon_light_client_optimistic_update)
@@ -4948,6 +4994,7 @@ pub fn serve<T: BeaconChainTypes>(
                     .uor(post_lighthouse_compaction)
                     .uor(post_lighthouse_add_peer)
                     .uor(post_lighthouse_remove_peer)
+                    .uor(post_lighthouse_custody_backfill)
                     .recover(warp_utils::reject::handle_rejection),
             ),
         )

beacon_node/http_api/src/proposer_duties.rs

@@ -60,13 +60,13 @@ pub fn proposer_duties<T: BeaconChainTypes>(
             .safe_add(1)
             .map_err(warp_utils::reject::arith_error)?
     {
-        let (proposers, dependent_root, execution_status, _fork) =
+        let (proposers, _dependent_root, legacy_dependent_root, execution_status, _fork) =
             compute_proposer_duties_from_head(request_epoch, chain)
                 .map_err(warp_utils::reject::unhandled_error)?;
         convert_to_api_response(
             chain,
             request_epoch,
-            dependent_root,
+            legacy_dependent_root,
             execution_status.is_optimistic_or_invalid(),
             proposers,
         )
@@ -116,6 +116,11 @@ fn try_proposer_duties_from_cache<T: BeaconChainTypes>(
         .beacon_state
         .proposer_shuffling_decision_root_at_epoch(request_epoch, head_block_root, &chain.spec)
         .map_err(warp_utils::reject::beacon_state_error)?;
+    let legacy_dependent_root = head
+        .snapshot
+        .beacon_state
+        .legacy_proposer_shuffling_decision_root_at_epoch(request_epoch, head_block_root)
+        .map_err(warp_utils::reject::beacon_state_error)?;
     let execution_optimistic = chain
         .is_optimistic_or_invalid_head_block(head_block)
         .map_err(warp_utils::reject::unhandled_error)?;
@@ -129,7 +134,7 @@ fn try_proposer_duties_from_cache<T: BeaconChainTypes>(
             convert_to_api_response(
                 chain,
                 request_epoch,
-                head_decision_root,
+                legacy_dependent_root,
                 execution_optimistic,
                 indices.to_vec(),
             )
@@ -151,7 +156,7 @@ fn compute_and_cache_proposer_duties<T: BeaconChainTypes>(
     current_epoch: Epoch,
     chain: &BeaconChain<T>,
 ) -> Result<ApiDuties, warp::reject::Rejection> {
-    let (indices, dependent_root, execution_status, fork) =
+    let (indices, dependent_root, legacy_dependent_root, execution_status, fork) =
         compute_proposer_duties_from_head(current_epoch, chain)
             .map_err(warp_utils::reject::unhandled_error)?;
 
@@ -166,7 +171,7 @@ fn compute_and_cache_proposer_duties<T: BeaconChainTypes>(
     convert_to_api_response(
         chain,
         current_epoch,
-        dependent_root,
+        legacy_dependent_root,
         execution_status.is_optimistic_or_invalid(),
         indices,
     )
@@ -229,12 +234,18 @@ fn compute_historic_proposer_duties<T: BeaconChainTypes>(
 
     // We can supply the genesis block root as the block root since we know that the only block that
     // decides its own root is the genesis block.
-    let dependent_root = state
-        .proposer_shuffling_decision_root(chain.genesis_block_root, &chain.spec)
+    let legacy_dependent_root = state
+        .legacy_proposer_shuffling_decision_root_at_epoch(epoch, chain.genesis_block_root)
         .map_err(BeaconChainError::from)
         .map_err(warp_utils::reject::unhandled_error)?;
 
-    convert_to_api_response(chain, epoch, dependent_root, execution_optimistic, indices)
+    convert_to_api_response(
+        chain,
+        epoch,
+        legacy_dependent_root,
+        execution_optimistic,
+        indices,
+    )
 }
 
 /// Converts the internal representation of proposer duties into one that is compatible with the

beacon_node/http_api/src/test_utils.rs

@@ -1,6 +1,7 @@
 use crate::{Config, Context};
 use beacon_chain::{
     BeaconChain, BeaconChainTypes,
+    custody_context::NodeCustodyType,
     test_utils::{BeaconChainHarness, BoxedMutator, Builder, EphemeralHarnessType},
 };
 use beacon_processor::{
@@ -67,6 +68,20 @@ impl<E: EthSpec> InteractiveTester<E> {
             None,
             Config::default(),
             true,
+            NodeCustodyType::Fullnode,
+        )
+        .await
+    }
+
+    pub async fn new_supernode(spec: Option<ChainSpec>, validator_count: usize) -> Self {
+        Self::new_with_initializer_and_mutator(
+            spec,
+            validator_count,
+            None,
+            None,
+            Config::default(),
+            true,
+            NodeCustodyType::Supernode,
         )
         .await
     }
@@ -78,6 +93,7 @@ impl<E: EthSpec> InteractiveTester<E> {
         mutator: Option<Mutator<E>>,
         config: Config,
         use_mock_builder: bool,
+        node_custody_type: NodeCustodyType,
     ) -> Self {
         let mut harness_builder = BeaconChainHarness::builder(E::default())
             .spec_or_default(spec.map(Arc::new))
@@ -93,6 +109,8 @@ impl<E: EthSpec> InteractiveTester<E> {
                 .fresh_ephemeral_store()
         };
 
+        harness_builder = harness_builder.node_custody_type(node_custody_type);
+
         // Add a mutator for the beacon chain builder which will be called in
         // `HarnessBuilder::build`.
         if let Some(mutator) = mutator {