Implement PeerDAS subnet decoupling (aka custody groups) (#6736)

* Implement PeerDAS subnet decoupling (aka custody groups).

* Merge branch 'unstable' into decouple-subnets

* Refactor feature testing for spec tests (#6737)

Squashed commit of the following:

commit 898d05ee17
Merge: ffbd25e2b 7e0cddef3
Author: Jimmy Chen <jchen.tc@gmail.com>
Date:   Tue Dec 24 14:41:19 2024 +1100

    Merge branch 'unstable' into refactor-ef-tests-features

commit ffbd25e2be
Author: Jimmy Chen <jchen.tc@gmail.com>
Date:   Tue Dec 24 14:40:38 2024 +1100

    Fix `SszStatic` tests for PeerDAS: exclude eip7594 test vectors when testing Electra types.

commit aa593cf35c
Author: Jimmy Chen <jchen.tc@gmail.com>
Date:   Fri Dec 20 12:08:54 2024 +1100

    Refactor spec testing for features and simplify usage.

* Fix build.

* Add input validation and improve arithmetic handling when calculating custody groups.

* Address review comments re code style consistency.

* Merge branch 'unstable' into decouple-subnets

# Conflicts:
#	beacon_node/beacon_chain/src/kzg_utils.rs
#	beacon_node/beacon_chain/src/observed_data_sidecars.rs
#	beacon_node/lighthouse_network/src/discovery/subnet_predicate.rs
#	common/eth2_network_config/built_in_network_configs/chiado/config.yaml
#	common/eth2_network_config/built_in_network_configs/gnosis/config.yaml
#	common/eth2_network_config/built_in_network_configs/holesky/config.yaml
#	common/eth2_network_config/built_in_network_configs/mainnet/config.yaml
#	common/eth2_network_config/built_in_network_configs/sepolia/config.yaml
#	consensus/types/src/chain_spec.rs

* Update consensus/types/src/chain_spec.rs

Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com>

* Merge remote-tracking branch 'origin/unstable' into decouple-subnets

* Update error handling.

* Address review comment.

* Merge remote-tracking branch 'origin/unstable' into decouple-subnets

# Conflicts:
#	consensus/types/src/chain_spec.rs

* Update PeerDAS spec tests to `1.5.0-beta.0` and fix failing unit tests.

* Merge remote-tracking branch 'origin/unstable' into decouple-subnets

# Conflicts:
#	beacon_node/lighthouse_network/src/peer_manager/mod.rs

testing/ef_tests/src/cases.rs

@@ -13,12 +13,13 @@ mod bls_fast_aggregate_verify;
 mod bls_sign_msg;
 mod bls_verify_msg;
 mod common;
+mod compute_columns_for_custody_groups;
 mod epoch_processing;
 mod fork;
 mod fork_choice;
 mod genesis_initialization;
 mod genesis_validity;
-mod get_custody_columns;
+mod get_custody_groups;
 mod kzg_blob_to_kzg_commitment;
 mod kzg_compute_blob_kzg_proof;
 mod kzg_compute_cells_and_kzg_proofs;
@@ -49,11 +50,12 @@ pub use bls_fast_aggregate_verify::*;
 pub use bls_sign_msg::*;
 pub use bls_verify_msg::*;
 pub use common::SszStaticType;
+pub use compute_columns_for_custody_groups::*;
 pub use epoch_processing::*;
 pub use fork::ForkTest;
 pub use genesis_initialization::*;
 pub use genesis_validity::*;
-pub use get_custody_columns::*;
+pub use get_custody_groups::*;
 pub use kzg_blob_to_kzg_commitment::*;
 pub use kzg_compute_blob_kzg_proof::*;
 pub use kzg_compute_cells_and_kzg_proofs::*;
@@ -89,18 +91,18 @@ pub use transition::TransitionTest;
 ///     to return `true` for the feature in order for the feature test vector to be tested.
 #[derive(Debug, PartialEq, Clone, Copy)]
 pub enum FeatureName {
-    Eip7594,
+    Fulu,
 }
 
 impl FeatureName {
     pub fn list_all() -> Vec<FeatureName> {
-        vec![FeatureName::Eip7594]
+        vec![FeatureName::Fulu]
     }
 
     /// `ForkName` to use when running the feature tests.
     pub fn fork_name(&self) -> ForkName {
         match self {
-            FeatureName::Eip7594 => ForkName::Deneb,
+            FeatureName::Fulu => ForkName::Electra,
         }
     }
 }
@@ -108,7 +110,7 @@ impl FeatureName {
 impl Display for FeatureName {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         match self {
-            FeatureName::Eip7594 => f.write_str("eip7594"),
+            FeatureName::Fulu => f.write_str("fulu"),
         }
     }
 }

testing/ef_tests/src/cases/compute_columns_for_custody_groups.rs

@@ -0,0 +1,47 @@
+use super::*;
+use serde::Deserialize;
+use std::marker::PhantomData;
+use types::data_column_custody_group::{compute_columns_for_custody_group, CustodyIndex};
+
+#[derive(Debug, Clone, Deserialize)]
+#[serde(bound = "E: EthSpec", deny_unknown_fields)]
+pub struct ComputeColumnsForCustodyGroups<E: EthSpec> {
+    /// The custody group index.
+    pub custody_group: CustodyIndex,
+    /// The list of resulting custody columns.
+    pub result: Vec<u64>,
+    #[serde(skip)]
+    _phantom: PhantomData<E>,
+}
+
+impl<E: EthSpec> LoadCase for ComputeColumnsForCustodyGroups<E> {
+    fn load_from_dir(path: &Path, _fork_name: ForkName) -> Result<Self, Error> {
+        decode::yaml_decode_file(path.join("meta.yaml").as_path())
+    }
+}
+
+impl<E: EthSpec> Case for ComputeColumnsForCustodyGroups<E> {
+    fn is_enabled_for_fork(_fork_name: ForkName) -> bool {
+        false
+    }
+
+    fn is_enabled_for_feature(feature_name: FeatureName) -> bool {
+        feature_name == FeatureName::Fulu
+    }
+
+    fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
+        let spec = E::default_spec();
+        let computed_columns = compute_columns_for_custody_group(self.custody_group, &spec)
+            .expect("should compute custody columns from group")
+            .collect::<Vec<_>>();
+
+        let expected = &self.result;
+        if computed_columns == *expected {
+            Ok(())
+        } else {
+            Err(Error::NotEqual(format!(
+                "Got {computed_columns:?}\nExpected {expected:?}"
+            )))
+        }
+    }
+}

testing/ef_tests/src/cases/get_custody_groups.rs

@@ -2,31 +2,34 @@ use super::*;
 use alloy_primitives::U256;
 use serde::Deserialize;
 use std::marker::PhantomData;
-use types::DataColumnSubnetId;
+use types::data_column_custody_group::get_custody_groups;
 
 #[derive(Debug, Clone, Deserialize)]
 #[serde(bound = "E: EthSpec", deny_unknown_fields)]
-pub struct GetCustodyColumns<E: EthSpec> {
+pub struct GetCustodyGroups<E: EthSpec> {
+    /// The NodeID input.
     pub node_id: String,
-    pub custody_subnet_count: u64,
+    /// The count of custody groups.
+    pub custody_group_count: u64,
+    /// The list of resulting custody groups.
     pub result: Vec<u64>,
     #[serde(skip)]
     _phantom: PhantomData<E>,
 }
 
-impl<E: EthSpec> LoadCase for GetCustodyColumns<E> {
+impl<E: EthSpec> LoadCase for GetCustodyGroups<E> {
     fn load_from_dir(path: &Path, _fork_name: ForkName) -> Result<Self, Error> {
         decode::yaml_decode_file(path.join("meta.yaml").as_path())
     }
 }
 
-impl<E: EthSpec> Case for GetCustodyColumns<E> {
+impl<E: EthSpec> Case for GetCustodyGroups<E> {
     fn is_enabled_for_fork(_fork_name: ForkName) -> bool {
         false
     }
 
     fn is_enabled_for_feature(feature_name: FeatureName) -> bool {
-        feature_name == FeatureName::Eip7594
+        feature_name == FeatureName::Fulu
     }
 
     fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
@@ -34,13 +37,10 @@ impl<E: EthSpec> Case for GetCustodyColumns<E> {
         let node_id = U256::from_str_radix(&self.node_id, 10)
             .map_err(|e| Error::FailedToParseTest(format!("{e:?}")))?;
         let raw_node_id = node_id.to_be_bytes::<32>();
-        let computed = DataColumnSubnetId::compute_custody_columns::<E>(
-            raw_node_id,
-            self.custody_subnet_count,
-            &spec,
-        )
-        .expect("should compute custody columns")
-        .collect::<Vec<_>>();
+        let mut computed = get_custody_groups(raw_node_id, self.custody_group_count, &spec)
+            .map(|set| set.into_iter().collect::<Vec<_>>())
+            .expect("should compute custody groups");
+        computed.sort();
 
         let expected = &self.result;
         if computed == *expected {

testing/ef_tests/src/cases/kzg_compute_cells_and_kzg_proofs.rs

@@ -31,7 +31,7 @@ impl<E: EthSpec> Case for KZGComputeCellsAndKZGProofs<E> {
     }
 
     fn is_enabled_for_feature(feature_name: FeatureName) -> bool {
-        feature_name == FeatureName::Eip7594
+        feature_name == FeatureName::Fulu
     }
 
     fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {

testing/ef_tests/src/cases/kzg_recover_cells_and_kzg_proofs.rs

@@ -32,7 +32,7 @@ impl<E: EthSpec> Case for KZGRecoverCellsAndKZGProofs<E> {
     }
 
     fn is_enabled_for_feature(feature_name: FeatureName) -> bool {
-        feature_name == FeatureName::Eip7594
+        feature_name == FeatureName::Fulu
     }
 
     fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {

testing/ef_tests/src/cases/kzg_verify_cell_kzg_proof_batch.rs

@@ -34,7 +34,7 @@ impl<E: EthSpec> Case for KZGVerifyCellKZGProofBatch<E> {
     }
 
     fn is_enabled_for_feature(feature_name: FeatureName) -> bool {
-        feature_name == FeatureName::Eip7594
+        feature_name == FeatureName::Fulu
     }
 
     fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {

testing/ef_tests/src/handler.rs

@@ -39,6 +39,10 @@ pub trait Handler {
             }
         }
 
+        // Run feature tests for future forks that are not yet added to `ForkName`.
+        // This runs tests in the directory named by the feature instead of the fork name.
+        // e.g. consensus-spec-tests/tests/general/[feature_name]/[runner_name]
+        // e.g. consensus-spec-tests/tests/general/peerdas/ssz_static
         for feature_name in FeatureName::list_all() {
             if self.is_enabled_for_feature(feature_name) {
                 self.run_for_feature(feature_name);
@@ -350,23 +354,20 @@ where
         self.supported_forks.contains(&fork_name)
     }
 
-    fn is_enabled_for_feature(&self, _feature_name: FeatureName) -> bool {
-        // This ensures we only run the tests **once** for `Eip7594`, using the types matching the
-        // correct fork, e.g. `Eip7594` uses SSZ types from `Deneb` as of spec test version
-        // `v1.5.0-alpha.8`, therefore the `Eip7594` tests should get included when testing Deneb types.
+    fn is_enabled_for_feature(&self, feature_name: FeatureName) -> bool {
+        // This ensures we only run the tests **once** for the feature, using the types matching the
+        // correct fork, e.g. `Fulu` uses SSZ types from `Electra` fork as of spec test version
+        // `v1.5.0-beta.0`, therefore the `Fulu` tests should get included when testing Electra types.
         //
-        // e.g. Eip7594 test vectors are executed in the first line below, but excluded in the 2nd
+        // e.g. Fulu test vectors are executed in the first line below, but excluded in the 2nd
         // line when testing the type `AttestationElectra`:
         //
         // ```
         // SszStaticHandler::<AttestationBase<MainnetEthSpec>, MainnetEthSpec>::pre_electra().run();
         // SszStaticHandler::<AttestationElectra<MainnetEthSpec>, MainnetEthSpec>::electra_only().run();
         // ```
-        /* TODO(das): re-enable
-        feature_name == FeatureName::Eip7594
+        feature_name == FeatureName::Fulu
             && self.supported_forks.contains(&feature_name.fork_name())
-        */
-        false
     }
 }
 
@@ -388,10 +389,8 @@ where
         BeaconState::<E>::name().into()
     }
 
-    fn is_enabled_for_feature(&self, _feature_name: FeatureName) -> bool {
-        // TODO(das): re-enable
-        // feature_name == FeatureName::Eip7594
-        false
+    fn is_enabled_for_feature(&self, feature_name: FeatureName) -> bool {
+        feature_name == FeatureName::Fulu
     }
 }
 
@@ -415,10 +414,8 @@ where
         T::name().into()
     }
 
-    fn is_enabled_for_feature(&self, _feature_name: FeatureName) -> bool {
-        // TODO(das): re-enable
-        // feature_name == FeatureName::Eip7594
-        false
+    fn is_enabled_for_feature(&self, feature_name: FeatureName) -> bool {
+        feature_name == FeatureName::Fulu
     }
 }
 
@@ -877,10 +874,10 @@ impl<E: EthSpec> Handler for KZGVerifyKZGProofHandler<E> {
 
 #[derive(Derivative)]
 #[derivative(Default(bound = ""))]
-pub struct GetCustodyColumnsHandler<E>(PhantomData<E>);
+pub struct GetCustodyGroupsHandler<E>(PhantomData<E>);
 
-impl<E: EthSpec + TypeName> Handler for GetCustodyColumnsHandler<E> {
-    type Case = cases::GetCustodyColumns<E>;
+impl<E: EthSpec + TypeName> Handler for GetCustodyGroupsHandler<E> {
+    type Case = cases::GetCustodyGroups<E>;
 
     fn config_name() -> &'static str {
         E::name()
@@ -891,7 +888,27 @@ impl<E: EthSpec + TypeName> Handler for GetCustodyColumnsHandler<E> {
     }
 
     fn handler_name(&self) -> String {
-        "get_custody_columns".into()
+        "get_custody_groups".into()
+    }
+}
+
+#[derive(Derivative)]
+#[derivative(Default(bound = ""))]
+pub struct ComputeColumnsForCustodyGroupHandler<E>(PhantomData<E>);
+
+impl<E: EthSpec + TypeName> Handler for ComputeColumnsForCustodyGroupHandler<E> {
+    type Case = cases::ComputeColumnsForCustodyGroups<E>;
+
+    fn config_name() -> &'static str {
+        E::name()
+    }
+
+    fn runner_name() -> &'static str {
+        "networking"
+    }
+
+    fn handler_name(&self) -> String {
+        "compute_columns_for_custody_group".into()
     }
 }
 
@@ -1002,10 +1019,8 @@ impl<E: EthSpec + TypeName> Handler for KzgInclusionMerkleProofValidityHandler<E
         fork_name.deneb_enabled()
     }
 
-    fn is_enabled_for_feature(&self, _feature_name: FeatureName) -> bool {
-        // TODO(das): re-enable this
-        // feature_name == FeatureName::Eip7594
-        false
+    fn is_enabled_for_feature(&self, feature_name: FeatureName) -> bool {
+        feature_name == FeatureName::Fulu
     }
 }