gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-05-07 16:55:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement PeerDAS subnet decoupling (aka custody groups) (#6736)

Browse Source 
* Implement PeerDAS subnet decoupling (aka custody groups).

* Merge branch 'unstable' into decouple-subnets

* Refactor feature testing for spec tests (#6737)

Squashed commit of the following:

commit 898d05ee17
Merge: ffbd25e2b 7e0cddef3
Author: Jimmy Chen <jchen.tc@gmail.com>
Date:   Tue Dec 24 14:41:19 2024 +1100

    Merge branch 'unstable' into refactor-ef-tests-features

commit ffbd25e2be
Author: Jimmy Chen <jchen.tc@gmail.com>
Date:   Tue Dec 24 14:40:38 2024 +1100

    Fix `SszStatic` tests for PeerDAS: exclude eip7594 test vectors when testing Electra types.

commit aa593cf35c
Author: Jimmy Chen <jchen.tc@gmail.com>
Date:   Fri Dec 20 12:08:54 2024 +1100

    Refactor spec testing for features and simplify usage.

* Fix build.

* Add input validation and improve arithmetic handling when calculating custody groups.

* Address review comments re code style consistency.

* Merge branch 'unstable' into decouple-subnets

# Conflicts:
#	beacon_node/beacon_chain/src/kzg_utils.rs
#	beacon_node/beacon_chain/src/observed_data_sidecars.rs
#	beacon_node/lighthouse_network/src/discovery/subnet_predicate.rs
#	common/eth2_network_config/built_in_network_configs/chiado/config.yaml
#	common/eth2_network_config/built_in_network_configs/gnosis/config.yaml
#	common/eth2_network_config/built_in_network_configs/holesky/config.yaml
#	common/eth2_network_config/built_in_network_configs/mainnet/config.yaml
#	common/eth2_network_config/built_in_network_configs/sepolia/config.yaml
#	consensus/types/src/chain_spec.rs

* Update consensus/types/src/chain_spec.rs

Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com>

* Merge remote-tracking branch 'origin/unstable' into decouple-subnets

* Update error handling.

* Address review comment.

* Merge remote-tracking branch 'origin/unstable' into decouple-subnets

# Conflicts:
#	consensus/types/src/chain_spec.rs

* Update PeerDAS spec tests to `1.5.0-beta.0` and fix failing unit tests.

* Merge remote-tracking branch 'origin/unstable' into decouple-subnets

# Conflicts:
#	beacon_node/lighthouse_network/src/peer_manager/mod.rs
This commit is contained in:
Jimmy Chen
2025-01-15 18:40:26 +11:00
committed by GitHub
parent dd7591f712
commit e98209d118
39 changed files with 552 additions and 430 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
consensus/types/src/chain_spec.rs
View File

@@ -204,10 +204,11 @@ pub struct ChainSpec {
* DAS params
*/
pub eip7594_fork_epoch: Option<Epoch>,
pub custody_requirement: u64,
pub number_of_columns: u64,
pub number_of_custody_groups: u64,
pub data_column_sidecar_subnet_count: u64,
pub number_of_columns: usize,
pub samples_per_slot: u64,
pub custody_requirement: u64,
/*
* Networking
@@ -237,7 +238,7 @@ pub struct ChainSpec {
pub max_request_data_column_sidecars: u64,
pub min_epochs_for_blob_sidecars_requests: u64,
pub blob_sidecar_subnet_count: u64,
max_blobs_per_block: u64,
pub max_blobs_per_block: u64,
/*
* Networking Electra
@@ -646,10 +647,33 @@ impl ChainSpec {
}
}
pub fn data_columns_per_subnet(&self) -> usize {
/// Returns the number of data columns per custody group.
pub fn data_columns_per_group(&self) -> u64 {
self.number_of_columns
.safe_div(self.data_column_sidecar_subnet_count as usize)
.expect("Subnet count must be greater than 0")
.safe_div(self.number_of_custody_groups)
.expect("Custody group count must be greater than 0")
}
/// Returns the number of column sidecars to sample per slot.
pub fn sampling_size(&self, custody_group_count: u64) -> Result<u64, String> {
let columns_per_custody_group = self
.number_of_columns
.safe_div(self.number_of_custody_groups)
.map_err(|_| "number_of_custody_groups must be greater than 0")?;
let custody_column_count = columns_per_custody_group
.safe_mul(custody_group_count)
.map_err(|_| "Computing sampling size should not overflow")?;
Ok(std::cmp::max(custody_column_count, self.samples_per_slot))
}
pub fn custody_group_count(&self, is_supernode: bool) -> u64 {
if is_supernode {
self.number_of_custody_groups
} else {
self.custody_requirement
}
}
/// Returns a `ChainSpec` compatible with the Ethereum Foundation specification.
@@ -856,10 +880,11 @@ impl ChainSpec {
* DAS params
*/
eip7594_fork_epoch: None,
custody_requirement: 4,
data_column_sidecar_subnet_count: 128,
number_of_columns: 128,
number_of_custody_groups: 128,
data_column_sidecar_subnet_count: 128,
samples_per_slot: 8,
custody_requirement: 4,
/*
* Network specific
@@ -1193,10 +1218,12 @@ impl ChainSpec {
* DAS params
*/
eip7594_fork_epoch: None,
custody_requirement: 4,
data_column_sidecar_subnet_count: 128,
number_of_columns: 128,
number_of_custody_groups: 128,
data_column_sidecar_subnet_count: 128,
samples_per_slot: 8,
custody_requirement: 4,
/*
* Network specific
*/
@@ -1454,18 +1481,21 @@ pub struct Config {
#[serde(with = "serde_utils::quoted_u64")]
max_request_blob_sidecars_electra: u64,
#[serde(default = "default_custody_requirement")]
#[serde(with = "serde_utils::quoted_u64")]
custody_requirement: u64,
#[serde(default = "default_data_column_sidecar_subnet_count")]
#[serde(with = "serde_utils::quoted_u64")]
data_column_sidecar_subnet_count: u64,
#[serde(default = "default_number_of_columns")]
#[serde(with = "serde_utils::quoted_u64")]
number_of_columns: u64,
#[serde(default = "default_number_of_custody_groups")]
#[serde(with = "serde_utils::quoted_u64")]
number_of_custody_groups: u64,
#[serde(default = "default_data_column_sidecar_subnet_count")]
#[serde(with = "serde_utils::quoted_u64")]
data_column_sidecar_subnet_count: u64,
#[serde(default = "default_samples_per_slot")]
#[serde(with = "serde_utils::quoted_u64")]
samples_per_slot: u64,
#[serde(default = "default_custody_requirement")]
#[serde(with = "serde_utils::quoted_u64")]
custody_requirement: u64,
}
fn default_bellatrix_fork_version() -> [u8; 4] {
@@ -1627,6 +1657,10 @@ const fn default_number_of_columns() -> u64 {
128
}
const fn default_number_of_custody_groups() -> u64 {
128
}
const fn default_samples_per_slot() -> u64 {
8
}
@@ -1830,10 +1864,11 @@ impl Config {
blob_sidecar_subnet_count_electra: spec.blob_sidecar_subnet_count_electra,
max_request_blob_sidecars_electra: spec.max_request_blob_sidecars_electra,
custody_requirement: spec.custody_requirement,
number_of_columns: spec.number_of_columns,
number_of_custody_groups: spec.number_of_custody_groups,
data_column_sidecar_subnet_count: spec.data_column_sidecar_subnet_count,
number_of_columns: spec.number_of_columns as u64,
samples_per_slot: spec.samples_per_slot,
custody_requirement: spec.custody_requirement,
}
}
@@ -1909,10 +1944,11 @@ impl Config {
max_blobs_per_block_electra,
blob_sidecar_subnet_count_electra,
max_request_blob_sidecars_electra,
custody_requirement,
data_column_sidecar_subnet_count,
number_of_columns,
number_of_custody_groups,
data_column_sidecar_subnet_count,
samples_per_slot,
custody_requirement,
} = self;
if preset_base != E::spec_name().to_string().as_str() {
@@ -1992,10 +2028,11 @@ impl Config {
max_request_data_column_sidecars,
),
custody_requirement,
number_of_columns,
number_of_custody_groups,
data_column_sidecar_subnet_count,
number_of_columns: number_of_columns as usize,
samples_per_slot,
custody_requirement,
..chain_spec.clone()
})

142
consensus/types/src/data_column_custody_group.rs Normal file
View File

@@ -0,0 +1,142 @@
use crate::{ChainSpec, ColumnIndex, DataColumnSubnetId};
use alloy_primitives::U256;
use itertools::Itertools;
use maplit::hashset;
use safe_arith::{ArithError, SafeArith};
use std::collections::HashSet;
pub type CustodyIndex = u64;
#[derive(Debug)]
pub enum DataColumnCustodyGroupError {
InvalidCustodyGroup(CustodyIndex),
InvalidCustodyGroupCount(u64),
ArithError(ArithError),
}
/// The `get_custody_groups` function is used to determine the custody groups that a node is
/// assigned to.
///
/// spec: https://github.com/ethereum/consensus-specs/blob/8e0d0d48e81d6c7c5a8253ab61340f5ea5bac66a/specs/fulu/das-core.md#get_custody_groups
pub fn get_custody_groups(
raw_node_id: [u8; 32],
custody_group_count: u64,
spec: &ChainSpec,
) -> Result<HashSet<CustodyIndex>, DataColumnCustodyGroupError> {
if custody_group_count > spec.number_of_custody_groups {
return Err(DataColumnCustodyGroupError::InvalidCustodyGroupCount(
custody_group_count,
));
}
let mut custody_groups: HashSet<u64> = hashset![];
let mut current_id = U256::from_be_slice(&raw_node_id);
while custody_groups.len() < custody_group_count as usize {
let mut node_id_bytes = [0u8; 32];
node_id_bytes.copy_from_slice(current_id.as_le_slice());
let hash = ethereum_hashing::hash_fixed(&node_id_bytes);
let hash_prefix: [u8; 8] = hash[0..8]
.try_into()
.expect("hash_fixed produces a 32 byte array");
let hash_prefix_u64 = u64::from_le_bytes(hash_prefix);
let custody_group = hash_prefix_u64
.safe_rem(spec.number_of_custody_groups)
.expect("spec.number_of_custody_groups must not be zero");
custody_groups.insert(custody_group);
current_id = current_id.wrapping_add(U256::from(1u64));
}
Ok(custody_groups)
}
/// Returns the columns that are associated with a given custody group.
///
/// spec: https://github.com/ethereum/consensus-specs/blob/8e0d0d48e81d6c7c5a8253ab61340f5ea5bac66a/specs/fulu/das-core.md#compute_columns_for_custody_group
pub fn compute_columns_for_custody_group(
custody_group: CustodyIndex,
spec: &ChainSpec,
) -> Result<impl Iterator<Item = ColumnIndex>, DataColumnCustodyGroupError> {
let number_of_custody_groups = spec.number_of_custody_groups;
if custody_group >= number_of_custody_groups {
return Err(DataColumnCustodyGroupError::InvalidCustodyGroup(
custody_group,
));
}
let mut columns = Vec::new();
for i in 0..spec.data_columns_per_group() {
let column = number_of_custody_groups
.safe_mul(i)
.and_then(|v| v.safe_add(custody_group))
.map_err(DataColumnCustodyGroupError::ArithError)?;
columns.push(column);
}
Ok(columns.into_iter())
}
pub fn compute_subnets_for_node(
raw_node_id: [u8; 32],
custody_group_count: u64,
spec: &ChainSpec,
) -> Result<HashSet<DataColumnSubnetId>, DataColumnCustodyGroupError> {
let custody_groups = get_custody_groups(raw_node_id, custody_group_count, spec)?;
let mut subnets = HashSet::new();
for custody_group in custody_groups {
let custody_group_subnets = compute_subnets_from_custody_group(custody_group, spec)?;
subnets.extend(custody_group_subnets);
}
Ok(subnets)
}
/// Returns the subnets that are associated with a given custody group.
pub fn compute_subnets_from_custody_group(
custody_group: CustodyIndex,
spec: &ChainSpec,
) -> Result<impl Iterator<Item = DataColumnSubnetId> + '_, DataColumnCustodyGroupError> {
let result = compute_columns_for_custody_group(custody_group, spec)?
.map(|column_index| DataColumnSubnetId::from_column_index(column_index, spec))
.unique();
Ok(result)
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn test_compute_columns_for_custody_group() {
let mut spec = ChainSpec::mainnet();
spec.number_of_custody_groups = 64;
spec.number_of_columns = 128;
let columns_per_custody_group = spec.number_of_columns / spec.number_of_custody_groups;
for custody_group in 0..spec.number_of_custody_groups {
let columns = compute_columns_for_custody_group(custody_group, &spec)
.unwrap()
.collect::<Vec<_>>();
assert_eq!(columns.len(), columns_per_custody_group as usize);
}
}
#[test]
fn test_compute_subnets_from_custody_group() {
let mut spec = ChainSpec::mainnet();
spec.number_of_custody_groups = 64;
spec.number_of_columns = 256;
spec.data_column_sidecar_subnet_count = 128;
let subnets_per_custody_group =
spec.data_column_sidecar_subnet_count / spec.number_of_custody_groups;
for custody_group in 0..spec.number_of_custody_groups {
let subnets = compute_subnets_from_custody_group(custody_group, &spec)
.unwrap()
.collect::<Vec<_>>();
assert_eq!(subnets.len(), subnets_per_custody_group as usize);
}
}
}

160
consensus/types/src/data_column_subnet_id.rs
View File

@@ -1,11 +1,8 @@
//! Identifies each data column subnet by an integer identifier.
use crate::data_column_sidecar::ColumnIndex;
use crate::{ChainSpec, EthSpec};
use alloy_primitives::U256;
use itertools::Itertools;
use crate::ChainSpec;
use safe_arith::{ArithError, SafeArith};
use serde::{Deserialize, Serialize};
use std::collections::HashSet;
use std::fmt::{self, Display};
use std::ops::{Deref, DerefMut};
@@ -18,76 +15,14 @@ impl DataColumnSubnetId {
id.into()
}
pub fn from_column_index<E: EthSpec>(column_index: usize, spec: &ChainSpec) -> Self {
(column_index
.safe_rem(spec.data_column_sidecar_subnet_count as usize)
pub fn from_column_index(column_index: ColumnIndex, spec: &ChainSpec) -> Self {
column_index
.safe_rem(spec.data_column_sidecar_subnet_count)
.expect(
"data_column_sidecar_subnet_count should never be zero if this function is called",
) as u64)
)
.into()
}
#[allow(clippy::arithmetic_side_effects)]
pub fn columns<E: EthSpec>(&self, spec: &ChainSpec) -> impl Iterator<Item = ColumnIndex> {
let subnet = self.0;
let data_column_sidecar_subnet = spec.data_column_sidecar_subnet_count;
let columns_per_subnet = spec.data_columns_per_subnet() as u64;
(0..columns_per_subnet).map(move |i| data_column_sidecar_subnet * i + subnet)
}
/// Compute required subnets to subscribe to given the node id.
#[allow(clippy::arithmetic_side_effects)]
pub fn compute_custody_subnets<E: EthSpec>(
raw_node_id: [u8; 32],
custody_subnet_count: u64,
spec: &ChainSpec,
) -> Result<impl Iterator<Item = DataColumnSubnetId>, Error> {
if custody_subnet_count > spec.data_column_sidecar_subnet_count {
return Err(Error::InvalidCustodySubnetCount(custody_subnet_count));
}
let mut subnets: HashSet<u64> = HashSet::new();
let mut current_id = U256::from_be_slice(&raw_node_id);
while (subnets.len() as u64) < custody_subnet_count {
let mut node_id_bytes = [0u8; 32];
node_id_bytes.copy_from_slice(current_id.as_le_slice());
let hash = ethereum_hashing::hash_fixed(&node_id_bytes);
let hash_prefix: [u8; 8] = hash[0..8]
.try_into()
.expect("hash_fixed produces a 32 byte array");
let hash_prefix_u64 = u64::from_le_bytes(hash_prefix);
let subnet = hash_prefix_u64 % spec.data_column_sidecar_subnet_count;
if !subnets.contains(&subnet) {
subnets.insert(subnet);
}
if current_id == U256::MAX {
current_id = U256::ZERO
}
current_id += U256::from(1u64)
}
Ok(subnets.into_iter().map(DataColumnSubnetId::new))
}
/// Compute the custody subnets for a given node id with the default `custody_requirement`.
/// This operation should be infallable, and empty iterator is returned if it fails unexpectedly.
pub fn compute_custody_requirement_subnets<E: EthSpec>(
node_id: [u8; 32],
spec: &ChainSpec,
) -> impl Iterator<Item = DataColumnSubnetId> {
Self::compute_custody_subnets::<E>(node_id, spec.custody_requirement, spec)
.expect("should compute default custody subnets")
}
pub fn compute_custody_columns<E: EthSpec>(
raw_node_id: [u8; 32],
custody_subnet_count: u64,
spec: &ChainSpec,
) -> Result<impl Iterator<Item = ColumnIndex>, Error> {
Self::compute_custody_subnets::<E>(raw_node_id, custody_subnet_count, spec)
.map(|subnet| subnet.flat_map(|subnet| subnet.columns::<E>(spec)).sorted())
}
}
impl Display for DataColumnSubnetId {
@@ -139,88 +74,3 @@ impl From<ArithError> for Error {
Error::ArithError(e)
}
}
#[cfg(test)]
mod test {
use crate::data_column_subnet_id::DataColumnSubnetId;
use crate::MainnetEthSpec;
use crate::Uint256;
use crate::{EthSpec, GnosisEthSpec, MinimalEthSpec};
type E = MainnetEthSpec;
#[test]
fn test_compute_subnets_for_data_column() {
let spec = E::default_spec();
let node_ids = [
"0",
"88752428858350697756262172400162263450541348766581994718383409852729519486397",
"18732750322395381632951253735273868184515463718109267674920115648614659369468",
"27726842142488109545414954493849224833670205008410190955613662332153332462900",
"39755236029158558527862903296867805548949739810920318269566095185775868999998",
"31899136003441886988955119620035330314647133604576220223892254902004850516297",
"58579998103852084482416614330746509727562027284701078483890722833654510444626",
"28248042035542126088870192155378394518950310811868093527036637864276176517397",
"60930578857433095740782970114409273483106482059893286066493409689627770333527",
"103822458477361691467064888613019442068586830412598673713899771287914656699997",
]
.into_iter()
.map(|v| Uint256::from_str_radix(v, 10).unwrap().to_be_bytes::<32>())
.collect::<Vec<_>>();
let custody_requirement = 4;
for node_id in node_ids {
let computed_subnets = DataColumnSubnetId::compute_custody_subnets::<E>(
node_id,
custody_requirement,
&spec,
)
.unwrap();
let computed_subnets: Vec<_> = computed_subnets.collect();
// the number of subnets is equal to the custody requirement
assert_eq!(computed_subnets.len() as u64, custody_requirement);
let subnet_count = spec.data_column_sidecar_subnet_count;
for subnet in computed_subnets {
let columns: Vec<_> = subnet.columns::<E>(&spec).collect();
// the number of columns is equal to the specified number of columns per subnet
assert_eq!(columns.len(), spec.data_columns_per_subnet());
for pair in columns.windows(2) {
// each successive column index is offset by the number of subnets
assert_eq!(pair[1] - pair[0], subnet_count);
}
}
}
}
#[test]
fn test_compute_custody_requirement_subnets_never_panics() {
let node_id = [1u8; 32];
test_compute_custody_requirement_subnets_with_spec::<MainnetEthSpec>(node_id);
test_compute_custody_requirement_subnets_with_spec::<MinimalEthSpec>(node_id);
test_compute_custody_requirement_subnets_with_spec::<GnosisEthSpec>(node_id);
}
fn test_compute_custody_requirement_subnets_with_spec<E: EthSpec>(node_id: [u8; 32]) {
let _ = DataColumnSubnetId::compute_custody_requirement_subnets::<E>(
node_id,
&E::default_spec(),
);
}
#[test]
fn test_columns_subnet_conversion() {
let spec = E::default_spec();
for subnet in 0..spec.data_column_sidecar_subnet_count {
let subnet_id = DataColumnSubnetId::new(subnet);
for column_index in subnet_id.columns::<E>(&spec) {
assert_eq!(
subnet_id,
DataColumnSubnetId::from_column_index::<E>(column_index as usize, &spec)
);
}
}
}
}

1
consensus/types/src/lib.rs
View File

@@ -104,6 +104,7 @@ pub mod slot_data;
pub mod sqlite;
pub mod blob_sidecar;
pub mod data_column_custody_group;
pub mod data_column_sidecar;
pub mod data_column_subnet_id;
pub mod light_client_header;