diff --git a/Cargo.lock b/Cargo.lock index 622ff88183..913382fe66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1608,9 +1608,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" dependencies = [ "serde", ] @@ -3098,7 +3098,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4729,7 +4729,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6291,7 +6291,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7199,7 +7199,7 @@ dependencies = [ "once_cell", "socket2 0.6.1", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -7716,7 +7716,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -8781,7 +8781,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.2", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -10132,7 +10132,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 2726af13a1..78c63875d3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -116,7 +116,7 @@ bincode = "1" bitvec = "1" bls = { path = "crypto/bls" } byteorder = "1" -bytes = "1" +bytes = "1.11.1" # Turn off c-kzg's default features which include `blst/portable`. We can turn on blst's portable # feature ourselves when desired. c-kzg = { version = "2.1", default-features = false } diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index 0cbcc28819..c095b57849 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -1218,7 +1218,6 @@ fn build_data_columns_from_blobs( if block.fork_name_unchecked().gloas_enabled() { build_data_column_sidecars_gloas( - kzg_commitments, block.message().tree_hash_root(), block.slot(), blob_cells_and_proofs_vec, diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 7f4848f006..666ba7cc41 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -1120,7 +1120,7 @@ mod test { let invalid_sidecar = DataColumnSidecar::Fulu(DataColumnSidecarFulu { column: DataColumn::::empty(), index: *d.index(), - kzg_commitments: d.kzg_commitments().clone(), + kzg_commitments: d.kzg_commitments().unwrap().clone(), kzg_proofs: d.kzg_proofs().clone(), signed_block_header: d.signed_block_header().unwrap().clone(), kzg_commitments_inclusion_proof: d diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index 03bdde6114..cf3385ec5b 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -573,12 +573,18 @@ fn verify_data_column_sidecar( *data_column.index(), )); } - if data_column.kzg_commitments().is_empty() { + + // TODO(gloas): implement Gloas verification that takes kzg_commitments from block as parameter + let commitments_len = match data_column { + DataColumnSidecar::Fulu(dc) => dc.kzg_commitments.len(), + DataColumnSidecar::Gloas(_) => return Err(GossipDataColumnError::InvalidVariant), + }; + + if commitments_len == 0 { return Err(GossipDataColumnError::UnexpectedDataColumn); } let cells_len = data_column.column().len(); - let commitments_len = data_column.kzg_commitments().len(); let proofs_len = data_column.kzg_proofs().len(); let max_blobs_per_block = spec.max_blobs_per_block(data_column.epoch()) as usize; diff --git a/beacon_node/beacon_chain/src/kzg_utils.rs b/beacon_node/beacon_chain/src/kzg_utils.rs index bef96836f1..33b3260361 100644 --- a/beacon_node/beacon_chain/src/kzg_utils.rs +++ b/beacon_node/beacon_chain/src/kzg_utils.rs @@ -75,7 +75,21 @@ where proofs.push(Bytes48::from(proof)); } - for &commitment in data_column.kzg_commitments() { + // In Gloas, commitments come from the block's ExecutionPayloadBid, not the sidecar. + // This function requires Fulu sidecars with embedded commitments. + let kzg_commitments = match data_column.as_ref() { + DataColumnSidecar::Fulu(dc) => &dc.kzg_commitments, + DataColumnSidecar::Gloas(_) => { + return Err(( + Some(col_index), + KzgError::InconsistentArrayLength( + "Gloas data columns require commitments from block".to_string(), + ), + )); + } + }; + + for &commitment in kzg_commitments.iter() { commitments.push(Bytes48::from(commitment)); } @@ -209,7 +223,6 @@ pub fn blobs_to_data_column_sidecars( if block.fork_name_unchecked().gloas_enabled() { build_data_column_sidecars_gloas( - kzg_commitments.clone(), signed_block_header.message.tree_hash_root(), block.slot(), blob_cells_and_proofs_vec, @@ -321,7 +334,6 @@ pub(crate) fn build_data_column_sidecars_fulu( } pub(crate) fn build_data_column_sidecars_gloas( - kzg_commitments: KzgCommitments, beacon_block_root: Hash256, slot: Slot, blob_cells_and_proofs_vec: Vec, @@ -374,7 +386,6 @@ pub(crate) fn build_data_column_sidecars_gloas( index: index as u64, column: DataColumn::::try_from(col) .map_err(|e| format!("MaxBlobCommitmentsPerBlock exceeded: {e:?}"))?, - kzg_commitments: kzg_commitments.clone(), kzg_proofs: VariableList::try_from(proofs) .map_err(|e| format!("MaxBlobCommitmentsPerBlock exceeded: {e:?}"))?, beacon_block_root, @@ -412,7 +423,12 @@ pub fn reconstruct_blobs( let blob_indices: Vec = match blob_indices_opt { Some(indices) => indices.into_iter().map(|i| i as usize).collect(), None => { - let num_of_blobs = first_data_column.kzg_commitments().len(); + // TODO(gloas): support blob reconstruction for Gloas + // https://github.com/sigp/lighthouse/issues/7413 + let num_of_blobs = first_data_column + .kzg_commitments() + .map_err(|_| "Gloas blob reconstruction not yet supported".to_string())? + .len(); (0..num_of_blobs).collect() } }; @@ -497,7 +513,16 @@ pub fn reconstruct_data_columns( "data_columns should have at least one element".to_string(), ))?; - let num_of_blobs = first_data_column.kzg_commitments().len(); + // TODO(gloas): support data column reconstruction for Gloas + // https://github.com/sigp/lighthouse/issues/7413 + let num_of_blobs = first_data_column + .kzg_commitments() + .map_err(|_| { + KzgError::InconsistentArrayLength( + "Gloas data column reconstruction not yet supported".to_string(), + ) + })? + .len(); let blob_cells_and_proofs_vec = (0..num_of_blobs) .into_par_iter() @@ -530,7 +555,6 @@ pub fn reconstruct_data_columns( .map_err(KzgError::ReconstructFailed) } DataColumnSidecar::Gloas(first_column) => build_data_column_sidecars_gloas( - first_column.kzg_commitments.clone(), first_column.beacon_block_root, first_column.slot, blob_cells_and_proofs_vec, @@ -629,11 +653,14 @@ mod test { for (idx, col_sidecar) in column_sidecars.iter().enumerate() { assert_eq!(*col_sidecar.index(), idx as u64); - assert_eq!(col_sidecar.kzg_commitments().len(), num_of_blobs); + assert_eq!(col_sidecar.kzg_commitments().unwrap().len(), num_of_blobs); assert_eq!(col_sidecar.column().len(), num_of_blobs); assert_eq!(col_sidecar.kzg_proofs().len(), num_of_blobs); - assert_eq!(col_sidecar.kzg_commitments().clone(), block_kzg_commitments); + assert_eq!( + col_sidecar.kzg_commitments().unwrap().clone(), + block_kzg_commitments + ); assert_eq!( col_sidecar .kzg_commitments_inclusion_proof() diff --git a/beacon_node/beacon_chain/src/observed_data_sidecars.rs b/beacon_node/beacon_chain/src/observed_data_sidecars.rs index b53eb3955c..894b8d3444 100644 --- a/beacon_node/beacon_chain/src/observed_data_sidecars.rs +++ b/beacon_node/beacon_chain/src/observed_data_sidecars.rs @@ -312,7 +312,6 @@ mod tests { Arc::new(DataColumnSidecar::Gloas(DataColumnSidecarGloas { index, column: vec![].try_into().unwrap(), - kzg_commitments: vec![].try_into().unwrap(), kzg_proofs: vec![].try_into().unwrap(), slot: slot.into(), beacon_block_root, diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index f4e4c37234..dcf8ee4f8e 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -3517,7 +3517,6 @@ pub fn generate_data_column_sidecars_from_block( vec![(cells.try_into().unwrap(), proofs.try_into().unwrap()); kzg_commitments.len()]; build_data_column_sidecars_gloas( - kzg_commitments.clone(), signed_block_header.message.tree_hash_root(), signed_block_header.message.slot, blob_cells_and_proofs_vec, diff --git a/beacon_node/beacon_chain/tests/block_verification.rs b/beacon_node/beacon_chain/tests/block_verification.rs index 440c0be3e4..417d2811dd 100644 --- a/beacon_node/beacon_chain/tests/block_verification.rs +++ b/beacon_node/beacon_chain/tests/block_verification.rs @@ -285,7 +285,7 @@ fn update_data_column_signed_header( let new_column_sidecar = Arc::new(DataColumnSidecar::Fulu(DataColumnSidecarFulu { index: *old_column_sidecar.index(), column: old_column_sidecar.column().clone(), - kzg_commitments: old_column_sidecar.kzg_commitments().clone(), + kzg_commitments: old_column_sidecar.kzg_commitments().unwrap().clone(), kzg_proofs: old_column_sidecar.kzg_proofs().clone(), signed_block_header: signed_block.signed_block_header(), kzg_commitments_inclusion_proof: signed_block diff --git a/beacon_node/lighthouse_network/tests/rpc_tests.rs b/beacon_node/lighthouse_network/tests/rpc_tests.rs index 553cfa6f0d..53939687d3 100644 --- a/beacon_node/lighthouse_network/tests/rpc_tests.rs +++ b/beacon_node/lighthouse_network/tests/rpc_tests.rs @@ -1017,7 +1017,6 @@ fn test_tcp_columns_by_root_chunked_rpc_for_fork(fork_name: ForkName) { column: vec![vec![0; E::bytes_per_cell()].try_into().unwrap()] .try_into() .unwrap(), - kzg_commitments: vec![KzgCommitment::empty_for_testing()].try_into().unwrap(), kzg_proofs: vec![KzgProof::empty()].try_into().unwrap(), })) } else { @@ -1188,7 +1187,6 @@ fn test_tcp_columns_by_range_chunked_rpc_for_fork(fork_name: ForkName) { column: vec![vec![0; E::bytes_per_cell()].try_into().unwrap()] .try_into() .unwrap(), - kzg_commitments: vec![KzgCommitment::empty_for_testing()].try_into().unwrap(), kzg_proofs: vec![KzgProof::empty()].try_into().unwrap(), })) } else { diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 149b342a8c..853afe9ece 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -1016,7 +1016,11 @@ impl SseDataColumnSidecar { pub fn from_data_column_sidecar( data_column_sidecar: &DataColumnSidecar, ) -> SseDataColumnSidecar { - let kzg_commitments = data_column_sidecar.kzg_commitments().to_vec(); + // TODO(gloas): fetch kzg_commitments from block for Gloas SSE events + let kzg_commitments: Vec = match data_column_sidecar { + DataColumnSidecar::Fulu(dc) => dc.kzg_commitments.to_vec(), + DataColumnSidecar::Gloas(_) => vec![], + }; let versioned_hashes = kzg_commitments .iter() .map(|c| c.calculate_versioned_hash()) diff --git a/consensus/state_processing/src/per_block_processing/withdrawals.rs b/consensus/state_processing/src/per_block_processing/withdrawals.rs index 39ad4efc5c..72c3339b10 100644 --- a/consensus/state_processing/src/per_block_processing/withdrawals.rs +++ b/consensus/state_processing/src/per_block_processing/withdrawals.rs @@ -84,11 +84,8 @@ pub fn get_builder_withdrawals( return Ok(None); }; - // TODO(gloas): this has already changed on `master`, we need to update at next spec release - let withdrawals_limit = E::max_withdrawals_per_payload(); + let withdrawals_limit = E::max_withdrawals_per_payload().safe_sub(1)?; - // TODO(gloas): this assert is from `master`, remove this comment once it is part of the tested - // spec version. block_verify!( withdrawals.len() <= withdrawals_limit, BlockProcessingError::WithdrawalsLimitExceeded { @@ -138,8 +135,6 @@ pub fn get_pending_partial_withdrawals( E::max_withdrawals_per_payload().safe_sub(1)?, ); - // TODO(gloas): this assert is from `master`, remove this comment once it is part of the tested - // spec version. block_verify!( withdrawals.len() <= withdrawals_limit, BlockProcessingError::WithdrawalsLimitExceeded { @@ -205,11 +200,8 @@ pub fn get_builders_sweep_withdrawals( let epoch = state.current_epoch(); let builders_limit = std::cmp::min(builders.len(), E::max_builders_per_withdrawals_sweep()); - // TODO(gloas): this has already changed on `master`, we should update at the next spec release - let withdrawals_limit = E::max_withdrawals_per_payload(); + let withdrawals_limit = E::max_withdrawals_per_payload().safe_sub(1)?; - // TODO(gloas): this assert is from `master`, remove this comment once it is part of the tested - // spec version. block_verify!( withdrawals.len() <= withdrawals_limit, BlockProcessingError::WithdrawalsLimitExceeded { diff --git a/consensus/types/src/data/data_column_sidecar.rs b/consensus/types/src/data/data_column_sidecar.rs index d98470297a..c8a49e346a 100644 --- a/consensus/types/src/data/data_column_sidecar.rs +++ b/consensus/types/src/data/data_column_sidecar.rs @@ -81,7 +81,9 @@ pub struct DataColumnSidecar { pub index: ColumnIndex, #[serde(with = "ssz_types::serde_utils::list_of_hex_fixed_vec")] pub column: DataColumn, - /// All the KZG commitments and proofs associated with the block, used for verifying sample cells. + /// All the KZG commitments associated with the block, used for verifying sample cells. + /// In Gloas, commitments come from `block.body.signed_execution_payload_bid.message.blob_kzg_commitments`. + #[superstruct(only(Fulu))] pub kzg_commitments: KzgCommitments, pub kzg_proofs: VariableList, #[superstruct(only(Fulu))] @@ -210,7 +212,6 @@ impl DataColumnSidecarGloas { Self { index: 0, column: VariableList::new(vec![Cell::::default()]).unwrap(), - kzg_commitments: VariableList::new(vec![KzgCommitment::empty_for_testing()]).unwrap(), kzg_proofs: VariableList::new(vec![KzgProof::empty()]).unwrap(), slot: Slot::new(0), beacon_block_root: Hash256::ZERO, @@ -223,11 +224,6 @@ impl DataColumnSidecarGloas { Self { index: 0, column: VariableList::new(vec![Cell::::default(); max_blobs_per_block]).unwrap(), - kzg_commitments: VariableList::new(vec![ - KzgCommitment::empty_for_testing(); - max_blobs_per_block - ]) - .unwrap(), kzg_proofs: VariableList::new(vec![KzgProof::empty(); max_blobs_per_block]).unwrap(), slot: Slot::new(0), beacon_block_root: Hash256::ZERO, diff --git a/testing/ef_tests/Makefile b/testing/ef_tests/Makefile index a881718e02..ee0cebd478 100644 --- a/testing/ef_tests/Makefile +++ b/testing/ef_tests/Makefile @@ -1,6 +1,10 @@ # To download/extract nightly tests, run: # CONSENSUS_SPECS_TEST_VERSION=nightly make +<<<<<<< HEAD CONSENSUS_SPECS_TEST_VERSION ?= nightly-21573464110 +======= +CONSENSUS_SPECS_TEST_VERSION ?= v1.7.0-alpha.2 +>>>>>>> 1dd0f7bcbb9e476028ec9146a149e121ad28ff9e REPO_NAME := consensus-spec-tests OUTPUT_DIR := ./$(REPO_NAME) diff --git a/testing/ef_tests/check_all_files_accessed.py b/testing/ef_tests/check_all_files_accessed.py index 7787af64f0..97c1c4f4f9 100755 --- a/testing/ef_tests/check_all_files_accessed.py +++ b/testing/ef_tests/check_all_files_accessed.py @@ -59,6 +59,8 @@ excluded_paths = [ # Ignore full epoch tests for now (just test the sub-transitions). "tests/.*/.*/epoch_processing/.*/pre_epoch.ssz_snappy", "tests/.*/.*/epoch_processing/.*/post_epoch.ssz_snappy", + # Ignore inactivity_scores tests for now (should implement soon). + "tests/.*/.*/rewards/inactivity_scores/.*", # Ignore gloas tests for now "tests/.*/gloas/.*", # Ignore KZG tests that target internal kzg library functions diff --git a/testing/ef_tests/src/cases/operations.rs b/testing/ef_tests/src/cases/operations.rs index 2c7a385bd5..e778300879 100644 --- a/testing/ef_tests/src/cases/operations.rs +++ b/testing/ef_tests/src/cases/operations.rs @@ -45,7 +45,7 @@ struct ExecutionMetadata { /// Newtype for testing withdrawals. #[derive(Debug, Clone, Deserialize)] pub struct WithdrawalsPayload { - payload: ExecutionPayload, + payload: Option>, } #[derive(Debug, Clone)] @@ -405,10 +405,17 @@ impl Operation for WithdrawalsPayload { } fn decode(path: &Path, fork_name: ForkName, _spec: &ChainSpec) -> Result { - ssz_decode_file_with(path, |bytes| { - ExecutionPayload::from_ssz_bytes_by_fork(bytes, fork_name) - }) - .map(|payload| WithdrawalsPayload { payload }) + if fork_name.gloas_enabled() { + // No payload present or required for Gloas tests. + Ok(WithdrawalsPayload { payload: None }) + } else { + ssz_decode_file_with(path, |bytes| { + ExecutionPayload::from_ssz_bytes_by_fork(bytes, fork_name) + }) + .map(|payload| WithdrawalsPayload { + payload: Some(payload), + }) + } } fn apply_to( @@ -420,7 +427,7 @@ impl Operation for WithdrawalsPayload { if state.fork_name_unchecked().gloas_enabled() { withdrawals::gloas::process_withdrawals(state, spec) } else { - let full_payload = FullPayload::from(self.payload.clone()); + let full_payload = FullPayload::from(self.payload.clone().unwrap()); withdrawals::capella_electra::process_withdrawals::<_, FullPayload<_>>( state, full_payload.to_ref(), diff --git a/testing/ef_tests/src/handler.rs b/testing/ef_tests/src/handler.rs index 86d317b564..b0fc90b169 100644 --- a/testing/ef_tests/src/handler.rs +++ b/testing/ef_tests/src/handler.rs @@ -333,6 +333,10 @@ impl SszStaticHandler { Self::for_forks(ForkName::list_all()[6..].to_vec()) } + pub fn gloas_and_later() -> Self { + Self::for_forks(ForkName::list_all()[7..].to_vec()) + } + pub fn pre_electra() -> Self { Self::for_forks(ForkName::list_all()[0..5].to_vec()) } @@ -397,10 +401,7 @@ where } fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool { - // TODO(gloas): DataColumnSidecar tests are disabled until we update the DataColumnSidecar - // type. self.supported_forks.contains(&fork_name) - && !(fork_name == ForkName::Gloas && T::name() == "DataColumnSidecar") } } diff --git a/testing/ef_tests/src/type_name.rs b/testing/ef_tests/src/type_name.rs index b24be6a943..ae00727fc3 100644 --- a/testing/ef_tests/src/type_name.rs +++ b/testing/ef_tests/src/type_name.rs @@ -87,6 +87,8 @@ type_name_generic!(ExecutionPayloadHeaderCapella, "ExecutionPayloadHeader"); type_name_generic!(ExecutionPayloadHeaderDeneb, "ExecutionPayloadHeader"); type_name_generic!(ExecutionPayloadHeaderElectra, "ExecutionPayloadHeader"); type_name_generic!(ExecutionPayloadHeaderFulu, "ExecutionPayloadHeader"); +type_name_generic!(ExecutionPayloadBid); +type_name_generic!(SignedExecutionPayloadBid); type_name_generic!(ExecutionRequests); type_name_generic!(BlindedPayload, "ExecutionPayloadHeader"); type_name!(Fork); diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index b8c69285d9..505693c31d 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -600,6 +600,10 @@ mod ssz_static { .run(); SszStaticHandler::, MinimalEthSpec>::fulu_only().run(); SszStaticHandler::, MainnetEthSpec>::fulu_only().run(); + SszStaticHandler::, MainnetEthSpec>::gloas_only() + .run(); + SszStaticHandler::, MainnetEthSpec>::gloas_only() + .run(); } #[test] @@ -626,6 +630,20 @@ mod ssz_static { .run(); } + #[test] + fn execution_payload_bid() { + SszStaticHandler::, MinimalEthSpec>::gloas_and_later() + .run(); + SszStaticHandler::, MainnetEthSpec>::gloas_and_later() + .run(); + } + + #[test] + fn signed_execution_payload_bid() { + SszStaticHandler::, MinimalEthSpec>::gloas_and_later().run(); + SszStaticHandler::, MainnetEthSpec>::gloas_and_later().run(); + } + #[test] fn withdrawal() { SszStaticHandler::::capella_and_later().run();