chore: update and enable PeerDAS tests to latest spec test release (1.5.0-alpha-5) (#6312)

* enable DAS tests

* update spec testing code

* Update PeerDAS kzg tests input structures to latest spec.

* Update `ef_tests` ignore files.

testing/ef_tests/check_all_files_accessed.py

@@ -20,8 +20,6 @@ tests_dir_filename = sys.argv[2]
 # following regular expressions, we will assume they are to be ignored (i.e., we are purposefully
 # *not* running the spec tests).
 excluded_paths = [
-    # TODO(das): ignore until new spec test release with column subnet count = 64.
-    "tests/.*/.*/.*/get_custody_columns/",
     # Eth1Block and PowBlock
     #
     # Intentionally omitted, as per https://github.com/sigp/lighthouse/issues/1835
@@ -35,15 +33,10 @@ excluded_paths = [
     "tests/.*/.*/ssz_static/LightClientStore",
     # LightClientSnapshot
     "tests/.*/.*/ssz_static/LightClientSnapshot",
-    # Unused container for das
-    "tests/.*/.*/ssz_static/MatrixEntry",
-    # Unused kzg methods
-    "tests/.*/.*/kzg/verify_cell_kzg_proof",
     # One of the EF researchers likes to pack the tarballs on a Mac
     ".*\\.DS_Store.*",
     # More Mac weirdness.
     "tests/mainnet/bellatrix/operations/deposit/pyspec_tests/deposit_with_previous_fork_version__valid_ineffective/._meta.yaml",
-    "tests/mainnet/eip7594/networking/get_custody_columns/pyspec_tests/get_custody_columns__short_node_id/._meta.yaml",
     # bls tests are moved to bls12-381-tests directory
     "tests/general/phase0/bls",
     # some bls tests are not included now

testing/ef_tests/src/cases/kzg_recover_cells_and_kzg_proofs.rs

@@ -1,8 +1,7 @@
 use super::*;
 use crate::case_result::compare_result;
-use kzg::{CellsAndKzgProofs, KzgProof};
+use kzg::CellsAndKzgProofs;
 use serde::Deserialize;
-use std::convert::Infallible;
 use std::marker::PhantomData;
 
 #[derive(Debug, Clone, Deserialize)]
@@ -10,7 +9,6 @@ use std::marker::PhantomData;
 pub struct KZGRecoverCellsAndKzgProofsInput {
     pub cell_indices: Vec<u64>,
     pub cells: Vec<String>,
-    pub proofs: Vec<String>,
 }
 
 #[derive(Debug, Clone, Deserialize)]
@@ -35,29 +33,17 @@ impl<E: EthSpec> Case for KZGRecoverCellsAndKZGProofs<E> {
 
     fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
         let parse_input = |input: &KZGRecoverCellsAndKzgProofsInput| {
-            // Proofs are not used for `recover_cells_and_compute_kzg_proofs`, they are only checked
-            // to satisfy the spec tests.
-            if input.proofs.len() != input.cell_indices.len() {
-                return Err(Error::SkippedKnownFailure);
-            }
-
-            let proofs = input
-                .proofs
-                .iter()
-                .map(|s| parse_proof(s))
-                .collect::<Result<Vec<_>, Error>>()?;
-
             let cells = input
                 .cells
                 .iter()
                 .map(|s| parse_cell(s))
                 .collect::<Result<Vec<_>, Error>>()?;
 
-            Ok((proofs, cells, input.cell_indices.clone()))
+            Ok((cells, input.cell_indices.clone()))
         };
 
-        let result =
-            parse_input(&self.input).and_then(|(input_proofs, input_cells, cell_indices)| {
+        let result: Result<_, Error> =
+            parse_input(&self.input).and_then(|(input_cells, cell_indices)| {
                 let input_cells_ref: Vec<_> = input_cells.iter().map(|cell| &**cell).collect();
                 let kzg = get_kzg();
                 let (cells, proofs) = kzg
@@ -71,18 +57,6 @@ impl<E: EthSpec> Case for KZGRecoverCellsAndKZGProofs<E> {
                         ))
                     })?;
 
-                // Check recovered proofs matches inputs proofs. This is done only to satisfy the
-                // spec tests, as the ckzg library recomputes all proofs and does not require
-                // proofs to recover.
-                for (input_proof, cell_id) in input_proofs.iter().zip(cell_indices) {
-                    if let Err(e) = compare_result::<KzgProof, Infallible>(
-                        &Ok(*input_proof),
-                        &proofs.get(cell_id as usize).cloned(),
-                    ) {
-                        return Err(e);
-                    }
-                }
-
                 Ok((cells, proofs))
             });
 

testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof.rs

@@ -13,8 +13,7 @@ static KZG: LazyLock<Arc<Kzg>> = LazyLock::new(|| {
     let trusted_setup: TrustedSetup = serde_json::from_reader(TRUSTED_SETUP_BYTES)
         .map_err(|e| Error::InternalError(format!("Failed to initialize trusted setup: {:?}", e)))
         .expect("failed to initialize trusted setup");
-    // TODO(das): need to enable these tests when rayon issues in rust_eth_kzg are fixed
-    let kzg = Kzg::new_from_trusted_setup(trusted_setup)
+    let kzg = Kzg::new_from_trusted_setup_das_enabled(trusted_setup)
         .map_err(|e| Error::InternalError(format!("Failed to initialize kzg: {:?}", e)))
         .expect("failed to initialize kzg");
     Arc::new(kzg)

testing/ef_tests/src/cases/kzg_verify_cell_kzg_proof_batch.rs

@@ -7,9 +7,8 @@ use std::marker::PhantomData;
 #[derive(Debug, Clone, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub struct KZGVerifyCellKZGProofBatchInput {
-    pub row_commitments: Vec<String>,
-    pub row_indices: Vec<usize>,
-    pub column_indices: Vec<usize>,
+    pub commitments: Vec<String>,
+    pub cell_indices: Vec<u64>,
     pub cells: Vec<String>,
     pub proofs: Vec<String>,
 }
@@ -37,32 +36,22 @@ impl<E: EthSpec> Case for KZGVerifyCellKZGProofBatch<E> {
     fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
         let parse_input = |input: &KZGVerifyCellKZGProofBatchInput| -> Result<_, Error> {
             let (cells, proofs) = parse_cells_and_proofs(&input.cells, &input.proofs)?;
-            let row_commitments = input
-                .row_commitments
+            let commitments = input
+                .commitments
                 .iter()
                 .map(|s| parse_commitment(s))
                 .collect::<Result<Vec<_>, _>>()?;
-            let coordinates = input
-                .row_indices
-                .iter()
-                .zip(&input.column_indices)
-                .map(|(&row, &col)| (row as u64, col as u64))
-                .collect::<Vec<_>>();
 
-            Ok((cells, proofs, coordinates, row_commitments))
+            Ok((cells, proofs, input.cell_indices.clone(), commitments))
         };
 
         let result =
-            parse_input(&self.input).and_then(|(cells, proofs, coordinates, commitments)| {
+            parse_input(&self.input).and_then(|(cells, proofs, cell_indices, commitments)| {
                 let proofs: Vec<Bytes48> = proofs.iter().map(|&proof| proof.into()).collect();
                 let commitments: Vec<Bytes48> = commitments.iter().map(|&c| c.into()).collect();
                 let cells = cells.iter().map(|c| c.as_ref()).collect::<Vec<_>>();
-                let column_indices = coordinates
-                    .into_iter()
-                    .map(|(_row, col)| col)
-                    .collect::<Vec<_>>();
                 let kzg = get_kzg();
-                match kzg.verify_cell_proof_batch(&cells, &proofs, column_indices, &commitments) {
+                match kzg.verify_cell_proof_batch(&cells, &proofs, cell_indices, &commitments) {
                     Ok(_) => Ok(true),
                     Err(KzgError::KzgVerificationFailed) => Ok(false),
                     Err(e) => Err(Error::InternalError(format!(

testing/ef_tests/tests/tests.rs

@@ -901,7 +901,6 @@ fn kzg_verify_kzg_proof() {
     KZGVerifyKZGProofHandler::<MainnetEthSpec>::default().run();
 }
 
-/* TODO(das): enable these tests
 #[test]
 fn kzg_compute_cells_and_proofs() {
     KZGComputeCellsAndKZGProofHandler::<MainnetEthSpec>::default()
@@ -919,7 +918,6 @@ fn kzg_recover_cells_and_proofs() {
     KZGRecoverCellsAndKZGProofHandler::<MainnetEthSpec>::default()
         .run_for_feature(ForkName::Deneb, FeatureName::Eip7594);
 }
-*/
 
 #[test]
 fn merkle_proof_validity() {