Update criterion (#5886)

* Start updating criterion

* Update consensus benches

Cargo.lock

@@ -18,7 +18,7 @@ version = "0.3.5"
 dependencies = [
  "account_utils",
  "bls",
- "clap 4.5.4",
+ "clap",
  "clap_utils",
  "directory",
  "environment",
@@ -280,6 +280,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "anes"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
+
 [[package]]
 name = "anstream"
 version = "0.6.12"
@@ -851,7 +857,7 @@ name = "beacon_node"
 version = "5.1.3"
 dependencies = [
  "beacon_chain",
- "clap 4.5.4",
+ "clap",
  "clap_utils",
  "client",
  "directory",
@@ -1057,7 +1063,7 @@ name = "boot_node"
 version = "5.1.3"
 dependencies = [
  "beacon_node",
- "clap 4.5.4",
+ "clap",
  "clap_utils",
  "eth2_network_config",
  "ethereum_ssz",
@@ -1282,6 +1288,33 @@ dependencies = [
  "windows-targets 0.52.5",
 ]
 
+[[package]]
+name = "ciborium"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e"
+dependencies = [
+ "ciborium-io",
+ "ciborium-ll",
+ "serde",
+]
+
+[[package]]
+name = "ciborium-io"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757"
+
+[[package]]
+name = "ciborium-ll"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9"
+dependencies = [
+ "ciborium-io",
+ "half",
+]
+
 [[package]]
 name = "cipher"
 version = "0.3.0"
@@ -1313,17 +1346,6 @@ dependencies = [
  "libloading",
 ]
 
-[[package]]
-name = "clap"
-version = "2.34.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
-dependencies = [
- "bitflags 1.3.2",
- "textwrap",
- "unicode-width",
-]
-
 [[package]]
 name = "clap"
 version = "4.5.4"
@@ -1356,7 +1378,7 @@ checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce"
 name = "clap_utils"
 version = "0.1.0"
 dependencies = [
- "clap 4.5.4",
+ "clap",
  "dirs",
  "eth2_network_config",
  "ethereum-types 0.14.1",
@@ -1525,24 +1547,24 @@ dependencies = [
 
 [[package]]
 name = "criterion"
-version = "0.3.6"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b01d6de93b2b6c65e17c634a26653a29d107b3c98c607c765bf38d041531cd8f"
+checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f"
 dependencies = [
- "atty",
+ "anes",
  "cast",
- "clap 2.34.0",
+ "ciborium",
+ "clap",
  "criterion-plot",
- "csv",
+ "is-terminal",
  "itertools",
- "lazy_static",
  "num-traits",
+ "once_cell",
  "oorandom",
  "plotters",
  "rayon",
  "regex",
  "serde",
- "serde_cbor",
  "serde_derive",
  "serde_json",
  "tinytemplate",
@@ -1551,9 +1573,9 @@ dependencies = [
 
 [[package]]
 name = "criterion-plot"
-version = "0.4.5"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2673cc8207403546f45f5fd319a974b1e6983ad1a3ee7e6041650013be041876"
+checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
 dependencies = [
  "cast",
  "itertools",
@@ -1654,27 +1676,6 @@ dependencies = [
  "subtle",
 ]
 
-[[package]]
-name = "csv"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe"
-dependencies = [
- "csv-core",
- "itoa",
- "ryu",
- "serde",
-]
-
-[[package]]
-name = "csv-core"
-version = "0.1.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "ctr"
 version = "0.7.0"
@@ -1833,7 +1834,7 @@ version = "0.1.0"
 dependencies = [
  "beacon_chain",
  "beacon_node",
- "clap 4.5.4",
+ "clap",
  "clap_utils",
  "environment",
  "hex",
@@ -2023,7 +2024,7 @@ dependencies = [
 name = "directory"
 version = "0.1.0"
 dependencies = [
- "clap 4.5.4",
+ "clap",
  "clap_utils",
  "eth2_network_config",
 ]
@@ -3433,9 +3434,13 @@ dependencies = [
 
 [[package]]
 name = "half"
-version = "1.8.3"
+version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b43ede17f21864e81be2fa654110bf1e793774238d86ef8555c37e6519c0403"
+checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888"
+dependencies = [
+ "cfg-if",
+ "crunchy",
+]
 
 [[package]]
 name = "hash-db"
@@ -4322,7 +4327,7 @@ dependencies = [
  "account_utils",
  "beacon_chain",
  "bls",
- "clap 4.5.4",
+ "clap",
  "clap_utils",
  "deposit_contract",
  "env_logger 0.9.3",
@@ -4896,7 +4901,7 @@ dependencies = [
  "beacon_processor",
  "bls",
  "boot_node",
- "clap 4.5.4",
+ "clap",
  "clap_utils",
  "database_manager",
  "directory",
@@ -7362,16 +7367,6 @@ dependencies = [
  "serde_urlencoded",
 ]
 
-[[package]]
-name = "serde_cbor"
-version = "0.11.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5"
-dependencies = [
- "half",
- "serde",
-]
-
 [[package]]
 name = "serde_derive"
 version = "1.0.200"
@@ -7598,7 +7593,7 @@ dependencies = [
 name = "simulator"
 version = "0.2.0"
 dependencies = [
- "clap 4.5.4",
+ "clap",
  "env_logger 0.9.3",
  "eth2_network_config",
  "execution_layer",
@@ -8260,15 +8255,6 @@ dependencies = [
  "sha2 0.10.8",
 ]
 
-[[package]]
-name = "textwrap"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
-dependencies = [
- "unicode-width",
-]
-
 [[package]]
 name = "thiserror"
 version = "1.0.59"
@@ -8895,12 +8881,6 @@ dependencies = [
  "tinyvec",
 ]
 
-[[package]]
-name = "unicode-width"
-version = "0.1.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68f5e5f3158ecfd4b8ff6fe086db7c8467a2dfdac97fe420f2b7c4aa97af66d6"
-
 [[package]]
 name = "unicode-xid"
 version = "0.2.4"
@@ -9014,7 +8994,7 @@ dependencies = [
  "account_utils",
  "bincode",
  "bls",
- "clap 4.5.4",
+ "clap",
  "clap_utils",
  "deposit_contract",
  "directory",
@@ -9086,7 +9066,7 @@ name = "validator_manager"
 version = "0.1.0"
 dependencies = [
  "account_utils",
- "clap 4.5.4",
+ "clap",
  "clap_utils",
  "environment",
  "eth2",
@@ -9325,7 +9305,7 @@ dependencies = [
  "beacon_chain",
  "beacon_node",
  "bls",
- "clap 4.5.4",
+ "clap",
  "clap_utils",
  "diesel",
  "diesel_migrations",

Cargo.toml

@@ -107,7 +107,7 @@ clap = { version = "4.5.4", features = ["cargo", "wrap_help"] }
 # feature ourselves when desired.
 c-kzg = { version = "1", default-features =  false }
 compare_fields_derive = { path = "common/compare_fields_derive" }
-criterion = "0.3"
+criterion = "0.5"
 delay_map = "0.3"
 derivative = "2"
 dirs = "3"

consensus/swap_or_not_shuffle/benches/benches.rs

@@ -1,7 +1,4 @@
-#![allow(deprecated)]
-
-use criterion::Criterion;
-use criterion::{black_box, criterion_group, criterion_main, Benchmark};
+use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
 use swap_or_not_shuffle::{compute_shuffled_index, shuffle_list as fast_shuffle};
 
 const SHUFFLE_ROUND_COUNT: u8 = 90;
@@ -25,70 +22,32 @@ fn shuffles(c: &mut Criterion) {
         b.iter(|| black_box(shuffle_list(&seed, 8)))
     });
 
-    c.bench(
-        "whole list shuffle",
-        Benchmark::new("8 elements", move |b| {
-            let seed = vec![42; 32];
-            b.iter(|| black_box(shuffle_list(&seed, 8)))
-        }),
-    );
+    for size in [8, 16, 512, 16_384] {
+        c.bench_with_input(
+            BenchmarkId::new("whole list shuffle", format!("{size} elements")),
+            &size,
+            move |b, &n| {
+                let seed = vec![42; 32];
+                b.iter(|| black_box(shuffle_list(&seed, n)))
+            },
+        );
+    }
 
-    c.bench(
-        "whole list shuffle",
-        Benchmark::new("16 elements", move |b| {
-            let seed = vec![42; 32];
-            b.iter(|| black_box(shuffle_list(&seed, 16)))
-        }),
-    );
-
-    c.bench(
-        "whole list shuffle",
-        Benchmark::new("512 elements", move |b| {
-            let seed = vec![42; 32];
-            b.iter(|| black_box(shuffle_list(&seed, 512)))
-        })
-        .sample_size(10),
-    );
-
-    c.bench(
-        "_fast_ whole list shuffle",
-        Benchmark::new("512 elements", move |b| {
-            let seed = vec![42; 32];
-            let list: Vec<usize> = (0..512).collect();
-            b.iter(|| black_box(fast_shuffle(list.clone(), SHUFFLE_ROUND_COUNT, &seed, true)))
-        })
-        .sample_size(10),
-    );
-
-    c.bench(
-        "whole list shuffle",
-        Benchmark::new("16384 elements", move |b| {
-            let seed = vec![42; 32];
-            b.iter(|| black_box(shuffle_list(&seed, 16_384)))
-        })
-        .sample_size(10),
-    );
-
-    c.bench(
-        "_fast_ whole list shuffle",
-        Benchmark::new("16384 elements", move |b| {
-            let seed = vec![42; 32];
-            let list: Vec<usize> = (0..16384).collect();
-            b.iter(|| black_box(fast_shuffle(list.clone(), SHUFFLE_ROUND_COUNT, &seed, true)))
-        })
-        .sample_size(10),
-    );
-
-    c.bench(
-        "_fast_ whole list shuffle",
-        Benchmark::new("4m elements", move |b| {
-            let seed = vec![42; 32];
-            let list: Vec<usize> = (0..4_000_000).collect();
-            b.iter(|| black_box(fast_shuffle(list.clone(), SHUFFLE_ROUND_COUNT, &seed, true)))
-        })
-        .sample_size(10),
-    );
+    let mut group = c.benchmark_group("fast");
+    group.sample_size(10);
+    for size in [512, 16_384, 4_000_000] {
+        group.bench_with_input(
+            BenchmarkId::new("whole list shuffle", format!("{size} elements")),
+            &size,
+            move |b, &n| {
+                let seed = vec![42; 32];
+                let list: Vec<usize> = (0..n).collect();
+                b.iter(|| black_box(fast_shuffle(list.clone(), SHUFFLE_ROUND_COUNT, &seed, true)))
+            },
+        );
+    }
+    group.finish();
 }
 
-criterion_group!(benches, shuffles,);
+criterion_group!(benches, shuffles);
 criterion_main!(benches);

consensus/types/benches/benches.rs

@@ -1,7 +1,4 @@
-#![allow(deprecated)]
-
-use criterion::Criterion;
-use criterion::{black_box, criterion_group, criterion_main, Benchmark};
+use criterion::{black_box, criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion};
 use milhouse::List;
 use rayon::prelude::*;
 use ssz::Encode;
@@ -53,75 +50,82 @@ fn all_benches(c: &mut Criterion) {
     let validator_count = 16_384;
     let spec = Arc::new(MainnetEthSpec::default_spec());
 
+    let mut g = c.benchmark_group("types");
+    g.sample_size(10);
+
     let mut state = get_state::<MainnetEthSpec>(validator_count);
     state.build_caches(&spec).expect("should build caches");
     let state_bytes = state.as_ssz_bytes();
 
     let inner_state = state.clone();
-    c.bench(
-        &format!("{}_validators", validator_count),
-        Benchmark::new("encode/beacon_state", move |b| {
+    g.bench_with_input(
+        BenchmarkId::new("encode/beacon_state", validator_count),
+        &inner_state,
+        |b, state| {
             b.iter_batched_ref(
-                || inner_state.clone(),
+                || state.clone(),
                 |state| black_box(state.as_ssz_bytes()),
-                criterion::BatchSize::SmallInput,
+                BatchSize::SmallInput,
             )
-        })
-        .sample_size(10),
+        },
     );
 
-    c.bench(
-        &format!("{}_validators", validator_count),
-        Benchmark::new("decode/beacon_state", move |b| {
+    g.bench_with_input(
+        BenchmarkId::new("decode/beacon_state", validator_count),
+        &(state_bytes.clone(), spec.clone()),
+        |b, (bytes, spec)| {
             b.iter_batched_ref(
-                || (state_bytes.clone(), spec.clone()),
+                || (bytes.clone(), spec.clone()),
                 |(bytes, spec)| {
                     let state: BeaconState<MainnetEthSpec> =
                         BeaconState::from_ssz_bytes(&bytes, &spec).expect("should decode");
                     black_box(state)
                 },
-                criterion::BatchSize::SmallInput,
+                BatchSize::SmallInput,
             )
-        })
-        .sample_size(10),
+        },
     );
 
     let inner_state = state.clone();
-    c.bench(
-        &format!("{}_validators", validator_count),
-        Benchmark::new("clone/beacon_state", move |b| {
+    g.bench_with_input(
+        BenchmarkId::new("clone/beacon_state", validator_count),
+        &inner_state,
+        |b, state| {
             b.iter_batched_ref(
-                || inner_state.clone(),
+                || state.clone(),
                 |state| black_box(state.clone()),
-                criterion::BatchSize::SmallInput,
+                BatchSize::SmallInput,
             )
-        })
-        .sample_size(10),
+        },
     );
 
     let inner_state = state.clone();
-    c.bench(
-        &format!("{}_validators", validator_count),
-        Benchmark::new(
+    g.bench_with_input(
+        BenchmarkId::new(
             "initialized_cached_tree_hash_without_changes/beacon_state",
-            move |b| {
-                b.iter_batched_ref(
-                    || inner_state.clone(),
-                    |state| black_box(state.update_tree_hash_cache()),
-                    criterion::BatchSize::SmallInput,
-                )
-            },
-        )
-        .sample_size(10),
+            validator_count,
+        ),
+        &inner_state,
+        |b, state| {
+            b.iter_batched_ref(
+                || state.clone(),
+                |state| black_box(state.update_tree_hash_cache()),
+                BatchSize::SmallInput,
+            )
+        },
     );
 
     let mut inner_state = state.clone();
     inner_state.drop_all_caches().unwrap();
-    c.bench(
-        &format!("{}_validators", validator_count),
-        Benchmark::new("non_initialized_cached_tree_hash/beacon_state", move |b| {
+    g.bench_with_input(
+        BenchmarkId::new(
+            "non_initialized_cached_tree_hash/beacon_state",
+            validator_count,
+        ),
+        &inner_state,
+        |b, state| {
             b.iter_batched_ref(
-                || inner_state.clone(),
+                || state.clone(),
                 |state| {
                     black_box(
                         state
@@ -129,41 +133,40 @@ fn all_benches(c: &mut Criterion) {
                             .expect("should update tree hash"),
                     )
                 },
-                criterion::BatchSize::SmallInput,
+                BatchSize::SmallInput,
             )
-        })
-        .sample_size(10),
+        },
     );
 
     let inner_state = state.clone();
-    c.bench(
-        &format!("{}_validators", validator_count),
-        Benchmark::new(
+    g.bench_with_input(
+        BenchmarkId::new(
             "initialized_cached_tree_hash_with_new_validators/beacon_state",
-            move |b| {
-                b.iter_batched_ref(
-                    || {
-                        let mut state = inner_state.clone();
-                        for _ in 0..16 {
-                            state
-                                .validators_mut()
-                                .push(Validator::default())
-                                .expect("should push validatorj");
-                            state
-                                .balances_mut()
-                                .push(32_000_000_000)
-                                .expect("should push balance");
-                        }
+            validator_count,
+        ),
+        &inner_state,
+        |b, state| {
+            b.iter_batched_ref(
+                || {
+                    let mut state = state.clone();
+                    for _ in 0..16 {
                         state
-                    },
-                    |state| black_box(state.update_tree_hash_cache()),
-                    criterion::BatchSize::SmallInput,
-                )
-            },
-        )
-        .sample_size(10),
+                            .validators_mut()
+                            .push(Validator::default())
+                            .expect("should push validator");
+                        state
+                            .balances_mut()
+                            .push(32_000_000_000)
+                            .expect("should push balance");
+                    }
+                    state
+                },
+                |state| black_box(state.update_tree_hash_cache()),
+                BatchSize::SmallInput,
+            )
+        },
     );
 }
 
-criterion_group!(benches, all_benches,);
+criterion_group!(benches, all_benches);
 criterion_main!(benches);