diff --git a/Cargo.lock b/Cargo.lock
index ab4652f785..c470e6c640 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3728,7 +3728,6 @@ dependencies = [
 [[package]]
 name = "milhouse"
 version = "0.1.0"
-source = "git+https://github.com/sigp/milhouse?branch=main#30c87f256deacc381fb15e39ccc4a281445f98da"
 dependencies = [
  "derivative",
  "eth2_hashing 0.2.0",
@@ -3741,6 +3740,7 @@ dependencies = [
  "tree_hash",
  "triomphe",
  "typenum",
+ "vec_map",
 ]
 
 [[package]]
@@ -5964,6 +5964,7 @@ dependencies = [
  "smallvec",
  "tree_hash",
  "types",
+ "vec_map",
 ]
 
 [[package]]
diff --git a/consensus/state_processing/Cargo.toml b/consensus/state_processing/Cargo.toml
index 51fb749625..b265283d2b 100644
--- a/consensus/state_processing/Cargo.toml
+++ b/consensus/state_processing/Cargo.toml
@@ -26,6 +26,7 @@ arbitrary = { version = "1.0", features = ["derive"], optional = true }
 lighthouse_metrics = { path = "../../common/lighthouse_metrics", optional = true }
 lazy_static = { version = "1.4.0", optional = true }
 rustc-hash = "1.1.0"
+vec_map = "0.8.2"
 
 [features]
 default = ["legacy-arith", "metrics"]
diff --git a/consensus/state_processing/src/per_epoch_processing/altair/participation_cache.rs b/consensus/state_processing/src/per_epoch_processing/altair/participation_cache.rs
index 6360074a8b..0fa944242d 100644
--- a/consensus/state_processing/src/per_epoch_processing/altair/participation_cache.rs
+++ b/consensus/state_processing/src/per_epoch_processing/altair/participation_cache.rs
@@ -13,7 +13,7 @@
 
 use crate::common::altair::get_base_reward;
 use safe_arith::{ArithError, SafeArith};
-use std::collections::BTreeMap;
+use types::milhouse::update_map::{MaxMap, UpdateMap};
 use types::{
     consts::altair::{
         NUM_FLAG_INDICES, TIMELY_HEAD_FLAG_INDEX, TIMELY_SOURCE_FLAG_INDEX,
@@ -22,6 +22,7 @@ use types::{
     BeaconState, BeaconStateError, ChainSpec, Epoch, EthSpec, ParticipationFlags, RelativeEpoch,
     Unsigned, Validator,
 };
+use vec_map::VecMap;
 
 #[derive(Debug, PartialEq)]
 pub enum Error {
@@ -198,7 +199,7 @@ pub struct ParticipationCache {
     /// `process_slashings`.
     process_slashings_indices: Vec<(usize, u64)>,
     /// Updates to the inactivity scores if we are definitely not in an inactivity leak.
-    pub inactivity_score_updates: Option<BTreeMap<usize, u64>>,
+    pub inactivity_score_updates: Option<MaxMap<VecMap<u64>>>,
 }
 
 impl ParticipationCache {
@@ -237,7 +238,7 @@ impl ParticipationCache {
         let definitely_not_in_inactivity_leak =
             state.finalized_checkpoint().epoch + spec.min_epochs_to_inactivity_penalty + 1
                 >= state.current_epoch();
-        let mut inactivity_score_updates = BTreeMap::new();
+        let mut inactivity_score_updates = MaxMap::default();
 
         // Iterate through all validators, updating:
         //
diff --git a/testing/ef_tests/src/cases/epoch_processing.rs b/testing/ef_tests/src/cases/epoch_processing.rs
index 98da036b18..ccc6b5e6f9 100644
--- a/testing/ef_tests/src/cases/epoch_processing.rs
+++ b/testing/ef_tests/src/cases/epoch_processing.rs
@@ -213,7 +213,7 @@ impl<E: EthSpec> EpochTransition<E> for InactivityUpdates {
             BeaconState::Base(_) => Ok(()),
             BeaconState::Altair(_) | BeaconState::Merge(_) => altair::process_inactivity_updates(
                 state,
-                &altair::ParticipationCache::new(state, spec).unwrap(),
+                &mut altair::ParticipationCache::new(state, spec).unwrap(),
                 spec,
             ),
         }
@@ -278,19 +278,22 @@ impl<E: EthSpec, T: EpochTransition<E>> Case for EpochProcessing<E, T> {
     fn result(&self, _case_index: usize, fork_name: ForkName) -> Result<(), Error> {
         self.metadata.bls_setting.unwrap_or_default().check()?;
 
-        let mut state = self.pre.clone();
+        let spec = &testing_spec::<E>(fork_name);
+        let mut pre_state = self.pre.clone();
+
+        // Processing requires the committee caches.
+        pre_state.build_all_committee_caches(spec).unwrap();
+
+        let mut state = pre_state.clone();
         let mut expected = self.post.clone();
 
-        let spec = &testing_spec::<E>(fork_name);
+        expected.as_mut().map(|post_state| {
+            post_state.build_all_committee_caches(spec).unwrap();
+        });
 
-        let mut result = (|| {
-            // Processing requires the committee caches.
-            state.build_all_committee_caches(spec)?;
+        let mut result = T::run(&mut state, spec).map(|_| state);
 
-            T::run(&mut state, spec).map(|_| state)
-        })();
-
-        compare_beacon_state_results_without_caches(&mut result, &mut expected)?;
-        check_state_diff(&self.pre, &self.post)
+        check_state_diff(&pre_state, &expected)?;
+        compare_beacon_state_results_without_caches(&mut result, &mut expected)
     }
 }
diff --git a/testing/ef_tests/src/cases/operations.rs b/testing/ef_tests/src/cases/operations.rs
index 82a2a12d61..e2b43e5c78 100644
--- a/testing/ef_tests/src/cases/operations.rs
+++ b/testing/ef_tests/src/cases/operations.rs
@@ -323,14 +323,20 @@ impl<E: EthSpec, O: Operation<E>> Case for Operations<E, O> {
 
     fn result(&self, _case_index: usize, fork_name: ForkName) -> Result<(), Error> {
         let spec = &testing_spec::<E>(fork_name);
-        let mut state = self.pre.clone();
-        let mut expected = self.post.clone();
 
+        let mut pre_state = self.pre.clone();
         // Processing requires the committee caches.
-        state
+        pre_state
             .build_all_committee_caches(spec)
             .expect("committee caches OK");
 
+        let mut state = pre_state.clone();
+        let mut expected = self.post.clone();
+
+        expected
+            .as_mut()
+            .map(|post_state| post_state.build_all_committee_caches(spec).unwrap());
+
         let mut result = self
             .operation
             .as_ref()
@@ -338,7 +344,7 @@ impl<E: EthSpec, O: Operation<E>> Case for Operations<E, O> {
             .apply_to(&mut state, spec, self)
             .map(|()| state);
 
-        compare_beacon_state_results_without_caches(&mut result, &mut expected)?;
-        check_state_diff(&self.pre, &self.post)
+        check_state_diff(&pre_state, &expected)?;
+        compare_beacon_state_results_without_caches(&mut result, &mut expected)
     }
 }