State cache tweaks (#7095)

Backport of:

- https://github.com/sigp/lighthouse/pull/7067

For:

- https://github.com/sigp/lighthouse/issues/7039


  - Prevent writing to state cache when migrating the database
- Add `state-cache-headroom` flag to control pruning
- Prune old epoch boundary states ahead of mid-epoch states
- Never prune head block's state
- Avoid caching ancestor states unless they are on an epoch boundary
- Log when states enter/exit the cache

Co-authored-by: Eitan Seri-Levi <eserilev@ucsc.edu>

beacon_node/http_api/src/block_rewards.rs

@@ -43,8 +43,10 @@ pub fn get_block_rewards<T: BeaconChainTypes>(
         .map_err(unhandled_error)?
         .ok_or_else(|| custom_bad_request(format!("prior state at slot {} unknown", prior_slot)))?;
 
+    // This branch is reached from the HTTP API. We assume the user wants
+    // to cache states so that future calls are faster.
     let mut state = chain
-        .get_state(&state_root, Some(prior_slot))
+        .get_state(&state_root, Some(prior_slot), true)
         .and_then(|maybe_state| maybe_state.ok_or(BeaconChainError::MissingBeaconState(state_root)))
         .map_err(unhandled_error)?;
 
@@ -133,8 +135,10 @@ pub fn compute_block_rewards<T: BeaconChainTypes>(
                     ))
                 })?;
 
+            // This branch is reached from the HTTP API. We assume the user wants
+            // to cache states so that future calls are faster.
             let parent_state = chain
-                .get_state(&parent_block.state_root(), Some(parent_block.slot()))
+                .get_state(&parent_block.state_root(), Some(parent_block.slot()), true)
                 .map_err(unhandled_error)?
                 .ok_or_else(|| {
                     custom_bad_request(format!(