State cache tweaks (#7095)

Backport of:

- https://github.com/sigp/lighthouse/pull/7067

For:

- https://github.com/sigp/lighthouse/issues/7039


  - Prevent writing to state cache when migrating the database
- Add `state-cache-headroom` flag to control pruning
- Prune old epoch boundary states ahead of mid-epoch states
- Never prune head block's state
- Avoid caching ancestor states unless they are on an epoch boundary
- Log when states enter/exit the cache

Co-authored-by: Eitan Seri-Levi <eserilev@ucsc.edu>

beacon_node/beacon_chain/tests/rewards.rs

@@ -20,6 +20,9 @@ use types::{ChainSpec, ForkName, Slot};
 
 pub const VALIDATOR_COUNT: usize = 64;
 
+// When set to true, cache any states fetched from the db.
+pub const CACHE_STATE_IN_TESTS: bool = true;
+
 type E = MinimalEthSpec;
 
 static KEYPAIRS: LazyLock<Vec<Keypair>> =
@@ -116,8 +119,13 @@ async fn test_sync_committee_rewards() {
         .get_blinded_block(&block.parent_root())
         .unwrap()
         .unwrap();
+
     let parent_state = chain
-        .get_state(&parent_block.state_root(), Some(parent_block.slot()))
+        .get_state(
+            &parent_block.state_root(),
+            Some(parent_block.slot()),
+            CACHE_STATE_IN_TESTS,
+        )
         .unwrap()
         .unwrap();