Prevent writing to state cache when migrating the database (#7067)

* add an update_cache flag to get_state to have more granular control over when we write to the cache

* State cache tweaks

- add state-cache-headroom flag to control pruning
- prune old epoch boundary states ahead of mid-epoch states
- never prune head block's state
- avoid caching ancestor states unless they are on an epoch boundary

---------

Co-authored-by: Michael Sproul <michael@sigmaprime.io>

beacon_node/http_api/src/attestation_performance.rs

@@ -127,7 +127,7 @@ pub fn get_attestation_performance<T: BeaconChainTypes>(
     // Load state for block replay.
     let state_root = prior_block.state_root();
     let state = chain
-        .get_state(&state_root, Some(prior_slot))
+        .get_state(&state_root, Some(prior_slot), true)
         .and_then(|maybe_state| maybe_state.ok_or(BeaconChainError::MissingBeaconState(state_root)))
         .map_err(unhandled_error)?;
 

beacon_node/http_api/src/block_packing_efficiency.rs

@@ -286,7 +286,7 @@ pub fn get_block_packing_efficiency<T: BeaconChainTypes>(
     let starting_state_root = first_block.state_root();
 
     let starting_state = chain
-        .get_state(&starting_state_root, Some(prior_slot))
+        .get_state(&starting_state_root, Some(prior_slot), true)
         .and_then(|maybe_state| {
             maybe_state.ok_or(BeaconChainError::MissingBeaconState(starting_state_root))
         })

beacon_node/http_api/src/block_rewards.rs

@@ -44,7 +44,7 @@ pub fn get_block_rewards<T: BeaconChainTypes>(
         .ok_or_else(|| custom_bad_request(format!("prior state at slot {} unknown", prior_slot)))?;
 
     let mut state = chain
-        .get_state(&state_root, Some(prior_slot))
+        .get_state(&state_root, Some(prior_slot), true)
         .and_then(|maybe_state| maybe_state.ok_or(BeaconChainError::MissingBeaconState(state_root)))
         .map_err(unhandled_error)?;
 
@@ -134,7 +134,7 @@ pub fn compute_block_rewards<T: BeaconChainTypes>(
                 })?;
 
             let parent_state = chain
-                .get_state(&parent_block.state_root(), Some(parent_block.slot()))
+                .get_state(&parent_block.state_root(), Some(parent_block.slot()), true)
                 .map_err(unhandled_error)?
                 .ok_or_else(|| {
                     custom_bad_request(format!(

beacon_node/http_api/src/state_id.rs

@@ -190,7 +190,7 @@ impl StateId {
         };
 
         let state = chain
-            .get_state(&state_root, slot_opt)
+            .get_state(&state_root, slot_opt, true)
             .map_err(warp_utils::reject::unhandled_error)
             .and_then(|opt| {
                 opt.ok_or_else(|| {

beacon_node/http_api/src/sync_committee_rewards.rs

@@ -59,7 +59,7 @@ pub fn get_state_before_applying_block<T: BeaconChainTypes>(
         .map_err(|e| custom_not_found(format!("Parent block is not available! {:?}", e)))?;
 
     let parent_state = chain
-        .get_state(&parent_block.state_root(), Some(parent_block.slot()))
+        .get_state(&parent_block.state_root(), Some(parent_block.slot()), true)
         .and_then(|maybe_state| {
             maybe_state
                 .ok_or_else(|| BeaconChainError::MissingBeaconState(parent_block.state_root()))