Prevent writing to state cache when migrating the database (#7067)

* add an update_cache flag to get_state to have more granular control over when we write to the cache

* State cache tweaks

- add state-cache-headroom flag to control pruning
- prune old epoch boundary states ahead of mid-epoch states
- never prune head block's state
- avoid caching ancestor states unless they are on an epoch boundary

---------

Co-authored-by: Michael Sproul <michael@sigmaprime.io>

beacon_node/src/cli.rs

@@ -792,6 +792,15 @@ pub fn cli_app() -> Command {
                 .action(ArgAction::Set)
                 .display_order(0)
         )
+        .arg(
+            Arg::new("state-cache-headroom")
+                .long("state-cache-headroom")
+                .value_name("N")
+                .help("Minimum number of states to cull from the state cache when it gets full")
+                .default_value("1")
+                .action(ArgAction::Set)
+                .display_order(0)
+        )
         .arg(
             Arg::new("block-cache-size")
                 .long("block-cache-size")

beacon_node/src/config.rs

@@ -466,6 +466,12 @@ pub fn get_config<E: EthSpec>(
         client_config.chain.epochs_per_migration = epochs_per_migration;
     }
 
+    if let Some(state_cache_headroom) =
+        clap_utils::parse_optional(cli_args, "state-cache-headroom")?
+    {
+        client_config.store.state_cache_headroom = state_cache_headroom;
+    }
+
     if let Some(prune_blobs) = clap_utils::parse_optional(cli_args, "prune-blobs")? {
         client_config.store.prune_blobs = prune_blobs;
     }