Shrink persisted fork choice data (#7805)

Closes:

- https://github.com/sigp/lighthouse/issues/7760


  - [x] Remove `balances_cache` from `PersistedForkChoiceStore` (~65 MB saving on mainnet)
- [x] Remove `justified_balances` from `PersistedForkChoiceStore` (~16 MB saving on mainnet)
- [x] Remove `balances` from `ProtoArray`/`SszContainer`.
- [x] Implement zstd compression for votes
- [x] Fix bug in justified state usage
- [x] Bump schema version to V28 and implement migration.

consensus/proto_array/src/fork_choice_test_definition.rs

@@ -306,8 +306,8 @@ fn get_checkpoint(i: u64) -> Checkpoint {
 
 fn check_bytes_round_trip(original: &ProtoArrayForkChoice) {
     let bytes = original.as_bytes();
-    let decoded =
-        ProtoArrayForkChoice::from_bytes(&bytes).expect("fork choice should decode from bytes");
+    let decoded = ProtoArrayForkChoice::from_bytes(&bytes, original.balances.clone())
+        .expect("fork choice should decode from bytes");
     assert!(
         *original == decoded,
         "fork choice should encode and decode without change"

consensus/proto_array/src/lib.rs

@@ -16,5 +16,5 @@ pub use error::Error;
 pub mod core {
     pub use super::proto_array::{ProposerBoost, ProtoArray, ProtoNode};
     pub use super::proto_array_fork_choice::VoteTracker;
-    pub use super::ssz_container::{SszContainer, SszContainerV17};
+    pub use super::ssz_container::{SszContainer, SszContainerV17, SszContainerV28};
 }

consensus/proto_array/src/proto_array_fork_choice.rs

@@ -866,14 +866,25 @@ impl ProtoArrayForkChoice {
         self.proto_array.iter_block_roots(block_root)
     }
 
+    pub fn as_ssz_container(&self) -> SszContainer {
+        SszContainer::from(self)
+    }
+
     pub fn as_bytes(&self) -> Vec<u8> {
         SszContainer::from(self).as_ssz_bytes()
     }
 
-    pub fn from_bytes(bytes: &[u8]) -> Result<Self, String> {
+    pub fn from_bytes(bytes: &[u8], balances: JustifiedBalances) -> Result<Self, String> {
         let container = SszContainer::from_ssz_bytes(bytes)
             .map_err(|e| format!("Failed to decode ProtoArrayForkChoice: {:?}", e))?;
-        container
+        Self::from_container(container, balances)
+    }
+
+    pub fn from_container(
+        container: SszContainer,
+        balances: JustifiedBalances,
+    ) -> Result<Self, String> {
+        (container, balances)
             .try_into()
             .map_err(|e| format!("Failed to initialize ProtoArrayForkChoice: {e:?}"))
     }

consensus/proto_array/src/ssz_container.rs

@@ -14,16 +14,20 @@ use types::{Checkpoint, Hash256};
 // selector.
 four_byte_option_impl!(four_byte_option_checkpoint, Checkpoint);
 
-pub type SszContainer = SszContainerV17;
+pub type SszContainer = SszContainerV28;
 
-#[superstruct(variants(V17), variant_attributes(derive(Encode, Decode)), no_enum)]
+#[superstruct(
+    variants(V17, V28),
+    variant_attributes(derive(Encode, Decode, Clone)),
+    no_enum
+)]
 pub struct SszContainer {
     pub votes: Vec<VoteTracker>,
+    #[superstruct(only(V17))]
     pub balances: Vec<u64>,
     pub prune_threshold: usize,
     pub justified_checkpoint: Checkpoint,
     pub finalized_checkpoint: Checkpoint,
-    #[superstruct(only(V17))]
     pub nodes: Vec<ProtoNodeV17>,
     pub indices: Vec<(Hash256, usize)>,
     pub previous_proposer_boost: ProposerBoost,
@@ -35,7 +39,6 @@ impl From<&ProtoArrayForkChoice> for SszContainer {
 
         Self {
             votes: from.votes.0.clone(),
-            balances: from.balances.effective_balances.clone(),
             prune_threshold: proto_array.prune_threshold,
             justified_checkpoint: proto_array.justified_checkpoint,
             finalized_checkpoint: proto_array.finalized_checkpoint,
@@ -46,10 +49,10 @@ impl From<&ProtoArrayForkChoice> for SszContainer {
     }
 }
 
-impl TryFrom<SszContainer> for ProtoArrayForkChoice {
+impl TryFrom<(SszContainer, JustifiedBalances)> for ProtoArrayForkChoice {
     type Error = Error;
 
-    fn try_from(from: SszContainer) -> Result<Self, Error> {
+    fn try_from((from, balances): (SszContainer, JustifiedBalances)) -> Result<Self, Error> {
         let proto_array = ProtoArray {
             prune_threshold: from.prune_threshold,
             justified_checkpoint: from.justified_checkpoint,
@@ -62,7 +65,38 @@ impl TryFrom<SszContainer> for ProtoArrayForkChoice {
         Ok(Self {
             proto_array,
             votes: ElasticList(from.votes),
-            balances: JustifiedBalances::from_effective_balances(from.balances)?,
+            balances,
         })
     }
 }
+
+// Convert V17 to V28 by dropping balances.
+impl From<SszContainerV17> for SszContainerV28 {
+    fn from(v17: SszContainerV17) -> Self {
+        Self {
+            votes: v17.votes,
+            prune_threshold: v17.prune_threshold,
+            justified_checkpoint: v17.justified_checkpoint,
+            finalized_checkpoint: v17.finalized_checkpoint,
+            nodes: v17.nodes,
+            indices: v17.indices,
+            previous_proposer_boost: v17.previous_proposer_boost,
+        }
+    }
+}
+
+// Convert V28 to V17 by re-adding balances.
+impl From<(SszContainerV28, JustifiedBalances)> for SszContainerV17 {
+    fn from((v28, balances): (SszContainerV28, JustifiedBalances)) -> Self {
+        Self {
+            votes: v28.votes,
+            balances: balances.effective_balances.clone(),
+            prune_threshold: v28.prune_threshold,
+            justified_checkpoint: v28.justified_checkpoint,
+            finalized_checkpoint: v28.finalized_checkpoint,
+            nodes: v28.nodes,
+            indices: v28.indices,
+            previous_proposer_boost: v28.previous_proposer_boost,
+        }
+    }
+}