Shrink persisted fork choice data (#7805)

Closes:

- https://github.com/sigp/lighthouse/issues/7760


  - [x] Remove `balances_cache` from `PersistedForkChoiceStore` (~65 MB saving on mainnet)
- [x] Remove `justified_balances` from `PersistedForkChoiceStore` (~16 MB saving on mainnet)
- [x] Remove `balances` from `ProtoArray`/`SszContainer`.
- [x] Implement zstd compression for votes
- [x] Fix bug in justified state usage
- [x] Bump schema version to V28 and implement migration.

beacon_node/store/src/config.rs

@@ -4,12 +4,12 @@ use crate::{DBColumn, Error, StoreItem};
 use serde::{Deserialize, Serialize};
 use ssz::{Decode, Encode};
 use ssz_derive::{Decode, Encode};
-use std::io::Write;
+use std::io::{Read, Write};
 use std::num::NonZeroUsize;
 use strum::{Display, EnumString, EnumVariantNames};
 use types::EthSpec;
 use types::non_zero_usize::new_non_zero_usize;
-use zstd::Encoder;
+use zstd::{Decoder, Encoder};
 
 #[cfg(all(feature = "redb", not(feature = "leveldb")))]
 pub const DEFAULT_BACKEND: DatabaseBackend = DatabaseBackend::Redb;
@@ -194,15 +194,23 @@ impl StoreConfig {
         }
     }
 
-    pub fn compress_bytes(&self, ssz_bytes: &[u8]) -> Result<Vec<u8>, Error> {
+    /// Compress bytes using zstd and the compression level from `self`.
+    pub fn compress_bytes(&self, ssz_bytes: &[u8]) -> Result<Vec<u8>, std::io::Error> {
         let mut compressed_value =
             Vec::with_capacity(self.estimate_compressed_size(ssz_bytes.len()));
-        let mut encoder = Encoder::new(&mut compressed_value, self.compression_level)
-            .map_err(Error::Compression)?;
-        encoder.write_all(ssz_bytes).map_err(Error::Compression)?;
-        encoder.finish().map_err(Error::Compression)?;
+        let mut encoder = Encoder::new(&mut compressed_value, self.compression_level)?;
+        encoder.write_all(ssz_bytes)?;
+        encoder.finish()?;
         Ok(compressed_value)
     }
+
+    /// Decompress bytes compressed using zstd.
+    pub fn decompress_bytes(&self, input: &[u8]) -> Result<Vec<u8>, std::io::Error> {
+        let mut out = Vec::with_capacity(self.estimate_decompressed_size(input.len()));
+        let mut decoder = Decoder::new(input)?;
+        decoder.read_to_end(&mut out)?;
+        Ok(out)
+    }
 }
 
 impl StoreItem for OnDiskStoreConfig {

beacon_node/store/src/hdiff.rs

@@ -6,14 +6,12 @@ use serde::{Deserialize, Serialize};
 use ssz::{Decode, Encode};
 use ssz_derive::{Decode, Encode};
 use std::cmp::Ordering;
-use std::io::{Read, Write};
 use std::ops::RangeInclusive;
 use std::str::FromStr;
 use std::sync::LazyLock;
 use superstruct::superstruct;
 use types::historical_summary::HistoricalSummary;
 use types::{BeaconState, ChainSpec, Epoch, EthSpec, Hash256, List, Slot, Validator};
-use zstd::{Decoder, Encoder};
 
 static EMPTY_PUBKEY: LazyLock<PublicKeyBytes> = LazyLock::new(PublicKeyBytes::empty);
 
@@ -395,13 +393,17 @@ impl CompressedU64Diff {
             .collect();
 
         Ok(CompressedU64Diff {
-            bytes: compress_bytes(&uncompressed_bytes, config)?,
+            bytes: config
+                .compress_bytes(&uncompressed_bytes)
+                .map_err(Error::Compression)?,
         })
     }
 
     pub fn apply(&self, xs: &mut Vec<u64>, config: &StoreConfig) -> Result<(), Error> {
         // Decompress balances diff.
-        let balances_diff_bytes = uncompress_bytes(&self.bytes, config)?;
+        let balances_diff_bytes = config
+            .decompress_bytes(&self.bytes)
+            .map_err(Error::Compression)?;
 
         for (i, diff_bytes) in balances_diff_bytes
             .chunks(u64::BITS as usize / 8)
@@ -428,22 +430,6 @@ impl CompressedU64Diff {
     }
 }
 
-fn compress_bytes(input: &[u8], config: &StoreConfig) -> Result<Vec<u8>, Error> {
-    let compression_level = config.compression_level;
-    let mut out = Vec::with_capacity(config.estimate_compressed_size(input.len()));
-    let mut encoder = Encoder::new(&mut out, compression_level).map_err(Error::Compression)?;
-    encoder.write_all(input).map_err(Error::Compression)?;
-    encoder.finish().map_err(Error::Compression)?;
-    Ok(out)
-}
-
-fn uncompress_bytes(input: &[u8], config: &StoreConfig) -> Result<Vec<u8>, Error> {
-    let mut out = Vec::with_capacity(config.estimate_decompressed_size(input.len()));
-    let mut decoder = Decoder::new(input).map_err(Error::Compression)?;
-    decoder.read_to_end(&mut out).map_err(Error::Compression)?;
-    Ok(out)
-}
-
 impl ValidatorsDiff {
     pub fn compute(
         xs: &[Validator],
@@ -534,12 +520,16 @@ impl ValidatorsDiff {
             .collect::<Vec<u8>>();
 
         Ok(Self {
-            bytes: compress_bytes(&uncompressed_bytes, config)?,
+            bytes: config
+                .compress_bytes(&uncompressed_bytes)
+                .map_err(Error::Compression)?,
         })
     }
 
     pub fn apply(&self, xs: &mut Vec<Validator>, config: &StoreConfig) -> Result<(), Error> {
-        let validator_diff_bytes = uncompress_bytes(&self.bytes, config)?;
+        let validator_diff_bytes = config
+            .decompress_bytes(&self.bytes)
+            .map_err(Error::Compression)?;
 
         for diff_bytes in
             validator_diff_bytes.chunks(<ValidatorDiffEntry as Decode>::ssz_fixed_len())

beacon_node/store/src/metadata.rs

@@ -4,7 +4,7 @@ use ssz::{Decode, Encode};
 use ssz_derive::{Decode, Encode};
 use types::{Hash256, Slot};
 
-pub const CURRENT_SCHEMA_VERSION: SchemaVersion = SchemaVersion(27);
+pub const CURRENT_SCHEMA_VERSION: SchemaVersion = SchemaVersion(28);
 
 // All the keys that get stored under the `BeaconMeta` column.
 //