Implement SSZ union type (#2579)

## Issue Addressed

NA

## Proposed Changes

Implements the "union" type from the SSZ spec for `ssz`, `ssz_derive`, `tree_hash` and `tree_hash_derive` so it may be derived for `enums`:

https://github.com/ethereum/consensus-specs/blob/v1.1.0-beta.3/ssz/simple-serialize.md#union

The union type is required for the merge, since the `Transaction` type is defined as a single-variant union `Union[OpaqueTransaction]`.

### Crate Updates

This PR will (hopefully) cause CI to publish new versions for the following crates:

- `eth2_ssz_derive`: `0.2.1` -> `0.3.0`
- `eth2_ssz`: `0.3.0` -> `0.4.0`
- `eth2_ssz_types`: `0.2.0` -> `0.2.1`
- `tree_hash`: `0.3.0` -> `0.4.0`
- `tree_hash_derive`: `0.3.0` -> `0.4.0`

These these crates depend on each other, I've had to add a workspace-level `[patch]` for these crates. A follow-up PR will need to remove this patch, ones the new versions are published.

### Union Behaviors

We already had SSZ `Encode` and `TreeHash` derive for enums, however it just did a "transparent" pass-through of the inner value. Since the "union" decoding from the spec is in conflict with the transparent method, I've required that all `enum` have exactly one of the following enum-level attributes:

#### SSZ

-  `#[ssz(enum_behaviour = "union")]`
    - matches the spec used for the merge
-  `#[ssz(enum_behaviour = "transparent")]`
    - maintains existing functionality
    - not supported for `Decode` (never was)
    
#### TreeHash

-  `#[tree_hash(enum_behaviour = "union")]`
    - matches the spec used for the merge
-  `#[tree_hash(enum_behaviour = "transparent")]`
    - maintains existing functionality

This means that we can maintain the existing transparent behaviour, but all existing users will get a compile-time error until they explicitly opt-in to being transparent.

### Legacy Option Encoding

Before this PR, we already had a union-esque encoding for `Option<T>`. However, this was with the *old* SSZ spec where the union selector was 4 bytes. During merge specification, the spec was changed to use 1 byte for the selector.

Whilst the 4-byte `Option` encoding was never used in the spec, we used it in our database. Writing a migrate script for all occurrences of `Option` in the database would be painful, especially since it's used in the `CommitteeCache`. To avoid the migrate script, I added a serde-esque `#[ssz(with = "module")]` field-level attribute to `ssz_derive` so that we can opt into the 4-byte encoding on a field-by-field basis.

The `ssz::legacy::four_byte_impl!` macro allows a one-liner to define the module required for the `#[ssz(with = "module")]` for some `Option<T> where T: Encode + Decode`.

Notably, **I have removed `Encode` and `Decode` impls for `Option`**. I've done this to force a break on downstream users. Like I mentioned, `Option` isn't used in the spec so I don't think it'll be *that* annoying. I think it's nicer than quietly having two different union implementations or quietly breaking the existing `Option` impl.

### Crate Publish Ordering

I've modified the order in which CI publishes crates to ensure that we don't publish a crate without ensuring we already published a crate that it depends upon.

## TODO

- [ ] Queue a follow-up `[patch]`-removing PR.

consensus/types/src/beacon_block.rs

@@ -28,14 +28,19 @@ use tree_hash_derive::TreeHash;
             TestRandom
         ),
         serde(bound = "T: EthSpec", deny_unknown_fields),
-        cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary))
+        cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary)),
     ),
-    ref_attributes(derive(Debug, PartialEq, TreeHash))
+    ref_attributes(
+        derive(Debug, PartialEq, TreeHash),
+        tree_hash(enum_behaviour = "transparent")
+    )
 )]
 #[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, TreeHash)]
 #[serde(untagged)]
 #[serde(bound = "T: EthSpec")]
 #[cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary))]
+#[tree_hash(enum_behaviour = "transparent")]
+#[ssz(enum_behaviour = "transparent")]
 pub struct BeaconBlock<T: EthSpec> {
     #[superstruct(getter(copy))]
     pub slot: Slot,

consensus/types/src/beacon_state.rs

@@ -197,6 +197,8 @@ impl From<BeaconStateHash> for Hash256 {
 #[serde(untagged)]
 #[serde(bound = "T: EthSpec")]
 #[cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary))]
+#[tree_hash(enum_behaviour = "transparent")]
+#[ssz(enum_behaviour = "transparent")]
 pub struct BeaconState<T>
 where
     T: EthSpec,
@@ -275,36 +277,31 @@ where
 
     // Caching (not in the spec)
     #[serde(skip_serializing, skip_deserializing)]
-    #[ssz(skip_serializing)]
-    #[ssz(skip_deserializing)]
+    #[ssz(skip_serializing, skip_deserializing)]
     #[tree_hash(skip_hashing)]
     #[test_random(default)]
     #[derivative(Clone(clone_with = "clone_default"))]
     pub total_active_balance: Option<(Epoch, u64)>,
     #[serde(skip_serializing, skip_deserializing)]
-    #[ssz(skip_serializing)]
-    #[ssz(skip_deserializing)]
+    #[ssz(skip_serializing, skip_deserializing)]
     #[tree_hash(skip_hashing)]
     #[test_random(default)]
     #[derivative(Clone(clone_with = "clone_default"))]
     pub committee_caches: [CommitteeCache; CACHED_EPOCHS],
     #[serde(skip_serializing, skip_deserializing)]
-    #[ssz(skip_serializing)]
-    #[ssz(skip_deserializing)]
+    #[ssz(skip_serializing, skip_deserializing)]
     #[tree_hash(skip_hashing)]
     #[test_random(default)]
     #[derivative(Clone(clone_with = "clone_default"))]
     pub pubkey_cache: PubkeyCache,
     #[serde(skip_serializing, skip_deserializing)]
-    #[ssz(skip_serializing)]
-    #[ssz(skip_deserializing)]
+    #[ssz(skip_serializing, skip_deserializing)]
     #[tree_hash(skip_hashing)]
     #[test_random(default)]
     #[derivative(Clone(clone_with = "clone_default"))]
     pub exit_cache: ExitCache,
     #[serde(skip_serializing, skip_deserializing)]
-    #[ssz(skip_serializing)]
-    #[ssz(skip_deserializing)]
+    #[ssz(skip_serializing, skip_deserializing)]
     #[tree_hash(skip_hashing)]
     #[test_random(default)]
     #[derivative(Clone(clone_with = "clone_default"))]

consensus/types/src/beacon_state/committee_cache.rs

@@ -5,19 +5,26 @@ use crate::*;
 use core::num::NonZeroUsize;
 use safe_arith::SafeArith;
 use serde_derive::{Deserialize, Serialize};
+use ssz::{four_byte_option_impl, Decode, DecodeError, Encode};
 use ssz_derive::{Decode, Encode};
 use std::ops::Range;
 use swap_or_not_shuffle::shuffle_list;
 
 mod tests;
 
+// Define "legacy" implementations of `Option<Epoch>`, `Option<NonZeroUsize>` which use four bytes
+// for encoding the union selector.
+four_byte_option_impl!(four_byte_option_epoch, Epoch);
+four_byte_option_impl!(four_byte_option_non_zero_usize, NonZeroUsize);
+
 /// Computes and stores the shuffling for an epoch. Provides various getters to allow callers to
 /// read the committees for the given epoch.
 #[derive(Debug, Default, PartialEq, Clone, Serialize, Deserialize, Encode, Decode)]
 pub struct CommitteeCache {
+    #[ssz(with = "four_byte_option_epoch")]
     initialized_epoch: Option<Epoch>,
     shuffling: Vec<usize>,
-    shuffling_positions: Vec<Option<NonZeroUsize>>,
+    shuffling_positions: Vec<NonZeroUsizeOption>,
     committees_per_slot: u64,
     slots_per_epoch: u64,
 }
@@ -63,11 +70,11 @@ impl CommitteeCache {
             return Err(Error::TooManyValidators);
         }
 
-        let mut shuffling_positions = vec![None; state.validators().len()];
+        let mut shuffling_positions = vec![<_>::default(); state.validators().len()];
         for (i, &v) in shuffling.iter().enumerate() {
             *shuffling_positions
                 .get_mut(v)
-                .ok_or(Error::ShuffleIndexOutOfBounds(v))? = NonZeroUsize::new(i + 1);
+                .ok_or(Error::ShuffleIndexOutOfBounds(v))? = NonZeroUsize::new(i + 1).into();
         }
 
         Ok(CommitteeCache {
@@ -258,7 +265,8 @@ impl CommitteeCache {
     pub fn shuffled_position(&self, validator_index: usize) -> Option<usize> {
         self.shuffling_positions
             .get(validator_index)?
-            .and_then(|p| Some(p.get() - 1))
+            .0
+            .map(|p| p.get() - 1)
     }
 }
 
@@ -324,3 +332,52 @@ impl arbitrary::Arbitrary for CommitteeCache {
         Ok(Self::default())
     }
 }
+
+/// This is a shim struct to ensure that we can encode a `Vec<Option<NonZeroUsize>>` an SSZ union
+/// with a four-byte selector. The SSZ specification changed from four bytes to one byte during 2021
+/// and we use this shim to avoid breaking the Lighthouse database.
+#[derive(Debug, Default, PartialEq, Clone, Serialize, Deserialize)]
+#[serde(transparent)]
+struct NonZeroUsizeOption(Option<NonZeroUsize>);
+
+impl From<Option<NonZeroUsize>> for NonZeroUsizeOption {
+    fn from(opt: Option<NonZeroUsize>) -> Self {
+        Self(opt)
+    }
+}
+
+impl Encode for NonZeroUsizeOption {
+    fn is_ssz_fixed_len() -> bool {
+        four_byte_option_non_zero_usize::encode::is_ssz_fixed_len()
+    }
+
+    fn ssz_fixed_len() -> usize {
+        four_byte_option_non_zero_usize::encode::ssz_fixed_len()
+    }
+
+    fn ssz_bytes_len(&self) -> usize {
+        four_byte_option_non_zero_usize::encode::ssz_bytes_len(&self.0)
+    }
+
+    fn ssz_append(&self, buf: &mut Vec<u8>) {
+        four_byte_option_non_zero_usize::encode::ssz_append(&self.0, buf)
+    }
+
+    fn as_ssz_bytes(&self) -> Vec<u8> {
+        four_byte_option_non_zero_usize::encode::as_ssz_bytes(&self.0)
+    }
+}
+
+impl Decode for NonZeroUsizeOption {
+    fn is_ssz_fixed_len() -> bool {
+        four_byte_option_non_zero_usize::decode::is_ssz_fixed_len()
+    }
+
+    fn ssz_fixed_len() -> usize {
+        four_byte_option_non_zero_usize::decode::ssz_fixed_len()
+    }
+
+    fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
+        four_byte_option_non_zero_usize::decode::from_ssz_bytes(bytes).map(Self)
+    }
+}

consensus/types/src/signed_beacon_block.rs

@@ -57,6 +57,8 @@ impl From<SignedBeaconBlockHash> for Hash256 {
 #[serde(untagged)]
 #[serde(bound = "E: EthSpec")]
 #[cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary))]
+#[tree_hash(enum_behaviour = "transparent")]
+#[ssz(enum_behaviour = "transparent")]
 pub struct SignedBeaconBlock<E: EthSpec> {
     #[superstruct(only(Base), partial_getter(rename = "message_base"))]
     pub message: BeaconBlockBase<E>,