Implement SSZ union type (#2579)

## Issue Addressed

NA

## Proposed Changes

Implements the "union" type from the SSZ spec for `ssz`, `ssz_derive`, `tree_hash` and `tree_hash_derive` so it may be derived for `enums`:

https://github.com/ethereum/consensus-specs/blob/v1.1.0-beta.3/ssz/simple-serialize.md#union

The union type is required for the merge, since the `Transaction` type is defined as a single-variant union `Union[OpaqueTransaction]`.

### Crate Updates

This PR will (hopefully) cause CI to publish new versions for the following crates:

- `eth2_ssz_derive`: `0.2.1` -> `0.3.0`
- `eth2_ssz`: `0.3.0` -> `0.4.0`
- `eth2_ssz_types`: `0.2.0` -> `0.2.1`
- `tree_hash`: `0.3.0` -> `0.4.0`
- `tree_hash_derive`: `0.3.0` -> `0.4.0`

These these crates depend on each other, I've had to add a workspace-level `[patch]` for these crates. A follow-up PR will need to remove this patch, ones the new versions are published.

### Union Behaviors

We already had SSZ `Encode` and `TreeHash` derive for enums, however it just did a "transparent" pass-through of the inner value. Since the "union" decoding from the spec is in conflict with the transparent method, I've required that all `enum` have exactly one of the following enum-level attributes:

#### SSZ

-  `#[ssz(enum_behaviour = "union")]`
    - matches the spec used for the merge
-  `#[ssz(enum_behaviour = "transparent")]`
    - maintains existing functionality
    - not supported for `Decode` (never was)
    
#### TreeHash

-  `#[tree_hash(enum_behaviour = "union")]`
    - matches the spec used for the merge
-  `#[tree_hash(enum_behaviour = "transparent")]`
    - maintains existing functionality

This means that we can maintain the existing transparent behaviour, but all existing users will get a compile-time error until they explicitly opt-in to being transparent.

### Legacy Option Encoding

Before this PR, we already had a union-esque encoding for `Option<T>`. However, this was with the *old* SSZ spec where the union selector was 4 bytes. During merge specification, the spec was changed to use 1 byte for the selector.

Whilst the 4-byte `Option` encoding was never used in the spec, we used it in our database. Writing a migrate script for all occurrences of `Option` in the database would be painful, especially since it's used in the `CommitteeCache`. To avoid the migrate script, I added a serde-esque `#[ssz(with = "module")]` field-level attribute to `ssz_derive` so that we can opt into the 4-byte encoding on a field-by-field basis.

The `ssz::legacy::four_byte_impl!` macro allows a one-liner to define the module required for the `#[ssz(with = "module")]` for some `Option<T> where T: Encode + Decode`.

Notably, **I have removed `Encode` and `Decode` impls for `Option`**. I've done this to force a break on downstream users. Like I mentioned, `Option` isn't used in the spec so I don't think it'll be *that* annoying. I think it's nicer than quietly having two different union implementations or quietly breaking the existing `Option` impl.

### Crate Publish Ordering

I've modified the order in which CI publishes crates to ensure that we don't publish a crate without ensuring we already published a crate that it depends upon.

## TODO

- [ ] Queue a follow-up `[patch]`-removing PR.

beacon_node/Cargo.toml

@@ -36,7 +36,7 @@ task_executor = { path = "../common/task_executor" }
 genesis = { path = "genesis" }
 eth2_network_config = { path = "../common/eth2_network_config" }
 eth2_libp2p = { path = "./eth2_libp2p" }
-eth2_ssz = "0.3.0"
+eth2_ssz = "0.4.0"
 serde = "1.0.116"
 clap_utils = { path = "../common/clap_utils" }
 hyper = "0.14.4"

beacon_node/beacon_chain/Cargo.toml

@@ -34,11 +34,11 @@ slog = { version = "2.5.2", features = ["max_level_trace"] }
 sloggers = "2.0.2"
 slot_clock = { path = "../../common/slot_clock" }
 eth2_hashing = "0.2.0"
-eth2_ssz = "0.3.0"
-eth2_ssz_types = "0.2.0"
-eth2_ssz_derive = "0.2.1"
+eth2_ssz = "0.4.0"
+eth2_ssz_types = "0.2.1"
+eth2_ssz_derive = "0.3.0"
 state_processing = { path = "../../consensus/state_processing" }
-tree_hash = "0.3.0"
+tree_hash = "0.4.0"
 types = { path = "../../consensus/types" }
 tokio = "1.10.0"
 eth1 = { path = "../eth1" }

beacon_node/client/Cargo.toml

@@ -17,7 +17,7 @@ eth2_libp2p = { path = "../eth2_libp2p" }
 parking_lot = "0.11.0"
 prometheus = "0.11.0"
 types = { path = "../../consensus/types" }
-tree_hash = "0.3.0"
+tree_hash = "0.4.0"
 eth2_config = { path = "../../common/eth2_config" }
 slot_clock = { path = "../../common/slot_clock" }
 serde = "1.0.116"
@@ -37,7 +37,7 @@ sensitive_url = { path = "../../common/sensitive_url" }
 genesis = { path = "../genesis" }
 task_executor = { path = "../../common/task_executor" }
 environment = { path = "../../lighthouse/environment" }
-eth2_ssz = "0.3.0"
+eth2_ssz = "0.4.0"
 lazy_static = "1.4.0"
 lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
 time = "0.2.22"

beacon_node/eth1/Cargo.toml

@@ -19,9 +19,9 @@ serde = { version = "1.0.116", features = ["derive"] }
 hex = "0.4.2"
 types = { path = "../../consensus/types"}
 merkle_proof = { path = "../../consensus/merkle_proof"}
-eth2_ssz = "0.3.0"
-eth2_ssz_derive = "0.2.1"
-tree_hash = "0.3.0"
+eth2_ssz = "0.4.0"
+eth2_ssz_derive = "0.3.0"
+tree_hash = "0.4.0"
 eth2_hashing = "0.2.0"
 parking_lot = "0.11.0"
 slog = "2.5.2"

beacon_node/eth1/src/inner.rs

@@ -5,11 +5,16 @@ use crate::{
     service::EndpointsCache,
 };
 use parking_lot::RwLock;
+use ssz::four_byte_option_impl;
 use ssz::{Decode, Encode};
 use ssz_derive::{Decode, Encode};
 use std::sync::Arc;
 use types::ChainSpec;
 
+// Define "legacy" implementations of `Option<u64>` which use four bytes for encoding the union
+// selector.
+four_byte_option_impl!(four_byte_option_u64, u64);
+
 #[derive(Default)]
 pub struct DepositUpdater {
     pub cache: DepositCache,
@@ -69,6 +74,7 @@ impl Inner {
 pub struct SszEth1Cache {
     block_cache: BlockCache,
     deposit_cache: SszDepositCache,
+    #[ssz(with = "four_byte_option_u64")]
     last_processed_block: Option<u64>,
 }
 

beacon_node/eth2_libp2p/Cargo.toml

@@ -10,11 +10,11 @@ discv5 = { git = "https://github.com/sigp/discv5", rev="10247bbd299227fef20233f2
 unsigned-varint = { version = "0.6.0", features = ["codec"] }
 types = { path =  "../../consensus/types" }
 hashset_delay = { path = "../../common/hashset_delay" }
-eth2_ssz_types = "0.2.0"
+eth2_ssz_types = "0.2.1"
 serde = { version = "1.0.116", features = ["derive"] }
 serde_derive = "1.0.116"
-eth2_ssz = "0.3.0"
-eth2_ssz_derive = "0.2.1"
+eth2_ssz = "0.4.0"
+eth2_ssz_derive = "0.3.0"
 slog = { version = "2.5.2", features = ["max_level_trace"] }
 lighthouse_version = { path = "../../common/lighthouse_version" }
 tokio = { version = "1.10.0", features = ["time", "macros"] }

beacon_node/eth2_libp2p/src/rpc/methods.rs

@@ -103,6 +103,7 @@ pub struct Ping {
 )]
 #[derive(Clone, Debug, PartialEq, Serialize, Encode)]
 #[serde(bound = "T: EthSpec")]
+#[ssz(enum_behaviour = "transparent")]
 pub struct MetaData<T: EthSpec> {
     /// A sequential counter indicating when data gets modified.
     pub seq_number: u64,

beacon_node/genesis/Cargo.toml

@@ -16,9 +16,9 @@ eth1 = { path = "../eth1"}
 rayon = "1.4.1"
 state_processing = { path = "../../consensus/state_processing" }
 merkle_proof = { path = "../../consensus/merkle_proof" }
-eth2_ssz = "0.3.0"
+eth2_ssz = "0.4.0"
 eth2_hashing = "0.2.0"
-tree_hash = "0.3.0"
+tree_hash = "0.4.0"
 tokio = { version = "1.10.0", features = ["full"] }
 parking_lot = "0.11.0"
 slog = "2.5.2"

beacon_node/http_api/Cargo.toml

@@ -27,14 +27,14 @@ lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
 lazy_static = "1.4.0"
 warp_utils = { path = "../../common/warp_utils" }
 slot_clock = { path = "../../common/slot_clock" }
-eth2_ssz = "0.3.0"
+eth2_ssz = "0.4.0"
 bs58 = "0.4.0"
 futures = "0.3.8"
 
 [dev-dependencies]
 store = { path = "../store" }
 environment = { path = "../../lighthouse/environment" }
-tree_hash = "0.3.0"
+tree_hash = "0.4.0"
 sensitive_url = { path = "../../common/sensitive_url" }
 
 [[test]]

beacon_node/network/Cargo.toml

@@ -25,9 +25,9 @@ state_processing = { path = "../../consensus/state_processing" }
 slot_clock = { path = "../../common/slot_clock" }
 slog = { version = "2.5.2", features = ["max_level_trace"] }
 hex = "0.4.2"
-eth2_ssz = "0.3.0"
-eth2_ssz_types = "0.2.0"
-tree_hash = "0.3.0"
+eth2_ssz = "0.4.0"
+eth2_ssz_types = "0.2.1"
+tree_hash = "0.4.0"
 futures = "0.3.7"
 error-chain = "0.12.4"
 tokio = { version = "1.10.0", features = ["full"] }

beacon_node/operation_pool/Cargo.toml

@@ -13,8 +13,8 @@ lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
 parking_lot = "0.11.0"
 types = { path = "../../consensus/types" }
 state_processing = { path = "../../consensus/state_processing" }
-eth2_ssz = "0.3.0"
-eth2_ssz_derive = "0.2.1"
+eth2_ssz = "0.4.0"
+eth2_ssz_derive = "0.3.0"
 rayon = "1.5.0"
 serde = "1.0.116"
 serde_derive = "1.0.116"

beacon_node/operation_pool/src/persistence.rs

@@ -28,6 +28,7 @@ type PersistedSyncContributions<T> = Vec<(SyncAggregateId, Vec<SyncCommitteeCont
 #[derive(PartialEq, Debug, Serialize, Deserialize, Encode)]
 #[serde(untagged)]
 #[serde(bound = "T: EthSpec")]
+#[ssz(enum_behaviour = "transparent")]
 pub struct PersistedOperationPool<T: EthSpec> {
     /// Mapping from attestation ID to attestation mappings.
     // We could save space by not storing the attestation ID, but it might

beacon_node/store/Cargo.toml

@@ -13,9 +13,9 @@ db-key = "0.0.5"
 leveldb = { version = "0.8.6", default-features = false }
 parking_lot = "0.11.0"
 itertools = "0.10.0"
-eth2_ssz = "0.3.0"
-eth2_ssz_derive = "0.2.1"
-tree_hash = "0.3.0"
+eth2_ssz = "0.4.0"
+eth2_ssz_derive = "0.3.0"
+tree_hash = "0.4.0"
 types = { path =  "../../consensus/types" }
 state_processing = { path = "../../consensus/state_processing" }
 slog = "2.5.2"

beacon_node/store/src/partial_beacon_state.rs

@@ -15,9 +15,10 @@ use types::*;
 /// Utilises lazy-loading from separate storage for its vector fields.
 #[superstruct(
     variants(Base, Altair),
-    variant_attributes(derive(Debug, PartialEq, Clone, Encode, Decode))
+    variant_attributes(derive(Debug, PartialEq, Clone, Encode, Decode),)
 )]
 #[derive(Debug, PartialEq, Clone, Encode)]
+#[ssz(enum_behaviour = "transparent")]
 pub struct PartialBeaconState<T>
 where
     T: EthSpec,
@@ -32,15 +33,12 @@ where
     // History
     pub latest_block_header: BeaconBlockHeader,
 
-    #[ssz(skip_serializing)]
-    #[ssz(skip_deserializing)]
+    #[ssz(skip_serializing, skip_deserializing)]
     pub block_roots: Option<FixedVector<Hash256, T::SlotsPerHistoricalRoot>>,
-    #[ssz(skip_serializing)]
-    #[ssz(skip_deserializing)]
+    #[ssz(skip_serializing, skip_deserializing)]
     pub state_roots: Option<FixedVector<Hash256, T::SlotsPerHistoricalRoot>>,
 
-    #[ssz(skip_serializing)]
-    #[ssz(skip_deserializing)]
+    #[ssz(skip_serializing, skip_deserializing)]
     pub historical_roots: Option<VariableList<Hash256, T::HistoricalRootsLimit>>,
 
     // Ethereum 1.0 chain data
@@ -55,8 +53,7 @@ where
     // Shuffling
     /// Randao value from the current slot, for patching into the per-epoch randao vector.
     pub latest_randao_value: Hash256,
-    #[ssz(skip_serializing)]
-    #[ssz(skip_deserializing)]
+    #[ssz(skip_serializing, skip_deserializing)]
     pub randao_mixes: Option<FixedVector<Hash256, T::EpochsPerHistoricalVector>>,
 
     // Slashings