Altair consensus changes and refactors (#2279)

## Proposed Changes

Implement the consensus changes necessary for the upcoming Altair hard fork.

## Additional Info

This is quite a heavy refactor, with pivotal types like the `BeaconState` and `BeaconBlock` changing from structs to enums. This ripples through the whole codebase with field accesses changing to methods, e.g. `state.slot` => `state.slot()`.


Co-authored-by: realbigsean <seananderson33@gmail.com>

testing/ef_tests/src/cases/operations.rs

@@ -1,20 +1,24 @@
 use super::*;
 use crate::bls_setting::BlsSetting;
 use crate::case_result::compare_beacon_state_results_without_caches;
-use crate::decode::{ssz_decode_file, yaml_decode_file};
+use crate::decode::{ssz_decode_file, ssz_decode_file_with, ssz_decode_state, yaml_decode_file};
+use crate::testing_spec;
 use crate::type_name::TypeName;
 use serde_derive::Deserialize;
-use ssz::Decode;
 use state_processing::per_block_processing::{
-    errors::BlockProcessingError, process_attestations, process_attester_slashings,
-    process_block_header, process_deposits, process_exits, process_proposer_slashings,
-    VerifySignatures,
+    errors::BlockProcessingError,
+    process_block_header,
+    process_operations::{
+        altair, base, process_attester_slashings, process_deposits, process_exits,
+        process_proposer_slashings,
+    },
+    process_sync_aggregate, VerifySignatures,
 };
 use std::fmt::Debug;
 use std::path::Path;
 use types::{
-    Attestation, AttesterSlashing, BeaconBlock, BeaconState, ChainSpec, Deposit, EthSpec,
-    ProposerSlashing, SignedVoluntaryExit,
+    Attestation, AttesterSlashing, BeaconBlock, BeaconState, ChainSpec, Deposit, EthSpec, ForkName,
+    ProposerSlashing, SignedVoluntaryExit, SyncAggregate,
 };
 
 #[derive(Debug, Clone, Default, Deserialize)]
@@ -31,15 +35,21 @@ pub struct Operations<E: EthSpec, O: Operation<E>> {
     pub post: Option<BeaconState<E>>,
 }
 
-pub trait Operation<E: EthSpec>: Decode + TypeName + Debug + Sync {
+pub trait Operation<E: EthSpec>: TypeName + Debug + Sync + Sized {
     fn handler_name() -> String {
         Self::name().to_lowercase()
     }
 
     fn filename() -> String {
-        format!("{}.ssz", Self::handler_name())
+        format!("{}.ssz_snappy", Self::handler_name())
     }
 
+    fn is_enabled_for_fork(_fork_name: ForkName) -> bool {
+        true
+    }
+
+    fn decode(path: &Path, spec: &ChainSpec) -> Result<Self, Error>;
+
     fn apply_to(
         &self,
         state: &mut BeaconState<E>,
@@ -48,12 +58,23 @@ pub trait Operation<E: EthSpec>: Decode + TypeName + Debug + Sync {
 }
 
 impl<E: EthSpec> Operation<E> for Attestation<E> {
+    fn decode(path: &Path, _spec: &ChainSpec) -> Result<Self, Error> {
+        ssz_decode_file(path)
+    }
+
     fn apply_to(
         &self,
         state: &mut BeaconState<E>,
         spec: &ChainSpec,
     ) -> Result<(), BlockProcessingError> {
-        process_attestations(state, &[self.clone()], VerifySignatures::True, spec)
+        match state {
+            BeaconState::Base(_) => {
+                base::process_attestations(state, &[self.clone()], VerifySignatures::True, spec)
+            }
+            BeaconState::Altair(_) => {
+                altair::process_attestation(state, self, 0, VerifySignatures::True, spec)
+            }
+        }
     }
 }
 
@@ -62,6 +83,10 @@ impl<E: EthSpec> Operation<E> for AttesterSlashing<E> {
         "attester_slashing".into()
     }
 
+    fn decode(path: &Path, _spec: &ChainSpec) -> Result<Self, Error> {
+        ssz_decode_file(path)
+    }
+
     fn apply_to(
         &self,
         state: &mut BeaconState<E>,
@@ -72,6 +97,10 @@ impl<E: EthSpec> Operation<E> for AttesterSlashing<E> {
 }
 
 impl<E: EthSpec> Operation<E> for Deposit {
+    fn decode(path: &Path, _spec: &ChainSpec) -> Result<Self, Error> {
+        ssz_decode_file(path)
+    }
+
     fn apply_to(
         &self,
         state: &mut BeaconState<E>,
@@ -86,6 +115,10 @@ impl<E: EthSpec> Operation<E> for ProposerSlashing {
         "proposer_slashing".into()
     }
 
+    fn decode(path: &Path, _spec: &ChainSpec) -> Result<Self, Error> {
+        ssz_decode_file(path)
+    }
+
     fn apply_to(
         &self,
         state: &mut BeaconState<E>,
@@ -100,6 +133,10 @@ impl<E: EthSpec> Operation<E> for SignedVoluntaryExit {
         "voluntary_exit".into()
     }
 
+    fn decode(path: &Path, _spec: &ChainSpec) -> Result<Self, Error> {
+        ssz_decode_file(path)
+    }
+
     fn apply_to(
         &self,
         state: &mut BeaconState<E>,
@@ -115,7 +152,11 @@ impl<E: EthSpec> Operation<E> for BeaconBlock<E> {
     }
 
     fn filename() -> String {
-        "block.ssz".into()
+        "block.ssz_snappy".into()
+    }
+
+    fn decode(path: &Path, spec: &ChainSpec) -> Result<Self, Error> {
+        ssz_decode_file_with(path, |bytes| BeaconBlock::from_ssz_bytes(bytes, spec))
     }
 
     fn apply_to(
@@ -123,12 +164,41 @@ impl<E: EthSpec> Operation<E> for BeaconBlock<E> {
         state: &mut BeaconState<E>,
         spec: &ChainSpec,
     ) -> Result<(), BlockProcessingError> {
-        Ok(process_block_header(state, self, spec)?)
+        process_block_header(state, self.to_ref(), spec)?;
+        Ok(())
+    }
+}
+
+impl<E: EthSpec> Operation<E> for SyncAggregate<E> {
+    fn handler_name() -> String {
+        "sync_aggregate".into()
+    }
+
+    fn filename() -> String {
+        "sync_aggregate.ssz_snappy".into()
+    }
+
+    fn is_enabled_for_fork(fork_name: ForkName) -> bool {
+        fork_name != ForkName::Base
+    }
+
+    fn decode(path: &Path, _spec: &ChainSpec) -> Result<Self, Error> {
+        ssz_decode_file(path)
+    }
+
+    fn apply_to(
+        &self,
+        state: &mut BeaconState<E>,
+        spec: &ChainSpec,
+    ) -> Result<(), BlockProcessingError> {
+        let proposer_index = state.get_beacon_proposer_index(state.slot(), spec)? as u64;
+        process_sync_aggregate(state, self, proposer_index, spec)
     }
 }
 
 impl<E: EthSpec, O: Operation<E>> LoadCase for Operations<E, O> {
-    fn load_from_dir(path: &Path) -> Result<Self, Error> {
+    fn load_from_dir(path: &Path, fork_name: ForkName) -> Result<Self, Error> {
+        let spec = &testing_spec::<E>(fork_name);
         let metadata_path = path.join("meta.yaml");
         let metadata: Metadata = if metadata_path.is_file() {
             yaml_decode_file(&metadata_path)?
@@ -136,12 +206,12 @@ impl<E: EthSpec, O: Operation<E>> LoadCase for Operations<E, O> {
             Metadata::default()
         };
 
-        let pre = ssz_decode_file(&path.join("pre.ssz"))?;
+        let pre = ssz_decode_state(&path.join("pre.ssz_snappy"), spec)?;
 
         // Check BLS setting here before SSZ deserialization, as most types require signatures
         // to be valid.
         let (operation, bls_error) = if metadata.bls_setting.unwrap_or_default().check().is_ok() {
-            match ssz_decode_file(&path.join(O::filename())) {
+            match O::decode(&path.join(O::filename()), spec) {
                 Ok(op) => (Some(op), None),
                 Err(Error::InvalidBLSInput(error)) => (None, Some(error)),
                 Err(e) => return Err(e),
@@ -149,12 +219,12 @@ impl<E: EthSpec, O: Operation<E>> LoadCase for Operations<E, O> {
         } else {
             (None, None)
         };
-        let post_filename = path.join("post.ssz");
+        let post_filename = path.join("post.ssz_snappy");
         let post = if post_filename.is_file() {
             if let Some(bls_error) = bls_error {
                 panic!("input is unexpectedly invalid: {}", bls_error);
             }
-            Some(ssz_decode_file(&post_filename)?)
+            Some(ssz_decode_state(&post_filename, spec)?)
         } else {
             None
         };
@@ -176,8 +246,12 @@ impl<E: EthSpec, O: Operation<E>> Case for Operations<E, O> {
             .unwrap_or_else(String::new)
     }
 
-    fn result(&self, _case_index: usize) -> Result<(), Error> {
-        let spec = &E::default_spec();
+    fn is_enabled_for_fork(fork_name: ForkName) -> bool {
+        O::is_enabled_for_fork(fork_name)
+    }
+
+    fn result(&self, _case_index: usize, fork_name: ForkName) -> Result<(), Error> {
+        let spec = &testing_spec::<E>(fork_name);
         let mut state = self.pre.clone();
         let mut expected = self.post.clone();