Altair consensus changes and refactors (#2279)

## Proposed Changes

Implement the consensus changes necessary for the upcoming Altair hard fork.

## Additional Info

This is quite a heavy refactor, with pivotal types like the `BeaconState` and `BeaconBlock` changing from structs to enums. This ripples through the whole codebase with field accesses changing to methods, e.g. `state.slot` => `state.slot()`.


Co-authored-by: realbigsean <seananderson33@gmail.com>

beacon_node/http_api/src/lib.rs

@@ -37,8 +37,9 @@ use std::sync::Arc;
 use tokio::sync::mpsc::UnboundedSender;
 use tokio_stream::{wrappers::BroadcastStream, StreamExt};
 use types::{
-    Attestation, AttesterSlashing, CommitteeCache, Epoch, EthSpec, ProposerSlashing, RelativeEpoch,
-    SignedAggregateAndProof, SignedBeaconBlock, SignedVoluntaryExit, Slot, YamlConfig,
+    Attestation, AttesterSlashing, CommitteeCache, ConfigAndPreset, Epoch, EthSpec,
+    ProposerSlashing, RelativeEpoch, SignedAggregateAndProof, SignedBeaconBlock,
+    SignedVoluntaryExit, Slot,
 };
 use warp::http::StatusCode;
 use warp::sse::Event;
@@ -75,6 +76,7 @@ pub struct Config {
     pub listen_addr: Ipv4Addr,
     pub listen_port: u16,
     pub allow_origin: Option<String>,
+    pub serve_legacy_spec: bool,
 }
 
 impl Default for Config {
@@ -84,6 +86,7 @@ impl Default for Config {
             listen_addr: Ipv4Addr::new(127, 0, 0, 1),
             listen_port: 5052,
             allow_origin: None,
+            serve_legacy_spec: true,
         }
     }
 }
@@ -332,7 +335,8 @@ pub fn serve<T: BeaconChainTypes>(
         .untuple_one();
 
     // Create a `warp` filter that provides access to the logger.
-    let log_filter = warp::any().map(move || ctx.log.clone());
+    let inner_ctx = ctx.clone();
+    let log_filter = warp::any().map(move || inner_ctx.log.clone());
 
     /*
      *
@@ -407,9 +411,9 @@ pub fn serve<T: BeaconChainTypes>(
                 state_id
                     .map_state(&chain, |state| {
                         Ok(api_types::FinalityCheckpointsData {
-                            previous_justified: state.previous_justified_checkpoint,
-                            current_justified: state.current_justified_checkpoint,
-                            finalized: state.finalized_checkpoint,
+                            previous_justified: state.previous_justified_checkpoint(),
+                            current_justified: state.current_justified_checkpoint(),
+                            finalized: state.finalized_checkpoint(),
                         })
                     })
                     .map(api_types::GenericResponse::from)
@@ -430,9 +434,9 @@ pub fn serve<T: BeaconChainTypes>(
                     state_id
                         .map_state(&chain, |state| {
                             Ok(state
-                                .validators
+                                .validators()
                                 .iter()
-                                .zip(state.balances.iter())
+                                .zip(state.balances().iter())
                                 .enumerate()
                                 // filter by validator id(s) if provided
                                 .filter(|(index, (validator, _))| {
@@ -475,9 +479,9 @@ pub fn serve<T: BeaconChainTypes>(
                             let far_future_epoch = chain.spec.far_future_epoch;
 
                             Ok(state
-                                .validators
+                                .validators()
                                 .iter()
-                                .zip(state.balances.iter())
+                                .zip(state.balances().iter())
                                 .enumerate()
                                 // filter by validator id(s) if provided
                                 .filter(|(index, (validator, _))| {
@@ -541,15 +545,15 @@ pub fn serve<T: BeaconChainTypes>(
                         .map_state(&chain, |state| {
                             let index_opt = match &validator_id {
                                 ValidatorId::PublicKey(pubkey) => {
-                                    state.validators.iter().position(|v| v.pubkey == *pubkey)
+                                    state.validators().iter().position(|v| v.pubkey == *pubkey)
                                 }
                                 ValidatorId::Index(index) => Some(*index as usize),
                             };
 
                             index_opt
                                 .and_then(|index| {
-                                    let validator = state.validators.get(index)?;
-                                    let balance = *state.balances.get(index)?;
+                                    let validator = state.validators().get(index)?;
+                                    let balance = *state.balances().get(index)?;
                                     let epoch = state.current_epoch();
                                     let far_future_epoch = chain.spec.far_future_epoch;
 
@@ -591,7 +595,7 @@ pub fn serve<T: BeaconChainTypes>(
 
                 blocking_json_task(move || {
                     query_state_id.map_state(&chain, |state| {
-                        let epoch = state.slot.epoch(T::EthSpec::slots_per_epoch());
+                        let epoch = state.slot().epoch(T::EthSpec::slots_per_epoch());
 
                         let committee_cache = if state
                             .committee_cache_is_initialized(RelativeEpoch::Current)
@@ -725,8 +729,8 @@ pub fn serve<T: BeaconChainTypes>(
                         root,
                         canonical: true,
                         header: api_types::BlockHeaderAndSignature {
-                            message: block.message.block_header(),
-                            signature: block.signature.into(),
+                            message: block.message().block_header(),
+                            signature: block.signature().clone().into(),
                         },
                     };
 
@@ -760,8 +764,8 @@ pub fn serve<T: BeaconChainTypes>(
                     root,
                     canonical,
                     header: api_types::BlockHeaderAndSignature {
-                        message: block.message.block_header(),
-                        signature: block.signature.into(),
+                        message: block.message().block_header(),
+                        signature: block.signature().clone().into(),
                     },
                 };
 
@@ -799,7 +803,7 @@ pub fn serve<T: BeaconChainTypes>(
 
                     // Determine the delay after the start of the slot, register it with metrics.
                     let delay =
-                        get_block_delay_ms(seen_timestamp, &block.message, &chain.slot_clock);
+                        get_block_delay_ms(seen_timestamp, block.message(), &chain.slot_clock);
                     metrics::observe_duration(
                         &metrics::HTTP_API_BLOCK_BROADCAST_DELAY_TIMES,
                         delay,
@@ -817,7 +821,7 @@ pub fn serve<T: BeaconChainTypes>(
                             // Notify the validator monitor.
                             chain.validator_monitor.read().register_api_block(
                                 seen_timestamp,
-                                &block.message,
+                                block.message(),
                                 root,
                                 &chain.slot_clock,
                             );
@@ -935,7 +939,8 @@ pub fn serve<T: BeaconChainTypes>(
             blocking_json_task(move || {
                 block_id
                     .block(&chain)
-                    .map(|block| block.message.body.attestations)
+                    // FIXME(altair): could avoid clone with by-value accessor
+                    .map(|block| block.message().body().attestations().clone())
                     .map(api_types::GenericResponse::from)
             })
         });
@@ -1266,17 +1271,19 @@ pub fn serve<T: BeaconChainTypes>(
         });
 
     // GET config/spec
+    let serve_legacy_spec = ctx.config.serve_legacy_spec;
     let get_config_spec = config_path
         .and(warp::path("spec"))
         .and(warp::path::end())
         .and(chain_filter.clone())
-        .and_then(|chain: Arc<BeaconChain<T>>| {
+        .and_then(move |chain: Arc<BeaconChain<T>>| {
             blocking_json_task(move || {
-                Ok(api_types::GenericResponse::from(YamlConfig::from_spec::<
-                    T::EthSpec,
-                >(
-                    &chain.spec
-                )))
+                let mut config_and_preset =
+                    ConfigAndPreset::from_chain_spec::<T::EthSpec>(&chain.spec);
+                if serve_legacy_spec {
+                    config_and_preset.make_backwards_compat(&chain.spec);
+                }
+                Ok(api_types::GenericResponse::from(config_and_preset))
             })
         });
 

beacon_node/http_api/src/proposer_duties.rs

@@ -148,7 +148,7 @@ fn compute_and_cache_proposer_duties<T: BeaconChainTypes>(
             state.current_epoch(),
             dependent_root,
             indices.clone(),
-            state.fork,
+            state.fork(),
         )
         .map_err(BeaconChainError::from)
         .map_err(warp_utils::reject::beacon_chain_error)?;

beacon_node/http_api/src/state_id.rs

@@ -57,7 +57,7 @@ impl StateId {
         &self,
         chain: &BeaconChain<T>,
     ) -> Result<Fork, warp::Rejection> {
-        self.map_state(chain, |state| Ok(state.fork))
+        self.map_state(chain, |state| Ok(state.fork()))
     }
 
     /// Return the `BeaconState` identified by `self`.

beacon_node/http_api/src/validator_inclusion.rs

@@ -20,7 +20,7 @@ pub fn global_validator_inclusion_data<T: BeaconChainTypes>(
     let mut validator_statuses = ValidatorStatuses::new(&state, &chain.spec)
         .map_err(warp_utils::reject::beacon_state_error)?;
     validator_statuses
-        .process_attestations(&state, &chain.spec)
+        .process_attestations(&state)
         .map_err(warp_utils::reject::beacon_state_error)?;
 
     let totals = validator_statuses.total_balances;
@@ -49,7 +49,7 @@ pub fn validator_inclusion_data<T: BeaconChainTypes>(
     let mut validator_statuses = ValidatorStatuses::new(&state, &chain.spec)
         .map_err(warp_utils::reject::beacon_state_error)?;
     validator_statuses
-        .process_attestations(&state, &chain.spec)
+        .process_attestations(&state)
         .map_err(warp_utils::reject::beacon_state_error)?;
 
     state

beacon_node/http_api/tests/tests.rs

@@ -23,11 +23,9 @@ use sensitive_url::SensitiveUrl;
 use slot_clock::SlotClock;
 use state_processing::per_slot_processing;
 use std::convert::TryInto;
-use std::iter::Iterator;
 use std::net::Ipv4Addr;
 use std::sync::Arc;
-use tokio::sync::mpsc;
-use tokio::sync::oneshot;
+use tokio::sync::{mpsc, oneshot};
 use tokio::time::Duration;
 use tree_hash::TreeHash;
 use types::{
@@ -77,6 +75,7 @@ impl ApiTester {
     pub fn new() -> Self {
         let mut harness = BeaconChainHarness::new(
             MainnetEthSpec,
+            None,
             generate_deterministic_keypairs(VALIDATOR_COUNT),
         );
 
@@ -189,6 +188,7 @@ impl ApiTester {
                 listen_addr: Ipv4Addr::new(127, 0, 0, 1),
                 listen_port: 0,
                 allow_origin: None,
+                serve_legacy_spec: true,
             },
             chain: Some(chain.clone()),
             network_tx: Some(network_tx),
@@ -235,6 +235,7 @@ impl ApiTester {
     pub fn new_from_genesis() -> Self {
         let harness = BeaconChainHarness::new(
             MainnetEthSpec,
+            None,
             generate_deterministic_keypairs(VALIDATOR_COUNT),
         );
 
@@ -301,6 +302,7 @@ impl ApiTester {
                 listen_addr: Ipv4Addr::new(127, 0, 0, 1),
                 listen_port: 0,
                 allow_origin: None,
+                serve_legacy_spec: true,
             },
             chain: Some(chain.clone()),
             network_tx: Some(network_tx),
@@ -445,8 +447,8 @@ impl ApiTester {
 
         let state = self.chain.head().unwrap().beacon_state;
         let expected = GenesisData {
-            genesis_time: state.genesis_time,
-            genesis_validators_root: state.genesis_validators_root,
+            genesis_time: state.genesis_time(),
+            genesis_validators_root: state.genesis_validators_root(),
             genesis_fork_version: self.chain.spec.genesis_fork_version,
         };
 
@@ -508,7 +510,7 @@ impl ApiTester {
                 .unwrap()
                 .map(|res| res.data);
 
-            let expected = self.get_state(state_id).map(|state| state.fork);
+            let expected = self.get_state(state_id).map(|state| state.fork());
 
             assert_eq!(result, expected, "{:?}", state_id);
         }
@@ -528,9 +530,9 @@ impl ApiTester {
             let expected = self
                 .get_state(state_id)
                 .map(|state| FinalityCheckpointsData {
-                    previous_justified: state.previous_justified_checkpoint,
-                    current_justified: state.current_justified_checkpoint,
-                    finalized: state.finalized_checkpoint,
+                    previous_justified: state.previous_justified_checkpoint(),
+                    current_justified: state.current_justified_checkpoint(),
+                    finalized: state.finalized_checkpoint(),
                 });
 
             assert_eq!(result, expected, "{:?}", state_id);
@@ -544,7 +546,7 @@ impl ApiTester {
             for validator_indices in self.interesting_validator_indices() {
                 let state_opt = self.get_state(state_id);
                 let validators: Vec<Validator> = match state_opt.as_ref() {
-                    Some(state) => state.validators.clone().into(),
+                    Some(state) => state.validators().clone().into(),
                     None => vec![],
                 };
                 let validator_index_ids = validator_indices
@@ -587,10 +589,10 @@ impl ApiTester {
                     let mut validators = Vec::with_capacity(validator_indices.len());
 
                     for i in validator_indices {
-                        if i < state.balances.len() as u64 {
+                        if i < state.balances().len() as u64 {
                             validators.push(ValidatorBalanceData {
                                 index: i as u64,
-                                balance: state.balances[i as usize],
+                                balance: state.balances()[i as usize],
                             });
                         }
                     }
@@ -612,7 +614,7 @@ impl ApiTester {
                 for validator_indices in self.interesting_validator_indices() {
                     let state_opt = self.get_state(state_id);
                     let validators: Vec<Validator> = match state_opt.as_ref() {
-                        Some(state) => state.validators.clone().into(),
+                        Some(state) => state.validators().clone().into(),
                         None => vec![],
                     };
                     let validator_index_ids = validator_indices
@@ -661,10 +663,10 @@ impl ApiTester {
                         let mut validators = Vec::with_capacity(validator_indices.len());
 
                         for i in validator_indices {
-                            if i >= state.validators.len() as u64 {
+                            if i >= state.validators().len() as u64 {
                                 continue;
                             }
-                            let validator = state.validators[i as usize].clone();
+                            let validator = state.validators()[i as usize].clone();
                             let status = ValidatorStatus::from_validator(
                                 &validator,
                                 epoch,
@@ -676,7 +678,7 @@ impl ApiTester {
                             {
                                 validators.push(ValidatorData {
                                     index: i as u64,
-                                    balance: state.balances[i as usize],
+                                    balance: state.balances()[i as usize],
                                     status,
                                     validator,
                                 });
@@ -699,7 +701,7 @@ impl ApiTester {
         for state_id in self.interesting_state_ids() {
             let state_opt = self.get_state(state_id);
             let validators = match state_opt.as_ref() {
-                Some(state) => state.validators.clone().into(),
+                Some(state) => state.validators().clone().into(),
                 None => vec![],
             };
 
@@ -729,7 +731,7 @@ impl ApiTester {
 
                         ValidatorData {
                             index: i as u64,
-                            balance: state.balances[i],
+                            balance: state.balances()[i],
                             status: ValidatorStatus::from_validator(
                                 &validator,
                                 epoch,
@@ -846,8 +848,8 @@ impl ApiTester {
                 root,
                 canonical: true,
                 header: BlockHeaderAndSignature {
-                    message: block.message.block_header(),
-                    signature: block.signature.into(),
+                    message: block.message().block_header(),
+                    signature: block.signature().clone().into(),
                 },
             };
             let expected = vec![header];
@@ -927,13 +929,13 @@ impl ApiTester {
             assert_eq!(result.root, block_root, "{:?}", block_id);
             assert_eq!(
                 result.header.message,
-                block.message.block_header(),
+                block.message().block_header(),
                 "{:?}",
                 block_id
             );
             assert_eq!(
                 result.header.signature,
-                block.signature.into(),
+                block.signature().clone().into(),
                 "{:?}",
                 block_id
             );
@@ -980,7 +982,7 @@ impl ApiTester {
 
     pub async fn test_post_beacon_blocks_invalid(mut self) -> Self {
         let mut next_block = self.next_block.clone();
-        next_block.message.proposer_index += 1;
+        *next_block.message_mut().proposer_index_mut() += 1;
 
         assert!(self.client.post_beacon_blocks(&next_block).await.is_err());
 
@@ -1012,7 +1014,11 @@ impl ApiTester {
                 .map(|res| res.data);
             assert_eq!(json_result, expected, "{:?}", block_id);
 
-            let ssz_result = self.client.get_beacon_blocks_ssz(block_id).await.unwrap();
+            let ssz_result = self
+                .client
+                .get_beacon_blocks_ssz(block_id, &self.chain.spec)
+                .await
+                .unwrap();
             assert_eq!(ssz_result, expected, "{:?}", block_id);
         }
 
@@ -1030,7 +1036,7 @@ impl ApiTester {
 
             let expected = self
                 .get_block(block_id)
-                .map(|block| block.message.body.attestations.into());
+                .map(|block| block.message().body().attestations().clone().into());
 
             if let BlockId::Slot(slot) = block_id {
                 if expected.is_none() {
@@ -1264,7 +1270,8 @@ impl ApiTester {
     pub async fn test_get_config_spec(self) -> Self {
         let result = self.client.get_config_spec().await.unwrap().data;
 
-        let expected = YamlConfig::from_spec::<E>(&self.chain.spec);
+        let mut expected = ConfigAndPreset::from_chain_spec::<E>(&self.chain.spec);
+        expected.make_backwards_compat(&self.chain.spec);
 
         assert_eq!(result, expected);
 
@@ -1432,7 +1439,7 @@ impl ApiTester {
         for state_id in self.interesting_state_ids() {
             let result_ssz = self
                 .client
-                .get_debug_beacon_states_ssz(state_id)
+                .get_debug_beacon_states_ssz(state_id, &self.chain.spec)
                 .await
                 .unwrap();
             let result_json = self
@@ -1471,7 +1478,7 @@ impl ApiTester {
     }
 
     fn validator_count(&self) -> usize {
-        self.chain.head().unwrap().beacon_state.validators.len()
+        self.chain.head().unwrap().beacon_state.validators().len()
     }
 
     fn interesting_validator_indices(&self) -> Vec<Vec<u64>> {
@@ -1575,7 +1582,7 @@ impl ApiTester {
 
                 let expected_len = indices
                     .iter()
-                    .filter(|i| **i < state.validators.len() as u64)
+                    .filter(|i| **i < state.validators().len() as u64)
                     .count();
 
                 assert_eq!(result_duties.len(), expected_len);
@@ -1586,7 +1593,7 @@ impl ApiTester {
                         .unwrap()
                     {
                         let expected = AttesterData {
-                            pubkey: state.validators[i as usize].pubkey.clone().into(),
+                            pubkey: state.validators()[i as usize].pubkey.clone().into(),
                             validator_index: i,
                             committees_at_slot: duty.committees_at_slot,
                             committee_index: duty.index,
@@ -1691,7 +1698,7 @@ impl ApiTester {
                     let index = state
                         .get_beacon_proposer_index(slot, &self.chain.spec)
                         .unwrap();
-                    let pubkey = state.validators[index].pubkey.clone().into();
+                    let pubkey = state.validators()[index].pubkey.clone().into();
 
                     ProposerData {
                         pubkey,
@@ -1849,7 +1856,7 @@ impl ApiTester {
 
     pub async fn test_get_validator_attestation_data(self) -> Self {
         let mut state = self.chain.head_beacon_state().unwrap();
-        let slot = state.slot;
+        let slot = state.slot();
         state
             .build_committee_cache(RelativeEpoch::Current, &self.chain.spec)
             .unwrap();
@@ -1879,9 +1886,9 @@ impl ApiTester {
             .chain
             .head_beacon_block()
             .unwrap()
-            .message
-            .body
-            .attestations[0]
+            .message()
+            .body()
+            .attestations()[0]
             .clone();
 
         let result = self
@@ -1915,7 +1922,7 @@ impl ApiTester {
             .unwrap();
 
         let committee_len = head.beacon_state.get_committee_count_at_slot(slot).unwrap();
-        let fork = head.beacon_state.fork;
+        let fork = head.beacon_state.fork();
         let genesis_validators_root = self.chain.genesis_validators_root;
 
         let duties = self
@@ -2118,7 +2125,7 @@ impl ApiTester {
         for state_id in self.interesting_state_ids() {
             let result = self
                 .client
-                .get_lighthouse_beacon_states_ssz(&state_id)
+                .get_lighthouse_beacon_states_ssz(&state_id, &self.chain.spec)
                 .await
                 .unwrap();