Merge branch 'blop-pool' into validator-enhancements

eth2/operation_pool/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "operation_pool"
+version = "0.1.0"
+authors = ["Michael Sproul <michael@sigmaprime.io>"]
+edition = "2018"
+
+[dependencies]
+int_to_bytes = { path = "../utils/int_to_bytes" }
+itertools = "0.8"
+parking_lot = "0.7"
+types = { path = "../types" }
+state_processing = { path = "../state_processing" }
+ssz = { path = "../utils/ssz" }

eth2/operation_pool/src/lib.rs

@@ -0,0 +1,939 @@
+use int_to_bytes::int_to_bytes8;
+use itertools::Itertools;
+use parking_lot::RwLock;
+use ssz::ssz_encode;
+use state_processing::per_block_processing::errors::{
+    AttestationValidationError, AttesterSlashingValidationError, DepositValidationError,
+    ExitValidationError, ProposerSlashingValidationError, TransferValidationError,
+};
+use state_processing::per_block_processing::{
+    gather_attester_slashing_indices_modular, validate_attestation,
+    validate_attestation_time_independent_only, verify_attester_slashing, verify_deposit,
+    verify_exit, verify_exit_time_independent_only, verify_proposer_slashing, verify_transfer,
+    verify_transfer_time_independent_only,
+};
+use std::collections::{btree_map::Entry, hash_map, BTreeMap, HashMap, HashSet};
+use types::chain_spec::Domain;
+use types::{
+    Attestation, AttestationData, AttesterSlashing, BeaconState, ChainSpec, Deposit, Epoch,
+    ProposerSlashing, Transfer, Validator, VoluntaryExit,
+};
+
+#[cfg(test)]
+const VERIFY_DEPOSIT_PROOFS: bool = false;
+#[cfg(not(test))]
+const VERIFY_DEPOSIT_PROOFS: bool = false; // TODO: enable this
+
+#[derive(Default)]
+pub struct OperationPool {
+    /// Map from attestation ID (see below) to vectors of attestations.
+    attestations: RwLock<HashMap<AttestationId, Vec<Attestation>>>,
+    /// Map from deposit index to deposit data.
+    // NOTE: We assume that there is only one deposit per index
+    // because the Eth1 data is updated (at most) once per epoch,
+    // and the spec doesn't seem to accomodate for re-orgs on a time-frame
+    // longer than an epoch
+    deposits: RwLock<BTreeMap<u64, Deposit>>,
+    /// Map from two attestation IDs to a slashing for those IDs.
+    attester_slashings: RwLock<HashMap<(AttestationId, AttestationId), AttesterSlashing>>,
+    /// Map from proposer index to slashing.
+    proposer_slashings: RwLock<HashMap<u64, ProposerSlashing>>,
+    /// Map from exiting validator to their exit data.
+    voluntary_exits: RwLock<HashMap<u64, VoluntaryExit>>,
+    /// Set of transfers.
+    transfers: RwLock<HashSet<Transfer>>,
+}
+
+/// Serialized `AttestationData` augmented with a domain to encode the fork info.
+#[derive(PartialEq, Eq, Clone, Hash, Debug)]
+struct AttestationId(Vec<u8>);
+
+/// Number of domain bytes that the end of an attestation ID is padded with.
+const DOMAIN_BYTES_LEN: usize = 8;
+
+impl AttestationId {
+    fn from_data(attestation: &AttestationData, state: &BeaconState, spec: &ChainSpec) -> Self {
+        let mut bytes = ssz_encode(attestation);
+        let epoch = attestation.slot.epoch(spec.slots_per_epoch);
+        bytes.extend_from_slice(&AttestationId::compute_domain_bytes(epoch, state, spec));
+        AttestationId(bytes)
+    }
+
+    fn compute_domain_bytes(epoch: Epoch, state: &BeaconState, spec: &ChainSpec) -> Vec<u8> {
+        int_to_bytes8(spec.get_domain(epoch, Domain::Attestation, &state.fork))
+    }
+
+    fn domain_bytes_match(&self, domain_bytes: &[u8]) -> bool {
+        &self.0[self.0.len() - DOMAIN_BYTES_LEN..] == domain_bytes
+    }
+}
+
+/// Compute a fitness score for an attestation.
+///
+/// The score is calculated by determining the number of *new* attestations that
+/// the aggregate attestation introduces, and is proportional to the size of the reward we will
+/// receive for including it in a block.
+// TODO: this could be optimised with a map from validator index to whether that validator has
+// attested in the *current* epoch. Alternatively, we could cache an index that allows us to
+// quickly look up the attestations in the current epoch for a given shard.
+fn attestation_score(attestation: &Attestation, state: &BeaconState) -> usize {
+    // Bitfield of validators whose attestations are new/fresh.
+    let mut new_validators = attestation.aggregation_bitfield.clone();
+
+    state
+        .current_epoch_attestations
+        .iter()
+        .filter(|current_attestation| current_attestation.data.shard == attestation.data.shard)
+        .for_each(|current_attestation| {
+            // Remove the validators who have signed the existing attestation (they are not new)
+            new_validators.difference_inplace(&current_attestation.aggregation_bitfield);
+        });
+
+    new_validators.num_set_bits()
+}
+
+#[derive(Debug, PartialEq, Clone)]
+pub enum DepositInsertStatus {
+    /// The deposit was not already in the pool.
+    Fresh,
+    /// The deposit already existed in the pool.
+    Duplicate,
+    /// The deposit conflicted with an existing deposit, which was replaced.
+    Replaced(Box<Deposit>),
+}
+
+impl OperationPool {
+    /// Create a new operation pool.
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    /// Insert an attestation into the pool, aggregating it with existing attestations if possible.
+    pub fn insert_attestation(
+        &self,
+        attestation: Attestation,
+        state: &BeaconState,
+        spec: &ChainSpec,
+    ) -> Result<(), AttestationValidationError> {
+        // Check that attestation signatures are valid.
+        validate_attestation_time_independent_only(state, &attestation, spec)?;
+
+        let id = AttestationId::from_data(&attestation.data, state, spec);
+
+        // Take a write lock on the attestations map.
+        let mut attestations = self.attestations.write();
+
+        let existing_attestations = match attestations.entry(id) {
+            hash_map::Entry::Vacant(entry) => {
+                entry.insert(vec![attestation]);
+                return Ok(());
+            }
+            hash_map::Entry::Occupied(entry) => entry.into_mut(),
+        };
+
+        let mut aggregated = false;
+        for existing_attestation in existing_attestations.iter_mut() {
+            if existing_attestation.signers_disjoint_from(&attestation) {
+                existing_attestation.aggregate(&attestation);
+                aggregated = true;
+            } else if *existing_attestation == attestation {
+                aggregated = true;
+            }
+        }
+
+        if !aggregated {
+            existing_attestations.push(attestation);
+        }
+
+        Ok(())
+    }
+
+    /// Total number of attestations in the pool, including attestations for the same data.
+    pub fn num_attestations(&self) -> usize {
+        self.attestations
+            .read()
+            .values()
+            .map(|atts| atts.len())
+            .sum()
+    }
+
+    /// Get a list of attestations for inclusion in a block.
+    pub fn get_attestations(&self, state: &BeaconState, spec: &ChainSpec) -> Vec<Attestation> {
+        // Attestations for the current fork, which may be from the current or previous epoch.
+        let prev_epoch = state.previous_epoch(spec);
+        let current_epoch = state.current_epoch(spec);
+        let prev_domain_bytes = AttestationId::compute_domain_bytes(prev_epoch, state, spec);
+        let curr_domain_bytes = AttestationId::compute_domain_bytes(current_epoch, state, spec);
+        self.attestations
+            .read()
+            .iter()
+            .filter(|(key, _)| {
+                key.domain_bytes_match(&prev_domain_bytes)
+                    || key.domain_bytes_match(&curr_domain_bytes)
+            })
+            .flat_map(|(_, attestations)| attestations)
+            // That are valid...
+            .filter(|attestation| validate_attestation(state, attestation, spec).is_ok())
+            // Scored by the number of new attestations they introduce (descending)
+            // TODO: need to consider attestations introduced in THIS block
+            .map(|att| (att, attestation_score(att, state)))
+            .sorted_by_key(|&(_, score)| std::cmp::Reverse(score))
+            // Limited to the maximum number of attestations per block
+            .take(spec.max_attestations as usize)
+            .map(|(att, _)| att)
+            .cloned()
+            .collect()
+    }
+
+    /// Remove attestations which are too old to be included in a block.
+    // TODO: we could probably prune other attestations here:
+    // - ones that are completely covered by attestations included in the state
+    // - maybe ones invalidated by the confirmation of one fork over another
+    pub fn prune_attestations(&self, finalized_state: &BeaconState, spec: &ChainSpec) {
+        self.attestations.write().retain(|_, attestations| {
+            // All the attestations in this bucket have the same data, so we only need to
+            // check the first one.
+            attestations.first().map_or(false, |att| {
+                finalized_state.slot < att.data.slot + spec.slots_per_epoch
+            })
+        });
+    }
+
+    /// Add a deposit to the pool.
+    ///
+    /// No two distinct deposits should be added with the same index.
+    pub fn insert_deposit(
+        &self,
+        deposit: Deposit,
+        state: &BeaconState,
+        spec: &ChainSpec,
+    ) -> Result<DepositInsertStatus, DepositValidationError> {
+        use DepositInsertStatus::*;
+
+        match self.deposits.write().entry(deposit.index) {
+            Entry::Vacant(entry) => {
+                verify_deposit(state, &deposit, VERIFY_DEPOSIT_PROOFS, spec)?;
+                entry.insert(deposit);
+                Ok(Fresh)
+            }
+            Entry::Occupied(mut entry) => {
+                if entry.get() == &deposit {
+                    Ok(Duplicate)
+                } else {
+                    verify_deposit(state, &deposit, VERIFY_DEPOSIT_PROOFS, spec)?;
+                    Ok(Replaced(Box::new(entry.insert(deposit))))
+                }
+            }
+        }
+    }
+
+    /// Get an ordered list of deposits for inclusion in a block.
+    ///
+    /// Take at most the maximum number of deposits, beginning from the current deposit index.
+    pub fn get_deposits(&self, state: &BeaconState, spec: &ChainSpec) -> Vec<Deposit> {
+        let start_idx = state.deposit_index;
+        (start_idx..start_idx + spec.max_deposits)
+            .map(|idx| self.deposits.read().get(&idx).cloned())
+            .take_while(Option::is_some)
+            .flatten()
+            .collect()
+    }
+
+    /// Remove all deposits with index less than the deposit index of the latest finalised block.
+    pub fn prune_deposits(&self, state: &BeaconState) -> BTreeMap<u64, Deposit> {
+        let deposits_keep = self.deposits.write().split_off(&state.deposit_index);
+        std::mem::replace(&mut self.deposits.write(), deposits_keep)
+    }
+
+    /// The number of deposits stored in the pool.
+    pub fn num_deposits(&self) -> usize {
+        self.deposits.read().len()
+    }
+
+    /// Insert a proposer slashing into the pool.
+    pub fn insert_proposer_slashing(
+        &self,
+        slashing: ProposerSlashing,
+        state: &BeaconState,
+        spec: &ChainSpec,
+    ) -> Result<(), ProposerSlashingValidationError> {
+        // TODO: should maybe insert anyway if the proposer is unknown in the validator index,
+        // because they could *become* known later
+        verify_proposer_slashing(&slashing, state, spec)?;
+        self.proposer_slashings
+            .write()
+            .insert(slashing.proposer_index, slashing);
+        Ok(())
+    }
+
+    /// Compute the tuple ID that is used to identify an attester slashing.
+    ///
+    /// Depends on the fork field of the state, but not on the state's epoch.
+    fn attester_slashing_id(
+        slashing: &AttesterSlashing,
+        state: &BeaconState,
+        spec: &ChainSpec,
+    ) -> (AttestationId, AttestationId) {
+        (
+            AttestationId::from_data(&slashing.slashable_attestation_1.data, state, spec),
+            AttestationId::from_data(&slashing.slashable_attestation_2.data, state, spec),
+        )
+    }
+
+    /// Insert an attester slashing into the pool.
+    pub fn insert_attester_slashing(
+        &self,
+        slashing: AttesterSlashing,
+        state: &BeaconState,
+        spec: &ChainSpec,
+    ) -> Result<(), AttesterSlashingValidationError> {
+        verify_attester_slashing(state, &slashing, true, spec)?;
+        let id = Self::attester_slashing_id(&slashing, state, spec);
+        self.attester_slashings.write().insert(id, slashing);
+        Ok(())
+    }
+
+    /// Get proposer and attester slashings for inclusion in a block.
+    ///
+    /// This function computes both types of slashings together, because
+    /// attester slashings may be invalidated by proposer slashings included
+    /// earlier in the block.
+    pub fn get_slashings(
+        &self,
+        state: &BeaconState,
+        spec: &ChainSpec,
+    ) -> (Vec<ProposerSlashing>, Vec<AttesterSlashing>) {
+        let proposer_slashings = filter_limit_operations(
+            self.proposer_slashings.read().values(),
+            |slashing| {
+                state
+                    .validator_registry
+                    .get(slashing.proposer_index as usize)
+                    .map_or(false, |validator| !validator.slashed)
+            },
+            spec.max_proposer_slashings,
+        );
+
+        // Set of validators to be slashed, so we don't attempt to construct invalid attester
+        // slashings.
+        let mut to_be_slashed = proposer_slashings
+            .iter()
+            .map(|s| s.proposer_index)
+            .collect::<HashSet<_>>();
+
+        let attester_slashings = self
+            .attester_slashings
+            .read()
+            .iter()
+            .filter(|(id, slashing)| {
+                // Check the fork.
+                Self::attester_slashing_id(slashing, state, spec) == **id
+            })
+            .filter(|(_, slashing)| {
+                // Take all slashings that will slash 1 or more validators.
+                let slashed_validators = gather_attester_slashing_indices_modular(
+                    state,
+                    slashing,
+                    |index, validator| validator.slashed || to_be_slashed.contains(&index),
+                    spec,
+                );
+
+                // Extend the `to_be_slashed` set so subsequent iterations don't try to include
+                // useless slashings.
+                if let Ok(validators) = slashed_validators {
+                    to_be_slashed.extend(validators);
+                    true
+                } else {
+                    false
+                }
+            })
+            .take(spec.max_attester_slashings as usize)
+            .map(|(_, slashing)| slashing.clone())
+            .collect();
+
+        (proposer_slashings, attester_slashings)
+    }
+
+    /// Prune proposer slashings for all slashed or withdrawn validators.
+    pub fn prune_proposer_slashings(&self, finalized_state: &BeaconState, spec: &ChainSpec) {
+        prune_validator_hash_map(
+            &mut self.proposer_slashings.write(),
+            |validator| {
+                validator.slashed
+                    || validator.is_withdrawable_at(finalized_state.current_epoch(spec))
+            },
+            finalized_state,
+        );
+    }
+
+    /// Prune attester slashings for all slashed or withdrawn validators, or attestations on another
+    /// fork.
+    pub fn prune_attester_slashings(&self, finalized_state: &BeaconState, spec: &ChainSpec) {
+        self.attester_slashings.write().retain(|id, slashing| {
+            let fork_ok = &Self::attester_slashing_id(slashing, finalized_state, spec) == id;
+            let curr_epoch = finalized_state.current_epoch(spec);
+            let slashing_ok = gather_attester_slashing_indices_modular(
+                finalized_state,
+                slashing,
+                |_, validator| validator.slashed || validator.is_withdrawable_at(curr_epoch),
+                spec,
+            )
+            .is_ok();
+            fork_ok && slashing_ok
+        });
+    }
+
+    /// Insert a voluntary exit, validating it almost-entirely (future exits are permitted).
+    pub fn insert_voluntary_exit(
+        &self,
+        exit: VoluntaryExit,
+        state: &BeaconState,
+        spec: &ChainSpec,
+    ) -> Result<(), ExitValidationError> {
+        verify_exit_time_independent_only(state, &exit, spec)?;
+        self.voluntary_exits
+            .write()
+            .insert(exit.validator_index, exit);
+        Ok(())
+    }
+
+    /// Get a list of voluntary exits for inclusion in a block.
+    pub fn get_voluntary_exits(&self, state: &BeaconState, spec: &ChainSpec) -> Vec<VoluntaryExit> {
+        filter_limit_operations(
+            self.voluntary_exits.read().values(),
+            |exit| verify_exit(state, exit, spec).is_ok(),
+            spec.max_voluntary_exits,
+        )
+    }
+
+    /// Prune if validator has already exited at the last finalized state.
+    pub fn prune_voluntary_exits(&self, finalized_state: &BeaconState, spec: &ChainSpec) {
+        prune_validator_hash_map(
+            &mut self.voluntary_exits.write(),
+            |validator| validator.is_exited_at(finalized_state.current_epoch(spec)),
+            finalized_state,
+        );
+    }
+
+    /// Insert a transfer into the pool, checking it for validity in the process.
+    pub fn insert_transfer(
+        &self,
+        transfer: Transfer,
+        state: &BeaconState,
+        spec: &ChainSpec,
+    ) -> Result<(), TransferValidationError> {
+        // The signature of the transfer isn't hashed, but because we check
+        // it before we insert into the HashSet, we can't end up with duplicate
+        // transactions.
+        verify_transfer_time_independent_only(state, &transfer, spec)?;
+        self.transfers.write().insert(transfer);
+        Ok(())
+    }
+
+    /// Get a list of transfers for inclusion in a block.
+    // TODO: improve the economic optimality of this function by accounting for
+    // dependencies between transfers in the same block e.g. A pays B, B pays C
+    pub fn get_transfers(&self, state: &BeaconState, spec: &ChainSpec) -> Vec<Transfer> {
+        self.transfers
+            .read()
+            .iter()
+            .filter(|transfer| verify_transfer(state, transfer, spec).is_ok())
+            .sorted_by_key(|transfer| std::cmp::Reverse(transfer.fee))
+            .take(spec.max_transfers as usize)
+            .cloned()
+            .collect()
+    }
+
+    /// Prune the set of transfers by removing all those whose slot has already passed.
+    pub fn prune_transfers(&self, finalized_state: &BeaconState) {
+        self.transfers
+            .write()
+            .retain(|transfer| transfer.slot > finalized_state.slot)
+    }
+
+    /// Prune all types of transactions given the latest finalized state.
+    pub fn prune_all(&self, finalized_state: &BeaconState, spec: &ChainSpec) {
+        self.prune_attestations(finalized_state, spec);
+        self.prune_deposits(finalized_state);
+        self.prune_proposer_slashings(finalized_state, spec);
+        self.prune_attester_slashings(finalized_state, spec);
+        self.prune_voluntary_exits(finalized_state, spec);
+        self.prune_transfers(finalized_state);
+    }
+}
+
+/// Filter up to a maximum number of operations out of an iterator.
+fn filter_limit_operations<'a, T: 'a, I, F>(operations: I, filter: F, limit: u64) -> Vec<T>
+where
+    I: IntoIterator<Item = &'a T>,
+    F: Fn(&T) -> bool,
+    T: Clone,
+{
+    operations
+        .into_iter()
+        .filter(|x| filter(*x))
+        .take(limit as usize)
+        .cloned()
+        .collect()
+}
+
+/// Remove all entries from the given hash map for which `prune_if` returns true.
+///
+/// The keys in the map should be validator indices, which will be looked up
+/// in the state's validator registry and then passed to `prune_if`.
+/// Entries for unknown validators will be kept.
+fn prune_validator_hash_map<T, F>(
+    map: &mut HashMap<u64, T>,
+    prune_if: F,
+    finalized_state: &BeaconState,
+) where
+    F: Fn(&Validator) -> bool,
+{
+    map.retain(|&validator_index, _| {
+        finalized_state
+            .validator_registry
+            .get(validator_index as usize)
+            .map_or(true, |validator| !prune_if(validator))
+    });
+}
+
+#[cfg(test)]
+mod tests {
+    use super::DepositInsertStatus::*;
+    use super::*;
+    use types::test_utils::*;
+    use types::*;
+
+    #[test]
+    fn insert_deposit() {
+        let rng = &mut XorShiftRng::from_seed([42; 16]);
+        let (ref spec, ref state) = test_state(rng);
+        let op_pool = OperationPool::new();
+        let deposit1 = make_deposit(rng, state, spec);
+        let mut deposit2 = make_deposit(rng, state, spec);
+        deposit2.index = deposit1.index;
+
+        assert_eq!(
+            op_pool.insert_deposit(deposit1.clone(), state, spec),
+            Ok(Fresh)
+        );
+        assert_eq!(
+            op_pool.insert_deposit(deposit1.clone(), state, spec),
+            Ok(Duplicate)
+        );
+        assert_eq!(
+            op_pool.insert_deposit(deposit2, state, spec),
+            Ok(Replaced(Box::new(deposit1)))
+        );
+    }
+
+    #[test]
+    fn get_deposits_max() {
+        let rng = &mut XorShiftRng::from_seed([42; 16]);
+        let (spec, mut state) = test_state(rng);
+        let op_pool = OperationPool::new();
+        let start = 10000;
+        let max_deposits = spec.max_deposits;
+        let extra = 5;
+        let offset = 1;
+        assert!(offset <= extra);
+
+        let deposits = dummy_deposits(rng, &state, &spec, start, max_deposits + extra);
+
+        for deposit in &deposits {
+            assert_eq!(
+                op_pool.insert_deposit(deposit.clone(), &state, &spec),
+                Ok(Fresh)
+            );
+        }
+
+        state.deposit_index = start + offset;
+        let deposits_for_block = op_pool.get_deposits(&state, &spec);
+
+        assert_eq!(deposits_for_block.len() as u64, max_deposits);
+        assert_eq!(
+            deposits_for_block[..],
+            deposits[offset as usize..(offset + max_deposits) as usize]
+        );
+    }
+
+    #[test]
+    fn prune_deposits() {
+        let rng = &mut XorShiftRng::from_seed([42; 16]);
+        let (spec, state) = test_state(rng);
+        let op_pool = OperationPool::new();
+
+        let start1 = 100;
+        // test is super slow in debug mode if this parameter is too high
+        let count = 5;
+        let gap = 25;
+        let start2 = start1 + count + gap;
+
+        let deposits1 = dummy_deposits(rng, &state, &spec, start1, count);
+        let deposits2 = dummy_deposits(rng, &state, &spec, start2, count);
+
+        for d in deposits1.into_iter().chain(deposits2) {
+            assert!(op_pool.insert_deposit(d, &state, &spec).is_ok());
+        }
+
+        assert_eq!(op_pool.num_deposits(), 2 * count as usize);
+
+        let mut state = BeaconState::random_for_test(rng);
+        state.deposit_index = start1;
+
+        // Pruning the first bunch of deposits in batches of 5 should work.
+        let step = 5;
+        let mut pool_size = step + 2 * count as usize;
+        for i in (start1..=(start1 + count)).step_by(step) {
+            state.deposit_index = i;
+            op_pool.prune_deposits(&state);
+            pool_size -= step;
+            assert_eq!(op_pool.num_deposits(), pool_size);
+        }
+        assert_eq!(pool_size, count as usize);
+        // Pruning in the gap should do nothing.
+        for i in (start1 + count..start2).step_by(step) {
+            state.deposit_index = i;
+            op_pool.prune_deposits(&state);
+            assert_eq!(op_pool.num_deposits(), count as usize);
+        }
+        // Same again for the later deposits.
+        pool_size += step;
+        for i in (start2..=(start2 + count)).step_by(step) {
+            state.deposit_index = i;
+            op_pool.prune_deposits(&state);
+            pool_size -= step;
+            assert_eq!(op_pool.num_deposits(), pool_size);
+        }
+        assert_eq!(op_pool.num_deposits(), 0);
+    }
+
+    // Create a random deposit (with a valid proof of posession)
+    fn make_deposit(rng: &mut XorShiftRng, state: &BeaconState, spec: &ChainSpec) -> Deposit {
+        let keypair = Keypair::random();
+        let mut deposit = Deposit::random_for_test(rng);
+        let mut deposit_input = DepositInput {
+            pubkey: keypair.pk.clone(),
+            withdrawal_credentials: Hash256::zero(),
+            proof_of_possession: Signature::empty_signature(),
+        };
+        deposit_input.proof_of_possession = deposit_input.create_proof_of_possession(
+            &keypair.sk,
+            state.slot.epoch(spec.slots_per_epoch),
+            &state.fork,
+            spec,
+        );
+        deposit.deposit_data.deposit_input = deposit_input;
+        deposit
+    }
+
+    // Create `count` dummy deposits with sequential deposit IDs beginning from `start`.
+    fn dummy_deposits(
+        rng: &mut XorShiftRng,
+        state: &BeaconState,
+        spec: &ChainSpec,
+        start: u64,
+        count: u64,
+    ) -> Vec<Deposit> {
+        let proto_deposit = make_deposit(rng, state, spec);
+        (start..start + count)
+            .map(|index| {
+                let mut deposit = proto_deposit.clone();
+                deposit.index = index;
+                deposit
+            })
+            .collect()
+    }
+
+    fn test_state(rng: &mut XorShiftRng) -> (ChainSpec, BeaconState) {
+        let spec = ChainSpec::foundation();
+        let mut state = BeaconState::random_for_test(rng);
+        state.fork = Fork::genesis(&spec);
+
+        (spec, state)
+    }
+
+    /// Create a signed attestation for use in tests.
+    /// Signed by all validators in `committee[signing_range]` and `committee[extra_signer]`.
+    fn signed_attestation<R: std::slice::SliceIndex<[usize], Output = [usize]>>(
+        committee: &CrosslinkCommittee,
+        keypairs: &[Keypair],
+        signing_range: R,
+        slot: Slot,
+        state: &BeaconState,
+        spec: &ChainSpec,
+        extra_signer: Option<usize>,
+    ) -> Attestation {
+        let mut builder = TestingAttestationBuilder::new(
+            state,
+            &committee.committee,
+            slot,
+            committee.shard,
+            spec,
+        );
+        let signers = &committee.committee[signing_range];
+        let committee_keys = signers.iter().map(|&i| &keypairs[i].sk).collect::<Vec<_>>();
+        builder.sign(signers, &committee_keys, &state.fork, spec);
+        extra_signer.map(|c_idx| {
+            let validator_index = committee.committee[c_idx];
+            builder.sign(
+                &[validator_index],
+                &[&keypairs[validator_index].sk],
+                &state.fork,
+                spec,
+            )
+        });
+        builder.build()
+    }
+
+    /// Test state for attestation-related tests.
+    fn attestation_test_state(
+        spec: &ChainSpec,
+        num_committees: usize,
+    ) -> (BeaconState, Vec<Keypair>) {
+        let num_validators =
+            num_committees * (spec.slots_per_epoch * spec.target_committee_size) as usize;
+        let mut state_builder =
+            TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(num_validators, spec);
+        let slot_offset = 100000;
+        let slot = spec.genesis_slot + slot_offset;
+        state_builder.teleport_to_slot(slot, spec);
+        state_builder.build_caches(spec).unwrap();
+        state_builder.build()
+    }
+
+    /// Set the latest crosslink in the state to match the attestation.
+    fn fake_latest_crosslink(att: &Attestation, state: &mut BeaconState, spec: &ChainSpec) {
+        state.latest_crosslinks[att.data.shard as usize] = Crosslink {
+            crosslink_data_root: att.data.crosslink_data_root,
+            epoch: att.data.slot.epoch(spec.slots_per_epoch),
+        };
+    }
+
+    #[test]
+    fn test_attestation_score() {
+        let spec = &ChainSpec::foundation();
+        let (ref mut state, ref keypairs) = attestation_test_state(spec, 1);
+        let slot = state.slot - 1;
+        let committees = state
+            .get_crosslink_committees_at_slot(slot, spec)
+            .unwrap()
+            .clone();
+
+        for committee in committees {
+            let att1 = signed_attestation(&committee, keypairs, ..2, slot, state, spec, None);
+            let att2 = signed_attestation(&committee, keypairs, .., slot, state, spec, None);
+
+            assert_eq!(
+                att1.aggregation_bitfield.num_set_bits(),
+                attestation_score(&att1, state)
+            );
+
+            state
+                .current_epoch_attestations
+                .push(PendingAttestation::from_attestation(&att1, state.slot));
+
+            assert_eq!(
+                committee.committee.len() - 2,
+                attestation_score(&att2, &state)
+            );
+        }
+    }
+
+    /// End-to-end test of basic attestation handling.
+    #[test]
+    fn attestation_aggregation_insert_get_prune() {
+        let spec = &ChainSpec::foundation();
+        let (ref mut state, ref keypairs) = attestation_test_state(spec, 1);
+        let op_pool = OperationPool::new();
+
+        let slot = state.slot - 1;
+        let committees = state
+            .get_crosslink_committees_at_slot(slot, spec)
+            .unwrap()
+            .clone();
+
+        assert_eq!(
+            committees.len(),
+            1,
+            "we expect just one committee with this many validators"
+        );
+
+        for committee in &committees {
+            let step_size = 2;
+            for i in (0..committee.committee.len()).step_by(step_size) {
+                let att = signed_attestation(
+                    committee,
+                    keypairs,
+                    i..i + step_size,
+                    slot,
+                    state,
+                    spec,
+                    None,
+                );
+                fake_latest_crosslink(&att, state, spec);
+                op_pool.insert_attestation(att, state, spec).unwrap();
+            }
+        }
+
+        assert_eq!(op_pool.attestations.read().len(), committees.len());
+        assert_eq!(op_pool.num_attestations(), committees.len());
+
+        // Before the min attestation inclusion delay, get_attestations shouldn't return anything.
+        assert_eq!(op_pool.get_attestations(state, spec).len(), 0);
+
+        // Then once the delay has elapsed, we should get a single aggregated attestation.
+        state.slot += spec.min_attestation_inclusion_delay;
+
+        let block_attestations = op_pool.get_attestations(state, spec);
+        assert_eq!(block_attestations.len(), committees.len());
+
+        let agg_att = &block_attestations[0];
+        assert_eq!(
+            agg_att.aggregation_bitfield.num_set_bits(),
+            spec.target_committee_size as usize
+        );
+
+        // Prune attestations shouldn't do anything at this point.
+        op_pool.prune_attestations(state, spec);
+        assert_eq!(op_pool.num_attestations(), committees.len());
+
+        // But once we advance to an epoch after the attestation, it should prune it out of
+        // existence.
+        state.slot = slot + spec.slots_per_epoch;
+        op_pool.prune_attestations(state, spec);
+        assert_eq!(op_pool.num_attestations(), 0);
+    }
+
+    /// Adding an attestation already in the pool should not increase the size of the pool.
+    #[test]
+    fn attestation_duplicate() {
+        let spec = &ChainSpec::foundation();
+        let (ref mut state, ref keypairs) = attestation_test_state(spec, 1);
+        let op_pool = OperationPool::new();
+
+        let slot = state.slot - 1;
+        let committees = state
+            .get_crosslink_committees_at_slot(slot, spec)
+            .unwrap()
+            .clone();
+
+        for committee in &committees {
+            let att = signed_attestation(committee, keypairs, .., slot, state, spec, None);
+            fake_latest_crosslink(&att, state, spec);
+            op_pool
+                .insert_attestation(att.clone(), state, spec)
+                .unwrap();
+            op_pool.insert_attestation(att, state, spec).unwrap();
+        }
+
+        assert_eq!(op_pool.num_attestations(), committees.len());
+    }
+
+    /// Adding lots of attestations that only intersect pairwise should lead to two aggregate
+    /// attestations.
+    #[test]
+    fn attestation_pairwise_overlapping() {
+        let spec = &ChainSpec::foundation();
+        let (ref mut state, ref keypairs) = attestation_test_state(spec, 1);
+        let op_pool = OperationPool::new();
+
+        let slot = state.slot - 1;
+        let committees = state
+            .get_crosslink_committees_at_slot(slot, spec)
+            .unwrap()
+            .clone();
+
+        let step_size = 2;
+        for committee in &committees {
+            // Create attestations that overlap on `step_size` validators, like:
+            // {0,1,2,3}, {2,3,4,5}, {4,5,6,7}, ...
+            for i in (0..committee.committee.len() - step_size).step_by(step_size) {
+                let att = signed_attestation(
+                    committee,
+                    keypairs,
+                    i..i + 2 * step_size,
+                    slot,
+                    state,
+                    spec,
+                    None,
+                );
+                fake_latest_crosslink(&att, state, spec);
+                op_pool.insert_attestation(att, state, spec).unwrap();
+            }
+        }
+
+        // The attestations should get aggregated into two attestations that comprise all
+        // validators.
+        assert_eq!(op_pool.attestations.read().len(), committees.len());
+        assert_eq!(op_pool.num_attestations(), 2 * committees.len());
+    }
+
+    /// Create a bunch of attestations signed by a small number of validators, and another
+    /// bunch signed by a larger number, such that there are at least `max_attestations`
+    /// signed by the larger number. Then, check that `get_attestations` only returns the
+    /// high-quality attestations. To ensure that no aggregation occurs, ALL attestations
+    /// are also signed by the 0th member of the committee.
+    #[test]
+    fn attestation_get_max() {
+        let spec = &ChainSpec::foundation();
+        let small_step_size = 2;
+        let big_step_size = 4;
+        let (ref mut state, ref keypairs) = attestation_test_state(spec, big_step_size);
+        let op_pool = OperationPool::new();
+
+        let slot = state.slot - 1;
+        let committees = state
+            .get_crosslink_committees_at_slot(slot, spec)
+            .unwrap()
+            .clone();
+
+        let max_attestations = spec.max_attestations as usize;
+        let target_committee_size = spec.target_committee_size as usize;
+
+        let mut insert_attestations = |committee, step_size| {
+            for i in (0..target_committee_size).step_by(step_size) {
+                let att = signed_attestation(
+                    committee,
+                    keypairs,
+                    i..i + step_size,
+                    slot,
+                    state,
+                    spec,
+                    if i == 0 { None } else { Some(0) },
+                );
+                fake_latest_crosslink(&att, state, spec);
+                op_pool.insert_attestation(att, state, spec).unwrap();
+            }
+        };
+
+        for committee in &committees {
+            assert_eq!(committee.committee.len(), target_committee_size);
+            // Attestations signed by only 2-3 validators
+            insert_attestations(committee, small_step_size);
+            // Attestations signed by 4+ validators
+            insert_attestations(committee, big_step_size);
+        }
+
+        let num_small = target_committee_size / small_step_size;
+        let num_big = target_committee_size / big_step_size;
+
+        assert_eq!(op_pool.attestations.read().len(), committees.len());
+        assert_eq!(
+            op_pool.num_attestations(),
+            (num_small + num_big) * committees.len()
+        );
+        assert!(op_pool.num_attestations() > max_attestations);
+
+        state.slot += spec.min_attestation_inclusion_delay;
+        let best_attestations = op_pool.get_attestations(state, spec);
+        assert_eq!(best_attestations.len(), max_attestations);
+
+        // All the best attestations should be signed by at least `big_step_size` (4) validators.
+        for att in &best_attestations {
+            assert!(att.aggregation_bitfield.num_set_bits() >= big_step_size);
+        }
+    }
+
+    // TODO: more tests
+}

eth2/state_processing/src/per_block_processing.rs

@@ -1,4 +1,3 @@
-use self::verify_proposer_slashing::verify_proposer_slashing;
 use crate::common::slash_validator;
 use errors::{BlockInvalid as Invalid, BlockProcessingError as Error, IntoWithIndex};
 use rayon::prelude::*;
@@ -6,13 +5,20 @@ use ssz::{SignedRoot, TreeHash};
 use types::*;
 
 pub use self::verify_attester_slashing::{
-    gather_attester_slashing_indices, verify_attester_slashing,
+    gather_attester_slashing_indices, gather_attester_slashing_indices_modular,
+    verify_attester_slashing,
+};
+pub use self::verify_proposer_slashing::verify_proposer_slashing;
+pub use validate_attestation::{
+    validate_attestation, validate_attestation_time_independent_only,
+    validate_attestation_without_signature,
 };
-pub use validate_attestation::{validate_attestation, validate_attestation_without_signature};
 pub use verify_deposit::{get_existing_validator_index, verify_deposit, verify_deposit_index};
-pub use verify_exit::verify_exit;
+pub use verify_exit::{verify_exit, verify_exit_time_independent_only};
 pub use verify_slashable_attestation::verify_slashable_attestation;
-pub use verify_transfer::{execute_transfer, verify_transfer};
+pub use verify_transfer::{
+    execute_transfer, verify_transfer, verify_transfer_time_independent_only,
+};
 
 pub mod errors;
 mod validate_attestation;
@@ -316,13 +322,7 @@ pub fn process_attestations(
 
     // Update the state in series.
     for attestation in attestations {
-        let pending_attestation = PendingAttestation {
-            data: attestation.data.clone(),
-            aggregation_bitfield: attestation.aggregation_bitfield.clone(),
-            custody_bitfield: attestation.custody_bitfield.clone(),
-            inclusion_slot: state.slot,
-        };
-
+        let pending_attestation = PendingAttestation::from_attestation(attestation, state.slot);
         let attestation_epoch = attestation.data.slot.epoch(spec.slots_per_epoch);
 
         if attestation_epoch == state.current_epoch(spec) {

eth2/state_processing/src/per_block_processing/errors.rs

@@ -390,6 +390,11 @@ pub enum TransferInvalid {
     ///
     /// (state_slot, transfer_slot)
     StateSlotMismatch(Slot, Slot),
+    /// The `transfer.slot` is in the past relative to the state slot.
+    ///
+    ///
+    /// (state_slot, transfer_slot)
+    TransferSlotInPast(Slot, Slot),
     /// The `transfer.from` validator has been activated and is not withdrawable.
     ///
     /// (from_validator)

eth2/state_processing/src/per_block_processing/validate_attestation.rs

@@ -14,7 +14,16 @@ pub fn validate_attestation(
     attestation: &Attestation,
     spec: &ChainSpec,
 ) -> Result<(), Error> {
-    validate_attestation_signature_optional(state, attestation, spec, true)
+    validate_attestation_parametric(state, attestation, spec, true, false)
+}
+
+/// Like `validate_attestation` but doesn't run checks which may become true in future states.
+pub fn validate_attestation_time_independent_only(
+    state: &BeaconState,
+    attestation: &Attestation,
+    spec: &ChainSpec,
+) -> Result<(), Error> {
+    validate_attestation_parametric(state, attestation, spec, true, true)
 }
 
 /// Indicates if an `Attestation` is valid to be included in a block in the current epoch of the
@@ -28,7 +37,7 @@ pub fn validate_attestation_without_signature(
     attestation: &Attestation,
     spec: &ChainSpec,
 ) -> Result<(), Error> {
-    validate_attestation_signature_optional(state, attestation, spec, false)
+    validate_attestation_parametric(state, attestation, spec, false, false)
 }
 
 /// Indicates if an `Attestation` is valid to be included in a block in the current epoch of the
@@ -36,15 +45,13 @@ pub fn validate_attestation_without_signature(
 ///
 ///
 /// Spec v0.5.0
-fn validate_attestation_signature_optional(
+fn validate_attestation_parametric(
     state: &BeaconState,
     attestation: &Attestation,
     spec: &ChainSpec,
     verify_signature: bool,
+    time_independent_only: bool,
 ) -> Result<(), Error> {
-    let state_epoch = state.slot.epoch(spec.slots_per_epoch);
-    let attestation_epoch = attestation.data.slot.epoch(spec.slots_per_epoch);
-
     // Can't submit pre-historic attestations.
     verify!(
         attestation.data.slot >= spec.genesis_slot,
@@ -65,7 +72,8 @@ fn validate_attestation_signature_optional(
 
     // Can't submit attestation too quickly.
     verify!(
-        attestation.data.slot + spec.min_attestation_inclusion_delay <= state.slot,
+        time_independent_only
+            || attestation.data.slot + spec.min_attestation_inclusion_delay <= state.slot,
         Invalid::IncludedTooEarly {
             state: state.slot,
             delay: spec.min_attestation_inclusion_delay,
@@ -74,40 +82,8 @@ fn validate_attestation_signature_optional(
     );
 
     // Verify the justified epoch and root is correct.
-    if attestation_epoch >= state_epoch {
-        verify!(
-            attestation.data.source_epoch == state.current_justified_epoch,
-            Invalid::WrongJustifiedEpoch {
-                state: state.current_justified_epoch,
-                attestation: attestation.data.source_epoch,
-                is_current: true,
-            }
-        );
-        verify!(
-            attestation.data.source_root == state.current_justified_root,
-            Invalid::WrongJustifiedRoot {
-                state: state.current_justified_root,
-                attestation: attestation.data.source_root,
-                is_current: true,
-            }
-        );
-    } else {
-        verify!(
-            attestation.data.source_epoch == state.previous_justified_epoch,
-            Invalid::WrongJustifiedEpoch {
-                state: state.previous_justified_epoch,
-                attestation: attestation.data.source_epoch,
-                is_current: false,
-            }
-        );
-        verify!(
-            attestation.data.source_root == state.previous_justified_root,
-            Invalid::WrongJustifiedRoot {
-                state: state.previous_justified_root,
-                attestation: attestation.data.source_root,
-                is_current: true,
-            }
-        );
+    if !time_independent_only {
+        verify_justified_epoch_and_root(attestation, state, spec)?;
     }
 
     // Check that the crosslink data is valid.
@@ -188,6 +164,56 @@ fn validate_attestation_signature_optional(
     Ok(())
 }
 
+/// Verify that the `source_epoch` and `source_root` of an `Attestation` correctly
+/// match the current (or previous) justified epoch and root from the state.
+///
+/// Spec v0.5.0
+fn verify_justified_epoch_and_root(
+    attestation: &Attestation,
+    state: &BeaconState,
+    spec: &ChainSpec,
+) -> Result<(), Error> {
+    let state_epoch = state.slot.epoch(spec.slots_per_epoch);
+    let attestation_epoch = attestation.data.slot.epoch(spec.slots_per_epoch);
+
+    if attestation_epoch >= state_epoch {
+        verify!(
+            attestation.data.source_epoch == state.current_justified_epoch,
+            Invalid::WrongJustifiedEpoch {
+                state: state.current_justified_epoch,
+                attestation: attestation.data.source_epoch,
+                is_current: true,
+            }
+        );
+        verify!(
+            attestation.data.source_root == state.current_justified_root,
+            Invalid::WrongJustifiedRoot {
+                state: state.current_justified_root,
+                attestation: attestation.data.source_root,
+                is_current: true,
+            }
+        );
+    } else {
+        verify!(
+            attestation.data.source_epoch == state.previous_justified_epoch,
+            Invalid::WrongJustifiedEpoch {
+                state: state.previous_justified_epoch,
+                attestation: attestation.data.source_epoch,
+                is_current: false,
+            }
+        );
+        verify!(
+            attestation.data.source_root == state.previous_justified_root,
+            Invalid::WrongJustifiedRoot {
+                state: state.previous_justified_root,
+                attestation: attestation.data.source_root,
+                is_current: true,
+            }
+        );
+    }
+    Ok(())
+}
+
 /// Verifies an aggregate signature for some given `AttestationData`, returning `true` if the
 /// `aggregate_signature` is valid.
 ///

eth2/state_processing/src/per_block_processing/verify_attester_slashing.rs

@@ -47,6 +47,25 @@ pub fn gather_attester_slashing_indices(
     attester_slashing: &AttesterSlashing,
     spec: &ChainSpec,
 ) -> Result<Vec<u64>, Error> {
+    gather_attester_slashing_indices_modular(
+        state,
+        attester_slashing,
+        |_, validator| validator.slashed,
+        spec,
+    )
+}
+
+/// Same as `gather_attester_slashing_indices` but allows the caller to specify the criteria
+/// for determining whether a given validator should be considered slashed.
+pub fn gather_attester_slashing_indices_modular<F>(
+    state: &BeaconState,
+    attester_slashing: &AttesterSlashing,
+    is_slashed: F,
+    spec: &ChainSpec,
+) -> Result<Vec<u64>, Error>
+where
+    F: Fn(u64, &Validator) -> bool,
+{
     let slashable_attestation_1 = &attester_slashing.slashable_attestation_1;
     let slashable_attestation_2 = &attester_slashing.slashable_attestation_2;
 
@@ -57,7 +76,7 @@ pub fn gather_attester_slashing_indices(
             .get(*i as usize)
             .ok_or_else(|| Error::Invalid(Invalid::UnknownValidator(*i)))?;
 
-        if slashable_attestation_2.validator_indices.contains(&i) & !validator.slashed {
+        if slashable_attestation_2.validator_indices.contains(&i) & !is_slashed(*i, validator) {
             // TODO: verify that we should reject any slashable attestation which includes a
             // withdrawn validator. PH has asked the question on gitter, awaiting response.
             verify!(

eth2/state_processing/src/per_block_processing/verify_exit.rs

@@ -12,6 +12,25 @@ pub fn verify_exit(
     state: &BeaconState,
     exit: &VoluntaryExit,
     spec: &ChainSpec,
+) -> Result<(), Error> {
+    verify_exit_parametric(state, exit, spec, false)
+}
+
+/// Like `verify_exit` but doesn't run checks which may become true in future states.
+pub fn verify_exit_time_independent_only(
+    state: &BeaconState,
+    exit: &VoluntaryExit,
+    spec: &ChainSpec,
+) -> Result<(), Error> {
+    verify_exit_parametric(state, exit, spec, true)
+}
+
+/// Parametric version of `verify_exit` that skips some checks if `time_independent_only` is true.
+fn verify_exit_parametric(
+    state: &BeaconState,
+    exit: &VoluntaryExit,
+    spec: &ChainSpec,
+    time_independent_only: bool,
 ) -> Result<(), Error> {
     let validator = state
         .validator_registry
@@ -32,7 +51,7 @@ pub fn verify_exit(
 
     // Exits must specify an epoch when they become valid; they are not valid before then.
     verify!(
-        state.current_epoch(spec) >= exit.epoch,
+        time_independent_only || state.current_epoch(spec) >= exit.epoch,
         Invalid::FutureEpoch {
             state: state.current_epoch(spec),
             exit: exit.epoch

eth2/state_processing/src/per_block_processing/verify_transfer.rs

@@ -15,6 +15,25 @@ pub fn verify_transfer(
     state: &BeaconState,
     transfer: &Transfer,
     spec: &ChainSpec,
+) -> Result<(), Error> {
+    verify_transfer_parametric(state, transfer, spec, false)
+}
+
+/// Like `verify_transfer` but doesn't run checks which may become true in future states.
+pub fn verify_transfer_time_independent_only(
+    state: &BeaconState,
+    transfer: &Transfer,
+    spec: &ChainSpec,
+) -> Result<(), Error> {
+    verify_transfer_parametric(state, transfer, spec, true)
+}
+
+/// Parametric version of `verify_transfer` that allows some checks to be skipped.
+fn verify_transfer_parametric(
+    state: &BeaconState,
+    transfer: &Transfer,
+    spec: &ChainSpec,
+    time_independent_only: bool,
 ) -> Result<(), Error> {
     let sender_balance = *state
         .validator_balances
@@ -27,17 +46,18 @@ pub fn verify_transfer(
         .ok_or_else(|| Error::Invalid(Invalid::FeeOverflow(transfer.amount, transfer.fee)))?;
 
     verify!(
-        sender_balance >= transfer.amount,
+        time_independent_only || sender_balance >= transfer.amount,
         Invalid::FromBalanceInsufficient(transfer.amount, sender_balance)
     );
 
     verify!(
-        sender_balance >= transfer.fee,
+        time_independent_only || sender_balance >= transfer.fee,
         Invalid::FromBalanceInsufficient(transfer.fee, sender_balance)
     );
 
     verify!(
-        (sender_balance == total_amount)
+        time_independent_only
+            || (sender_balance == total_amount)
             || (sender_balance >= (total_amount + spec.min_deposit_amount)),
         Invalid::InvalidResultingFromBalance(
             sender_balance - total_amount,
@@ -45,10 +65,17 @@ pub fn verify_transfer(
         )
     );
 
-    verify!(
-        state.slot == transfer.slot,
-        Invalid::StateSlotMismatch(state.slot, transfer.slot)
-    );
+    if time_independent_only {
+        verify!(
+            state.slot <= transfer.slot,
+            Invalid::TransferSlotInPast(state.slot, transfer.slot)
+        );
+    } else {
+        verify!(
+            state.slot == transfer.slot,
+            Invalid::StateSlotMismatch(state.slot, transfer.slot)
+        );
+    }
 
     let sender_validator = state
         .validator_registry
@@ -57,7 +84,8 @@ pub fn verify_transfer(
     let epoch = state.slot.epoch(spec.slots_per_epoch);
 
     verify!(
-        sender_validator.is_withdrawable_at(epoch)
+        time_independent_only
+            || sender_validator.is_withdrawable_at(epoch)
             || sender_validator.activation_epoch == spec.far_future_epoch,
         Invalid::FromValidatorIneligableForTransfer(transfer.sender)
     );

eth2/types/Cargo.toml

@@ -8,6 +8,7 @@ edition = "2018"
 bls = { path = "../utils/bls" }
 boolean-bitfield = { path = "../utils/boolean-bitfield" }
 dirs = "1.0"
+derivative = "1.0"
 ethereum-types = "0.5"
 hashing = { path = "../utils/hashing" }
 hex = "0.3"

eth2/types/src/attestation.rs

@@ -28,6 +28,29 @@ pub struct Attestation {
     pub aggregate_signature: AggregateSignature,
 }
 
+impl Attestation {
+    /// Are the aggregation bitfields of these attestations disjoint?
+    pub fn signers_disjoint_from(&self, other: &Attestation) -> bool {
+        self.aggregation_bitfield
+            .intersection(&other.aggregation_bitfield)
+            .is_zero()
+    }
+
+    /// Aggregate another Attestation into this one.
+    ///
+    /// The aggregation bitfields must be disjoint, and the data must be the same.
+    pub fn aggregate(&mut self, other: &Attestation) {
+        debug_assert_eq!(self.data, other.data);
+        debug_assert!(self.signers_disjoint_from(other));
+
+        self.aggregation_bitfield
+            .union_inplace(&other.aggregation_bitfield);
+        self.custody_bitfield.union_inplace(&other.custody_bitfield);
+        self.aggregate_signature
+            .add_aggregate(&other.aggregate_signature);
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

eth2/types/src/attestation_duty.rs

@@ -1,7 +1,7 @@
 use crate::*;
 use serde_derive::{Deserialize, Serialize};
 
-#[derive(Debug, PartialEq, Clone, Default, Serialize, Deserialize)]
+#[derive(Debug, PartialEq, Clone, Copy, Default, Serialize, Deserialize)]
 pub struct AttestationDuty {
     pub slot: Slot,
     pub shard: Shard,

eth2/types/src/pending_attestation.rs

@@ -1,5 +1,5 @@
 use crate::test_utils::TestRandom;
-use crate::{AttestationData, Bitfield, Slot};
+use crate::{Attestation, AttestationData, Bitfield, Slot};
 use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode, TreeHash};
@@ -16,6 +16,18 @@ pub struct PendingAttestation {
     pub inclusion_slot: Slot,
 }
 
+impl PendingAttestation {
+    /// Create a `PendingAttestation` from an `Attestation`, at the given `inclusion_slot`.
+    pub fn from_attestation(attestation: &Attestation, inclusion_slot: Slot) -> Self {
+        PendingAttestation {
+            data: attestation.data.clone(),
+            aggregation_bitfield: attestation.aggregation_bitfield.clone(),
+            custody_bitfield: attestation.custody_bitfield.clone(),
+            inclusion_slot,
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

eth2/types/src/test_utils/generate_deterministic_keypairs.rs

@@ -19,7 +19,7 @@ pub fn generate_deterministic_keypairs(validator_count: usize) -> Vec<Keypair> {
         .collect::<Vec<usize>>()
         .par_iter()
         .map(|&i| {
-            let secret = int_to_bytes48(i as u64 + 1);
+            let secret = int_to_bytes48(i as u64 + 1000);
             let sk = SecretKey::from_bytes(&secret).unwrap();
             let pk = PublicKey::from_secret_key(&sk);
             Keypair { sk, pk }

eth2/types/src/transfer.rs

@@ -1,6 +1,7 @@
 use super::Slot;
 use crate::test_utils::TestRandom;
 use bls::{PublicKey, Signature};
+use derivative::Derivative;
 use rand::RngCore;
 use serde_derive::{Deserialize, Serialize};
 use ssz::TreeHash;
@@ -12,7 +13,6 @@ use test_random_derive::TestRandom;
 /// Spec v0.5.0
 #[derive(
     Debug,
-    PartialEq,
     Clone,
     Serialize,
     Deserialize,
@@ -21,7 +21,9 @@ use test_random_derive::TestRandom;
     TreeHash,
     TestRandom,
     SignedRoot,
+    Derivative,
 )]
+#[derivative(PartialEq, Eq, Hash)]
 pub struct Transfer {
     pub sender: u64,
     pub recipient: u64,
@@ -29,6 +31,7 @@ pub struct Transfer {
     pub fee: u64,
     pub slot: Slot,
     pub pubkey: PublicKey,
+    #[derivative(Hash = "ignore")]
     pub signature: Signature,
 }
 

eth2/utils/bls/src/aggregate_public_key.rs

@@ -1,7 +1,7 @@
 use super::PublicKey;
 use bls_aggregates::AggregatePublicKey as RawAggregatePublicKey;
 
-/// A single BLS signature.
+/// A BLS aggregate public key.
 ///
 /// This struct is a wrapper upon a base type and provides helper functions (e.g., SSZ
 /// serialization).
@@ -17,7 +17,7 @@ impl AggregatePublicKey {
         self.0.add(public_key.as_raw())
     }
 
-    /// Returns the underlying signature.
+    /// Returns the underlying public key.
     pub fn as_raw(&self) -> &RawAggregatePublicKey {
         &self.0
     }

eth2/utils/bls/src/aggregate_signature.rs

@@ -27,6 +27,11 @@ impl AggregateSignature {
         self.0.add(signature.as_raw())
     }
 
+    /// Add (aggregate) another `AggregateSignature`.
+    pub fn add_aggregate(&mut self, agg_signature: &AggregateSignature) {
+        self.0.add_aggregate(&agg_signature.0)
+    }
+
     /// Verify the `AggregateSignature` against an `AggregatePublicKey`.
     ///
     /// Only returns `true` if the set of keys in the `AggregatePublicKey` match the set of keys

eth2/utils/bls/src/fake_aggregate_signature.rs

@@ -35,6 +35,11 @@ impl FakeAggregateSignature {
         // Do nothing.
     }
 
+    /// Does glorious nothing.
+    pub fn add_aggregate(&mut self, _agg_sig: &FakeAggregateSignature) {
+        // Do nothing.
+    }
+
     /// _Always_ returns `true`.
     pub fn verify(
         &self,

eth2/utils/boolean-bitfield/src/lib.rs

@@ -89,6 +89,11 @@ impl BooleanBitfield {
         self.len() == 0
     }
 
+    /// Returns true if all bits are set to 0.
+    pub fn is_zero(&self) -> bool {
+        self.0.none()
+    }
+
     /// Returns the number of bytes required to represent this bitfield.
     pub fn num_bytes(&self) -> usize {
         self.to_bytes().len()
@@ -104,6 +109,44 @@ impl BooleanBitfield {
     pub fn to_bytes(&self) -> Vec<u8> {
         self.0.to_bytes()
     }
+
+    /// Compute the intersection (binary-and) of this bitfield with another. Lengths must match.
+    pub fn intersection(&self, other: &Self) -> Self {
+        let mut res = self.clone();
+        res.intersection_inplace(other);
+        res
+    }
+
+    /// Like `intersection` but in-place (updates `self`).
+    pub fn intersection_inplace(&mut self, other: &Self) {
+        self.0.intersect(&other.0);
+    }
+
+    /// Compute the union (binary-or) of this bitfield with another. Lengths must match.
+    pub fn union(&self, other: &Self) -> Self {
+        let mut res = self.clone();
+        res.union_inplace(other);
+        res
+    }
+
+    /// Like `union` but in-place (updates `self`).
+    pub fn union_inplace(&mut self, other: &Self) {
+        self.0.union(&other.0);
+    }
+
+    /// Compute the difference (binary-minus) of this bitfield with another. Lengths must match.
+    ///
+    /// Computes `self - other`.
+    pub fn difference(&self, other: &Self) -> Self {
+        let mut res = self.clone();
+        res.difference_inplace(other);
+        res
+    }
+
+    /// Like `difference` but in-place (updates `self`).
+    pub fn difference_inplace(&mut self, other: &Self) {
+        self.0.difference(&other.0);
+    }
 }
 
 impl default::Default for BooleanBitfield {
@@ -125,10 +168,11 @@ impl cmp::PartialEq for BooleanBitfield {
 /// Create a new bitfield that is a union of two other bitfields.
 ///
 /// For example `union(0101, 1000) == 1101`
-impl std::ops::BitAnd for BooleanBitfield {
+// TODO: length-independent intersection for BitAnd
+impl std::ops::BitOr for BooleanBitfield {
     type Output = Self;
 
-    fn bitand(self, other: Self) -> Self {
+    fn bitor(self, other: Self) -> Self {
         let (biggest, smallest) = if self.len() > other.len() {
             (&self, &other)
         } else {
@@ -421,10 +465,59 @@ mod tests {
     }
 
     #[test]
-    fn test_bitand() {
+    fn test_bitor() {
         let a = BooleanBitfield::from_bytes(&vec![2, 8, 1][..]);
         let b = BooleanBitfield::from_bytes(&vec![4, 8, 16][..]);
         let c = BooleanBitfield::from_bytes(&vec![6, 8, 17][..]);
-        assert_eq!(c, a & b);
+        assert_eq!(c, a | b);
+    }
+
+    #[test]
+    fn test_is_zero() {
+        let yes_data: &[&[u8]] = &[&[], &[0], &[0, 0], &[0, 0, 0]];
+        for bytes in yes_data {
+            assert!(BooleanBitfield::from_bytes(bytes).is_zero());
+        }
+        let no_data: &[&[u8]] = &[&[1], &[6], &[0, 1], &[0, 0, 1], &[0, 0, 255]];
+        for bytes in no_data {
+            assert!(!BooleanBitfield::from_bytes(bytes).is_zero());
+        }
+    }
+
+    #[test]
+    fn test_intersection() {
+        let a = BooleanBitfield::from_bytes(&[0b1100, 0b0001]);
+        let b = BooleanBitfield::from_bytes(&[0b1011, 0b1001]);
+        let c = BooleanBitfield::from_bytes(&[0b1000, 0b0001]);
+        assert_eq!(a.intersection(&b), c);
+        assert_eq!(b.intersection(&a), c);
+        assert_eq!(a.intersection(&c), c);
+        assert_eq!(b.intersection(&c), c);
+        assert_eq!(a.intersection(&a), a);
+        assert_eq!(b.intersection(&b), b);
+        assert_eq!(c.intersection(&c), c);
+    }
+
+    #[test]
+    fn test_union() {
+        let a = BooleanBitfield::from_bytes(&[0b1100, 0b0001]);
+        let b = BooleanBitfield::from_bytes(&[0b1011, 0b1001]);
+        let c = BooleanBitfield::from_bytes(&[0b1111, 0b1001]);
+        assert_eq!(a.union(&b), c);
+        assert_eq!(b.union(&a), c);
+        assert_eq!(a.union(&a), a);
+        assert_eq!(b.union(&b), b);
+        assert_eq!(c.union(&c), c);
+    }
+
+    #[test]
+    fn test_difference() {
+        let a = BooleanBitfield::from_bytes(&[0b1100, 0b0001]);
+        let b = BooleanBitfield::from_bytes(&[0b1011, 0b1001]);
+        let a_b = BooleanBitfield::from_bytes(&[0b0100, 0b0000]);
+        let b_a = BooleanBitfield::from_bytes(&[0b0011, 0b1000]);
+        assert_eq!(a.difference(&b), a_b);
+        assert_eq!(b.difference(&a), b_a);
+        assert!(a.difference(&a).is_zero());
     }
 }