Merge master and remove ssz length encoding from FakeBLS

beacon_node/rpc/src/beacon_attester.rs

@@ -0,0 +1,61 @@
+use futures::Future;
+use grpcio::{RpcContext, UnarySink};
+use protos::services::{
+    Attestation as AttestationProto, ProduceAttestation, ProduceAttestationResponse,
+    ProduceAttestationRequest, PublishAttestationResponse, PublishAttestationRequest,
+    PublishAttestation
+};
+use protos::services_grpc::BeaconBlockService;
+use slog::Logger;
+
+#[derive(Clone)]
+pub struct AttestationServiceInstance {
+    pub log: Logger,
+}
+
+impl AttestationService for AttestationServiceInstance {
+    /// Produce a `BeaconBlock` for signing by a validator.
+    fn produce_attestation(
+        &mut self,
+        ctx: RpcContext,
+        req: ProduceAttestationRequest,
+        sink: UnarySink<ProduceAttestationResponse>,
+    ) {
+        println!("producing attestation at slot {}", req.get_slot());
+
+        // TODO: build a legit block.
+        let mut attestation = AttestationProto::new();
+        attestation.set_slot(req.get_slot());
+        // TODO Set the shard to something legit.
+        attestation.set_shard(0);
+        attestation.set_block_root(b"cats".to_vec());
+
+        let mut resp = ProduceAttestationResponse::new();
+        resp.set_attestation_data(attestation);
+
+        let f = sink
+            .success(resp)
+            .map_err(move |e| println!("failed to reply {:?}: {:?}", req, e));
+        ctx.spawn(f)
+    }
+
+    /// Accept some fully-formed `BeaconBlock`, process and publish it.
+    fn publish_attestation(
+        &mut self,
+        ctx: RpcContext,
+        req: PublishAttestationRequest,
+        sink: UnarySink<PublishAttestationResponse>,
+    ) {
+        println!("publishing attestation {:?}", req.get_block());
+
+        // TODO: actually process the block.
+        let mut resp = PublishAttestationResponse::new();
+
+        resp.set_success(true);
+
+        let f = sink
+            .success(resp)
+            .map_err(move |e| println!("failed to reply {:?}: {:?}", req, e));
+        ctx.spawn(f)
+    }
+}

beacon_node/rpc/src/beacon_block.rs

@@ -1,14 +1,25 @@
+use crate::beacon_chain::BeaconChain;
+use crossbeam_channel;
+use eth2_libp2p::rpc::methods::BlockRootSlot;
+use eth2_libp2p::PubsubMessage;
 use futures::Future;
 use grpcio::{RpcContext, UnarySink};
+use network::NetworkMessage;
 use protos::services::{
     BeaconBlock as BeaconBlockProto, ProduceBeaconBlockRequest, ProduceBeaconBlockResponse,
     PublishBeaconBlockRequest, PublishBeaconBlockResponse,
 };
 use protos::services_grpc::BeaconBlockService;
 use slog::Logger;
+use slog::{debug, error, info, warn};
+use ssz::{Decodable, TreeHash};
+use std::sync::Arc;
+use types::{BeaconBlock, Hash256, Slot};
 
 #[derive(Clone)]
 pub struct BeaconBlockServiceInstance {
+    pub chain: Arc<BeaconChain>,
+    pub network_chan: crossbeam_channel::Sender<NetworkMessage>,
     pub log: Logger,
 }
 
@@ -24,8 +35,7 @@ impl BeaconBlockService for BeaconBlockServiceInstance {
 
         // TODO: build a legit block.
         let mut block = BeaconBlockProto::new();
-        block.set_slot(req.get_slot());
-        block.set_block_root(b"cats".to_vec());
+        block.set_ssz(b"cats".to_vec());
 
         let mut resp = ProduceBeaconBlockResponse::new();
         resp.set_block(block);
@@ -43,11 +53,88 @@ impl BeaconBlockService for BeaconBlockServiceInstance {
         req: PublishBeaconBlockRequest,
         sink: UnarySink<PublishBeaconBlockResponse>,
     ) {
-        println!("publishing {:?}", req.get_block());
-
-        // TODO: actually process the block.
         let mut resp = PublishBeaconBlockResponse::new();
-        resp.set_success(true);
+
+        let ssz_serialized_block = req.get_block().get_ssz();
+
+        match BeaconBlock::ssz_decode(ssz_serialized_block, 0) {
+            Ok((block, _i)) => {
+                let block_root = Hash256::from_slice(&block.hash_tree_root()[..]);
+
+                match self.chain.process_block(block.clone()) {
+                    Ok(outcome) => {
+                        if outcome.sucessfully_processed() {
+                            // Block was successfully processed.
+                            info!(
+                                self.log,
+                                "PublishBeaconBlock";
+                                "type" => "valid_block",
+                                "block_slot" => block.slot,
+                                "outcome" => format!("{:?}", outcome)
+                            );
+
+                            // TODO: Obtain topics from the network service properly.
+                            let topic =
+                                types::TopicBuilder::new("beacon_chain".to_string()).build();
+                            let message = PubsubMessage::Block(BlockRootSlot {
+                                block_root,
+                                slot: block.slot,
+                            });
+
+                            println!("Sending beacon block to gossipsub");
+                            self.network_chan.send(NetworkMessage::Publish {
+                                topics: vec![topic],
+                                message,
+                            });
+
+                            resp.set_success(true);
+                        } else if outcome.is_invalid() {
+                            // Block was invalid.
+                            warn!(
+                                self.log,
+                                "PublishBeaconBlock";
+                                "type" => "invalid_block",
+                                "outcome" => format!("{:?}", outcome)
+                            );
+
+                            resp.set_success(false);
+                            resp.set_msg(
+                                format!("InvalidBlock: {:?}", outcome).as_bytes().to_vec(),
+                            );
+                        } else {
+                            // Some failure during processing.
+                            warn!(
+                                self.log,
+                                "PublishBeaconBlock";
+                                "type" => "unable_to_import",
+                                "outcome" => format!("{:?}", outcome)
+                            );
+
+                            resp.set_success(false);
+                            resp.set_msg(format!("other: {:?}", outcome).as_bytes().to_vec());
+                        }
+                    }
+                    Err(e) => {
+                        // Some failure during processing.
+                        error!(
+                            self.log,
+                            "PublishBeaconBlock";
+                            "type" => "failed_to_process",
+                            "error" => format!("{:?}", e)
+                        );
+
+                        resp.set_success(false);
+                        resp.set_msg(format!("failed_to_process: {:?}", e).as_bytes().to_vec());
+                    }
+                }
+
+                resp.set_success(true);
+            }
+            Err(_) => {
+                resp.set_success(false);
+                resp.set_msg(b"Invalid SSZ".to_vec());
+            }
+        };
 
         let f = sink
             .success(resp)

beacon_node/rpc/src/beacon_chain.rs

@@ -0,0 +1,42 @@
+use beacon_chain::BeaconChain as RawBeaconChain;
+use beacon_chain::{
+    db::ClientDB,
+    fork_choice::ForkChoice,
+    parking_lot::RwLockReadGuard,
+    slot_clock::SlotClock,
+    types::{BeaconState, ChainSpec},
+};
+pub use beacon_chain::{BeaconChainError, BlockProcessingOutcome};
+use types::BeaconBlock;
+
+/// The RPC's API to the beacon chain.
+pub trait BeaconChain: Send + Sync {
+    fn get_spec(&self) -> &ChainSpec;
+
+    fn get_state(&self) -> RwLockReadGuard<BeaconState>;
+
+    fn process_block(&self, block: BeaconBlock)
+        -> Result<BlockProcessingOutcome, BeaconChainError>;
+}
+
+impl<T, U, F> BeaconChain for RawBeaconChain<T, U, F>
+where
+    T: ClientDB + Sized,
+    U: SlotClock,
+    F: ForkChoice,
+{
+    fn get_spec(&self) -> &ChainSpec {
+        &self.spec
+    }
+
+    fn get_state(&self) -> RwLockReadGuard<BeaconState> {
+        self.state.read()
+    }
+
+    fn process_block(
+        &self,
+        block: BeaconBlock,
+    ) -> Result<BlockProcessingOutcome, BeaconChainError> {
+        self.process_block(block)
+    }
+}

beacon_node/rpc/src/beacon_node.rs

@@ -0,0 +1,47 @@
+use crate::beacon_chain::BeaconChain;
+use futures::Future;
+use grpcio::{RpcContext, UnarySink};
+use protos::services::{Empty, Fork, NodeInfoResponse};
+use protos::services_grpc::BeaconNodeService;
+use slog::{trace, warn};
+use std::sync::Arc;
+
+#[derive(Clone)]
+pub struct BeaconNodeServiceInstance {
+    pub chain: Arc<BeaconChain>,
+    pub log: slog::Logger,
+}
+
+impl BeaconNodeService for BeaconNodeServiceInstance {
+    /// Provides basic node information.
+    fn info(&mut self, ctx: RpcContext, _req: Empty, sink: UnarySink<NodeInfoResponse>) {
+        trace!(self.log, "Node info requested via RPC");
+
+        // build the response
+        let mut node_info = NodeInfoResponse::new();
+        node_info.set_version(version::version());
+
+        // get the chain state
+        let state = self.chain.get_state();
+        let state_fork = state.fork.clone();
+        let genesis_time = state.genesis_time.clone();
+
+        // build the rpc fork struct
+        let mut fork = Fork::new();
+        fork.set_previous_version(state_fork.previous_version.to_vec());
+        fork.set_current_version(state_fork.current_version.to_vec());
+        fork.set_epoch(state_fork.epoch.into());
+
+        node_info.set_fork(fork);
+        node_info.set_genesis_time(genesis_time);
+        node_info.set_genesis_slot(self.chain.get_spec().genesis_slot.as_u64());
+        node_info.set_chain_id(self.chain.get_spec().chain_id as u32);
+
+        // send the node_info the requester
+        let error_log = self.log.clone();
+        let f = sink
+            .success(node_info)
+            .map_err(move |e| warn!(error_log, "failed to reply {:?}", e));
+        ctx.spawn(f)
+    }
+}

beacon_node/rpc/src/lib.rs

@@ -1,38 +1,85 @@
 mod beacon_block;
+pub mod beacon_chain;
+mod beacon_node;
 pub mod config;
 mod validator;
 
 use self::beacon_block::BeaconBlockServiceInstance;
+use self::beacon_chain::BeaconChain;
+use self::beacon_node::BeaconNodeServiceInstance;
 use self::validator::ValidatorServiceInstance;
 pub use config::Config as RPCConfig;
+use futures::{future, Future};
 use grpcio::{Environment, Server, ServerBuilder};
-use protos::services_grpc::{create_beacon_block_service, create_validator_service};
+use network::NetworkMessage;
+use protos::services_grpc::{
+    create_beacon_block_service, create_beacon_node_service, create_validator_service,
+};
+use slog::{info, o, warn};
 use std::sync::Arc;
+use tokio::runtime::TaskExecutor;
 
-use slog::{info, o};
-
-pub fn start_server(config: &RPCConfig, log: &slog::Logger) -> Server {
+pub fn start_server(
+    config: &RPCConfig,
+    executor: &TaskExecutor,
+    network_chan: crossbeam_channel::Sender<NetworkMessage>,
+    beacon_chain: Arc<BeaconChain>,
+    log: &slog::Logger,
+) -> exit_future::Signal {
     let log = log.new(o!("Service"=>"RPC"));
     let env = Arc::new(Environment::new(1));
 
+    // build a channel to kill the rpc server
+    let (rpc_exit_signal, rpc_exit) = exit_future::signal();
+
+    // build the individual rpc services
+    let beacon_node_service = {
+        let instance = BeaconNodeServiceInstance {
+            chain: beacon_chain.clone(),
+            log: log.clone(),
+        };
+        create_beacon_node_service(instance)
+    };
+
     let beacon_block_service = {
-        let instance = BeaconBlockServiceInstance { log: log.clone() };
+        let instance = BeaconBlockServiceInstance {
+            chain: beacon_chain.clone(),
+            network_chan,
+            log: log.clone(),
+        };
         create_beacon_block_service(instance)
     };
     let validator_service = {
-        let instance = ValidatorServiceInstance { log: log.clone() };
+        let instance = ValidatorServiceInstance {
+            chain: beacon_chain.clone(),
+            log: log.clone(),
+        };
         create_validator_service(instance)
     };
 
     let mut server = ServerBuilder::new(env)
         .register_service(beacon_block_service)
         .register_service(validator_service)
+        .register_service(beacon_node_service)
         .bind(config.listen_address.to_string(), config.port)
         .build()
         .unwrap();
-    server.start();
-    for &(ref host, port) in server.bind_addrs() {
-        info!(log, "gRPC listening on {}:{}", host, port);
-    }
-    server
+
+    let spawn_rpc = {
+        server.start();
+        for &(ref host, port) in server.bind_addrs() {
+            info!(log, "gRPC listening on {}:{}", host, port);
+        }
+        rpc_exit.and_then(move |_| {
+            info!(log, "RPC Server shutting down");
+            server
+                .shutdown()
+                .wait()
+                .map(|_| ())
+                .map_err(|e| warn!(log, "RPC server failed to shutdown: {:?}", e))?;
+            Ok(())
+        })
+    };
+    executor.spawn(spawn_rpc);
+    rpc_exit_signal
 }

beacon_node/rpc/src/validator.rs

@@ -1,60 +1,166 @@
+use crate::beacon_chain::BeaconChain;
 use bls::PublicKey;
 use futures::Future;
 use grpcio::{RpcContext, RpcStatus, RpcStatusCode, UnarySink};
-use protos::services::{
-    IndexResponse, ProposeBlockSlotRequest, ProposeBlockSlotResponse, PublicKey as PublicKeyRequest,
-};
+use protos::services::{ActiveValidator, GetDutiesRequest, GetDutiesResponse, ValidatorDuty};
 use protos::services_grpc::ValidatorService;
-use slog::{debug, Logger};
+use slog::{debug, info, warn, Logger};
 use ssz::decode;
+use std::sync::Arc;
+use types::{Epoch, RelativeEpoch};
 
 #[derive(Clone)]
 pub struct ValidatorServiceInstance {
+    pub chain: Arc<BeaconChain>,
     pub log: Logger,
 }
+//TODO: Refactor Errors
 
 impl ValidatorService for ValidatorServiceInstance {
-    fn validator_index(
+    /// For a list of validator public keys, this function returns the slot at which each
+    /// validator must propose a block, attest to a shard, their shard committee and the shard they
+    /// need to attest to.
+    fn get_validator_duties(
         &mut self,
         ctx: RpcContext,
-        req: PublicKeyRequest,
-        sink: UnarySink<IndexResponse>,
+        req: GetDutiesRequest,
+        sink: UnarySink<GetDutiesResponse>,
     ) {
-        if let Ok(public_key) = decode::<PublicKey>(req.get_public_key()) {
-            debug!(self.log, "RPC request"; "endpoint" => "ValidatorIndex", "public_key" => public_key.concatenated_hex_id());
+        let validators = req.get_validators();
+        debug!(self.log, "RPC request"; "endpoint" => "GetValidatorDuties", "epoch" => req.get_epoch());
 
-            let mut resp = IndexResponse::new();
+        let epoch = Epoch::from(req.get_epoch());
+        let mut resp = GetDutiesResponse::new();
+        let resp_validators = resp.mut_active_validators();
 
-            // TODO: return a legit value.
-            resp.set_index(1);
+        let spec = self.chain.get_spec();
+        let state = self.chain.get_state();
 
-            let f = sink
-                .success(resp)
-                .map_err(move |e| println!("failed to reply {:?}: {:?}", req, e));
-            ctx.spawn(f)
-        } else {
-            let f = sink
-                .fail(RpcStatus::new(
-                    RpcStatusCode::InvalidArgument,
-                    Some("Invalid public_key".to_string()),
-                ))
-                .map_err(move |e| println!("failed to reply {:?}: {:?}", req, e));
-            ctx.spawn(f)
+        let relative_epoch =
+            match RelativeEpoch::from_epoch(state.slot.epoch(spec.slots_per_epoch), epoch) {
+                Ok(v) => v,
+                Err(e) => {
+                    // incorrect epoch
+                    let log_clone = self.log.clone();
+                    let f = sink
+                        .fail(RpcStatus::new(
+                            RpcStatusCode::FailedPrecondition,
+                            Some(format!("Invalid epoch: {:?}", e)),
+                        ))
+                        .map_err(move |e| warn!(log_clone, "failed to reply {:?}: {:?}", req, e));
+                    return ctx.spawn(f);
+                }
+            };
+
+        let validator_proposers: Result<Vec<usize>, _> = epoch
+            .slot_iter(spec.slots_per_epoch)
+            .map(|slot| state.get_beacon_proposer_index(slot, relative_epoch, &spec))
+            .collect();
+        let validator_proposers = match validator_proposers {
+            Ok(v) => v,
+            Err(e) => {
+                // could not get the validator proposer index
+                let log_clone = self.log.clone();
+                let f = sink
+                    .fail(RpcStatus::new(
+                        RpcStatusCode::FailedPrecondition,
+                        Some(format!("Could not find beacon proposers: {:?}", e)),
+                    ))
+                    .map_err(move |e| warn!(log_clone, "failed to reply {:?} : {:?}", req, e));
+                return ctx.spawn(f);
+            }
+        };
+
+        // get the duties for each validator
+        for validator_pk in validators.get_public_keys() {
+            let mut active_validator = ActiveValidator::new();
+
+            let public_key = match decode::<PublicKey>(validator_pk) {
+                Ok(v) => v,
+                Err(_) => {
+                    let log_clone = self.log.clone();
+                    let f = sink
+                        .fail(RpcStatus::new(
+                            RpcStatusCode::InvalidArgument,
+                            Some("Invalid public_key".to_string()),
+                        ))
+                        .map_err(move |e| warn!(log_clone, "failed to reply {:?}", req));
+                    return ctx.spawn(f);
+                }
+            };
+
+            // get the validator index
+            let val_index = match state.get_validator_index(&public_key) {
+                Ok(Some(index)) => index,
+                Ok(None) => {
+                    // index not present in registry, set the duties for this key to None
+                    warn!(
+                        self.log,
+                        "RPC requested a public key that is not in the registry: {:?}", public_key
+                    );
+                    active_validator.set_none(false);
+                    resp_validators.push(active_validator);
+                    continue;
+                }
+                // the cache is not built, throw an error
+                Err(e) => {
+                    let log_clone = self.log.clone();
+                    let f = sink
+                        .fail(RpcStatus::new(
+                            RpcStatusCode::FailedPrecondition,
+                            Some(format!("Beacon state error {:?}", e)),
+                        ))
+                        .map_err(move |e| warn!(log_clone, "Failed to reply {:?}: {:?}", req, e));
+                    return ctx.spawn(f);
+                }
+            };
+
+            // get attestation duties and check if validator is active
+            let attestation_duties = match state.get_attestation_duties(val_index, &spec) {
+                Ok(Some(v)) => v,
+                Ok(_) => {
+                    // validator is inactive, go to the next validator
+                    warn!(
+                        self.log,
+                        "RPC requested an inactive validator key: {:?}", public_key
+                    );
+                    active_validator.set_none(false);
+                    resp_validators.push(active_validator);
+                    continue;
+                }
+                // the cache is not built, throw an error
+                Err(e) => {
+                    let log_clone = self.log.clone();
+                    let f = sink
+                        .fail(RpcStatus::new(
+                            RpcStatusCode::FailedPrecondition,
+                            Some(format!("Beacon state error {:?}", e)),
+                        ))
+                        .map_err(move |e| warn!(log_clone, "Failed to reply {:?}: {:?}", req, e));
+                    return ctx.spawn(f);
+                }
+            };
+
+            // we have an active validator, set its duties
+            let mut duty = ValidatorDuty::new();
+
+            // check if the validator needs to propose a block
+            if let Some(slot) = validator_proposers.iter().position(|&v| val_index == v) {
+                duty.set_block_production_slot(
+                    epoch.start_slot(spec.slots_per_epoch).as_u64() + slot as u64,
+                );
+            } else {
+                // no blocks to propose this epoch
+                duty.set_none(false)
+            }
+
+            duty.set_committee_index(attestation_duties.committee_index as u64);
+            duty.set_attestation_slot(attestation_duties.slot.as_u64());
+            duty.set_attestation_shard(attestation_duties.shard);
+
+            active_validator.set_duty(duty);
+            resp_validators.push(active_validator);
         }
-    }
-
-    fn propose_block_slot(
-        &mut self,
-        ctx: RpcContext,
-        req: ProposeBlockSlotRequest,
-        sink: UnarySink<ProposeBlockSlotResponse>,
-    ) {
-        debug!(self.log, "RPC request"; "endpoint" => "ProposeBlockSlot", "epoch" => req.get_epoch(), "validator_index" => req.get_validator_index());
-
-        let mut resp = ProposeBlockSlotResponse::new();
-
-        // TODO: return a legit value.
-        resp.set_slot(1);
 
         let f = sink
             .success(resp)