Add validator_node, restructure binaries, gRPC.

This is a massive commit which restructures the workspace, adds a very
basic, untested, validator client and some very basic, non-functioning
gRPC endpoints to the beacon-node.

beacon_node/src/beacon_chain/block_processing.rs

@@ -0,0 +1,72 @@
+use super::{BeaconChain, ClientDB, DBError, SlotClock};
+use slot_clock::TestingSlotClockError;
+use ssz::{ssz_encode, Encodable};
+use types::{readers::BeaconBlockReader, Hash256};
+
+#[derive(Debug, PartialEq)]
+pub enum Outcome {
+    FutureSlot,
+    Processed,
+
+    NewCanonicalBlock,
+    NewReorgBlock,
+    NewForkBlock,
+}
+
+#[derive(Debug, PartialEq)]
+pub enum Error {
+    DBError(String),
+    NotImplemented,
+    PresentSlotIsNone,
+}
+
+impl<T, U> BeaconChain<T, U>
+where
+    T: ClientDB,
+    U: SlotClock,
+    Error: From<<U as SlotClock>::Error>,
+{
+    pub fn process_block<V>(&mut self, block: &V) -> Result<(Outcome, Hash256), Error>
+    where
+        V: BeaconBlockReader + Encodable + Sized,
+    {
+        let block_root = block.canonical_root();
+
+        let present_slot = self
+            .slot_clock
+            .present_slot()?
+            .ok_or(Error::PresentSlotIsNone)?;
+
+        // Block from future slots (i.e., greater than the present slot) should not be processed.
+        if block.slot() > present_slot {
+            return Ok((Outcome::FutureSlot, block_root));
+        }
+
+        // TODO: block processing has been removed.
+        // https://github.com/sigp/lighthouse/issues/98
+
+        // Update leaf blocks.
+        self.block_store.put(&block_root, &ssz_encode(block)[..])?;
+        if self.leaf_blocks.contains(&block.parent_root()) {
+            self.leaf_blocks.remove(&block.parent_root());
+        }
+        if self.canonical_leaf_block == block.parent_root() {
+            self.canonical_leaf_block = block_root;
+        }
+        self.leaf_blocks.insert(block_root);
+
+        Ok((Outcome::Processed, block_root))
+    }
+}
+
+impl From<DBError> for Error {
+    fn from(e: DBError) -> Error {
+        Error::DBError(e.message)
+    }
+}
+
+impl From<TestingSlotClockError> for Error {
+    fn from(_: TestingSlotClockError) -> Error {
+        unreachable!(); // Testing clock never throws an error.
+    }
+}

beacon_node/src/beacon_chain/block_production.rs

@@ -0,0 +1,75 @@
+use super::{BeaconChain, ClientDB, DBError, SlotClock};
+use slot_clock::TestingSlotClockError;
+use types::{
+    readers::{BeaconBlockReader, BeaconStateReader},
+    BeaconBlock, BeaconState, Hash256,
+};
+
+#[derive(Debug, PartialEq)]
+pub enum Error {
+    DBError(String),
+    PresentSlotIsNone,
+}
+
+impl<T, U> BeaconChain<T, U>
+where
+    T: ClientDB,
+    U: SlotClock,
+    Error: From<<U as SlotClock>::Error>,
+{
+    pub fn produce_block(&mut self) -> Result<(BeaconBlock, BeaconState), Error> {
+        /*
+         * Important: this code is a big stub and only exists to ensure that tests pass.
+         *
+         * https://github.com/sigp/lighthouse/issues/107
+         */
+        let present_slot = self
+            .slot_clock
+            .present_slot()?
+            .ok_or(Error::PresentSlotIsNone)?;
+        let parent_root = self.canonical_leaf_block;
+        let parent_block_reader = self
+            .block_store
+            .get_reader(&parent_root)?
+            .ok_or_else(|| Error::DBError("Block not found.".to_string()))?;
+        let parent_state_reader = self
+            .state_store
+            .get_reader(&parent_block_reader.state_root())?
+            .ok_or_else(|| Error::DBError("State not found.".to_string()))?;
+
+        let parent_block = parent_block_reader
+            .into_beacon_block()
+            .ok_or_else(|| Error::DBError("Bad parent block SSZ.".to_string()))?;
+        let mut block = BeaconBlock {
+            slot: present_slot,
+            parent_root,
+            state_root: Hash256::zero(), // Updated after the state is calculated.
+            ..parent_block
+        };
+
+        let parent_state = parent_state_reader
+            .into_beacon_state()
+            .ok_or_else(|| Error::DBError("Bad parent block SSZ.".to_string()))?;
+        let state = BeaconState {
+            slot: present_slot,
+            ..parent_state
+        };
+        let state_root = state.canonical_root();
+
+        block.state_root = state_root;
+
+        Ok((block, state))
+    }
+}
+
+impl From<DBError> for Error {
+    fn from(e: DBError) -> Error {
+        Error::DBError(e.message)
+    }
+}
+
+impl From<TestingSlotClockError> for Error {
+    fn from(_: TestingSlotClockError) -> Error {
+        unreachable!(); // Testing clock never throws an error.
+    }
+}

beacon_node/src/beacon_chain/mod.rs

@@ -0,0 +1,81 @@
+mod block_processing;
+mod block_production;
+
+use db::{
+    stores::{BeaconBlockStore, BeaconStateStore},
+    ClientDB, DBError,
+};
+use genesis::{genesis_beacon_block, genesis_beacon_state, GenesisError};
+use slot_clock::SlotClock;
+use spec::ChainSpec;
+use ssz::ssz_encode;
+use std::collections::HashSet;
+use std::sync::Arc;
+use types::Hash256;
+
+pub use crate::block_processing::Outcome as BlockProcessingOutcome;
+
+#[derive(Debug, PartialEq)]
+pub enum BeaconChainError {
+    InsufficientValidators,
+    GenesisError(GenesisError),
+    DBError(String),
+}
+
+pub struct BeaconChain<T: ClientDB + Sized, U: SlotClock> {
+    pub block_store: Arc<BeaconBlockStore<T>>,
+    pub state_store: Arc<BeaconStateStore<T>>,
+    pub slot_clock: U,
+    pub leaf_blocks: HashSet<Hash256>,
+    pub canonical_leaf_block: Hash256,
+    pub spec: ChainSpec,
+}
+
+impl<T, U> BeaconChain<T, U>
+where
+    T: ClientDB,
+    U: SlotClock,
+{
+    pub fn genesis(
+        state_store: Arc<BeaconStateStore<T>>,
+        block_store: Arc<BeaconBlockStore<T>>,
+        slot_clock: U,
+        spec: ChainSpec,
+    ) -> Result<Self, BeaconChainError> {
+        if spec.initial_validators.is_empty() {
+            return Err(BeaconChainError::InsufficientValidators);
+        }
+
+        let genesis_state = genesis_beacon_state(&spec)?;
+        let state_root = genesis_state.canonical_root();
+        state_store.put(&state_root, &ssz_encode(&genesis_state)[..])?;
+
+        let genesis_block = genesis_beacon_block(state_root, &spec);
+        let block_root = genesis_block.canonical_root();
+        block_store.put(&block_root, &ssz_encode(&genesis_block)[..])?;
+
+        let mut leaf_blocks = HashSet::new();
+        leaf_blocks.insert(block_root);
+
+        Ok(Self {
+            block_store,
+            state_store,
+            slot_clock,
+            leaf_blocks,
+            canonical_leaf_block: block_root,
+            spec,
+        })
+    }
+}
+
+impl From<DBError> for BeaconChainError {
+    fn from(e: DBError) -> BeaconChainError {
+        BeaconChainError::DBError(e.message)
+    }
+}
+
+impl From<GenesisError> for BeaconChainError {
+    fn from(e: GenesisError) -> BeaconChainError {
+        BeaconChainError::GenesisError(e)
+    }
+}

beacon_node/src/beacon_chain/tests/chain_test.rs

@@ -0,0 +1,49 @@
+use chain::{BlockProcessingOutcome, BeaconChain};
+use db::{
+    stores::{BeaconBlockStore, BeaconStateStore},
+    MemoryDB,
+};
+use slot_clock::TestingSlotClock;
+use spec::ChainSpec;
+use std::sync::Arc;
+
+fn in_memory_test_stores() -> (
+    Arc<MemoryDB>,
+    Arc<BeaconBlockStore<MemoryDB>>,
+    Arc<BeaconStateStore<MemoryDB>>,
+) {
+    let db = Arc::new(MemoryDB::open());
+    let block_store = Arc::new(BeaconBlockStore::new(db.clone()));
+    let state_store = Arc::new(BeaconStateStore::new(db.clone()));
+    (db, block_store, state_store)
+}
+
+fn in_memory_test_chain(
+    spec: ChainSpec,
+) -> (Arc<MemoryDB>, BeaconChain<MemoryDB, TestingSlotClock>) {
+    let (db, block_store, state_store) = in_memory_test_stores();
+    let slot_clock = TestingSlotClock::new(0);
+
+    let chain = BeaconChain::genesis(state_store, block_store, slot_clock, spec);
+    (db, chain.unwrap())
+}
+
+#[test]
+fn it_constructs() {
+    let (_db, _chain) = in_memory_test_chain(ChainSpec::foundation());
+}
+
+#[test]
+fn it_produces() {
+    let (_db, mut chain) = in_memory_test_chain(ChainSpec::foundation());
+    let (_block, _state) = chain.produce_block().unwrap();
+}
+
+#[test]
+fn it_processes_a_block_it_produces() {
+    let (_db, mut chain) = in_memory_test_chain(ChainSpec::foundation());
+    let (block, _state) = chain.produce_block().unwrap();
+    let (outcome, new_block_hash) = chain.process_block(&block).unwrap();
+    assert_eq!(outcome, BlockProcessingOutcome::Processed);
+    assert_eq!(chain.canonical_leaf_block, new_block_hash);
+}

beacon_node/src/beacon_chain/transition.rs

@@ -0,0 +1,29 @@
+use super::BeaconChain;
+use db::ClientDB;
+use state_transition::{extend_active_state, StateTransitionError};
+use types::{ActiveState, BeaconBlock, CrystallizedState, Hash256};
+
+impl<T, U> BeaconChain<T, U>
+where
+    T: ClientDB + Sized,
+{
+    pub(crate) fn transition_states(
+        &self,
+        act_state: &ActiveState,
+        cry_state: &CrystallizedState,
+        block: &BeaconBlock,
+        block_hash: &Hash256,
+    ) -> Result<(ActiveState, Option<CrystallizedState>), StateTransitionError> {
+        let state_recalc_distance = block
+            .slot
+            .checked_sub(cry_state.last_state_recalculation_slot)
+            .ok_or(StateTransitionError::BlockSlotBeforeRecalcSlot)?;
+
+        if state_recalc_distance >= u64::from(self.spec.epoch_length) {
+            panic!("Not implemented!")
+        } else {
+            let new_act_state = extend_active_state(act_state, block, block_hash)?;
+            Ok((new_act_state, None))
+        }
+    }
+}

beacon_node/src/config/mod.rs

@@ -0,0 +1,30 @@
+use std::fs;
+use std::path::PathBuf;
+
+/// Stores the core configuration for this Lighthouse instance.
+/// This struct is general, other components may implement more
+/// specialized config structs.
+#[derive(Clone)]
+pub struct LighthouseConfig {
+    pub data_dir: PathBuf,
+    pub p2p_listen_port: u16,
+}
+
+const DEFAULT_LIGHTHOUSE_DIR: &str = ".lighthouse";
+
+impl LighthouseConfig {
+    /// Build a new lighthouse configuration from defaults.
+    pub fn default() -> Self {
+        let data_dir = {
+            let home = dirs::home_dir().expect("Unable to determine home dir.");
+            home.join(DEFAULT_LIGHTHOUSE_DIR)
+        };
+        fs::create_dir_all(&data_dir)
+            .unwrap_or_else(|_| panic!("Unable to create {:?}", &data_dir));
+        let p2p_listen_port = 0;
+        Self {
+            data_dir,
+            p2p_listen_port,
+        }
+    }
+}

beacon_node/src/main.rs

@@ -0,0 +1,66 @@
+extern crate slog;
+
+mod config;
+mod rpc;
+
+use std::path::PathBuf;
+
+use crate::config::LighthouseConfig;
+use crate::rpc::start_server;
+use clap::{App, Arg};
+use slog::{error, info, o, Drain};
+
+fn main() {
+    let decorator = slog_term::TermDecorator::new().build();
+    let drain = slog_term::CompactFormat::new(decorator).build().fuse();
+    let drain = slog_async::Async::new(drain).build().fuse();
+    let log = slog::Logger::root(drain, o!());
+
+    let matches = App::new("Lighthouse")
+        .version("0.0.1")
+        .author("Sigma Prime <paul@sigmaprime.io>")
+        .about("Eth 2.0 Client")
+        .arg(
+            Arg::with_name("datadir")
+                .long("datadir")
+                .value_name("DIR")
+                .help("Data directory for keys and databases.")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("port")
+                .long("port")
+                .value_name("PORT")
+                .help("Network listen port for p2p connections.")
+                .takes_value(true),
+        )
+        .get_matches();
+
+    let mut config = LighthouseConfig::default();
+
+    // Custom datadir
+    if let Some(dir) = matches.value_of("datadir") {
+        config.data_dir = PathBuf::from(dir.to_string());
+    }
+
+    // Custom p2p listen port
+    if let Some(port_str) = matches.value_of("port") {
+        if let Ok(port) = port_str.parse::<u16>() {
+            config.p2p_listen_port = port;
+        } else {
+            error!(log, "Invalid port"; "port" => port_str);
+            return;
+        }
+    }
+
+    // Log configuration
+    info!(log, "";
+          "data_dir" => &config.data_dir.to_str(),
+          "port" => &config.p2p_listen_port);
+
+    let _server = start_server(log.clone());
+
+    loop {}
+
+    // info!(log, "Exiting.");
+}

beacon_node/src/rpc/mod.rs

@@ -0,0 +1,79 @@
+use std::sync::Arc;
+
+use futures::Future;
+use grpcio::{Environment, RpcContext, Server, ServerBuilder, UnarySink};
+
+use protos::services::{
+    BeaconBlock as BeaconBlockProto, ProduceBeaconBlockRequest, ProduceBeaconBlockResponse,
+    PublishBeaconBlockRequest, PublishBeaconBlockResponse,
+};
+use protos::services_grpc::{create_beacon_block_service, BeaconBlockService};
+
+use slog::{info, Logger};
+
+#[derive(Clone)]
+struct BeaconBlockServiceInstance {
+    log: Logger,
+}
+
+impl BeaconBlockService for BeaconBlockServiceInstance {
+    /// Produce a `BeaconBlock` for signing by a validator.
+    fn produce_beacon_block(
+        &mut self,
+        ctx: RpcContext,
+        req: ProduceBeaconBlockRequest,
+        sink: UnarySink<ProduceBeaconBlockResponse>,
+    ) {
+        println!("producing at slot {}", req.get_slot());
+
+        // TODO: build a legit block.
+        let mut block = BeaconBlockProto::new();
+        block.set_slot(req.get_slot());
+        block.set_block_root("cats".as_bytes().to_vec());
+
+        let mut resp = ProduceBeaconBlockResponse::new();
+        resp.set_block(block);
+
+        let f = sink
+            .success(resp)
+            .map_err(move |e| println!("failed to reply {:?}: {:?}", req, e));
+        ctx.spawn(f)
+    }
+
+    /// Accept some fully-formed `BeaconBlock`, process and publish it.
+    fn publish_beacon_block(
+        &mut self,
+        ctx: RpcContext,
+        req: PublishBeaconBlockRequest,
+        sink: UnarySink<PublishBeaconBlockResponse>,
+    ) {
+        println!("publishing {:?}", req.get_block());
+
+        // TODO: actually process the block.
+        let mut resp = PublishBeaconBlockResponse::new();
+        resp.set_success(true);
+
+        let f = sink
+            .success(resp)
+            .map_err(move |e| println!("failed to reply {:?}: {:?}", req, e));
+        ctx.spawn(f)
+    }
+}
+
+pub fn start_server(log: Logger) -> Server {
+    let log_clone = log.clone();
+
+    let env = Arc::new(Environment::new(1));
+    let instance = BeaconBlockServiceInstance { log };
+    let service = create_beacon_block_service(instance);
+    let mut server = ServerBuilder::new(env)
+        .register_service(service)
+        .bind("127.0.0.1", 50_051)
+        .build()
+        .unwrap();
+    server.start();
+    for &(ref host, port) in server.bind_addrs() {
+        info!(log_clone, "gRPC listening on {}:{}", host, port);
+    }
+    server
+}