Start heavy refactor of validator client

- Block production is working

validator_client/src/block_service.rs

@@ -0,0 +1,245 @@
+use crate::{
+    duties_service::DutiesService, fork_service::ForkService, validator_store::ValidatorStore,
+};
+use environment::RuntimeContext;
+use exit_future::Signal;
+use futures::{stream, Future, IntoFuture, Stream};
+use remote_beacon_node::{PublishStatus, RemoteBeaconNode};
+use slog::{error, info, trace, warn};
+use slot_clock::SlotClock;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+use tokio::timer::Interval;
+use types::{ChainSpec, EthSpec};
+
+/// Delay this period of time after the slot starts. This allows the node to process the new slot.
+const TIME_DELAY_FROM_SLOT: Duration = Duration::from_millis(100);
+
+#[derive(Clone)]
+pub struct BlockServiceBuilder<T: Clone, E: EthSpec> {
+    fork_service: Option<ForkService<T, E>>,
+    duties_service: Option<DutiesService<T, E>>,
+    validator_store: Option<ValidatorStore<E>>,
+    slot_clock: Option<Arc<T>>,
+    beacon_node: Option<RemoteBeaconNode<E>>,
+    context: Option<RuntimeContext<E>>,
+}
+
+// TODO: clean trait bounds.
+impl<T: SlotClock + Clone + 'static, E: EthSpec> BlockServiceBuilder<T, E> {
+    pub fn new() -> Self {
+        Self {
+            fork_service: None,
+            duties_service: None,
+            validator_store: None,
+            slot_clock: None,
+            beacon_node: None,
+            context: None,
+        }
+    }
+
+    pub fn fork_service(mut self, service: ForkService<T, E>) -> Self {
+        self.fork_service = Some(service);
+        self
+    }
+
+    pub fn duties_service(mut self, service: DutiesService<T, E>) -> Self {
+        self.duties_service = Some(service);
+        self
+    }
+
+    pub fn validator_store(mut self, store: ValidatorStore<E>) -> Self {
+        self.validator_store = Some(store);
+        self
+    }
+
+    pub fn slot_clock(mut self, slot_clock: T) -> Self {
+        self.slot_clock = Some(Arc::new(slot_clock));
+        self
+    }
+
+    pub fn beacon_node(mut self, beacon_node: RemoteBeaconNode<E>) -> Self {
+        self.beacon_node = Some(beacon_node);
+        self
+    }
+
+    pub fn runtime_context(mut self, context: RuntimeContext<E>) -> Self {
+        self.context = Some(context);
+        self
+    }
+
+    pub fn build(self) -> Result<BlockService<T, E>, String> {
+        Ok(BlockService {
+            fork_service: self
+                .fork_service
+                .ok_or_else(|| "Cannot build BlockService without fork_service")?,
+            duties_service: self
+                .duties_service
+                .ok_or_else(|| "Cannot build BlockService without duties_service")?,
+            validator_store: self
+                .validator_store
+                .ok_or_else(|| "Cannot build BlockService without validator_store")?,
+            slot_clock: self
+                .slot_clock
+                .ok_or_else(|| "Cannot build BlockService without slot_clock")?,
+            beacon_node: self
+                .beacon_node
+                .ok_or_else(|| "Cannot build BlockService without beacon_node")?,
+            context: self
+                .context
+                .ok_or_else(|| "Cannot build BlockService without runtime_context")?,
+        })
+    }
+}
+
+#[derive(Clone)]
+pub struct BlockService<T: Clone, E: EthSpec> {
+    duties_service: DutiesService<T, E>,
+    fork_service: ForkService<T, E>,
+    validator_store: ValidatorStore<E>,
+    slot_clock: Arc<T>,
+    beacon_node: RemoteBeaconNode<E>,
+    context: RuntimeContext<E>,
+}
+
+// TODO: clean trait bounds.
+impl<T: SlotClock + Clone + 'static, E: EthSpec> BlockService<T, E> {
+    pub fn start_update_service(&self, spec: &ChainSpec) -> Result<Signal, String> {
+        let log = self.context.log.clone();
+
+        let duration_to_next_slot = self
+            .slot_clock
+            .duration_to_next_slot()
+            .ok_or_else(|| "Unable to determine duration to next slot".to_string())?;
+
+        let interval = {
+            let slot_duration = Duration::from_millis(spec.milliseconds_per_slot);
+            Interval::new(
+                Instant::now() + duration_to_next_slot + TIME_DELAY_FROM_SLOT,
+                slot_duration,
+            )
+        };
+
+        info!(
+            log,
+            "Waiting for next slot";
+            "seconds_to_wait" => duration_to_next_slot.as_secs()
+        );
+
+        let (exit_signal, exit_fut) = exit_future::signal();
+        let service = self.clone();
+
+        self.context.executor.spawn(
+            interval
+                .map_err(move |e| {
+                    error! {
+                        log,
+                        "Timer thread failed";
+                        "error" => format!("{}", e)
+                    }
+                })
+                .and_then(move |_| if exit_fut.is_live() { Ok(()) } else { Err(()) })
+                .for_each(move |_| service.clone().do_update()),
+        );
+
+        Ok(exit_signal)
+    }
+
+    fn do_update(self) -> impl Future<Item = (), Error = ()> {
+        let service = self.clone();
+        let log = self.context.log.clone();
+
+        self.slot_clock
+            .now()
+            .ok_or_else(move || {
+                error!(log, "Duties manager failed to read slot clock");
+            })
+            .into_future()
+            .and_then(move |slot| {
+                let iter = service.duties_service.block_producers(slot).into_iter();
+
+                stream::unfold(iter, move |mut block_producers| {
+                    let log_1 = service.context.log.clone();
+                    let log_2 = service.context.log.clone();
+                    let service_1 = service.clone();
+                    let service_2 = service.clone();
+                    let service_3 = service.clone();
+
+                    block_producers.next().map(move |validator_pubkey| {
+                        service_2
+                            .fork_service
+                            .fork()
+                            .ok_or_else(|| "Fork is unknown, unable to sign".to_string())
+                            .and_then(|fork| {
+                                service_1
+                                    .validator_store
+                                    .randao_reveal(
+                                        &validator_pubkey,
+                                        slot.epoch(E::slots_per_epoch()),
+                                        &fork,
+                                    )
+                                    .map(|randao_reveal| (fork, randao_reveal))
+                                    .ok_or_else(|| "Unable to produce randao reveal".to_string())
+                            })
+                            .into_future()
+                            .and_then(move |(fork, randao_reveal)| {
+                                service_1
+                                    .beacon_node
+                                    .http
+                                    .validator()
+                                    .produce_block(slot, randao_reveal)
+                                    .map(|block| (fork, block))
+                                    .map_err(|e| {
+                                        format!(
+                                            "Error from beacon node when producing block: {:?}",
+                                            e
+                                        )
+                                    })
+                            })
+                            .and_then(move |(fork, block)| {
+                                service_2
+                                    .validator_store
+                                    .sign_block(&validator_pubkey, block, &fork)
+                                    .ok_or_else(|| "Unable to sign block".to_string())
+                            })
+                            .and_then(move |block| {
+                                service_3
+                                    .beacon_node
+                                    .http
+                                    .validator()
+                                    .publish_block(block)
+                                    .map_err(|e| {
+                                        format!(
+                                            "Error from beacon node when publishing block: {:?}",
+                                            e
+                                        )
+                                    })
+                            })
+                            .map(move |publish_outcome| match publish_outcome {
+                                PublishStatus::Valid => {
+                                    info!(log_1, "Successfully published block")
+                                }
+                                PublishStatus::Invalid(msg) => error!(
+                                    log_1,
+                                    "Published block was invalid";
+                                    "message" => msg
+                                ),
+                                PublishStatus::Unknown => {
+                                    info!(log_1, "Unknown condition when publishing block")
+                                }
+                            })
+                            .map_err(move |e| {
+                                error!(
+                                    log_2,
+                                    "Error whilst producing block";
+                                    "message" => e
+                                )
+                            })
+                            .then(|_| Ok(((), block_producers)))
+                    })
+                })
+                .collect()
+                .map(|_| ())
+            })
+    }
+}

validator_client/src/duties_service.rs

@@ -0,0 +1,307 @@
+use crate::validator_store::ValidatorStore;
+use environment::RuntimeContext;
+use exit_future::Signal;
+use futures::{Future, IntoFuture, Stream};
+use parking_lot::RwLock;
+use remote_beacon_node::{RemoteBeaconNode, ValidatorDuty};
+use slog::{error, info, trace, warn};
+use slot_clock::SlotClock;
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+use tokio::timer::Interval;
+use types::{ChainSpec, Epoch, EthSpec, PublicKey, Slot};
+
+/// Delay this period of time after the slot starts. This allows the node to process the new slot.
+const TIME_DELAY_FROM_SLOT: Duration = Duration::from_millis(100);
+
+type BaseHashMap = HashMap<PublicKey, HashMap<Epoch, ValidatorDuty>>;
+
+enum InsertOutcome {
+    New,
+    Identical,
+    Replaced,
+}
+
+#[derive(Default)]
+pub struct DutiesStore {
+    store: RwLock<BaseHashMap>,
+}
+
+impl DutiesStore {
+    fn block_producers(&self, slot: Slot) -> Vec<PublicKey> {
+        self.store
+            .read()
+            .iter()
+            // As long as a `HashMap` iterator does not return duplicate keys, neither will this
+            // function.
+            .filter(|(_validator_pubkey, validator_map)| {
+                validator_map.iter().any(|(_epoch, duties)| {
+                    duties
+                        .block_proposal_slot
+                        .map(|proposal_slot| proposal_slot == slot)
+                        .unwrap_or_else(|| false)
+                })
+            })
+            .map(|(validator_pubkey, _validator_map)| validator_pubkey)
+            .cloned()
+            .collect()
+    }
+
+    fn insert(&self, epoch: Epoch, duties: ValidatorDuty) -> InsertOutcome {
+        let mut store = self.store.write();
+
+        if store.contains_key(&duties.validator_pubkey) {
+            let validator_map = store.get_mut(&duties.validator_pubkey).expect(
+                "Store is exclusively locked and this path is guarded to ensure the key exists.",
+            );
+
+            // TODO: validate that the slots in the duties are all in the given epoch.
+
+            if validator_map.contains_key(&epoch) {
+                let known_duties = validator_map.get_mut(&epoch).expect(
+                    "Validator map is exclusively mutable and this path is guarded to ensure the key exists.",
+                );
+
+                if *known_duties == duties {
+                    InsertOutcome::Identical
+                } else {
+                    *known_duties = duties;
+                    InsertOutcome::Replaced
+                }
+            } else {
+                validator_map.insert(epoch, duties);
+
+                InsertOutcome::New
+            }
+        } else {
+            let validator_pubkey = duties.validator_pubkey.clone();
+
+            let mut validator_map = HashMap::new();
+            validator_map.insert(epoch, duties);
+
+            store.insert(validator_pubkey, validator_map);
+
+            InsertOutcome::New
+        }
+    }
+
+    // TODO: call this.
+    fn prune(&self, prior_to: Epoch) {
+        self.store
+            .write()
+            .retain(|_validator_pubkey, validator_map| {
+                validator_map.retain(|epoch, _duties| *epoch >= prior_to);
+                !validator_map.is_empty()
+            });
+    }
+}
+
+#[derive(Clone)]
+pub struct DutiesServiceBuilder<T: Clone, E: EthSpec> {
+    store: Option<Arc<DutiesStore>>,
+    validator_store: Option<ValidatorStore<E>>,
+    slot_clock: Option<Arc<T>>,
+    beacon_node: Option<RemoteBeaconNode<E>>,
+    context: Option<RuntimeContext<E>>,
+}
+
+// TODO: clean trait bounds.
+impl<T: SlotClock + Clone + 'static, E: EthSpec> DutiesServiceBuilder<T, E> {
+    pub fn new() -> Self {
+        Self {
+            store: None,
+            validator_store: None,
+            slot_clock: None,
+            beacon_node: None,
+            context: None,
+        }
+    }
+
+    pub fn validator_store(mut self, store: ValidatorStore<E>) -> Self {
+        self.validator_store = Some(store);
+        self
+    }
+
+    pub fn slot_clock(mut self, slot_clock: T) -> Self {
+        self.slot_clock = Some(Arc::new(slot_clock));
+        self
+    }
+
+    pub fn beacon_node(mut self, beacon_node: RemoteBeaconNode<E>) -> Self {
+        self.beacon_node = Some(beacon_node);
+        self
+    }
+
+    pub fn runtime_context(mut self, context: RuntimeContext<E>) -> Self {
+        self.context = Some(context);
+        self
+    }
+
+    pub fn build(self) -> Result<DutiesService<T, E>, String> {
+        Ok(DutiesService {
+            store: Arc::new(DutiesStore::default()),
+            validator_store: self
+                .validator_store
+                .ok_or_else(|| "Cannot build DutiesService without validator_store")?,
+            slot_clock: self
+                .slot_clock
+                .ok_or_else(|| "Cannot build DutiesService without slot_clock")?,
+            beacon_node: self
+                .beacon_node
+                .ok_or_else(|| "Cannot build DutiesService without beacon_node")?,
+            context: self
+                .context
+                .ok_or_else(|| "Cannot build DutiesService without runtime_context")?,
+        })
+    }
+}
+
+#[derive(Clone)]
+pub struct DutiesService<T: Clone, E: EthSpec> {
+    store: Arc<DutiesStore>,
+    validator_store: ValidatorStore<E>,
+    slot_clock: Arc<T>,
+    beacon_node: RemoteBeaconNode<E>,
+    context: RuntimeContext<E>,
+}
+
+impl<T: SlotClock + Clone + 'static, E: EthSpec> DutiesService<T, E> {
+    /// Returns the pubkeys of the validators which are assigned to propose in the given slot.
+    ///
+    /// In normal cases, there should be 0 or 1 validators returned. In extreme cases (i.e., deep forking)
+    ///
+    /// It is possible that multiple validators have an identical proposal slot, however that is
+    /// likely the result of heavy forking (lol) or inconsistent beacon node connections.
+    pub fn block_producers(&self, slot: Slot) -> Vec<PublicKey> {
+        self.store.block_producers(slot)
+    }
+
+    pub fn start_update_service(&self, spec: &ChainSpec) -> Result<Signal, String> {
+        let log = self.context.log.clone();
+
+        let duration_to_next_slot = self
+            .slot_clock
+            .duration_to_next_slot()
+            .ok_or_else(|| "Unable to determine duration to next slot".to_string())?;
+
+        let interval = {
+            let slot_duration = Duration::from_millis(spec.milliseconds_per_slot);
+            Interval::new(
+                Instant::now() + duration_to_next_slot + TIME_DELAY_FROM_SLOT,
+                slot_duration,
+            )
+        };
+
+        info!(
+            log,
+            "Waiting for next slot";
+            "seconds_to_wait" => duration_to_next_slot.as_secs()
+        );
+
+        let (exit_signal, exit_fut) = exit_future::signal();
+        let service = self.clone();
+
+        // Run an immediate update before starting the updater service.
+        self.context.executor.spawn(service.clone().do_update());
+
+        self.context.executor.spawn(
+            interval
+                .map_err(move |e| {
+                    error! {
+                        log,
+                        "Timer thread failed";
+                        "error" => format!("{}", e)
+                    }
+                })
+                .and_then(move |_| if exit_fut.is_live() { Ok(()) } else { Err(()) })
+                .for_each(move |_| service.clone().do_update()),
+        );
+
+        Ok(exit_signal)
+    }
+
+    fn do_update(self) -> impl Future<Item = (), Error = ()> {
+        let slots_per_epoch = E::slots_per_epoch();
+        let service_1 = self.clone();
+        let service_2 = self.clone();
+        let log = self.context.log.clone();
+
+        self.slot_clock
+            .now()
+            .ok_or_else(move || {
+                error!(log, "Duties manager failed to read slot clock");
+            })
+            .into_future()
+            .map(move |slot| slot.epoch(slots_per_epoch))
+            .and_then(move |epoch| {
+                let log = service_1.context.log.clone();
+                service_1.update_epoch(epoch).then(move |result| {
+                    if let Err(e) = result {
+                        error!(
+                            log,
+                            "Failed to get current epoch duties";
+                            "http_error" => format!("{:?}", e)
+                        );
+                    }
+
+                    let log = service_2.context.log.clone();
+                    service_2.update_epoch(epoch + 1).map_err(move |e| {
+                        error!(
+                            log,
+                            "Failed to get next epoch duties";
+                            "http_error" => format!("{:?}", e)
+                        );
+                    })
+                })
+            })
+            .map(|_| ())
+            // Returning an error will stop the interval. This is not desired, a single failure
+            // should not stop all future attempts.
+            .then(|_| Ok(()))
+    }
+
+    fn update_epoch(self, epoch: Epoch) -> impl Future<Item = (), Error = String> {
+        let service_1 = self.clone();
+        let service_2 = self.clone();
+
+        let pubkeys = service_1.validator_store.voting_pubkeys();
+        service_1
+            .beacon_node
+            .http
+            .validator()
+            .get_duties(epoch, pubkeys.as_slice())
+            .map(move |all_duties| (epoch, all_duties))
+            .map_err(move |e| format!("Failed to get duties for epoch {}: {:?}", epoch, e))
+            .map(move |(epoch, all_duties)| {
+                let mut new = 0;
+                let mut identical = 0;
+                let mut replaced = 0;
+
+                all_duties.into_iter().for_each(|duties| {
+                    match service_2.store.insert(epoch, duties) {
+                        InsertOutcome::New => new += 1,
+                        InsertOutcome::Identical => identical += 1,
+                        InsertOutcome::Replaced => replaced += 1,
+                    };
+                });
+
+                trace!(
+                    service_2.context.log,
+                    "Performed duties update";
+                    "replaced_duties" => replaced,
+                    "identical_duties" => identical,
+                    "new_duties" => new,
+                    "epoch" => format!("{}", epoch)
+                );
+
+                if replaced > 0 {
+                    warn!(
+                        service_2.context.log,
+                        "Duties changed during routine update";
+                        "info" => "Chain re-org likely occurred."
+                    )
+                }
+            })
+    }
+}

validator_client/src/fork_service.rs

@@ -0,0 +1,157 @@
+use environment::RuntimeContext;
+use exit_future::Signal;
+use futures::{Future, Stream};
+use parking_lot::RwLock;
+use remote_beacon_node::RemoteBeaconNode;
+use slog::{error, info, trace};
+use slot_clock::SlotClock;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+use tokio::timer::Interval;
+use types::{ChainSpec, EthSpec, Fork};
+
+/// Delay this period of time after the slot starts. This allows the node to process the new slot.
+const TIME_DELAY_FROM_SLOT: Duration = Duration::from_millis(80);
+
+#[derive(Clone)]
+pub struct ForkServiceBuilder<T: Clone, E: EthSpec> {
+    fork: Option<Fork>,
+    slot_clock: Option<T>,
+    beacon_node: Option<RemoteBeaconNode<E>>,
+    context: Option<RuntimeContext<E>>,
+}
+
+// TODO: clean trait bounds.
+impl<T: SlotClock + Clone + 'static, E: EthSpec> ForkServiceBuilder<T, E> {
+    pub fn new() -> Self {
+        Self {
+            fork: None,
+            slot_clock: None,
+            beacon_node: None,
+            context: None,
+        }
+    }
+
+    pub fn slot_clock(mut self, slot_clock: T) -> Self {
+        self.slot_clock = Some(slot_clock);
+        self
+    }
+
+    pub fn beacon_node(mut self, beacon_node: RemoteBeaconNode<E>) -> Self {
+        self.beacon_node = Some(beacon_node);
+        self
+    }
+
+    pub fn runtime_context(mut self, context: RuntimeContext<E>) -> Self {
+        self.context = Some(context);
+        self
+    }
+
+    pub fn build(self) -> Result<ForkService<T, E>, String> {
+        Ok(ForkService {
+            inner: Arc::new(Inner {
+                fork: RwLock::new(self.fork),
+                slot_clock: self
+                    .slot_clock
+                    .ok_or_else(|| "Cannot build ForkService without slot_clock")?,
+                beacon_node: self
+                    .beacon_node
+                    .ok_or_else(|| "Cannot build ForkService without beacon_node")?,
+                context: self
+                    .context
+                    .ok_or_else(|| "Cannot build ForkService without runtime_context")?,
+            }),
+        })
+    }
+}
+
+struct Inner<T, E: EthSpec> {
+    fork: RwLock<Option<Fork>>,
+    beacon_node: RemoteBeaconNode<E>,
+    context: RuntimeContext<E>,
+    slot_clock: T,
+}
+
+#[derive(Clone)]
+pub struct ForkService<T, E: EthSpec> {
+    inner: Arc<Inner<T, E>>,
+}
+
+// TODO: clean trait bounds.
+impl<T: SlotClock + Clone + 'static, E: EthSpec> ForkService<T, E> {
+    pub fn fork(&self) -> Option<Fork> {
+        self.inner.fork.read().clone()
+    }
+
+    pub fn start_update_service(&self, spec: &ChainSpec) -> Result<Signal, String> {
+        let log = self.inner.context.log.clone();
+
+        let duration_to_next_epoch = self
+            .inner
+            .slot_clock
+            .duration_to_next_epoch(E::slots_per_epoch())
+            .ok_or_else(|| "Unable to determine duration to next epoch".to_string())?;
+
+        let interval = {
+            let slot_duration = Duration::from_millis(spec.milliseconds_per_slot);
+            Interval::new(
+                Instant::now() + duration_to_next_epoch + TIME_DELAY_FROM_SLOT,
+                slot_duration * E::slots_per_epoch() as u32,
+            )
+        };
+
+        info!(
+            log,
+            "Waiting for next slot";
+            "seconds_to_wait" => duration_to_next_epoch.as_secs()
+        );
+
+        let (exit_signal, exit_fut) = exit_future::signal();
+        let service = self.clone();
+
+        // Run an immediate update before starting the updater service.
+        self.inner
+            .context
+            .executor
+            .spawn(service.clone().do_update());
+
+        self.inner.context.executor.spawn(
+            interval
+                .map_err(move |e| {
+                    error! {
+                        log,
+                        "Timer thread failed";
+                        "error" => format!("{}", e)
+                    }
+                })
+                .and_then(move |_| if exit_fut.is_live() { Ok(()) } else { Err(()) })
+                .for_each(move |_| service.clone().do_update()),
+        );
+
+        Ok(exit_signal)
+    }
+
+    fn do_update(self) -> impl Future<Item = (), Error = ()> {
+        let service_1 = self.inner.clone();
+        let log_1 = service_1.context.log.clone();
+        let log_2 = service_1.context.log.clone();
+
+        self.inner
+            .beacon_node
+            .http
+            .beacon()
+            .get_fork()
+            .map(move |fork| *(service_1.fork.write()) = Some(fork))
+            .map(move |_| trace!(log_1, "Fork update success"))
+            .map_err(move |e| {
+                trace!(
+                    log_2,
+                    "Fork update failed";
+                    "error" => format!("Error retrieving fork: {:?}", e)
+                )
+            })
+            // Returning an error will stop the interval. This is not desired, a single failure
+            // should not stop all future attempts.
+            .then(|_| Ok(()))
+    }
+}

validator_client/src/lib.rs

@@ -1,42 +1,44 @@
-mod attestation_producer;
-mod block_producer;
+mod block_service;
 mod cli;
 mod config;
-mod duties;
+mod duties_service;
 mod error;
-mod service;
+mod fork_service;
 mod signer;
+mod validator_store;
+
 pub mod validator_directory;
 
 pub use cli::cli_app;
 pub use config::Config;
 
+use block_service::{BlockService, BlockServiceBuilder};
 use clap::ArgMatches;
 use config::{Config as ClientConfig, KeySource};
+use duties_service::{DutiesService, DutiesServiceBuilder};
 use environment::RuntimeContext;
 use eth2_config::Eth2Config;
 use exit_future::Signal;
-use futures::Stream;
+use fork_service::{ForkService, ForkServiceBuilder};
+use futures::{Future, IntoFuture};
 use lighthouse_bootstrap::Bootstrapper;
 use parking_lot::RwLock;
-use protos::services_grpc::ValidatorServiceClient;
-use service::Service;
-use slog::{error, info, warn, Logger};
+use remote_beacon_node::RemoteBeaconNode;
+use slog::{info, Logger};
 use slot_clock::SlotClock;
+use slot_clock::SystemTimeSlotClock;
 use std::path::PathBuf;
 use std::sync::Arc;
-use std::time::{Duration, Instant};
-use tokio::timer::Interval;
-use types::{EthSpec, Keypair};
-
-/// A fixed amount of time after a slot to perform operations. This gives the node time to complete
-/// per-slot processes.
-const TIME_DELAY_FROM_SLOT: Duration = Duration::from_millis(100);
+use std::time::Duration;
+use types::EthSpec;
+use validator_store::ValidatorStore;
 
 #[derive(Clone)]
 pub struct ProductionValidatorClient<T: EthSpec> {
     context: RuntimeContext<T>,
-    service: Arc<Service<ValidatorServiceClient, Keypair, T>>,
+    duties_service: DutiesService<SystemTimeSlotClock, T>,
+    fork_service: ForkService<SystemTimeSlotClock, T>,
+    block_service: BlockService<SystemTimeSlotClock, T>,
     exit_signals: Arc<RwLock<Vec<Signal>>>,
 }
 
@@ -46,97 +48,156 @@ impl<T: EthSpec> ProductionValidatorClient<T> {
     pub fn new_from_cli(
         mut context: RuntimeContext<T>,
         matches: &ArgMatches,
-    ) -> Result<Self, String> {
+    ) -> impl Future<Item = Self, Error = String> {
         let mut log = context.log.clone();
 
-        let (config, eth2_config) = get_configs(&matches, &mut log)
-            .map_err(|e| format!("Unable to initialize config: {}", e))?;
+        get_configs(&matches, &mut log)
+            .into_future()
+            .map_err(|e| format!("Unable to initialize config: {}", e))
+            .and_then(|(client_config, eth2_config)| {
+                // TODO: the eth2 config in the env is being completely ignored.
+                //
+                // See https://github.com/sigp/lighthouse/issues/602
+                context.eth2_config = eth2_config;
 
-        // TODO: the eth2 config in the env is being completely ignored.
-        //
-        // See https://github.com/sigp/lighthouse/issues/602
-        context.eth2_config = eth2_config;
-
-        Self::new(context, config)
+                Self::new(context, client_config)
+            })
     }
 
     /// Instantiates the validator client, _without_ starting the timers to trigger block
     /// and attestation production.
-    pub fn new(context: RuntimeContext<T>, config: Config) -> Result<Self, String> {
+    pub fn new(
+        mut context: RuntimeContext<T>,
+        client_config: ClientConfig,
+    ) -> impl Future<Item = Self, Error = String> {
         let log = context.log.clone();
 
         info!(
             log,
             "Starting validator client";
-            "datadir" => config.full_data_dir().expect("Unable to find datadir").to_str(),
+            "datadir" => client_config.full_data_dir().expect("Unable to find datadir").to_str(),
         );
 
-        let service: Service<ValidatorServiceClient, Keypair, T> =
-            Service::initialize_service(config, context.eth2_config.clone(), log.clone())
-                .map_err(|e| e.to_string())?;
+        format!(
+            "{}:{}",
+            client_config.server, client_config.server_http_port
+        )
+        .parse()
+        .map_err(|e| format!("Unable to parse server address: {:?}", e))
+        .into_future()
+        .and_then(|http_server_addr| {
+            RemoteBeaconNode::new(http_server_addr)
+                .map_err(|e| format!("Unable to init beacon node http client: {}", e))
+        })
+        .and_then(|beacon_node| {
+            // TODO: add loop function to retry if node not online.
+            beacon_node
+                .http
+                .spec()
+                .get_eth2_config()
+                .map(|eth2_config| (beacon_node, eth2_config))
+                .map_err(|e| format!("Unable to read eth2 config from beacon node: {:?}", e))
+        })
+        .and_then(|(beacon_node, eth2_config)| {
+            beacon_node
+                .http
+                .beacon()
+                .get_genesis_time()
+                .map(|genesis_time| (beacon_node, eth2_config, genesis_time))
+                .map_err(|e| format!("Unable to read genesis time from beacon node: {:?}", e))
+        })
+        .and_then(move |(beacon_node, remote_eth2_config, genesis_time)| {
+            // Do not permit a connection to a beacon node using different spec constants.
+            if context.eth2_config.spec_constants != remote_eth2_config.spec_constants {
+                return Err(format!(
+                    "Beacon node is using an incompatible spec. Got {}, expected {}",
+                    remote_eth2_config.spec_constants, context.eth2_config.spec_constants
+                ));
+            }
 
-        Ok(Self {
-            context,
-            service: Arc::new(service),
-            exit_signals: Arc::new(RwLock::new(vec![])),
+            // Note: here we just assume the spec variables of the remote node. This is very useful
+            // for testnets, but perhaps a security issue when it comes to mainnet.
+            //
+            // A damaging attack would be for a beacon node to convince the validator client of a
+            // different `SLOTS_PER_EPOCH` variable. This could result in slashable messages being
+            // produced. We are safe from this because `SLOTS_PER_EPOCH` is a type-level constant
+            // for Lighthouse.
+            context.eth2_config = remote_eth2_config;
+
+            let slot_clock = SystemTimeSlotClock::new(
+                context.eth2_config.spec.genesis_slot,
+                Duration::from_secs(genesis_time),
+                Duration::from_millis(context.eth2_config.spec.milliseconds_per_slot),
+            );
+
+            dbg!(context.eth2_config.spec.milliseconds_per_slot);
+
+            // TODO: fix expect.
+            let validator_store = ValidatorStore::load_from_disk(
+                client_config.full_data_dir().expect("Get rid of this."),
+                context.eth2_config.spec.clone(),
+                log.clone(),
+            )?;
+
+            info!(
+                log,
+                "Loaded validator keypair store";
+                "voting_validators" => validator_store.num_voting_validators()
+            );
+
+            let duties_service = DutiesServiceBuilder::new()
+                .slot_clock(slot_clock.clone())
+                .validator_store(validator_store.clone())
+                .beacon_node(beacon_node.clone())
+                .runtime_context(context.service_context("duties"))
+                .build()?;
+
+            let fork_service = ForkServiceBuilder::new()
+                .slot_clock(slot_clock.clone())
+                .beacon_node(beacon_node.clone())
+                .runtime_context(context.service_context("fork"))
+                .build()?;
+
+            let block_service = BlockServiceBuilder::new()
+                .duties_service(duties_service.clone())
+                .fork_service(fork_service.clone())
+                .slot_clock(slot_clock)
+                .validator_store(validator_store)
+                .beacon_node(beacon_node)
+                .runtime_context(context.service_context("block"))
+                .build()?;
+
+            Ok(Self {
+                context,
+                duties_service,
+                fork_service,
+                block_service,
+                exit_signals: Arc::new(RwLock::new(vec![])),
+            })
         })
     }
 
-    /// Starts the timers to trigger block and attestation production.
     pub fn start_service(&self) -> Result<(), String> {
-        let service = self.clone().service;
-        let log = self.context.log.clone();
+        let duties_exit = self
+            .duties_service
+            .start_update_service(&self.context.eth2_config.spec)
+            .map_err(|e| format!("Unable to start duties service: {}", e))?;
 
-        let duration_to_next_slot = service
-            .slot_clock
-            .duration_to_next_slot()
-            .ok_or_else(|| "Unable to determine duration to next slot. Exiting.".to_string())?;
+        self.exit_signals.write().push(duties_exit);
 
-        // set up the validator work interval - start at next slot and proceed every slot
-        let interval = {
-            // Set the interval to start at the next slot, and every slot after
-            let slot_duration = Duration::from_millis(service.spec.milliseconds_per_slot);
-            //TODO: Handle checked add correctly
-            Interval::new(Instant::now() + duration_to_next_slot, slot_duration)
-        };
+        let fork_exit = self
+            .fork_service
+            .start_update_service(&self.context.eth2_config.spec)
+            .map_err(|e| format!("Unable to start fork service: {}", e))?;
 
-        if service.slot_clock.now().is_none() {
-            warn!(
-                log,
-                "Starting node prior to genesis";
-            );
-        }
+        self.exit_signals.write().push(fork_exit);
 
-        info!(
-            log,
-            "Waiting for next slot";
-            "seconds_to_wait" => duration_to_next_slot.as_secs()
-        );
+        let block_exit = self
+            .block_service
+            .start_update_service(&self.context.eth2_config.spec)
+            .map_err(|e| format!("Unable to start block service: {}", e))?;
 
-        let (exit_signal, exit_fut) = exit_future::signal();
-
-        self.exit_signals.write().push(exit_signal);
-
-        /* kick off the core service */
-        self.context.executor.spawn(
-            interval
-                .map_err(move |e| {
-                    error! {
-                        log,
-                        "Timer thread failed";
-                        "error" => format!("{}", e)
-                    }
-                })
-                .and_then(move |_| if exit_fut.is_live() { Ok(()) } else { Err(()) })
-                .for_each(move |_| {
-                    // wait for node to process
-                    std::thread::sleep(TIME_DELAY_FROM_SLOT);
-                    // if a non-fatal error occurs, proceed to the next slot.
-                    let _ignore_error = service.per_slot_execution();
-                    // completed a slot process
-                    Ok(())
-                }),
-        );
+        self.exit_signals.write().push(block_exit);
 
         Ok(())
     }

validator_client/src/validator_store.rs

@@ -0,0 +1,109 @@
+use crate::validator_directory::ValidatorDirectory;
+use parking_lot::RwLock;
+use slog::{error, Logger};
+use std::collections::HashMap;
+use std::fs::read_dir;
+use std::iter::FromIterator;
+use std::marker::PhantomData;
+use std::path::PathBuf;
+use std::sync::Arc;
+use tree_hash::{SignedRoot, TreeHash};
+use types::{BeaconBlock, ChainSpec, Domain, Epoch, EthSpec, Fork, PublicKey, Signature};
+
+#[derive(Clone)]
+pub struct ValidatorStore<E> {
+    validators: Arc<RwLock<HashMap<PublicKey, ValidatorDirectory>>>,
+    spec: Arc<ChainSpec>,
+    _phantom: PhantomData<E>,
+}
+
+impl<E: EthSpec> ValidatorStore<E> {
+    pub fn load_from_disk(base_dir: PathBuf, spec: ChainSpec, log: Logger) -> Result<Self, String> {
+        let validator_iter = read_dir(&base_dir)
+            .map_err(|e| format!("Failed to read base directory: {:?}", e))?
+            .filter_map(|validator_dir| {
+                let path = validator_dir.ok()?.path();
+
+                if path.is_dir() {
+                    match ValidatorDirectory::load_for_signing(path.clone()) {
+                        Ok(validator_directory) => Some(validator_directory),
+                        Err(e) => {
+                            error!(
+                                log,
+                                "Failed to load a validator directory";
+                                "error" => e,
+                                "path" => path.to_str(),
+                            );
+                            None
+                        }
+                    }
+                } else {
+                    None
+                }
+            })
+            .filter_map(|validator_directory| {
+                validator_directory
+                    .voting_keypair
+                    .clone()
+                    .map(|voting_keypair| (voting_keypair.pk, validator_directory))
+            });
+
+        Ok(Self {
+            validators: Arc::new(RwLock::new(HashMap::from_iter(validator_iter))),
+            spec: Arc::new(spec),
+            _phantom: PhantomData,
+        })
+    }
+
+    pub fn voting_pubkeys(&self) -> Vec<PublicKey> {
+        self.validators
+            .read()
+            .iter()
+            .map(|(pubkey, _dir)| pubkey.clone())
+            .collect()
+    }
+
+    pub fn num_voting_validators(&self) -> usize {
+        self.validators.read().len()
+    }
+
+    pub fn randao_reveal(
+        &self,
+        validator_pubkey: &PublicKey,
+        epoch: Epoch,
+        fork: &Fork,
+    ) -> Option<Signature> {
+        // TODO: check this against the slot clock to make sure it's not an early reveal?
+        self.validators
+            .read()
+            .get(validator_pubkey)
+            .and_then(|validator_dir| {
+                validator_dir.voting_keypair.as_ref().map(|voting_keypair| {
+                    let message = epoch.tree_hash_root();
+                    let domain = self.spec.get_domain(epoch, Domain::Randao, &fork);
+                    Signature::new(&message, domain, &voting_keypair.sk)
+                })
+            })
+    }
+
+    pub fn sign_block(
+        &self,
+        validator_pubkey: &PublicKey,
+        mut block: BeaconBlock<E>,
+        fork: &Fork,
+    ) -> Option<BeaconBlock<E>> {
+        // TODO: check for slashing.
+        self.validators
+            .read()
+            .get(validator_pubkey)
+            .and_then(|validator_dir| {
+                validator_dir.voting_keypair.as_ref().map(|voting_keypair| {
+                    let epoch = block.slot.epoch(E::slots_per_epoch());
+                    let message = block.signed_root();
+                    let domain = self.spec.get_domain(epoch, Domain::BeaconProposer, &fork);
+                    block.signature = Signature::new(&message, domain, &voting_keypair.sk);
+                    block
+                })
+            })
+    }
+}