From 0052ea711e2e2baab3b3fba6557247284c651e5e Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Wed, 31 Jul 2019 18:29:41 +1000 Subject: [PATCH] First RESTful HTTP API (#399) * Added generated code for REST API. - Created a new crate rest_api, which will adapt the openapi generated code to Lighthouse - Committed automatically generated code from openapi-generator-cli (via docker). Should hopfully not have to modify this at all, and do all changes in the rest_api crate. * Removed openapi generated code, because it was the rust client, not the rust server. * Added the correct rust-server code, automatically generated from openapi. * Added generated code for REST API. - Created a new crate rest_api, which will adapt the openapi generated code to Lighthouse - Committed automatically generated code from openapi-generator-cli (via docker). Should hopfully not have to modify this at all, and do all changes in the rest_api crate. * Removed openapi generated code, because it was the rust client, not the rust server. * Added the correct rust-server code, automatically generated from openapi. * Included REST API in configuratuion. - Started adding the rest_api into the beacon node's dependencies. - Set up configuration file for rest_api and integrated into main client config - Added CLI flags for REST API. * Futher work on REST API. - Adding the dependencies to rest_api crate - Created a skeleton BeaconNodeService, which will handle /node requests. - Started the rest_api server definition, with the high level request handling logic. * Added generated code for REST API. - Created a new crate rest_api, which will adapt the openapi generated code to Lighthouse - Committed automatically generated code from openapi-generator-cli (via docker). Should hopfully not have to modify this at all, and do all changes in the rest_api crate. * Removed openapi generated code, because it was the rust client, not the rust server. * Added the correct rust-server code, automatically generated from openapi. * Included REST API in configuratuion. - Started adding the rest_api into the beacon node's dependencies. - Set up configuration file for rest_api and integrated into main client config - Added CLI flags for REST API. * Futher work on REST API. - Adding the dependencies to rest_api crate - Created a skeleton BeaconNodeService, which will handle /node requests. - Started the rest_api server definition, with the high level request handling logic. * WIP: Restructured REST API to use hyper_router and separate services. * WIP: Fixing rust for REST API * WIP: Fixed up many bugs in trying to get router to compile. * WIP: Got the beacon_node to compile with the REST changes * Basic API works! - Changed CLI flags from rest-api* to api* - Fixed port cli flag - Tested, works over HTTP * WIP: Moved things around so that we can get state inside the handlers. * WIP: Significant API updates. - Started writing a macro for getting the handler functions. - Added the BeaconChain into the type map, gives stateful access to the beacon state. - Created new generic error types (haven't figured out yet), to reduce code duplication. - Moved common stuff into lib.rs * WIP: Factored macros, defined API result and error. - did more logging when creating HTTP responses - Tried moving stuff into macros, but can't get macros in macros to compile. - Pulled out a lot of placeholder code. * Fixed macros so that things compile. * Cleaned up code. - Removed unused imports - Removed comments - Addressed all compiler warnings. - Ran cargo fmt. * Removed auto-generated OpenAPI code. * Addressed Paul's suggestions. - Fixed spelling mistake - Moved the simple macros into functions, since it doesn't make sense for them to be macros. - Removed redundant code & inclusions. * Removed redundant validate_request function. * Included graceful shutdown in Hyper server. * Fixing the dropped exit_signal, which prevented the API from starting. * Wrapped the exit signal, to get an API shutdown log line. --- Cargo.toml | 1 + beacon_node/client/Cargo.toml | 1 + beacon_node/client/src/config.rs | 3 + beacon_node/client/src/lib.rs | 22 ++++ beacon_node/rest_api/Cargo.toml | 22 ++++ beacon_node/rest_api/src/beacon_node.rs | 65 ++++++++++++ beacon_node/rest_api/src/config.rs | 46 +++++++++ beacon_node/rest_api/src/lib.rs | 132 ++++++++++++++++++++++++ beacon_node/rest_api/src/macros.rs | 23 +++++ beacon_node/src/main.rs | 23 +++++ 10 files changed, 338 insertions(+) create mode 100644 beacon_node/rest_api/Cargo.toml create mode 100644 beacon_node/rest_api/src/beacon_node.rs create mode 100644 beacon_node/rest_api/src/config.rs create mode 100644 beacon_node/rest_api/src/lib.rs create mode 100644 beacon_node/rest_api/src/macros.rs diff --git a/Cargo.toml b/Cargo.toml index 20c5b31755..c4034ad355 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ members = [ "beacon_node/store", "beacon_node/client", "beacon_node/http_server", + "beacon_node/rest_api", "beacon_node/network", "beacon_node/eth2-libp2p", "beacon_node/rpc", diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index 5bbd33b3da..3367b84ce5 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -9,6 +9,7 @@ beacon_chain = { path = "../beacon_chain" } network = { path = "../network" } http_server = { path = "../http_server" } rpc = { path = "../rpc" } +rest_api = { path = "../rest_api" } prometheus = "^0.6" types = { path = "../../eth2/types" } tree_hash = { path = "../../eth2/utils/tree_hash" } diff --git a/beacon_node/client/src/config.rs b/beacon_node/client/src/config.rs index 8645775593..9a9fed8023 100644 --- a/beacon_node/client/src/config.rs +++ b/beacon_node/client/src/config.rs @@ -17,6 +17,7 @@ pub struct Config { pub network: network::NetworkConfig, pub rpc: rpc::RPCConfig, pub http: HttpServerConfig, + pub rest_api: rest_api::APIConfig, } impl Default for Config { @@ -31,6 +32,7 @@ impl Default for Config { network: NetworkConfig::new(), rpc: rpc::RPCConfig::default(), http: HttpServerConfig::default(), + rest_api: rest_api::APIConfig::default(), } } } @@ -101,6 +103,7 @@ impl Config { self.network.apply_cli_args(args)?; self.rpc.apply_cli_args(args)?; self.http.apply_cli_args(args)?; + self.rest_api.apply_cli_args(args)?; if let Some(log_file) = args.value_of("logfile") { self.log_file = PathBuf::from(log_file); diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index 1b9f320be3..8138c7d47a 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -2,6 +2,7 @@ extern crate slog; mod beacon_chain_types; mod config; + pub mod error; pub mod notifier; @@ -39,6 +40,8 @@ pub struct Client { pub http_exit_signal: Option, /// Signal to terminate the slot timer. pub slot_timer_exit_signal: Option, + /// Signal to terminate the API + pub api_exit_signal: Option, /// The clients logger. log: slog::Logger, /// Marker to pin the beacon chain generics. @@ -143,6 +146,24 @@ where None }; + // Start the `rest_api` service + let api_exit_signal = if client_config.rest_api.enabled { + match rest_api::start_server( + &client_config.rest_api, + executor, + beacon_chain.clone(), + &log, + ) { + Ok(s) => Some(s), + Err(e) => { + error!(log, "API service failed to start."; "error" => format!("{:?}",e)); + None + } + } + } else { + None + }; + let (slot_timer_exit_signal, exit) = exit_future::signal(); if let Ok(Some(duration_to_next_slot)) = beacon_chain.slot_clock.duration_to_next_slot() { // set up the validator work interval - start at next slot and proceed every slot @@ -175,6 +196,7 @@ where http_exit_signal, rpc_exit_signal, slot_timer_exit_signal: Some(slot_timer_exit_signal), + api_exit_signal, log, network, phantom: PhantomData, diff --git a/beacon_node/rest_api/Cargo.toml b/beacon_node/rest_api/Cargo.toml new file mode 100644 index 0000000000..7a63ca0361 --- /dev/null +++ b/beacon_node/rest_api/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "rest_api" +version = "0.1.0" +authors = ["Luke Anderson "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[dependencies] +beacon_chain = { path = "../beacon_chain" } +version = { path = "../version" } +serde = { version = "1.0", features = ["derive"] } +serde_json = "^1.0" +slog = "^2.2.3" +slog-term = "^2.4.0" +slog-async = "^2.3.0" +clap = "2.32.0" +http = "^0.1.17" +hyper = "0.12.32" +hyper-router = "^0.5" +futures = "0.1" +exit-future = "0.1.3" +tokio = "0.1.17" diff --git a/beacon_node/rest_api/src/beacon_node.rs b/beacon_node/rest_api/src/beacon_node.rs new file mode 100644 index 0000000000..87d2d3cdcd --- /dev/null +++ b/beacon_node/rest_api/src/beacon_node.rs @@ -0,0 +1,65 @@ +use beacon_chain::{BeaconChain, BeaconChainTypes}; +use serde::Serialize; +use slog::info; +use std::sync::Arc; +use version; + +use super::{path_from_request, success_response, APIResult, APIService}; + +use hyper::{Body, Request, Response}; +use hyper_router::{Route, RouterBuilder}; + +#[derive(Clone)] +pub struct BeaconNodeServiceInstance { + pub marker: std::marker::PhantomData, +} + +/// A string which uniquely identifies the client implementation and its version; similar to [HTTP User-Agent](https://tools.ietf.org/html/rfc7231#section-5.5.3). +#[derive(Serialize)] +pub struct Version(String); +impl From for Version { + fn from(x: String) -> Self { + Version(x) + } +} + +/// The genesis_time configured for the beacon node, which is the unix time at which the Eth2.0 chain began. +#[derive(Serialize)] +pub struct GenesisTime(u64); +impl From for GenesisTime { + fn from(x: u64) -> Self { + GenesisTime(x) + } +} + +impl APIService for BeaconNodeServiceInstance { + fn add_routes(&mut self, router_builder: RouterBuilder) -> Result { + let router_builder = router_builder + .add(Route::get("/version").using(result_to_response!(get_version))) + .add(Route::get("/genesis_time").using(result_to_response!(get_genesis_time::))); + Ok(router_builder) + } +} + +/// Read the version string from the current Lighthouse build. +fn get_version(_req: Request) -> APIResult { + let ver = Version::from(version::version()); + let body = Body::from( + serde_json::to_string(&ver).expect("Version should always be serialializable as JSON."), + ); + Ok(success_response(body)) +} + +/// Read the genesis time from the current beacon chain state. +fn get_genesis_time(req: Request) -> APIResult { + let beacon_chain = req.extensions().get::>>().unwrap(); + let gen_time = { + let state = beacon_chain.current_state(); + state.genesis_time + }; + let body = Body::from( + serde_json::to_string(&gen_time) + .expect("Genesis should time always have a valid JSON serialization."), + ); + Ok(success_response(body)) +} diff --git a/beacon_node/rest_api/src/config.rs b/beacon_node/rest_api/src/config.rs new file mode 100644 index 0000000000..c4a9c738a0 --- /dev/null +++ b/beacon_node/rest_api/src/config.rs @@ -0,0 +1,46 @@ +use clap::ArgMatches; +use serde::{Deserialize, Serialize}; +use std::net::Ipv4Addr; + +/// HTTP REST API Configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Config { + /// Enable the REST API server. + pub enabled: bool, + /// The IPv4 address the REST API HTTP server will listen on. + pub listen_address: Ipv4Addr, + /// The port the REST API HTTP server will listen on. + pub port: u16, +} + +impl Default for Config { + fn default() -> Self { + Config { + enabled: true, // rest_api enabled by default + listen_address: Ipv4Addr::new(127, 0, 0, 1), + port: 1248, + } + } +} + +impl Config { + pub fn apply_cli_args(&mut self, args: &ArgMatches) -> Result<(), &'static str> { + if args.is_present("api") { + self.enabled = true; + } + + if let Some(rpc_address) = args.value_of("api-address") { + self.listen_address = rpc_address + .parse::() + .map_err(|_| "api-address is not a valid IPv4 address.")?; + } + + if let Some(rpc_port) = args.value_of("api-port") { + self.port = rpc_port + .parse::() + .map_err(|_| "api-port is not a valid u16.")?; + } + + Ok(()) + } +} diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs new file mode 100644 index 0000000000..0f78494492 --- /dev/null +++ b/beacon_node/rest_api/src/lib.rs @@ -0,0 +1,132 @@ +extern crate futures; +extern crate hyper; +#[macro_use] +mod macros; +mod beacon_node; +pub mod config; + +use beacon_chain::{BeaconChain, BeaconChainTypes}; +pub use config::Config as APIConfig; + +use slog::{info, o, warn}; +use std::sync::Arc; +use tokio::runtime::TaskExecutor; + +use crate::beacon_node::BeaconNodeServiceInstance; +use hyper::rt::Future; +use hyper::service::{service_fn, Service}; +use hyper::{Body, Request, Response, Server, StatusCode}; +use hyper_router::{RouterBuilder, RouterService}; + +pub enum APIError { + MethodNotAllowed { desc: String }, + ServerError { desc: String }, + NotImplemented { desc: String }, +} + +pub type APIResult = Result, APIError>; + +impl Into> for APIError { + fn into(self) -> Response { + let status_code: (StatusCode, String) = match self { + APIError::MethodNotAllowed { desc } => (StatusCode::METHOD_NOT_ALLOWED, desc), + APIError::ServerError { desc } => (StatusCode::INTERNAL_SERVER_ERROR, desc), + APIError::NotImplemented { desc } => (StatusCode::NOT_IMPLEMENTED, desc), + }; + Response::builder() + .status(status_code.0) + .body(Body::from(status_code.1)) + .expect("Response should always be created.") + } +} + +pub trait APIService { + fn add_routes(&mut self, router_builder: RouterBuilder) -> Result; +} + +pub fn start_server( + config: &APIConfig, + executor: &TaskExecutor, + beacon_chain: Arc>, + log: &slog::Logger, +) -> Result { + let log = log.new(o!("Service" => "API")); + + // build a channel to kill the HTTP server + let (exit_signal, exit) = exit_future::signal(); + + let exit_log = log.clone(); + let server_exit = exit.and_then(move |_| { + info!(exit_log, "API service shutdown"); + Ok(()) + }); + + // Get the address to bind to + let bind_addr = (config.listen_address, config.port).into(); + + // Clone our stateful objects, for use in service closure. + let server_log = log.clone(); + let server_bc = beacon_chain.clone(); + + // Create the service closure + let service = move || { + //TODO: This router must be moved out of this closure, so it isn't rebuilt for every connection. + let mut router = build_router_service::(); + + // Clone our stateful objects, for use in handler closure + let service_log = server_log.clone(); + let service_bc = server_bc.clone(); + + // Create a simple handler for the router, inject our stateful objects into the request. + service_fn(move |mut req| { + req.extensions_mut() + .insert::(service_log.clone()); + req.extensions_mut() + .insert::>>(service_bc.clone()); + router.call(req) + }) + }; + + let server = Server::bind(&bind_addr) + .serve(service) + .with_graceful_shutdown(server_exit) + .map_err(move |e| { + warn!( + log, + "API failed to start, Unable to bind"; "address" => format!("{:?}", e) + ) + }); + + executor.spawn(server); + + Ok(exit_signal) +} + +fn build_router_service() -> RouterService { + let mut router_builder = RouterBuilder::new(); + + let mut bn_service: BeaconNodeServiceInstance = BeaconNodeServiceInstance { + marker: std::marker::PhantomData, + }; + + router_builder = bn_service + .add_routes(router_builder) + .expect("The routes should always be made."); + + RouterService::new(router_builder.build()) +} + +fn path_from_request(req: &Request) -> String { + req.uri() + .path_and_query() + .as_ref() + .map(|pq| String::from(pq.as_str())) + .unwrap_or(String::new()) +} + +fn success_response(body: Body) -> Response { + Response::builder() + .status(StatusCode::OK) + .body(body) + .expect("We should always be able to make response from the success body.") +} diff --git a/beacon_node/rest_api/src/macros.rs b/beacon_node/rest_api/src/macros.rs new file mode 100644 index 0000000000..db9bfd8483 --- /dev/null +++ b/beacon_node/rest_api/src/macros.rs @@ -0,0 +1,23 @@ +macro_rules! result_to_response { + ($handler: path) => { + |req: Request| -> Response { + let log = req + .extensions() + .get::() + .expect("Our logger should be on req.") + .clone(); + let path = path_from_request(&req); + let result = $handler(req); + match result { + Ok(response) => { + info!(log, "Request successful: {:?}", path); + response + } + Err(e) => { + info!(log, "Request failure: {:?}", path); + e.into() + } + } + } + }; +} diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index 4ad544bb1a..2e0cbb67bc 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -127,6 +127,29 @@ fn main() { .help("Listen port for the HTTP server.") .takes_value(true), ) + // REST API related arguments + .arg( + Arg::with_name("api") + .long("api") + .value_name("API") + .help("Enable the RESTful HTTP API server.") + .takes_value(false), + ) + .arg( + Arg::with_name("api-address") + .long("api-address") + .value_name("APIADDRESS") + .help("Set the listen address for the RESTful HTTP API server.") + .takes_value(true), + ) + .arg( + Arg::with_name("api-port") + .long("api-port") + .value_name("APIPORT") + .help("Set the listen TCP port for the RESTful HTTP API server.") + .takes_value(true), + ) + // General arguments .arg( Arg::with_name("db") .long("db")