Add beacon.watch (#3362)

> This is currently a WIP and all features are subject to alteration or removal at any time.

## Overview

The successor to #2873.

Contains the backbone of `beacon.watch` including syncing code, the initial API, and several core database tables.

See `watch/README.md` for more information, requirements and usage.

watch/src/server/config.rs

@@ -0,0 +1,28 @@
+use serde::{Deserialize, Serialize};
+use std::net::IpAddr;
+
+pub const LISTEN_ADDR: &str = "127.0.0.1";
+
+pub const fn listen_port() -> u16 {
+    5059
+}
+fn listen_addr() -> IpAddr {
+    LISTEN_ADDR.parse().expect("Server address is not valid")
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Config {
+    #[serde(default = "listen_addr")]
+    pub listen_addr: IpAddr,
+    #[serde(default = "listen_port")]
+    pub listen_port: u16,
+}
+
+impl Default for Config {
+    fn default() -> Self {
+        Self {
+            listen_addr: listen_addr(),
+            listen_port: listen_port(),
+        }
+    }
+}

watch/src/server/error.rs

@@ -0,0 +1,50 @@
+use crate::database::Error as DbError;
+use axum::Error as AxumError;
+use axum::{http::StatusCode, response::IntoResponse, Json};
+use hyper::Error as HyperError;
+use serde_json::json;
+
+#[derive(Debug)]
+pub enum Error {
+    Axum(AxumError),
+    Hyper(HyperError),
+    Database(DbError),
+    BadRequest,
+    NotFound,
+    Other(String),
+}
+
+impl IntoResponse for Error {
+    fn into_response(self) -> axum::response::Response {
+        let (status, error_message) = match self {
+            Self::BadRequest => (StatusCode::BAD_REQUEST, "Bad Request"),
+            Self::NotFound => (StatusCode::NOT_FOUND, "Not Found"),
+            _ => (StatusCode::INTERNAL_SERVER_ERROR, "Internal Server Error"),
+        };
+        (status, Json(json!({ "error": error_message }))).into_response()
+    }
+}
+
+impl From<HyperError> for Error {
+    fn from(e: HyperError) -> Self {
+        Error::Hyper(e)
+    }
+}
+
+impl From<AxumError> for Error {
+    fn from(e: AxumError) -> Self {
+        Error::Axum(e)
+    }
+}
+
+impl From<DbError> for Error {
+    fn from(e: DbError) -> Self {
+        Error::Database(e)
+    }
+}
+
+impl From<String> for Error {
+    fn from(e: String) -> Self {
+        Error::Other(e)
+    }
+}

watch/src/server/handler.rs

@@ -0,0 +1,266 @@
+use crate::database::{
+    self, Error as DbError, PgPool, WatchBeaconBlock, WatchCanonicalSlot, WatchHash, WatchPK,
+    WatchProposerInfo, WatchSlot, WatchValidator,
+};
+use crate::server::Error;
+use axum::{
+    extract::{Path, Query},
+    Extension, Json,
+};
+use eth2::types::BlockId;
+use std::collections::HashMap;
+use std::str::FromStr;
+
+pub async fn get_slot(
+    Path(slot): Path<u64>,
+    Extension(pool): Extension<PgPool>,
+) -> Result<Json<Option<WatchCanonicalSlot>>, Error> {
+    let mut conn = database::get_connection(&pool).map_err(Error::Database)?;
+    Ok(Json(database::get_canonical_slot(
+        &mut conn,
+        WatchSlot::new(slot),
+    )?))
+}
+
+pub async fn get_slot_lowest(
+    Extension(pool): Extension<PgPool>,
+) -> Result<Json<Option<WatchCanonicalSlot>>, Error> {
+    let mut conn = database::get_connection(&pool).map_err(Error::Database)?;
+    Ok(Json(database::get_lowest_canonical_slot(&mut conn)?))
+}
+
+pub async fn get_slot_highest(
+    Extension(pool): Extension<PgPool>,
+) -> Result<Json<Option<WatchCanonicalSlot>>, Error> {
+    let mut conn = database::get_connection(&pool).map_err(Error::Database)?;
+    Ok(Json(database::get_highest_canonical_slot(&mut conn)?))
+}
+
+pub async fn get_slots_by_range(
+    Query(query): Query<HashMap<String, u64>>,
+    Extension(pool): Extension<PgPool>,
+) -> Result<Json<Option<Vec<WatchCanonicalSlot>>>, Error> {
+    let mut conn = database::get_connection(&pool).map_err(Error::Database)?;
+    if let Some(start_slot) = query.get("start_slot") {
+        if let Some(end_slot) = query.get("end_slot") {
+            if start_slot > end_slot {
+                Err(Error::BadRequest)
+            } else {
+                Ok(Json(database::get_canonical_slots_by_range(
+                    &mut conn,
+                    WatchSlot::new(*start_slot),
+                    WatchSlot::new(*end_slot),
+                )?))
+            }
+        } else {
+            Err(Error::BadRequest)
+        }
+    } else {
+        Err(Error::BadRequest)
+    }
+}
+
+pub async fn get_block(
+    Path(block_query): Path<String>,
+    Extension(pool): Extension<PgPool>,
+) -> Result<Json<Option<WatchBeaconBlock>>, Error> {
+    let mut conn = database::get_connection(&pool).map_err(Error::Database)?;
+    let block_id: BlockId = BlockId::from_str(&block_query).map_err(|_| Error::BadRequest)?;
+    match block_id {
+        BlockId::Slot(slot) => Ok(Json(database::get_beacon_block_by_slot(
+            &mut conn,
+            WatchSlot::from_slot(slot),
+        )?)),
+        BlockId::Root(root) => Ok(Json(database::get_beacon_block_by_root(
+            &mut conn,
+            WatchHash::from_hash(root),
+        )?)),
+        _ => Err(Error::BadRequest),
+    }
+}
+
+pub async fn get_block_lowest(
+    Extension(pool): Extension<PgPool>,
+) -> Result<Json<Option<WatchBeaconBlock>>, Error> {
+    let mut conn = database::get_connection(&pool).map_err(Error::Database)?;
+    Ok(Json(database::get_lowest_beacon_block(&mut conn)?))
+}
+
+pub async fn get_block_highest(
+    Extension(pool): Extension<PgPool>,
+) -> Result<Json<Option<WatchBeaconBlock>>, Error> {
+    let mut conn = database::get_connection(&pool).map_err(Error::Database)?;
+    Ok(Json(database::get_highest_beacon_block(&mut conn)?))
+}
+
+pub async fn get_block_previous(
+    Path(block_query): Path<String>,
+    Extension(pool): Extension<PgPool>,
+) -> Result<Json<Option<WatchBeaconBlock>>, Error> {
+    let mut conn = database::get_connection(&pool).map_err(Error::Database)?;
+    match BlockId::from_str(&block_query).map_err(|_| Error::BadRequest)? {
+        BlockId::Root(root) => {
+            if let Some(block) =
+                database::get_beacon_block_by_root(&mut conn, WatchHash::from_hash(root))?
+                    .map(|block| block.parent_root)
+            {
+                Ok(Json(database::get_beacon_block_by_root(&mut conn, block)?))
+            } else {
+                Err(Error::NotFound)
+            }
+        }
+        BlockId::Slot(slot) => Ok(Json(database::get_beacon_block_by_slot(
+            &mut conn,
+            WatchSlot::new(slot.as_u64().checked_sub(1_u64).ok_or(Error::NotFound)?),
+        )?)),
+        _ => Err(Error::BadRequest),
+    }
+}
+
+pub async fn get_block_next(
+    Path(block_query): Path<String>,
+    Extension(pool): Extension<PgPool>,
+) -> Result<Json<Option<WatchBeaconBlock>>, Error> {
+    let mut conn = database::get_connection(&pool).map_err(Error::Database)?;
+    match BlockId::from_str(&block_query).map_err(|_| Error::BadRequest)? {
+        BlockId::Root(root) => Ok(Json(database::get_beacon_block_with_parent(
+            &mut conn,
+            WatchHash::from_hash(root),
+        )?)),
+        BlockId::Slot(slot) => Ok(Json(database::get_beacon_block_by_slot(
+            &mut conn,
+            WatchSlot::from_slot(slot + 1_u64),
+        )?)),
+        _ => Err(Error::BadRequest),
+    }
+}
+
+pub async fn get_blocks_by_range(
+    Query(query): Query<HashMap<String, u64>>,
+    Extension(pool): Extension<PgPool>,
+) -> Result<Json<Option<Vec<WatchBeaconBlock>>>, Error> {
+    let mut conn = database::get_connection(&pool).map_err(Error::Database)?;
+    if let Some(start_slot) = query.get("start_slot") {
+        if let Some(end_slot) = query.get("end_slot") {
+            if start_slot > end_slot {
+                Err(Error::BadRequest)
+            } else {
+                Ok(Json(database::get_beacon_blocks_by_range(
+                    &mut conn,
+                    WatchSlot::new(*start_slot),
+                    WatchSlot::new(*end_slot),
+                )?))
+            }
+        } else {
+            Err(Error::BadRequest)
+        }
+    } else {
+        Err(Error::BadRequest)
+    }
+}
+
+pub async fn get_block_proposer(
+    Path(block_query): Path<String>,
+    Extension(pool): Extension<PgPool>,
+) -> Result<Json<Option<WatchProposerInfo>>, Error> {
+    let mut conn = database::get_connection(&pool).map_err(Error::Database)?;
+    match BlockId::from_str(&block_query).map_err(|_| Error::BadRequest)? {
+        BlockId::Root(root) => Ok(Json(database::get_proposer_info_by_root(
+            &mut conn,
+            WatchHash::from_hash(root),
+        )?)),
+        BlockId::Slot(slot) => Ok(Json(database::get_proposer_info_by_slot(
+            &mut conn,
+            WatchSlot::from_slot(slot),
+        )?)),
+        _ => Err(Error::BadRequest),
+    }
+}
+
+pub async fn get_validator(
+    Path(validator_query): Path<String>,
+    Extension(pool): Extension<PgPool>,
+) -> Result<Json<Option<WatchValidator>>, Error> {
+    let mut conn = database::get_connection(&pool).map_err(Error::Database)?;
+    if validator_query.starts_with("0x") {
+        let pubkey = WatchPK::from_str(&validator_query).map_err(|_| Error::BadRequest)?;
+        Ok(Json(database::get_validator_by_public_key(
+            &mut conn, pubkey,
+        )?))
+    } else {
+        let index = i32::from_str(&validator_query).map_err(|_| Error::BadRequest)?;
+        Ok(Json(database::get_validator_by_index(&mut conn, index)?))
+    }
+}
+
+pub async fn get_all_validators(
+    Extension(pool): Extension<PgPool>,
+) -> Result<Json<Vec<WatchValidator>>, Error> {
+    let mut conn = database::get_connection(&pool).map_err(Error::Database)?;
+    Ok(Json(database::get_all_validators(&mut conn)?))
+}
+
+pub async fn get_validator_latest_proposal(
+    Path(validator_query): Path<String>,
+    Extension(pool): Extension<PgPool>,
+) -> Result<Json<HashMap<i32, WatchProposerInfo>>, Error> {
+    let mut conn = database::get_connection(&pool).map_err(Error::Database)?;
+    if validator_query.starts_with("0x") {
+        let pubkey = WatchPK::from_str(&validator_query).map_err(|_| Error::BadRequest)?;
+        let validator =
+            database::get_validator_by_public_key(&mut conn, pubkey)?.ok_or(Error::NotFound)?;
+        Ok(Json(database::get_validators_latest_proposer_info(
+            &mut conn,
+            vec![validator.index],
+        )?))
+    } else {
+        let index = i32::from_str(&validator_query).map_err(|_| Error::BadRequest)?;
+        Ok(Json(database::get_validators_latest_proposer_info(
+            &mut conn,
+            vec![index],
+        )?))
+    }
+}
+
+pub async fn get_client_breakdown(
+    Extension(pool): Extension<PgPool>,
+    Extension(slots_per_epoch): Extension<u64>,
+) -> Result<Json<HashMap<String, usize>>, Error> {
+    let mut conn = database::get_connection(&pool).map_err(Error::Database)?;
+
+    if let Some(target_slot) = database::get_highest_canonical_slot(&mut conn)? {
+        Ok(Json(database::get_validators_clients_at_slot(
+            &mut conn,
+            target_slot.slot,
+            slots_per_epoch,
+        )?))
+    } else {
+        Err(Error::Database(DbError::Other(
+            "No slots found in database.".to_string(),
+        )))
+    }
+}
+
+pub async fn get_client_breakdown_percentages(
+    Extension(pool): Extension<PgPool>,
+    Extension(slots_per_epoch): Extension<u64>,
+) -> Result<Json<HashMap<String, f64>>, Error> {
+    let mut conn = database::get_connection(&pool).map_err(Error::Database)?;
+
+    let mut result = HashMap::new();
+    if let Some(target_slot) = database::get_highest_canonical_slot(&mut conn)? {
+        let total = database::count_validators_activated_before_slot(
+            &mut conn,
+            target_slot.slot,
+            slots_per_epoch,
+        )?;
+        let clients =
+            database::get_validators_clients_at_slot(&mut conn, target_slot.slot, slots_per_epoch)?;
+        for (client, number) in clients.iter() {
+            let percentage: f64 = *number as f64 / total as f64 * 100.0;
+            result.insert(client.to_string(), percentage);
+        }
+    }
+
+    Ok(Json(result))
+}

watch/src/server/mod.rs

@@ -0,0 +1,134 @@
+use crate::block_packing::block_packing_routes;
+use crate::block_rewards::block_rewards_routes;
+use crate::blockprint::blockprint_routes;
+use crate::config::Config as FullConfig;
+use crate::database::{self, PgPool};
+use crate::suboptimal_attestations::{attestation_routes, blockprint_attestation_routes};
+use axum::{
+    handler::Handler,
+    http::{StatusCode, Uri},
+    routing::get,
+    Extension, Json, Router,
+};
+use eth2::types::ErrorMessage;
+use log::info;
+use std::future::Future;
+use std::net::SocketAddr;
+use tokio::sync::oneshot;
+
+pub use config::Config;
+pub use error::Error;
+
+mod config;
+mod error;
+mod handler;
+
+pub async fn serve(config: FullConfig, shutdown: oneshot::Receiver<()>) -> Result<(), Error> {
+    let db = database::build_connection_pool(&config.database)?;
+    let (_, slots_per_epoch) = database::get_active_config(&mut database::get_connection(&db)?)?
+        .ok_or_else(|| {
+            Error::Other(
+                "Database not found. Please run the updater prior to starting the server"
+                    .to_string(),
+            )
+        })?;
+
+    let server = start_server(&config, slots_per_epoch as u64, db, async {
+        let _ = shutdown.await;
+    })?;
+
+    server.await?;
+
+    Ok(())
+}
+
+/// Creates a server that will serve requests using information from `config`.
+///
+/// The server will create its own connection pool to serve connections to the database.
+/// This is separate to the connection pool that is used for the `updater`.
+///
+/// The server will shut down gracefully when the `shutdown` future resolves.
+///
+/// ## Returns
+///
+/// This function will bind the server to the address specified in the config and then return a
+/// Future representing the actual server that will need to be awaited.
+///
+/// ## Errors
+///
+/// Returns an error if the server is unable to bind or there is another error during
+/// configuration.
+pub fn start_server(
+    config: &FullConfig,
+    slots_per_epoch: u64,
+    pool: PgPool,
+    shutdown: impl Future<Output = ()> + Send + Sync + 'static,
+) -> Result<impl Future<Output = Result<(), hyper::Error>> + 'static, Error> {
+    let mut routes = Router::new()
+        .route("/v1/slots", get(handler::get_slots_by_range))
+        .route("/v1/slots/:slot", get(handler::get_slot))
+        .route("/v1/slots/lowest", get(handler::get_slot_lowest))
+        .route("/v1/slots/highest", get(handler::get_slot_highest))
+        .route("/v1/slots/:slot/block", get(handler::get_block))
+        .route("/v1/blocks", get(handler::get_blocks_by_range))
+        .route("/v1/blocks/:block", get(handler::get_block))
+        .route("/v1/blocks/lowest", get(handler::get_block_lowest))
+        .route("/v1/blocks/highest", get(handler::get_block_highest))
+        .route(
+            "/v1/blocks/:block/previous",
+            get(handler::get_block_previous),
+        )
+        .route("/v1/blocks/:block/next", get(handler::get_block_next))
+        .route(
+            "/v1/blocks/:block/proposer",
+            get(handler::get_block_proposer),
+        )
+        .route("/v1/validators/:validator", get(handler::get_validator))
+        .route("/v1/validators/all", get(handler::get_all_validators))
+        .route(
+            "/v1/validators/:validator/latest_proposal",
+            get(handler::get_validator_latest_proposal),
+        )
+        .route("/v1/clients", get(handler::get_client_breakdown))
+        .route(
+            "/v1/clients/percentages",
+            get(handler::get_client_breakdown_percentages),
+        )
+        .merge(attestation_routes())
+        .merge(blockprint_routes())
+        .merge(block_packing_routes())
+        .merge(block_rewards_routes());
+
+    if config.blockprint.enabled && config.updater.attestations {
+        routes = routes.merge(blockprint_attestation_routes())
+    }
+
+    let app = routes
+        .fallback(route_not_found.into_service())
+        .layer(Extension(pool))
+        .layer(Extension(slots_per_epoch));
+
+    let addr = SocketAddr::new(config.server.listen_addr, config.server.listen_port);
+
+    let server = axum::Server::try_bind(&addr)?.serve(app.into_make_service());
+
+    let server = server.with_graceful_shutdown(async {
+        shutdown.await;
+    });
+
+    info!("HTTP server listening on {}", addr);
+
+    Ok(server)
+}
+
+// The default route indicating that no available routes matched the request.
+async fn route_not_found(uri: Uri) -> (StatusCode, Json<ErrorMessage>) {
+    (
+        StatusCode::METHOD_NOT_ALLOWED,
+        Json(ErrorMessage {
+            code: StatusCode::METHOD_NOT_ALLOWED.as_u16(),
+            message: format!("No route for {uri}"),
+            stacktraces: vec![],
+        }),
+    )
+}