resolve merge conflicts

validator_client/beacon_node_fallback/Cargo.toml

@@ -11,7 +11,7 @@ path = "src/lib.rs"
 [dependencies]
 bls = { workspace = true }
 clap = { workspace = true }
-eth2 = { workspace = true }
+eth2 = { workspace = true, features = ["events"] }
 futures = { workspace = true }
 itertools = { workspace = true }
 sensitive_url = { workspace = true }

validator_client/beacon_node_fallback/src/beacon_head_monitor.rs

@@ -0,0 +1,423 @@
+use crate::BeaconNodeFallback;
+use eth2::types::{EventKind, EventTopic, Hash256, SseHead};
+use futures::StreamExt;
+use slot_clock::SlotClock;
+use std::collections::HashMap;
+use std::sync::Arc;
+use tokio::sync::RwLock;
+use tracing::{debug, info, warn};
+use types::EthSpec;
+
+type CacheHashMap = HashMap<usize, SseHead>;
+
+// This is used to send the index derived from `CandidateBeaconNode` to the
+// `AttestationService` for further processing
+#[derive(Debug)]
+pub struct HeadEvent {
+    pub beacon_node_index: usize,
+    pub slot: types::Slot,
+    pub beacon_block_root: Hash256,
+}
+
+/// Cache to maintain the latest head received from each of the beacon nodes
+/// in the `BeaconNodeFallback`.
+#[derive(Debug)]
+pub struct BeaconHeadCache {
+    cache: RwLock<CacheHashMap>,
+}
+
+impl BeaconHeadCache {
+    /// Creates a new empty beacon head cache.
+    pub fn new() -> Self {
+        Self {
+            cache: RwLock::new(HashMap::new()),
+        }
+    }
+
+    /// Retrieves the cached head for a specific beacon node.
+    /// Returns `None` if no head has been cached for that node yet.
+    pub async fn get(&self, beacon_node_index: usize) -> Option<SseHead> {
+        self.cache.read().await.get(&beacon_node_index).cloned()
+    }
+
+    /// Stores or updates the head event for a specific beacon node.
+    /// Replaces any previously cached head for the given node.
+    pub async fn insert(&self, beacon_node_index: usize, head: SseHead) {
+        self.cache.write().await.insert(beacon_node_index, head);
+    }
+
+    /// Checks if the given head is the latest among all cached heads.
+    /// Returns `true` if the head's slot is >= all cached heads' slots.
+    pub async fn is_latest(&self, head: &SseHead) -> bool {
+        let cache = self.cache.read().await;
+        cache
+            .values()
+            .all(|cache_head| head.slot >= cache_head.slot)
+    }
+
+    /// Clears all cached heads, removing entries for all beacon nodes.
+    /// Useful when beacon node candidates are refreshed to avoid stale references.
+    pub async fn purge_cache(&self) {
+        self.cache.write().await.clear();
+    }
+}
+
+impl Default for BeaconHeadCache {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+// Runs a non-terminating loop to update the `BeaconHeadCache` with the latest head received
+// from the candidate beacon_nodes. This is an attempt to stream events to beacon nodes and
+// potential start attestation duties earlier as soon as latest head is receive from any of the
+// beacon node in contrast to attest at the 1/3rd mark in the slot.
+//
+//
+// The cache and the candidate BNs list are refresh/purged to avoid dangling reference conditions
+// that arise due to `update_candidates_list`.
+//
+// Starts the service to perpetually stream head events from connected beacon_nodes
+pub async fn poll_head_event_from_beacon_nodes<E: EthSpec, T: SlotClock + 'static>(
+    beacon_nodes: Arc<BeaconNodeFallback<T>>,
+) -> Result<(), String> {
+    let head_cache = beacon_nodes
+        .beacon_head_cache
+        .clone()
+        .ok_or("Unable to start head monitor without beacon_head_cache")?;
+    let head_monitor_send = beacon_nodes
+        .head_monitor_send
+        .clone()
+        .ok_or("Unable to start head monitor without head_monitor_send")?;
+
+    info!("Starting head monitoring service");
+    let candidates = {
+        let candidates_guard = beacon_nodes.candidates.read().await;
+        candidates_guard.clone()
+    };
+
+    // Clear the cache in case it contains stale data from a previous run. This function gets
+    // restarted if it fails (see monitoring in `start_fallback_updater_service`).
+    head_cache.purge_cache().await;
+
+    // Create Vec of streams, which we will select over.
+    let mut streams = vec![];
+
+    for candidate in &candidates {
+        let head_event_stream = candidate
+            .beacon_node
+            .get_events::<E>(&[EventTopic::Head])
+            .await;
+
+        let head_event_stream = match head_event_stream {
+            Ok(stream) => stream,
+            Err(e) => {
+                warn!(error = ?e, node_index = candidate.index, "Failed to get head event stream");
+                continue;
+            }
+        };
+
+        streams.push(head_event_stream.map(|event| (candidate.index, event)));
+    }
+
+    if streams.is_empty() {
+        return Err("No beacon nodes available for head event streaming".to_string());
+    }
+
+    // Combine streams into a single stream and poll events from any of them.
+    let mut combined_stream = futures::stream::select_all(streams);
+
+    while let Some((candidate_index, event_result)) = combined_stream.next().await {
+        match event_result {
+            Ok(EventKind::Head(head)) => {
+                debug!(
+                    candidate_index,
+                    block_root = ?head.block,
+                    slot = %head.slot,
+                    "New head from beacon node"
+                );
+
+                // Skip optimistic heads - the beacon node can't produce valid
+                // attestation data when its execution layer is not verified
+                if head.execution_optimistic {
+                    debug!(
+                        candidate_index,
+                        block_root = ?head.block,
+                        slot = %head.slot,
+                        "Skipping optimistic head"
+                    );
+                    continue;
+                }
+
+                head_cache.insert(candidate_index, head.clone()).await;
+
+                if !head_cache.is_latest(&head).await {
+                    debug!(
+                        candidate_index,
+                        block_root = ?head.block,
+                        slot = %head.slot,
+                        "Skipping stale head"
+                    );
+                    continue;
+                }
+
+                if head_monitor_send
+                    .send(HeadEvent {
+                        beacon_node_index: candidate_index,
+                        slot: head.slot,
+                        beacon_block_root: head.block,
+                    })
+                    .await
+                    .is_err()
+                {
+                    return Err("Head monitoring service channel closed".into());
+                }
+            }
+            Ok(event) => {
+                warn!(
+                    event_kind = event.topic_name(),
+                    candidate_index, "Received unexpected event from BN"
+                );
+                continue;
+            }
+            Err(e) => {
+                return Err(format!(
+                    "Head monitoring stream error, node: {candidate_index}, error: {e:?}"
+                ));
+            }
+        }
+    }
+
+    Err("Stream ended unexpectedly".into())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use bls::FixedBytesExtended;
+    use types::{Hash256, Slot};
+
+    fn create_sse_head(slot: u64, block_root: u8) -> SseHead {
+        SseHead {
+            slot: types::Slot::new(slot),
+            block: Hash256::from_low_u64_be(block_root as u64),
+            state: Hash256::from_low_u64_be(block_root as u64),
+            epoch_transition: false,
+            previous_duty_dependent_root: Hash256::from_low_u64_be(block_root as u64),
+            current_duty_dependent_root: Hash256::from_low_u64_be(block_root as u64),
+            execution_optimistic: false,
+        }
+    }
+
+    #[tokio::test]
+    async fn test_beacon_head_cache_insertion_and_retrieval() {
+        let cache = BeaconHeadCache::new();
+        let head_1 = create_sse_head(1, 1);
+        let head_2 = create_sse_head(2, 2);
+
+        cache.insert(0, head_1.clone()).await;
+        cache.insert(1, head_2.clone()).await;
+
+        assert_eq!(cache.get(0).await, Some(head_1));
+        assert_eq!(cache.get(1).await, Some(head_2));
+        assert_eq!(cache.get(2).await, None);
+    }
+
+    #[tokio::test]
+    async fn test_beacon_head_cache_update() {
+        let cache = BeaconHeadCache::new();
+        let head_old = create_sse_head(1, 1);
+        let head_new = create_sse_head(2, 2);
+
+        cache.insert(0, head_old).await;
+        cache.insert(0, head_new.clone()).await;
+
+        assert_eq!(cache.get(0).await, Some(head_new));
+    }
+
+    #[tokio::test]
+    async fn test_is_latest_with_higher_slot() {
+        let cache = BeaconHeadCache::new();
+        let head_1 = create_sse_head(1, 1);
+        let head_2 = create_sse_head(2, 2);
+        let head_3 = create_sse_head(3, 3);
+
+        cache.insert(0, head_1).await;
+        cache.insert(1, head_2).await;
+
+        assert!(cache.is_latest(&head_3).await);
+    }
+
+    #[tokio::test]
+    async fn test_is_latest_with_lower_slot() {
+        let cache = BeaconHeadCache::new();
+        let head_1 = create_sse_head(1, 1);
+        let head_2 = create_sse_head(2, 2);
+        let head_older = create_sse_head(1, 99);
+
+        cache.insert(0, head_1).await;
+        cache.insert(1, head_2).await;
+
+        assert!(!cache.is_latest(&head_older).await);
+    }
+
+    #[tokio::test]
+    async fn test_is_latest_with_equal_slot() {
+        let cache = BeaconHeadCache::new();
+        let head_1 = create_sse_head(5, 1);
+        let head_2 = create_sse_head(5, 2);
+        let head_equal = create_sse_head(5, 3);
+
+        cache.insert(0, head_1).await;
+        cache.insert(1, head_2).await;
+
+        assert!(cache.is_latest(&head_equal).await);
+    }
+
+    #[tokio::test]
+    async fn test_is_latest_empty_cache() {
+        let cache = BeaconHeadCache::new();
+        let head = create_sse_head(1, 1);
+
+        assert!(cache.is_latest(&head).await);
+    }
+
+    #[tokio::test]
+    async fn test_purge_cache_clears_all_entries() {
+        let cache = BeaconHeadCache::new();
+        let head_1 = create_sse_head(1, 1);
+        let head_2 = create_sse_head(2, 2);
+
+        cache.insert(0, head_1).await;
+        cache.insert(1, head_2).await;
+
+        assert!(cache.get(0).await.is_some());
+        assert!(cache.get(1).await.is_some());
+
+        cache.purge_cache().await;
+
+        assert!(cache.get(0).await.is_none());
+        assert!(cache.get(1).await.is_none());
+    }
+
+    #[tokio::test]
+    async fn test_head_event_creation() {
+        let block_root = Hash256::from_low_u64_be(99);
+        let event = HeadEvent {
+            beacon_node_index: 42,
+            slot: Slot::new(123),
+            beacon_block_root: block_root,
+        };
+        assert_eq!(event.beacon_node_index, 42);
+        assert_eq!(event.slot, Slot::new(123));
+        assert_eq!(event.beacon_block_root, block_root);
+    }
+
+    #[tokio::test]
+    async fn test_cache_caches_multiple_heads_from_different_nodes() {
+        let cache = BeaconHeadCache::new();
+        let head_1 = create_sse_head(10, 1);
+        let head_2 = create_sse_head(5, 2);
+        let head_3 = create_sse_head(8, 3);
+
+        cache.insert(0, head_1.clone()).await;
+        cache.insert(1, head_2.clone()).await;
+        cache.insert(2, head_3.clone()).await;
+
+        // Verify all are stored
+        assert_eq!(cache.get(0).await, Some(head_1));
+        assert_eq!(cache.get(1).await, Some(head_2));
+        assert_eq!(cache.get(2).await, Some(head_3));
+
+        // The latest should be slot 10
+        let head_10 = create_sse_head(10, 99);
+        assert!(cache.is_latest(&head_10).await);
+
+        // Anything with slot > 10 should be latest
+        let head_11 = create_sse_head(11, 99);
+        assert!(cache.is_latest(&head_11).await);
+
+        // Anything with slot < 10 should not be latest
+        let head_9 = create_sse_head(9, 99);
+        assert!(!cache.is_latest(&head_9).await);
+    }
+
+    #[tokio::test]
+    async fn test_cache_handles_concurrent_operations() {
+        let cache = Arc::new(BeaconHeadCache::new());
+        let mut handles = vec![];
+
+        // Spawn multiple tasks that insert heads concurrently
+        for i in 0..10 {
+            let cache_clone = cache.clone();
+            let handle = tokio::spawn(async move {
+                let head = create_sse_head(i as u64, (i % 256) as u8);
+                cache_clone.insert(i, head).await;
+            });
+            handles.push(handle);
+        }
+
+        // Wait for all tasks to complete
+        for handle in handles {
+            handle.await.unwrap();
+        }
+
+        // Verify all heads are cached
+        for i in 0..10 {
+            assert!(cache.get(i).await.is_some());
+        }
+    }
+
+    #[tokio::test]
+    async fn test_is_latest_after_cache_updates() {
+        let cache = BeaconHeadCache::new();
+
+        // Start with head at slot 5
+        let head_5 = create_sse_head(5, 1);
+        cache.insert(0, head_5.clone()).await;
+        assert!(cache.is_latest(&head_5).await);
+
+        // Add a higher slot
+        let head_10 = create_sse_head(10, 2);
+        cache.insert(1, head_10.clone()).await;
+
+        // head_5 should no longer be latest
+        assert!(!cache.is_latest(&head_5).await);
+        // head_10 should be latest
+        assert!(cache.is_latest(&head_10).await);
+
+        // Add an even higher slot
+        let head_15 = create_sse_head(15, 3);
+        cache.insert(2, head_15.clone()).await;
+
+        // head_10 should no longer be latest
+        assert!(!cache.is_latest(&head_10).await);
+        // head_15 should be latest
+        assert!(cache.is_latest(&head_15).await);
+    }
+
+    #[tokio::test]
+    async fn test_cache_default_is_empty() {
+        let cache = BeaconHeadCache::default();
+        assert!(cache.get(0).await.is_none());
+        assert!(cache.get(999).await.is_none());
+    }
+
+    #[tokio::test]
+    async fn test_is_latest_with_multiple_same_slot_heads() {
+        let cache = BeaconHeadCache::new();
+        let head_slot_5_node1 = create_sse_head(5, 1);
+        let head_slot_5_node2 = create_sse_head(5, 2);
+        let head_slot_5_node3 = create_sse_head(5, 3);
+
+        cache.insert(0, head_slot_5_node1).await;
+        cache.insert(1, head_slot_5_node2).await;
+
+        // All heads with slot 5 should be considered latest
+        assert!(cache.is_latest(&head_slot_5_node3).await);
+
+        // But heads with slot 4 should not be latest
+        let head_slot_4 = create_sse_head(4, 4);
+        assert!(!cache.is_latest(&head_slot_4).await);
+    }
+}

validator_client/beacon_node_fallback/src/lib.rs

@@ -2,7 +2,10 @@
 //! "fallback" behaviour; it will try a request on all of the nodes until one or none of them
 //! succeed.
 
+pub mod beacon_head_monitor;
 pub mod beacon_node_health;
+
+use beacon_head_monitor::{BeaconHeadCache, HeadEvent, poll_head_event_from_beacon_nodes};
 use beacon_node_health::{
     BeaconNodeHealth, BeaconNodeSyncDistanceTiers, ExecutionEngineHealth, IsOptimistic,
     SyncDistanceTier, check_node_health,
@@ -22,7 +25,10 @@ use std::time::{Duration, Instant};
 use std::vec::Vec;
 use strum::VariantNames;
 use task_executor::TaskExecutor;
-use tokio::{sync::RwLock, time::sleep};
+use tokio::{
+    sync::{RwLock, mpsc},
+    time::sleep,
+};
 use tracing::{debug, error, warn};
 use types::{ChainSpec, Config as ConfigSpec, EthSpec, Slot};
 use validator_metrics::{ENDPOINT_ERRORS, ENDPOINT_REQUESTS, inc_counter_vec};
@@ -68,6 +74,31 @@ pub fn start_fallback_updater_service<T: SlotClock + 'static, E: EthSpec>(
         return Err("Cannot start fallback updater without slot clock");
     }
 
+    let beacon_nodes_ref = beacon_nodes.clone();
+
+    // the existence of head_monitor_send is overloaded with the predicate of
+    // requirement of starting the head monitoring service or not.
+    if beacon_nodes_ref.head_monitor_send.is_some() {
+        let head_monitor_future = async move {
+            loop {
+                if let Err(error) =
+                    poll_head_event_from_beacon_nodes::<E, T>(beacon_nodes_ref.clone()).await
+                {
+                    warn!(error, "Head service failed retrying starting next slot");
+
+                    let sleep_time = beacon_nodes_ref
+                        .slot_clock
+                        .as_ref()
+                        .and_then(|slot_clock| slot_clock.duration_to_next_slot())
+                        .unwrap_or_else(|| beacon_nodes_ref.spec.get_slot_duration());
+                    sleep(sleep_time).await
+                }
+            }
+        };
+
+        executor.spawn(head_monitor_future, "head_monitoring");
+    }
+
     let future = async move {
         loop {
             beacon_nodes.update_all_candidates::<E>().await;
@@ -96,12 +127,15 @@ pub fn start_fallback_updater_service<T: SlotClock + 'static, E: EthSpec>(
 pub enum Error<T> {
     /// We attempted to contact the node but it failed.
     RequestFailed(T),
+    /// The beacon node with the requested index was not available.
+    CandidateIndexUnknown(usize),
 }
 
 impl<T> Error<T> {
     pub fn request_failure(&self) -> Option<&T> {
         match self {
             Error::RequestFailed(e) => Some(e),
+            Error::CandidateIndexUnknown(_) => None,
         }
     }
 }
@@ -380,6 +414,8 @@ pub struct BeaconNodeFallback<T> {
     pub candidates: Arc<RwLock<Vec<CandidateBeaconNode>>>,
     distance_tiers: BeaconNodeSyncDistanceTiers,
     slot_clock: Option<T>,
+    beacon_head_cache: Option<Arc<BeaconHeadCache>>,
+    head_monitor_send: Option<Arc<mpsc::Sender<HeadEvent>>>,
     broadcast_topics: Vec<ApiTopic>,
     spec: Arc<ChainSpec>,
 }
@@ -396,6 +432,8 @@ impl<T: SlotClock> BeaconNodeFallback<T> {
             candidates: Arc::new(RwLock::new(candidates)),
             distance_tiers,
             slot_clock: None,
+            beacon_head_cache: None,
+            head_monitor_send: None,
             broadcast_topics,
             spec,
         }
@@ -410,6 +448,15 @@ impl<T: SlotClock> BeaconNodeFallback<T> {
         self.slot_clock = Some(slot_clock);
     }
 
+    /// This the head monitor channel that streams events from all the beacon nodes that the
+    /// validator client is connected in the `BeaconNodeFallback`. This also initializes the
+    /// beacon_head_cache under the assumption the beacon_head_cache will always be needed when
+    /// head_monitor_send is set.
+    pub fn set_head_send(&mut self, head_monitor_send: Arc<mpsc::Sender<HeadEvent>>) {
+        self.head_monitor_send = Some(head_monitor_send);
+        self.beacon_head_cache = Some(Arc::new(BeaconHeadCache::new()));
+    }
+
     /// The count of candidates, regardless of their state.
     pub async fn num_total(&self) -> usize {
         self.candidates.read().await.len()
@@ -476,9 +523,9 @@ impl<T: SlotClock> BeaconNodeFallback<T> {
         }
 
         let timeouts: Timeouts = if new_list.len() == 1 || use_long_timeouts {
-            Timeouts::set_all(Duration::from_secs(self.spec.seconds_per_slot))
+            Timeouts::set_all(self.spec.get_slot_duration())
         } else {
-            Timeouts::use_optimized_timeouts(Duration::from_secs(self.spec.seconds_per_slot))
+            Timeouts::use_optimized_timeouts(self.spec.get_slot_duration())
         };
 
         let new_candidates: Vec<CandidateBeaconNode> = new_list
@@ -493,6 +540,10 @@ impl<T: SlotClock> BeaconNodeFallback<T> {
         let mut candidates = self.candidates.write().await;
         *candidates = new_candidates;
 
+        if let Some(cache) = &self.beacon_head_cache {
+            cache.purge_cache().await;
+        }
+
         Ok(new_list)
     }
 
@@ -646,6 +697,32 @@ impl<T: SlotClock> BeaconNodeFallback<T> {
         Err(Errors(errors))
     }
 
+    /// Try `func` on a specific beacon node by index.
+    ///
+    /// Returns immediately if the preferred node succeeds, otherwise return an error.
+    pub async fn run_on_candidate_index<F, O, Err, R>(
+        &self,
+        candidate_index: usize,
+        func: F,
+    ) -> Result<O, Error<Err>>
+    where
+        F: Fn(BeaconNodeHttpClient) -> R + Clone,
+        R: Future<Output = Result<O, Err>>,
+        Err: Debug,
+    {
+        // Find the requested beacon node or return an error.
+        let candidates = self.candidates.read().await;
+        let Some(candidate) = candidates.iter().find(|c| c.index == candidate_index) else {
+            return Err(Error::CandidateIndexUnknown(candidate_index));
+        };
+        let candidate_node = candidate.beacon_node.clone();
+        drop(candidates);
+
+        Self::run_on_candidate(candidate_node, &func)
+            .await
+            .map_err(|(_, err)| err)
+    }
+
     /// Run the future `func` on `candidate` while reporting metrics.
     async fn run_on_candidate<F, R, Err, O>(
         candidate: BeaconNodeHttpClient,
@@ -1080,4 +1157,60 @@ mod tests {
         mock1.expect(3).assert();
         mock2.expect(3).assert();
     }
+
+    #[tokio::test]
+    async fn run_on_candidate_index_success() {
+        let spec = Arc::new(MainnetEthSpec::default_spec());
+        let (mut mock_beacon_node_1, beacon_node_1) = new_mock_beacon_node(0, &spec).await;
+        let (mut mock_beacon_node_2, beacon_node_2) = new_mock_beacon_node(1, &spec).await;
+        let (mut mock_beacon_node_3, beacon_node_3) = new_mock_beacon_node(2, &spec).await;
+
+        let beacon_node_fallback = create_beacon_node_fallback(
+            vec![beacon_node_1, beacon_node_2, beacon_node_3],
+            vec![],
+            spec.clone(),
+        );
+
+        let mock1 = mock_beacon_node_1.mock_offline_node();
+        let _mock2 = mock_beacon_node_2.mock_online_node();
+        let mock3 = mock_beacon_node_3.mock_online_node();
+
+        // Request with preferred_index=1 (beacon_node_2)
+        let result = beacon_node_fallback
+            .run_on_candidate_index(1, |client| async move { client.get_node_version().await })
+            .await;
+
+        // Should succeed since beacon_node_2 is online
+        assert!(result.is_ok());
+
+        // mock1 should not be called since preferred node succeeds
+        mock1.expect(0).assert();
+        mock3.expect(0).assert();
+    }
+
+    #[tokio::test]
+    async fn run_on_candidate_index_error() {
+        let spec = Arc::new(MainnetEthSpec::default_spec());
+        let (mut mock_beacon_node_1, beacon_node_1) = new_mock_beacon_node(0, &spec).await;
+        let (mut mock_beacon_node_2, beacon_node_2) = new_mock_beacon_node(1, &spec).await;
+        let (mut mock_beacon_node_3, beacon_node_3) = new_mock_beacon_node(2, &spec).await;
+
+        let beacon_node_fallback = create_beacon_node_fallback(
+            vec![beacon_node_1, beacon_node_2, beacon_node_3],
+            vec![],
+            spec.clone(),
+        );
+
+        let _mock1 = mock_beacon_node_1.mock_online_node();
+        let _mock2 = mock_beacon_node_2.mock_offline_node();
+        let _mock3 = mock_beacon_node_3.mock_offline_node();
+
+        // Request with preferred_index=1 (beacon_node_2), but it's offline
+        let result = beacon_node_fallback
+            .run_on_candidate_index(1, |client| async move { client.get_node_version().await })
+            .await;
+
+        // Should fail.
+        assert!(result.is_err());
+    }
 }

validator_client/http_api/Cargo.toml

@@ -8,14 +8,17 @@ authors = ["Sigma Prime <contact@sigmaprime.io>"]
 name = "validator_http_api"
 path = "src/lib.rs"
 
+[features]
+testing = ["dep:deposit_contract", "dep:doppelganger_service", "dep:tempfile"]
+
 [dependencies]
 account_utils = { workspace = true }
 beacon_node_fallback = { workspace = true }
 bls = { workspace = true }
-deposit_contract = { workspace = true }
+deposit_contract = { workspace = true, optional = true }
 directory = { workspace = true }
 dirs = { workspace = true }
-doppelganger_service = { workspace = true }
+doppelganger_service = { workspace = true, optional = true }
 eth2 = { workspace = true, features = ["lighthouse"] }
 eth2_keystore = { workspace = true }
 ethereum_serde_utils = { workspace = true }
@@ -38,7 +41,7 @@ slot_clock = { workspace = true }
 sysinfo = { workspace = true }
 system_health = { workspace = true }
 task_executor = { workspace = true }
-tempfile = { workspace = true }
+tempfile = { workspace = true, optional = true }
 tokio = { workspace = true }
 tokio-stream = { workspace = true }
 tracing = { workspace = true }
@@ -53,7 +56,10 @@ warp_utils = { workspace = true }
 zeroize = { workspace = true }
 
 [dev-dependencies]
+deposit_contract = { workspace = true }
+doppelganger_service = { workspace = true }
 futures = { workspace = true }
 itertools = { workspace = true }
 rand = { workspace = true, features = ["small_rng"] }
 ssz_types = { workspace = true }
+tempfile = { workspace = true }

validator_client/http_api/src/keystores.rs

@@ -102,10 +102,8 @@ pub fn import<T: SlotClock + 'static, E: EthSpec>(
     // Import each keystore. Some keystores may fail to be imported, so we record a status for each.
     let mut statuses = Vec::with_capacity(request.keystores.len());
 
-    for (KeystoreJsonStr(keystore), password) in request
-        .keystores
-        .into_iter()
-        .zip(request.passwords.into_iter())
+    for (KeystoreJsonStr(keystore), password) in
+        request.keystores.into_iter().zip(request.passwords)
     {
         let pubkey_str = keystore.pubkey().to_string();
 

validator_client/http_api/src/lib.rs

@@ -1,3 +1,6 @@
+#[cfg(feature = "testing")]
+pub mod test_utils;
+
 mod api_secret;
 mod create_signed_voluntary_exit;
 mod create_validator;
@@ -6,7 +9,6 @@ mod keystores;
 mod remotekeys;
 mod tests;
 
-pub mod test_utils;
 pub use api_secret::PK_FILENAME;
 
 use graffiti::{delete_graffiti, get_graffiti, set_graffiti};

validator_client/http_api/src/tests/keystores.rs

@@ -9,6 +9,7 @@ use eth2::lighthouse_vc::{
     types::Web3SignerValidatorRequest,
 };
 use fixed_bytes::FixedBytesExtended;
+use futures::StreamExt;
 use itertools::Itertools;
 use lighthouse_validator_store::DEFAULT_GAS_LIMIT;
 use rand::rngs::StdRng;
@@ -19,6 +20,7 @@ use std::{collections::HashMap, path::Path};
 use tokio::runtime::Handle;
 use typenum::Unsigned;
 use types::{Address, attestation::AttestationBase};
+use validator_store::AttestationToSign;
 use validator_store::ValidatorStore;
 use zeroize::Zeroizing;
 
@@ -1099,14 +1101,23 @@ async fn generic_migration_test(
         check_keystore_import_response(&import_res, all_imported(keystores.len()));
 
         // Sign attestations on VC1.
-        for (validator_index, mut attestation) in first_vc_attestations {
+        for (validator_index, attestation) in first_vc_attestations {
             let public_key = keystore_pubkey(&keystores[validator_index]);
-            let current_epoch = attestation.data().target.epoch;
-            tester1
+            let stream = tester1
                 .validator_store
-                .sign_attestation(public_key, 0, &mut attestation, current_epoch)
-                .await
-                .unwrap();
+                .sign_attestations(vec![AttestationToSign {
+                    validator_index: 0,
+                    pubkey: public_key,
+                    validator_committee_index: 0,
+                    attestation: attestation.clone(),
+                }]);
+            tokio::pin!(stream);
+            let safe_attestations = stream.next().await.unwrap().unwrap();
+            assert_eq!(safe_attestations.len(), 1);
+            // Compare data only, ignoring signatures which are added during signing.
+            assert_eq!(safe_attestations[0].1.data(), attestation.data());
+            // Check that the signature is non-zero.
+            assert!(!safe_attestations[0].1.signature().is_infinity());
         }
 
         // Delete the selected keys from VC1.
@@ -1178,16 +1189,34 @@ async fn generic_migration_test(
         check_keystore_import_response(&import_res, all_imported(import_indices.len()));
 
         // Sign attestations on the second VC.
-        for (validator_index, mut attestation, should_succeed) in second_vc_attestations {
+        for (validator_index, attestation, should_succeed) in second_vc_attestations {
             let public_key = keystore_pubkey(&keystores[validator_index]);
-            let current_epoch = attestation.data().target.epoch;
-            match tester2
+            let stream = tester2
                 .validator_store
-                .sign_attestation(public_key, 0, &mut attestation, current_epoch)
-                .await
-            {
-                Ok(()) => assert!(should_succeed),
-                Err(e) => assert!(!should_succeed, "{:?}", e),
+                .sign_attestations(vec![AttestationToSign {
+                    validator_index: 0,
+                    pubkey: public_key,
+                    validator_committee_index: 0,
+                    attestation: attestation.clone(),
+                }]);
+            tokio::pin!(stream);
+            let result = stream.next().await.unwrap();
+            match result {
+                Ok(safe_attestations) => {
+                    if should_succeed {
+                        // Compare data only, ignoring signatures which are added during signing.
+                        assert_eq!(safe_attestations.len(), 1);
+                        assert_eq!(safe_attestations[0].1.data(), attestation.data());
+                        // Check that the signature is non-zero.
+                        assert!(!safe_attestations[0].1.signature().is_infinity());
+                    } else {
+                        assert!(safe_attestations.is_empty());
+                    }
+                }
+                Err(_) => {
+                    // Doppelganger protected or other error.
+                    assert!(!should_succeed);
+                }
             }
         }
     })
@@ -1313,11 +1342,16 @@ async fn delete_concurrent_with_signing() {
 
         let handle = handle.spawn(async move {
             for j in 0..num_attestations {
-                let mut att = make_attestation(j, j + 1);
-                for public_key in thread_pubkeys.iter() {
-                    let _ = validator_store
-                        .sign_attestation(*public_key, 0, &mut att, Epoch::new(j + 1))
-                        .await;
+                let att = make_attestation(j, j + 1);
+                for (validator_index, public_key) in thread_pubkeys.iter().enumerate() {
+                    let stream = validator_store.sign_attestations(vec![AttestationToSign {
+                        validator_index: validator_index as u64,
+                        pubkey: *public_key,
+                        validator_committee_index: 0,
+                        attestation: att.clone(),
+                    }]);
+                    tokio::pin!(stream);
+                    let _ = stream.next().await;
                 }
             }
         });

validator_client/http_metrics/src/lib.rs

@@ -197,6 +197,16 @@ pub fn gather_prometheus_metrics<E: EthSpec>(
                 &[NEXT_EPOCH],
                 duties_service.attester_count(next_epoch) as i64,
             );
+            set_int_gauge(
+                &PTC_COUNT,
+                &[CURRENT_EPOCH],
+                duties_service.ptc_count(current_epoch) as i64,
+            );
+            set_int_gauge(
+                &PTC_COUNT,
+                &[NEXT_EPOCH],
+                duties_service.ptc_count(next_epoch) as i64,
+            );
         }
     }
 

validator_client/initialized_validators/Cargo.toml

@@ -12,7 +12,9 @@ eth2_keystore = { workspace = true }
 filesystem = { workspace = true }
 lockfile = { workspace = true }
 metrics = { workspace = true }
+p12-keystore = "0.2"
 parking_lot = { workspace = true }
+pem = "3"
 rand = { workspace = true }
 reqwest = { workspace = true }
 serde = { workspace = true }

validator_client/initialized_validators/src/key_cache.rs

@@ -1,7 +1,7 @@
 use account_utils::write_file_via_temporary;
 use bls::{Keypair, PublicKey};
 use eth2_keystore::json_keystore::{
-    Aes128Ctr, ChecksumModule, Cipher, CipherModule, Crypto, EmptyMap, EmptyString, KdfModule,
+    Aes128Ctr, ChecksumModule, Cipher, CipherModule, Crypto, EmptyMap, EmptyString, Kdf, KdfModule,
     Sha256Checksum,
 };
 use eth2_keystore::{
@@ -65,10 +65,14 @@ impl KeyCache {
     }
 
     pub fn init_crypto() -> Crypto {
+        Self::build_crypto(default_kdf)
+    }
+
+    fn build_crypto(kdf_fn: fn(Vec<u8>) -> Kdf) -> Crypto {
         let salt = rand::rng().random::<[u8; SALT_SIZE]>();
         let iv = rand::rng().random::<[u8; IV_SIZE]>().to_vec().into();
 
-        let kdf = default_kdf(salt.to_vec());
+        let kdf = kdf_fn(salt.to_vec());
         let cipher = Cipher::Aes128Ctr(Aes128Ctr { iv });
 
         Crypto {
@@ -116,7 +120,11 @@ impl KeyCache {
     }
 
     fn encrypt(&mut self) -> Result<(), Error> {
-        self.crypto = Self::init_crypto();
+        self.encrypt_with(default_kdf)
+    }
+
+    fn encrypt_with(&mut self, kdf_fn: fn(Vec<u8>) -> Kdf) -> Result<(), Error> {
+        self.crypto = Self::build_crypto(kdf_fn);
         let secret_map: SerializedKeyMap = self
             .pairs
             .iter()
@@ -268,7 +276,19 @@ pub enum Error {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use eth2_keystore::json_keystore::HexBytes;
+    use eth2_keystore::json_keystore::{HexBytes, Scrypt};
+
+    /// Scrypt with minimal cost (n=1024) for fast test execution.
+    /// Production uses n=262144 which takes ~45s per derivation.
+    fn insecure_kdf(salt: Vec<u8>) -> Kdf {
+        Kdf::Scrypt(Scrypt {
+            dklen: 32,
+            n: 1024,
+            p: 1,
+            r: 8,
+            salt: salt.into(),
+        })
+    }
 
     #[tokio::test]
     async fn test_serialization() {
@@ -302,7 +322,7 @@ mod tests {
             key_cache.add(keypair.clone(), uuid, password.clone());
         }
 
-        key_cache.encrypt().unwrap();
+        key_cache.encrypt_with(insecure_kdf).unwrap();
         key_cache.state = State::DecryptedAndSaved;
 
         assert_eq!(&key_cache.uuids, &uuids);

validator_client/initialized_validators/src/lib.rs

@@ -397,6 +397,7 @@ pub fn load_pem_certificate<P: AsRef<Path>>(pem_path: P) -> Result<Certificate,
     Certificate::from_pem(&buf).map_err(Error::InvalidWeb3SignerRootCertificate)
 }
 
+// Read a PKCS12 identity certificate and parse it into a PEM certificate.
 pub fn load_pkcs12_identity<P: AsRef<Path>>(
     pkcs12_path: P,
     password: &str,
@@ -406,7 +407,29 @@ pub fn load_pkcs12_identity<P: AsRef<Path>>(
         .map_err(Error::InvalidWeb3SignerClientIdentityCertificateFile)?
         .read_to_end(&mut buf)
         .map_err(Error::InvalidWeb3SignerClientIdentityCertificateFile)?;
-    Identity::from_pkcs12_der(&buf, password)
+
+    let keystore = p12_keystore::KeyStore::from_pkcs12(&buf, password).map_err(|e| {
+        Error::InvalidWeb3SignerClientIdentityCertificateFile(io::Error::new(
+            io::ErrorKind::InvalidData,
+            format!("PKCS12 parse error: {e:?}"),
+        ))
+    })?;
+
+    let (_alias, key_chain) = keystore
+        .private_key_chain()
+        .ok_or(Error::MissingWeb3SignerClientIdentityCertificateFile)?;
+
+    let key_pem = pem::encode(&pem::Pem::new("PRIVATE KEY", key_chain.key()));
+    let certs_pem: String = key_chain
+        .chain()
+        .iter()
+        .map(|cert| pem::encode(&pem::Pem::new("CERTIFICATE", cert.as_der())))
+        .collect::<Vec<_>>()
+        .join("\n");
+
+    let combined_pem = format!("{key_pem}\n{certs_pem}");
+
+    Identity::from_pem(combined_pem.as_bytes())
         .map_err(Error::InvalidWeb3SignerClientIdentityCertificate)
 }
 

validator_client/lighthouse_validator_store/Cargo.toml

@@ -12,6 +12,7 @@ doppelganger_service = { workspace = true }
 either = { workspace = true }
 environment = { workspace = true }
 eth2 = { workspace = true }
+futures = { workspace = true }
 initialized_validators = { workspace = true }
 logging = { workspace = true }
 parking_lot = { workspace = true }

validator_client/lighthouse_validator_store/src/lib.rs

@@ -2,6 +2,7 @@ use account_utils::validator_definitions::{PasswordStorage, ValidatorDefinition}
 use bls::{PublicKeyBytes, Signature};
 use doppelganger_service::DoppelgangerService;
 use eth2::types::PublishBlockRequest;
+use futures::{Stream, future::join_all, stream};
 use initialized_validators::InitializedValidators;
 use logging::crit;
 use parking_lot::{Mutex, RwLock};
@@ -9,25 +10,27 @@ use serde::{Deserialize, Serialize};
 use signing_method::Error as SigningError;
 use signing_method::{SignableMessage, SigningContext, SigningMethod};
 use slashing_protection::{
-    InterchangeError, NotSafe, Safe, SlashingDatabase, interchange::Interchange,
+    CheckSlashability, InterchangeError, NotSafe, Safe, SlashingDatabase, interchange::Interchange,
 };
 use slot_clock::SlotClock;
 use std::marker::PhantomData;
 use std::path::Path;
 use std::sync::Arc;
 use task_executor::TaskExecutor;
-use tracing::{error, info, instrument, warn};
+use tracing::{Instrument, debug, error, info, info_span, instrument, warn};
 use types::{
     AbstractExecPayload, Address, AggregateAndProof, Attestation, BeaconBlock, BlindedPayload,
-    ChainSpec, ContributionAndProof, Domain, Epoch, EthSpec, Fork, Graffiti, Hash256,
-    InclusionList, SelectionProof, SignedAggregateAndProof, SignedBeaconBlock,
-    SignedContributionAndProof, SignedInclusionList, SignedRoot, SignedValidatorRegistrationData,
+    ChainSpec, ContributionAndProof, Domain, Epoch, EthSpec, ExecutionPayloadEnvelope, Fork,
+    FullPayload, Graffiti, Hash256, InclusionList, PayloadAttestationData, PayloadAttestationMessage,
+    SelectionProof, SignedAggregateAndProof, SignedBeaconBlock, SignedContributionAndProof,
+    SignedExecutionPayloadEnvelope, SignedInclusionList, SignedRoot, SignedValidatorRegistrationData,
     SignedVoluntaryExit, Slot, SyncAggregatorSelectionData, SyncCommitteeContribution,
     SyncCommitteeMessage, SyncSelectionProof, SyncSubnetId, ValidatorRegistrationData,
     VoluntaryExit, graffiti::GraffitiString,
 };
 use validator_store::{
-    DoppelgangerStatus, Error as ValidatorStoreError, ProposalData, SignedBlock, UnsignedBlock,
+    AggregateToSign, AttestationToSign, ContributionToSign, DoppelgangerStatus,
+    Error as ValidatorStoreError, ProposalData, SignedBlock, SyncMessageToSign, UnsignedBlock,
     ValidatorStore,
 };
 
@@ -52,7 +55,7 @@ pub struct Config {
 /// Number of epochs of slashing protection history to keep.
 ///
 /// This acts as a maximum safe-guard against clock drift.
-const SLASHING_PROTECTION_HISTORY_EPOCHS: u64 = 512;
+const SLASHING_PROTECTION_HISTORY_EPOCHS: u64 = 1;
 
 /// Currently used as the default gas limit in execution clients.
 ///
@@ -556,6 +559,253 @@ impl<T: SlotClock + 'static, E: EthSpec> LighthouseValidatorStore<T, E> {
             signature,
         })
     }
+
+    /// Sign an attestation without performing any slashing protection checks.
+    ///
+    /// THIS METHOD IS DANGEROUS AND SHOULD ONLY BE USED INTERNALLY IMMEDIATELY PRIOR TO A
+    /// SLASHING PROTECTION CHECK. See `slashing_protect_attestations`.
+    ///
+    /// This method DOES perform doppelganger protection checks.
+    #[instrument(level = "debug", skip_all)]
+    async fn sign_attestation_no_slashing_protection(
+        &self,
+        validator_pubkey: PublicKeyBytes,
+        validator_committee_position: usize,
+        attestation: &mut Attestation<E>,
+    ) -> Result<(), Error> {
+        // Get the signing method and check doppelganger protection.
+        let signing_method = self.doppelganger_checked_signing_method(validator_pubkey)?;
+
+        // Sign the attestation.
+        let signing_epoch = attestation.data().target.epoch;
+        let signing_context = self.signing_context(Domain::BeaconAttester, signing_epoch);
+
+        let signature = signing_method
+            .get_signature::<E, BlindedPayload<E>>(
+                SignableMessage::AttestationData(attestation.data()),
+                signing_context,
+                &self.spec,
+                &self.task_executor,
+            )
+            .await?;
+        attestation
+            .add_signature(&signature, validator_committee_position)
+            .map_err(Error::UnableToSignAttestation)?;
+
+        Ok(())
+    }
+
+    /// Provide slashing protection for `attestations`, safely updating the slashing protection DB.
+    ///
+    /// Return a vec of safe attestations which have passed slashing protection. Unsafe attestations
+    /// will be dropped and result in warning logs.
+    ///
+    /// This method SKIPS slashing protection for web3signer validators that have slashing
+    /// protection disabled at the Lighthouse layer. It is up to the user to ensure slashing
+    /// protection is enabled in web3signer instead.
+    #[instrument(level = "debug", skip_all)]
+    fn slashing_protect_attestations(
+        &self,
+        attestations: Vec<(u64, Attestation<E>, PublicKeyBytes)>,
+    ) -> Result<Vec<(u64, Attestation<E>)>, Error> {
+        let mut safe_attestations = Vec::with_capacity(attestations.len());
+        let mut attestations_to_check = Vec::with_capacity(attestations.len());
+
+        // Split attestations into de-facto safe attestations (checked by web3signer's slashing
+        // protection) and ones requiring checking against the slashing protection DB.
+        //
+        // All attestations are added to `attestation_to_check`, with skipped attestations having
+        // `CheckSlashability::No`.
+        for (_, attestation, validator_pubkey) in &attestations {
+            let signing_method = self.doppelganger_checked_signing_method(*validator_pubkey)?;
+            let signing_epoch = attestation.data().target.epoch;
+            let signing_context = self.signing_context(Domain::BeaconAttester, signing_epoch);
+            let domain_hash = signing_context.domain_hash(&self.spec);
+
+            let check_slashability = if signing_method
+                .requires_local_slashing_protection(self.enable_web3signer_slashing_protection)
+            {
+                CheckSlashability::Yes
+            } else {
+                CheckSlashability::No
+            };
+            attestations_to_check.push((
+                attestation.data(),
+                validator_pubkey,
+                domain_hash,
+                check_slashability,
+            ));
+        }
+
+        // Batch check the attestations against the slashing protection DB while preserving the
+        // order so we can zip the results against the original vec.
+        //
+        // If the DB transaction fails then we consider the entire batch slashable and discard it.
+        let results = self
+            .slashing_protection
+            .check_and_insert_attestations(&attestations_to_check)
+            .map_err(Error::Slashable)?;
+
+        for ((validator_index, attestation, validator_pubkey), slashing_status) in
+            attestations.into_iter().zip(results.into_iter())
+        {
+            match slashing_status {
+                Ok(Safe::Valid) => {
+                    safe_attestations.push((validator_index, attestation));
+                    validator_metrics::inc_counter_vec(
+                        &validator_metrics::SIGNED_ATTESTATIONS_TOTAL,
+                        &[validator_metrics::SUCCESS],
+                    );
+                }
+                Ok(Safe::SameData) => {
+                    warn!("Skipping previously signed attestation");
+                    validator_metrics::inc_counter_vec(
+                        &validator_metrics::SIGNED_ATTESTATIONS_TOTAL,
+                        &[validator_metrics::SAME_DATA],
+                    );
+                }
+                Err(NotSafe::UnregisteredValidator(pk)) => {
+                    warn!(
+                        msg = "Carefully consider running with --init-slashing-protection (see --help)",
+                        public_key = ?pk,
+                        "Not signing attestation for unregistered validator"
+                    );
+                    validator_metrics::inc_counter_vec(
+                        &validator_metrics::SIGNED_ATTESTATIONS_TOTAL,
+                        &[validator_metrics::UNREGISTERED],
+                    );
+                }
+                Err(e) => {
+                    warn!(
+                        slot = %attestation.data().slot,
+                        block_root = ?attestation.data().beacon_block_root,
+                        public_key = ?validator_pubkey,
+                        error = ?e,
+                        "Skipping signing of slashable attestation"
+                    );
+                    validator_metrics::inc_counter_vec(
+                        &validator_metrics::SIGNED_ATTESTATIONS_TOTAL,
+                        &[validator_metrics::SLASHABLE],
+                    );
+                }
+            }
+        }
+
+        Ok(safe_attestations)
+    }
+
+    /// Signs an `AggregateAndProof` for a given validator.
+    ///
+    /// The resulting `SignedAggregateAndProof` is sent on the aggregation channel and cannot be
+    /// modified by actors other than the signing validator.
+    pub async fn produce_signed_aggregate_and_proof(
+        &self,
+        validator_pubkey: PublicKeyBytes,
+        aggregator_index: u64,
+        aggregate: Attestation<E>,
+        selection_proof: SelectionProof,
+    ) -> Result<SignedAggregateAndProof<E>, Error> {
+        let signing_epoch = aggregate.data().target.epoch;
+        let signing_context = self.signing_context(Domain::AggregateAndProof, signing_epoch);
+
+        let message =
+            AggregateAndProof::from_attestation(aggregator_index, aggregate, selection_proof);
+
+        let signing_method = self.doppelganger_checked_signing_method(validator_pubkey)?;
+        let signature = signing_method
+            .get_signature::<E, BlindedPayload<E>>(
+                SignableMessage::SignedAggregateAndProof(message.to_ref()),
+                signing_context,
+                &self.spec,
+                &self.task_executor,
+            )
+            .await?;
+
+        validator_metrics::inc_counter_vec(
+            &validator_metrics::SIGNED_AGGREGATES_TOTAL,
+            &[validator_metrics::SUCCESS],
+        );
+
+        Ok(SignedAggregateAndProof::from_aggregate_and_proof(
+            message, signature,
+        ))
+    }
+
+    pub async fn produce_sync_committee_signature(
+        &self,
+        slot: Slot,
+        beacon_block_root: Hash256,
+        validator_index: u64,
+        validator_pubkey: &PublicKeyBytes,
+    ) -> Result<SyncCommitteeMessage, Error> {
+        let signing_epoch = slot.epoch(E::slots_per_epoch());
+        let signing_context = self.signing_context(Domain::SyncCommittee, signing_epoch);
+
+        // Bypass `with_validator_signing_method`: sync committee messages are not slashable.
+        let signing_method = self.doppelganger_bypassed_signing_method(*validator_pubkey)?;
+
+        let signature = signing_method
+            .get_signature::<E, BlindedPayload<E>>(
+                SignableMessage::SyncCommitteeSignature {
+                    beacon_block_root,
+                    slot,
+                },
+                signing_context,
+                &self.spec,
+                &self.task_executor,
+            )
+            .await
+            .map_err(Error::SpecificError)?;
+
+        validator_metrics::inc_counter_vec(
+            &validator_metrics::SIGNED_SYNC_COMMITTEE_MESSAGES_TOTAL,
+            &[validator_metrics::SUCCESS],
+        );
+
+        Ok(SyncCommitteeMessage {
+            slot,
+            beacon_block_root,
+            validator_index,
+            signature,
+        })
+    }
+
+    pub async fn produce_signed_contribution_and_proof(
+        &self,
+        aggregator_index: u64,
+        aggregator_pubkey: PublicKeyBytes,
+        contribution: SyncCommitteeContribution<E>,
+        selection_proof: SyncSelectionProof,
+    ) -> Result<SignedContributionAndProof<E>, Error> {
+        let signing_epoch = contribution.slot.epoch(E::slots_per_epoch());
+        let signing_context = self.signing_context(Domain::ContributionAndProof, signing_epoch);
+
+        // Bypass `with_validator_signing_method`: sync committee messages are not slashable.
+        let signing_method = self.doppelganger_bypassed_signing_method(aggregator_pubkey)?;
+
+        let message = ContributionAndProof {
+            aggregator_index,
+            contribution,
+            selection_proof: selection_proof.into(),
+        };
+
+        let signature = signing_method
+            .get_signature::<E, BlindedPayload<E>>(
+                SignableMessage::SignedContributionAndProof(&message),
+                signing_context,
+                &self.spec,
+                &self.task_executor,
+            )
+            .await
+            .map_err(Error::SpecificError)?;
+
+        validator_metrics::inc_counter_vec(
+            &validator_metrics::SIGNED_SYNC_COMMITTEE_CONTRIBUTIONS_TOTAL,
+            &[validator_metrics::SUCCESS],
+        );
+
+        Ok(SignedContributionAndProof { message, signature })
+    }
 }
 
 impl<T: SlotClock + 'static, E: EthSpec> ValidatorStore for LighthouseValidatorStore<T, E> {
@@ -747,103 +997,90 @@ impl<T: SlotClock + 'static, E: EthSpec> ValidatorStore for LighthouseValidatorS
         }
     }
 
-    #[instrument(skip_all)]
-    async fn sign_attestation(
-        &self,
-        validator_pubkey: PublicKeyBytes,
-        validator_committee_position: usize,
-        attestation: &mut Attestation<E>,
-        current_epoch: Epoch,
-    ) -> Result<(), Error> {
-        // Make sure the target epoch is not higher than the current epoch to avoid potential attacks.
-        if attestation.data().target.epoch > current_epoch {
-            return Err(Error::GreaterThanCurrentEpoch {
-                epoch: attestation.data().target.epoch,
-                current_epoch,
-            });
-        }
+    fn sign_attestations(
+        self: &Arc<Self>,
+        mut attestations: Vec<AttestationToSign<E>>,
+    ) -> impl Stream<Item = Result<Vec<(u64, Attestation<E>)>, Error>> + Send {
+        let store = self.clone();
+        stream::once(async move {
+            // Sign all attestations concurrently.
+            let signing_futures = attestations.iter_mut().map(
+                |AttestationToSign {
+                     pubkey,
+                     validator_committee_index,
+                     attestation,
+                     ..
+                 }| {
+                    let pubkey = *pubkey;
+                    let validator_committee_index = *validator_committee_index;
+                    let store = store.clone();
+                    async move {
+                        store
+                            .sign_attestation_no_slashing_protection(
+                                pubkey,
+                                validator_committee_index,
+                                attestation,
+                            )
+                            .await
+                    }
+                },
+            );
 
-        // Get the signing method and check doppelganger protection.
-        let signing_method = self.doppelganger_checked_signing_method(validator_pubkey)?;
+            // Execute all signing in parallel.
+            let results: Vec<_> = join_all(signing_futures).await;
 
-        // Checking for slashing conditions.
-        let signing_epoch = attestation.data().target.epoch;
-        let signing_context = self.signing_context(Domain::BeaconAttester, signing_epoch);
-        let domain_hash = signing_context.domain_hash(&self.spec);
-        let slashing_status = if signing_method
-            .requires_local_slashing_protection(self.enable_web3signer_slashing_protection)
-        {
-            self.slashing_protection.check_and_insert_attestation(
-                &validator_pubkey,
-                attestation.data(),
-                domain_hash,
-            )
-        } else {
-            Ok(Safe::Valid)
-        };
-
-        match slashing_status {
-            // We can safely sign this attestation.
-            Ok(Safe::Valid) => {
-                let signature = signing_method
-                    .get_signature::<E, BlindedPayload<E>>(
-                        SignableMessage::AttestationData(attestation.data()),
-                        signing_context,
-                        &self.spec,
-                        &self.task_executor,
-                    )
-                    .await?;
-                attestation
-                    .add_signature(&signature, validator_committee_position)
-                    .map_err(Error::UnableToSignAttestation)?;
-
-                validator_metrics::inc_counter_vec(
-                    &validator_metrics::SIGNED_ATTESTATIONS_TOTAL,
-                    &[validator_metrics::SUCCESS],
-                );
-
-                Ok(())
+            // Collect successfully signed attestations and log errors.
+            let mut signed_attestations = Vec::with_capacity(attestations.len());
+            for (result, att) in results.into_iter().zip(attestations) {
+                match result {
+                    Ok(()) => {
+                        signed_attestations.push((
+                            att.validator_index,
+                            att.attestation,
+                            att.pubkey,
+                        ));
+                    }
+                    Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
+                        warn!(
+                            info = "a validator may have recently been removed from this VC",
+                            ?pubkey,
+                            "Missing pubkey for attestation"
+                        );
+                    }
+                    Err(e) => {
+                        crit!(
+                            error = ?e,
+                            "Failed to sign attestation"
+                        );
+                    }
+                }
             }
-            Ok(Safe::SameData) => {
-                warn!("Skipping signing of previously signed attestation");
-                validator_metrics::inc_counter_vec(
-                    &validator_metrics::SIGNED_ATTESTATIONS_TOTAL,
-                    &[validator_metrics::SAME_DATA],
-                );
-                Err(Error::SameData)
+
+            if signed_attestations.is_empty() {
+                return Ok(vec![]);
             }
-            Err(NotSafe::UnregisteredValidator(pk)) => {
-                warn!(
-                    msg = "Carefully consider running with --init-slashing-protection (see --help)",
-                    public_key = format!("{:?}", pk),
-                    "Not signing attestation for unregistered validator"
-                );
-                validator_metrics::inc_counter_vec(
-                    &validator_metrics::SIGNED_ATTESTATIONS_TOTAL,
-                    &[validator_metrics::UNREGISTERED],
-                );
-                Err(Error::Slashable(NotSafe::UnregisteredValidator(pk)))
-            }
-            Err(e) => {
-                crit!(
-                    attestation = format!("{:?}", attestation.data()),
-                    error = format!("{:?}", e),
-                    "Not signing slashable attestation"
-                );
-                validator_metrics::inc_counter_vec(
-                    &validator_metrics::SIGNED_ATTESTATIONS_TOTAL,
-                    &[validator_metrics::SLASHABLE],
-                );
-                Err(Error::Slashable(e))
-            }
-        }
+
+            // Check slashing protection and insert into database. Use a dedicated blocking
+            // thread to avoid clogging the async executor with blocking database I/O.
+            let validator_store = store.clone();
+            let safe_attestations = store
+                .task_executor
+                .spawn_blocking_handle(
+                    move || validator_store.slashing_protect_attestations(signed_attestations),
+                    "slashing_protect_attestations",
+                )
+                .ok_or(Error::ExecutorError)?
+                .await
+                .map_err(|_| Error::ExecutorError)??;
+            Ok(safe_attestations)
+        })
     }
 
     async fn sign_validator_registration_data(
         &self,
         validator_registration_data: ValidatorRegistrationData,
     ) -> Result<SignedValidatorRegistrationData, Error> {
-        let domain_hash = self.spec.get_builder_domain();
+        let domain_hash = self.spec.get_builder_application_domain();
         let signing_root = validator_registration_data.signing_root(domain_hash);
 
         let signing_method =
@@ -868,43 +1105,6 @@ impl<T: SlotClock + 'static, E: EthSpec> ValidatorStore for LighthouseValidatorS
         })
     }
 
-    /// Signs an `AggregateAndProof` for a given validator.
-    ///
-    /// The resulting `SignedAggregateAndProof` is sent on the aggregation channel and cannot be
-    /// modified by actors other than the signing validator.
-    async fn produce_signed_aggregate_and_proof(
-        &self,
-        validator_pubkey: PublicKeyBytes,
-        aggregator_index: u64,
-        aggregate: Attestation<E>,
-        selection_proof: SelectionProof,
-    ) -> Result<SignedAggregateAndProof<E>, Error> {
-        let signing_epoch = aggregate.data().target.epoch;
-        let signing_context = self.signing_context(Domain::AggregateAndProof, signing_epoch);
-
-        let message =
-            AggregateAndProof::from_attestation(aggregator_index, aggregate, selection_proof);
-
-        let signing_method = self.doppelganger_checked_signing_method(validator_pubkey)?;
-        let signature = signing_method
-            .get_signature::<E, BlindedPayload<E>>(
-                SignableMessage::SignedAggregateAndProof(message.to_ref()),
-                signing_context,
-                &self.spec,
-                &self.task_executor,
-            )
-            .await?;
-
-        validator_metrics::inc_counter_vec(
-            &validator_metrics::SIGNED_AGGREGATES_TOTAL,
-            &[validator_metrics::SUCCESS],
-        );
-
-        Ok(SignedAggregateAndProof::from_aggregate_and_proof(
-            message, signature,
-        ))
-    }
-
     /// Produces a `SelectionProof` for the `slot`, signed by with corresponding secret key to
     /// `validator_pubkey`.
     async fn produce_selection_proof(
@@ -979,80 +1179,172 @@ impl<T: SlotClock + 'static, E: EthSpec> ValidatorStore for LighthouseValidatorS
         Ok(signature.into())
     }
 
-    async fn produce_sync_committee_signature(
-        &self,
-        slot: Slot,
-        beacon_block_root: Hash256,
-        validator_index: u64,
-        validator_pubkey: &PublicKeyBytes,
-    ) -> Result<SyncCommitteeMessage, Error> {
-        let signing_epoch = slot.epoch(E::slots_per_epoch());
-        let signing_context = self.signing_context(Domain::SyncCommittee, signing_epoch);
-
-        // Bypass `with_validator_signing_method`: sync committee messages are not slashable.
-        let signing_method = self.doppelganger_bypassed_signing_method(*validator_pubkey)?;
-
-        let signature = signing_method
-            .get_signature::<E, BlindedPayload<E>>(
-                SignableMessage::SyncCommitteeSignature {
-                    beacon_block_root,
-                    slot,
+    fn sign_aggregate_and_proofs(
+        self: &Arc<Self>,
+        aggregates: Vec<AggregateToSign<E>>,
+    ) -> impl Stream<Item = Result<Vec<SignedAggregateAndProof<E>>, Error>> + Send {
+        let store = self.clone();
+        let count = aggregates.len();
+        stream::once(async move {
+            let signing_futures = aggregates.into_iter().map(
+                |AggregateToSign {
+                     pubkey,
+                     aggregator_index,
+                     aggregate,
+                     selection_proof,
+                 }| {
+                    let store = store.clone();
+                    async move {
+                        let result = store
+                            .produce_signed_aggregate_and_proof(
+                                pubkey,
+                                aggregator_index,
+                                aggregate,
+                                selection_proof,
+                            )
+                            .await;
+                        (pubkey, result)
+                    }
                 },
-                signing_context,
-                &self.spec,
-                &self.task_executor,
-            )
-            .await
-            .map_err(Error::SpecificError)?;
+            );
 
-        validator_metrics::inc_counter_vec(
-            &validator_metrics::SIGNED_SYNC_COMMITTEE_MESSAGES_TOTAL,
-            &[validator_metrics::SUCCESS],
-        );
+            let results = join_all(signing_futures)
+                .instrument(info_span!("sign_aggregates", count))
+                .await;
 
-        Ok(SyncCommitteeMessage {
-            slot,
-            beacon_block_root,
-            validator_index,
-            signature,
+            let mut signed = Vec::with_capacity(results.len());
+            for (pubkey, result) in results {
+                match result {
+                    Ok(agg) => signed.push(agg),
+                    Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
+                        // A pubkey can be missing when a validator was recently
+                        // removed via the API.
+                        debug!(?pubkey, "Missing pubkey for aggregate");
+                    }
+                    Err(e) => {
+                        crit!(error = ?e, pubkey = ?pubkey, "Failed to sign aggregate");
+                    }
+                }
+            }
+            Ok(signed)
         })
     }
 
-    async fn produce_signed_contribution_and_proof(
-        &self,
-        aggregator_index: u64,
-        aggregator_pubkey: PublicKeyBytes,
-        contribution: SyncCommitteeContribution<E>,
-        selection_proof: SyncSelectionProof,
-    ) -> Result<SignedContributionAndProof<E>, Error> {
-        let signing_epoch = contribution.slot.epoch(E::slots_per_epoch());
-        let signing_context = self.signing_context(Domain::ContributionAndProof, signing_epoch);
+    fn sign_sync_committee_signatures(
+        self: &Arc<Self>,
+        messages: Vec<SyncMessageToSign>,
+    ) -> impl Stream<Item = Result<Vec<SyncCommitteeMessage>, Error>> + Send {
+        let store = self.clone();
+        let count = messages.len();
+        stream::once(async move {
+            let signing_futures = messages.into_iter().map(
+                |SyncMessageToSign {
+                     slot,
+                     beacon_block_root,
+                     validator_index,
+                     pubkey,
+                 }| {
+                    let store = store.clone();
+                    async move {
+                        let result = store
+                            .produce_sync_committee_signature(
+                                slot,
+                                beacon_block_root,
+                                validator_index,
+                                &pubkey,
+                            )
+                            .await;
+                        (pubkey, validator_index, slot, result)
+                    }
+                },
+            );
 
-        // Bypass `with_validator_signing_method`: sync committee messages are not slashable.
-        let signing_method = self.doppelganger_bypassed_signing_method(aggregator_pubkey)?;
+            let results = join_all(signing_futures)
+                .instrument(info_span!("sign_sync_signatures", count))
+                .await;
 
-        let message = ContributionAndProof {
-            aggregator_index,
-            contribution,
-            selection_proof: selection_proof.into(),
-        };
+            let mut signed = Vec::with_capacity(results.len());
+            for (_pubkey, validator_index, slot, result) in results {
+                match result {
+                    Ok(sig) => signed.push(sig),
+                    Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
+                        // A pubkey can be missing when a validator was recently
+                        // removed via the API.
+                        debug!(
+                            ?pubkey,
+                            validator_index,
+                            %slot,
+                            "Missing pubkey for sync committee signature"
+                        );
+                    }
+                    Err(e) => {
+                        crit!(
+                            validator_index,
+                            %slot,
+                            error = ?e,
+                            "Failed to sign sync committee signature"
+                        );
+                    }
+                }
+            }
+            Ok(signed)
+        })
+    }
 
-        let signature = signing_method
-            .get_signature::<E, BlindedPayload<E>>(
-                SignableMessage::SignedContributionAndProof(&message),
-                signing_context,
-                &self.spec,
-                &self.task_executor,
-            )
-            .await
-            .map_err(Error::SpecificError)?;
+    fn sign_sync_committee_contributions(
+        self: &Arc<Self>,
+        contributions: Vec<ContributionToSign<E>>,
+    ) -> impl Stream<Item = Result<Vec<SignedContributionAndProof<E>>, Error>> + Send {
+        let store = self.clone();
+        let count = contributions.len();
+        stream::once(async move {
+            let signing_futures = contributions.into_iter().map(
+                |ContributionToSign {
+                     aggregator_index,
+                     aggregator_pubkey,
+                     contribution,
+                     selection_proof,
+                 }| {
+                    let store = store.clone();
+                    let slot = contribution.slot;
+                    async move {
+                        let result = store
+                            .produce_signed_contribution_and_proof(
+                                aggregator_index,
+                                aggregator_pubkey,
+                                contribution,
+                                selection_proof,
+                            )
+                            .await;
+                        (slot, result)
+                    }
+                },
+            );
 
-        validator_metrics::inc_counter_vec(
-            &validator_metrics::SIGNED_SYNC_COMMITTEE_CONTRIBUTIONS_TOTAL,
-            &[validator_metrics::SUCCESS],
-        );
+            let results = join_all(signing_futures)
+                .instrument(info_span!("sign_sync_contributions", count))
+                .await;
 
-        Ok(SignedContributionAndProof { message, signature })
+            let mut signed = Vec::with_capacity(results.len());
+            for (slot, result) in results {
+                match result {
+                    Ok(contribution) => signed.push(contribution),
+                    Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
+                        // A pubkey can be missing when a validator was recently
+                        // removed via the API.
+                        debug!(?pubkey, %slot, "Missing pubkey for sync contribution");
+                    }
+                    Err(e) => {
+                        crit!(
+                            %slot,
+                            error = ?e,
+                            "Unable to sign sync committee contribution"
+                        );
+                    }
+                }
+            }
+            Ok(signed)
+        })
     }
 
     /// Prune the slashing protection database so that it remains performant.
@@ -1155,4 +1447,66 @@ impl<T: SlotClock + 'static, E: EthSpec> ValidatorStore for LighthouseValidatorS
             signature,
         })
     }
+
+    async fn sign_payload_attestation(
+        &self,
+        validator_pubkey: PublicKeyBytes,
+        data: PayloadAttestationData,
+    ) -> Result<PayloadAttestationMessage, Error> {
+        let signing_context =
+            self.signing_context(Domain::PTCAttester, data.slot.epoch(E::slots_per_epoch()));
+
+        let validator_index = self
+            .validator_index(&validator_pubkey)
+            .ok_or(ValidatorStoreError::UnknownPubkey(validator_pubkey))?;
+
+        let signing_method = self.doppelganger_bypassed_signing_method(validator_pubkey)?;
+
+        let signature = signing_method
+            .get_signature::<E, FullPayload<E>>(
+                SignableMessage::PayloadAttestationData(&data),
+                signing_context,
+                &self.spec,
+                &self.task_executor,
+            )
+            .await
+            .map_err(Error::SpecificError)?;
+
+        Ok(PayloadAttestationMessage {
+            validator_index,
+            data,
+            signature,
+        })
+    }
+
+    /// Sign an `ExecutionPayloadEnvelope` for Gloas (local building).
+    /// The proposer acts as the builder and signs with the BeaconBuilder domain.
+    async fn sign_execution_payload_envelope(
+        &self,
+        validator_pubkey: PublicKeyBytes,
+        envelope: ExecutionPayloadEnvelope<E>,
+    ) -> Result<SignedExecutionPayloadEnvelope<E>, Error> {
+        let signing_context = self.signing_context(
+            Domain::BeaconBuilder,
+            envelope.slot().epoch(E::slots_per_epoch()),
+        );
+
+        // Execution payload envelope signing is not slashable, bypass doppelganger protection.
+        let signing_method = self.doppelganger_bypassed_signing_method(validator_pubkey)?;
+
+        let signature = signing_method
+            .get_signature::<E, FullPayload<E>>(
+                SignableMessage::ExecutionPayloadEnvelope(&envelope),
+                signing_context,
+                &self.spec,
+                &self.task_executor,
+            )
+            .await
+            .map_err(Error::SpecificError)?;
+
+        Ok(SignedExecutionPayloadEnvelope {
+            message: envelope,
+            signature,
+        })
+    }
 }

validator_client/signing_method/src/lib.rs

@@ -10,7 +10,7 @@ use parking_lot::Mutex;
 use reqwest::{Client, header::ACCEPT};
 use std::path::PathBuf;
 use std::sync::Arc;
-use task_executor::TaskExecutor;
+use task_executor::{RayonPoolType, TaskExecutor};
 use tracing::instrument;
 use types::*;
 use url::Url;
@@ -50,6 +50,8 @@ pub enum SignableMessage<'a, E: EthSpec, Payload: AbstractExecPayload<E> = FullP
     ValidatorRegistration(&'a ValidatorRegistrationData),
     VoluntaryExit(&'a VoluntaryExit),
     InclusionList(&'a InclusionList<E>),
+    ExecutionPayloadEnvelope(&'a ExecutionPayloadEnvelope<E>),
+    PayloadAttestationData(&'a PayloadAttestationData),
 }
 
 impl<E: EthSpec, Payload: AbstractExecPayload<E>> SignableMessage<'_, E, Payload> {
@@ -72,6 +74,8 @@ impl<E: EthSpec, Payload: AbstractExecPayload<E>> SignableMessage<'_, E, Payload
             SignableMessage::ValidatorRegistration(v) => v.signing_root(domain),
             SignableMessage::VoluntaryExit(exit) => exit.signing_root(domain),
             SignableMessage::InclusionList(il) => il.signing_root(domain),
+            SignableMessage::ExecutionPayloadEnvelope(e) => e.signing_root(domain),
+            SignableMessage::PayloadAttestationData(d) => d.signing_root(domain),
         }
     }
 }
@@ -183,14 +187,16 @@ impl SigningMethod {
                 let voting_keypair = voting_keypair.clone();
                 // Spawn a blocking task to produce the signature. This avoids blocking the core
                 // tokio executor.
+                //
+                // We are using the Rayon high-priority pool which uses up to 80% of available
+                // threads. In future we could consider using 90-100% in the VC, seeing as we have
+                // very little other work to do aside from signing.
                 let signature = executor
-                    .spawn_blocking_handle(
-                        move || voting_keypair.sk.sign(signing_root),
-                        "local_keystore_signer",
-                    )
-                    .ok_or(Error::ShuttingDown)?
+                    .spawn_blocking_with_rayon_async(RayonPoolType::HighPriority, move || {
+                        voting_keypair.sk.sign(signing_root)
+                    })
                     .await
-                    .map_err(|e| Error::TokioJoin(e.to_string()))?;
+                    .map_err(|_| Error::ShuttingDown)?;
                 Ok(signature)
             }
             SigningMethod::Web3Signer {
@@ -234,6 +240,12 @@ impl SigningMethod {
                     }
                     SignableMessage::VoluntaryExit(e) => Web3SignerObject::VoluntaryExit(e),
                     SignableMessage::InclusionList(il) => Web3SignerObject::InclusionList(il),
+                    SignableMessage::ExecutionPayloadEnvelope(e) => {
+                        Web3SignerObject::ExecutionPayloadEnvelope(e)
+                    }
+                    SignableMessage::PayloadAttestationData(d) => {
+                        Web3SignerObject::PayloadAttestationData(d)
+                    }
                 };
 
                 // Determine the Web3Signer message type.

validator_client/signing_method/src/web3signer.rs

@@ -20,6 +20,9 @@ pub enum MessageType {
     SyncCommitteeContributionAndProof,
     ValidatorRegistration,
     InclusionList,
+    // TODO(gloas) verify w/ web3signer specs
+    ExecutionPayloadEnvelope,
+    PayloadAttestation,
 }
 
 #[derive(Debug, PartialEq, Copy, Clone, Serialize)]
@@ -78,6 +81,8 @@ pub enum Web3SignerObject<'a, E: EthSpec, Payload: AbstractExecPayload<E>> {
     ContributionAndProof(&'a ContributionAndProof<E>),
     ValidatorRegistration(&'a ValidatorRegistrationData),
     InclusionList(&'a InclusionList<E>),
+    ExecutionPayloadEnvelope(&'a ExecutionPayloadEnvelope<E>),
+    PayloadAttestationData(&'a PayloadAttestationData),
 }
 
 impl<'a, E: EthSpec, Payload: AbstractExecPayload<E>> Web3SignerObject<'a, E, Payload> {
@@ -149,6 +154,8 @@ impl<'a, E: EthSpec, Payload: AbstractExecPayload<E>> Web3SignerObject<'a, E, Pa
             }
             Web3SignerObject::ValidatorRegistration(_) => MessageType::ValidatorRegistration,
             Web3SignerObject::InclusionList(_) => MessageType::InclusionList,
+            Web3SignerObject::ExecutionPayloadEnvelope(_) => MessageType::ExecutionPayloadEnvelope,
+            Web3SignerObject::PayloadAttestationData(_) => MessageType::PayloadAttestation,
         }
     }
 }

validator_client/slashing_protection/Cargo.toml

@@ -6,24 +6,24 @@ edition = { workspace = true }
 autotests = false
 
 [features]
-arbitrary-fuzz = ["types/arbitrary-fuzz", "eip_3076/arbitrary-fuzz"]
+arbitrary = ["dep:arbitrary", "types/arbitrary", "eip_3076/arbitrary"]
 portable = ["types/portable"]
 
 [dependencies]
-arbitrary = { workspace = true, features = ["derive"] }
+arbitrary = { workspace = true, features = ["derive"], optional = true }
 bls = { workspace = true }
 eip_3076 = { workspace = true, features = ["json"] }
 ethereum_serde_utils = { workspace = true }
 filesystem = { workspace = true }
 fixed_bytes = { workspace = true }
 r2d2 = { workspace = true }
-r2d2_sqlite = "0.21.0"
+r2d2_sqlite = "0.32"
 rusqlite = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 tempfile = { workspace = true }
 tracing = { workspace = true }
-types = { workspace = true }
+types = { workspace = true, features = ["sqlite"] }
 
 [dev-dependencies]
 rayon = { workspace = true }

validator_client/slashing_protection/src/interchange_test.rs

@@ -11,7 +11,7 @@ use tempfile::tempdir;
 use types::{Epoch, Hash256, Slot};
 
 #[derive(Debug, Clone, Deserialize, Serialize)]
-#[cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary))]
+#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
 pub struct MultiTestCase {
     pub name: String,
     pub genesis_validators_root: Hash256,
@@ -19,7 +19,7 @@ pub struct MultiTestCase {
 }
 
 #[derive(Debug, Clone, Deserialize, Serialize)]
-#[cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary))]
+#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
 pub struct TestCase {
     pub should_succeed: bool,
     pub contains_slashable_data: bool,
@@ -29,7 +29,7 @@ pub struct TestCase {
 }
 
 #[derive(Debug, Clone, Deserialize, Serialize)]
-#[cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary))]
+#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
 pub struct TestBlock {
     pub pubkey: PublicKeyBytes,
     pub slot: Slot,
@@ -39,7 +39,7 @@ pub struct TestBlock {
 }
 
 #[derive(Debug, Clone, Deserialize, Serialize)]
-#[cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary))]
+#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
 pub struct TestAttestation {
     pub pubkey: PublicKeyBytes,
     pub source_epoch: Epoch,
@@ -135,12 +135,15 @@ impl MultiTestCase {
             }
 
             for (i, att) in test_case.attestations.iter().enumerate() {
-                match slashing_db.check_and_insert_attestation_signing_root(
-                    &att.pubkey,
-                    att.source_epoch,
-                    att.target_epoch,
-                    SigningRoot::from(att.signing_root),
-                ) {
+                match slashing_db.with_transaction(|txn| {
+                    slashing_db.check_and_insert_attestation_signing_root(
+                        &att.pubkey,
+                        att.source_epoch,
+                        att.target_epoch,
+                        SigningRoot::from(att.signing_root),
+                        txn,
+                    )
+                }) {
                     Ok(safe) if !att.should_succeed => {
                         panic!(
                             "attestation {} from `{}` succeeded when it should have failed: {:?}",

validator_client/slashing_protection/src/lib.rs

@@ -16,8 +16,8 @@ pub mod interchange {
 pub use crate::signed_attestation::{InvalidAttestation, SignedAttestation};
 pub use crate::signed_block::{InvalidBlock, SignedBlock};
 pub use crate::slashing_database::{
-    InterchangeError, InterchangeImportOutcome, SUPPORTED_INTERCHANGE_FORMAT_VERSION,
-    SlashingDatabase,
+    CheckSlashability, InterchangeError, InterchangeImportOutcome,
+    SUPPORTED_INTERCHANGE_FORMAT_VERSION, SlashingDatabase,
 };
 use bls::PublicKeyBytes;
 use rusqlite::Error as SQLError;

validator_client/slashing_protection/src/parallel_tests.rs

@@ -44,11 +44,14 @@ fn attestation_same_target() {
     let results = (0..num_attestations)
         .into_par_iter()
         .map(|i| {
-            slashing_db.check_and_insert_attestation(
-                &pk,
-                &attestation_data_builder(i, num_attestations),
-                DEFAULT_DOMAIN,
-            )
+            slashing_db.with_transaction(|txn| {
+                slashing_db.check_and_insert_attestation(
+                    &pk,
+                    &attestation_data_builder(i, num_attestations),
+                    DEFAULT_DOMAIN,
+                    txn,
+                )
+            })
         })
         .collect::<Vec<_>>();
 
@@ -73,7 +76,9 @@ fn attestation_surround_fest() {
         .into_par_iter()
         .map(|i| {
             let att = attestation_data_builder(i, 2 * num_attestations - i);
-            slashing_db.check_and_insert_attestation(&pk, &att, DEFAULT_DOMAIN)
+            slashing_db.with_transaction(|txn| {
+                slashing_db.check_and_insert_attestation(&pk, &att, DEFAULT_DOMAIN, txn)
+            })
         })
         .collect::<Vec<_>>();
 

validator_client/slashing_protection/src/slashing_database.rs

@@ -38,6 +38,17 @@ pub struct SlashingDatabase {
     conn_pool: Pool,
 }
 
+/// Whether to check slashability of a message.
+///
+/// The `No` variant MUST only be used if there is another source of slashing protection configured,
+/// e.g. web3signer's slashing protection.
+#[derive(Debug, Clone, Copy, Default)]
+pub enum CheckSlashability {
+    #[default]
+    Yes,
+    No,
+}
+
 impl SlashingDatabase {
     /// Open an existing database at the given `path`, or create one if none exists.
     pub fn open_or_create(path: &Path) -> Result<Self, NotSafe> {
@@ -183,7 +194,9 @@ impl SlashingDatabase {
         U: From<NotSafe>,
     {
         let mut conn = self.conn_pool.get().map_err(NotSafe::from)?;
-        let txn = conn.transaction().map_err(NotSafe::from)?;
+        let txn = conn
+            .transaction_with_behavior(TransactionBehavior::Exclusive)
+            .map_err(NotSafe::from)?;
         let value = f(&txn)?;
         txn.commit().map_err(NotSafe::from)?;
         Ok(value)
@@ -635,6 +648,43 @@ impl SlashingDatabase {
         self.check_block_proposal(&txn, validator_pubkey, slot, signing_root)
     }
 
+    #[instrument(name = "db_check_and_insert_attestations", level = "debug", skip_all)]
+    pub fn check_and_insert_attestations<'a>(
+        &self,
+        attestations: &'a [(
+            &'a AttestationData,
+            &'a PublicKeyBytes,
+            Hash256,
+            CheckSlashability,
+        )],
+    ) -> Result<Vec<Result<Safe, NotSafe>>, NotSafe> {
+        let mut conn = self.conn_pool.get()?;
+        let txn = conn.transaction_with_behavior(TransactionBehavior::Exclusive)?;
+
+        let mut results = Vec::with_capacity(attestations.len());
+        for (attestation, validator_pubkey, domain, check_slashability) in attestations {
+            match check_slashability {
+                CheckSlashability::No => {
+                    results.push(Ok(Safe::Valid));
+                }
+                CheckSlashability::Yes => {
+                    let attestation_signing_root = attestation.signing_root(*domain).into();
+                    results.push(self.check_and_insert_attestation_signing_root(
+                        validator_pubkey,
+                        attestation.source.epoch,
+                        attestation.target.epoch,
+                        attestation_signing_root,
+                        &txn,
+                    ));
+                }
+            }
+        }
+
+        txn.commit()?;
+
+        Ok(results)
+    }
+
     /// Check an attestation for slash safety, and if it is safe, record it in the database.
     ///
     /// The checking and inserting happen atomically and exclusively. We enforce exclusivity
@@ -647,6 +697,7 @@ impl SlashingDatabase {
         validator_pubkey: &PublicKeyBytes,
         attestation: &AttestationData,
         domain: Hash256,
+        txn: &Transaction,
     ) -> Result<Safe, NotSafe> {
         let attestation_signing_root = attestation.signing_root(domain).into();
         self.check_and_insert_attestation_signing_root(
@@ -654,6 +705,7 @@ impl SlashingDatabase {
             attestation.source.epoch,
             attestation.target.epoch,
             attestation_signing_root,
+            txn,
         )
     }
 
@@ -664,17 +716,15 @@ impl SlashingDatabase {
         att_source_epoch: Epoch,
         att_target_epoch: Epoch,
         att_signing_root: SigningRoot,
+        txn: &Transaction,
     ) -> Result<Safe, NotSafe> {
-        let mut conn = self.conn_pool.get()?;
-        let txn = conn.transaction_with_behavior(TransactionBehavior::Exclusive)?;
         let safe = self.check_and_insert_attestation_signing_root_txn(
             validator_pubkey,
             att_source_epoch,
             att_target_epoch,
             att_signing_root,
-            &txn,
+            txn,
         )?;
-        txn.commit()?;
         Ok(safe)
     }
 

validator_client/slashing_protection/src/test_utils.rs

@@ -1,3 +1,4 @@
+use crate::slashing_database::CheckSlashability;
 use crate::*;
 use tempfile::{TempDir, tempdir};
 use types::{AttestationData, BeaconBlockHeader, test_utils::generate_deterministic_keypair};
@@ -72,6 +73,12 @@ impl<T> Default for StreamTest<T> {
 
 impl StreamTest<AttestationData> {
     pub fn run(&self) {
+        self.run_solo();
+        self.run_batched();
+    }
+
+    // Run the test with every attestation processed individually.
+    pub fn run_solo(&self) {
         let dir = tempdir().unwrap();
         let slashing_db_file = dir.path().join("slashing_protection.sqlite");
         let slashing_db = SlashingDatabase::create(&slashing_db_file).unwrap();
@@ -84,7 +91,12 @@ impl StreamTest<AttestationData> {
 
         for (i, test) in self.cases.iter().enumerate() {
             assert_eq!(
-                slashing_db.check_and_insert_attestation(&test.pubkey, &test.data, test.domain),
+                slashing_db.with_transaction(|txn| slashing_db.check_and_insert_attestation(
+                    &test.pubkey,
+                    &test.data,
+                    test.domain,
+                    txn
+                )),
                 test.expected,
                 "attestation {} not processed as expected",
                 i
@@ -93,6 +105,48 @@ impl StreamTest<AttestationData> {
 
         roundtrip_database(&dir, &slashing_db, self.registered_validators.is_empty());
     }
+
+    // Run the test with all attestations processed by the slashing DB as part of a batch.
+    pub fn run_batched(&self) {
+        let dir = tempdir().unwrap();
+        let slashing_db_file = dir.path().join("slashing_protection.sqlite");
+        let slashing_db = SlashingDatabase::create(&slashing_db_file).unwrap();
+
+        for pubkey in &self.registered_validators {
+            slashing_db.register_validator(*pubkey).unwrap();
+        }
+
+        check_registration_invariants(&slashing_db, &self.registered_validators);
+
+        let attestations_to_check = self
+            .cases
+            .iter()
+            .map(|test| {
+                (
+                    &test.data,
+                    &test.pubkey,
+                    test.domain,
+                    CheckSlashability::Yes,
+                )
+            })
+            .collect::<Vec<_>>();
+
+        let results = slashing_db
+            .check_and_insert_attestations(&attestations_to_check)
+            .unwrap();
+
+        assert_eq!(results.len(), self.cases.len());
+
+        for ((i, test), result) in self.cases.iter().enumerate().zip(results) {
+            assert_eq!(
+                result, test.expected,
+                "attestation {} not processed as expected",
+                i
+            );
+        }
+
+        roundtrip_database(&dir, &slashing_db, self.registered_validators.is_empty());
+    }
 }
 
 impl StreamTest<BeaconBlockHeader> {

validator_client/src/cli.rs

@@ -476,6 +476,17 @@ pub struct ValidatorClient {
     )]
     pub beacon_nodes_sync_tolerances: Vec<u64>,
 
+    #[clap(
+        long,
+        help = "Disable the beacon head monitor which tries to attest as soon as any of the \
+                configured beacon nodes sends a head event. Leaving the service enabled is \
+                recommended, but disabling it can lead to reduced bandwidth and more predictable \
+                usage of the primary beacon node (rather than the fastest BN).",
+        display_order = 0,
+        help_heading = FLAG_HEADER
+    )]
+    pub disable_beacon_head_monitor: bool,
+
     #[clap(
         long,
         help = "Disable Lighthouse's slashing protection for all web3signer keys. This can \

validator_client/src/config.rs

@@ -82,6 +82,8 @@ pub struct Config {
     pub broadcast_topics: Vec<ApiTopic>,
     /// Enables a service which attempts to measure latency between the VC and BNs.
     pub enable_latency_measurement_service: bool,
+    /// Enables the beacon head monitor that reacts to head updates from connected beacon nodes.
+    pub enable_beacon_head_monitor: bool,
     /// Defines the number of validators per `validator/register_validator` request sent to the BN.
     pub validator_registration_batch_size: usize,
     /// Whether we are running with distributed network support.
@@ -132,6 +134,7 @@ impl Default for Config {
             builder_registration_timestamp_override: None,
             broadcast_topics: vec![ApiTopic::Subscriptions],
             enable_latency_measurement_service: true,
+            enable_beacon_head_monitor: true,
             validator_registration_batch_size: 500,
             distributed: false,
             initialized_validators: <_>::default(),
@@ -377,6 +380,7 @@ impl Config {
         config.validator_store.builder_boost_factor = validator_client_config.builder_boost_factor;
         config.enable_latency_measurement_service =
             !validator_client_config.disable_latency_measurement_service;
+        config.enable_beacon_head_monitor = !validator_client_config.disable_beacon_head_monitor;
 
         config.validator_registration_batch_size =
             validator_client_config.validator_registration_batch_size;

validator_client/src/lib.rs

@@ -9,19 +9,21 @@ use metrics::set_gauge;
 use monitoring_api::{MonitoringHttpClient, ProcessType};
 use sensitive_url::SensitiveUrl;
 use slashing_protection::{SLASHING_PROTECTION_FILENAME, SlashingDatabase};
+use tokio::sync::Mutex;
 
 use account_utils::validator_definitions::ValidatorDefinitions;
 use beacon_node_fallback::{
-    BeaconNodeFallback, CandidateBeaconNode, start_fallback_updater_service,
+    BeaconNodeFallback, CandidateBeaconNode, beacon_head_monitor::HeadEvent,
+    start_fallback_updater_service,
 };
 use clap::ArgMatches;
 use doppelganger_service::DoppelgangerService;
 use environment::RuntimeContext;
-use eth2::{BeaconNodeHttpClient, StatusCode, Timeouts, reqwest::ClientBuilder};
+use eth2::{BeaconNodeHttpClient, Timeouts};
 use initialized_validators::Error::UnableToOpenVotingKeystore;
 use lighthouse_validator_store::LighthouseValidatorStore;
 use parking_lot::RwLock;
-use reqwest::Certificate;
+use reqwest::{Certificate, ClientBuilder, StatusCode};
 use slot_clock::SlotClock;
 use slot_clock::SystemTimeSlotClock;
 use std::fs::File;
@@ -45,6 +47,7 @@ use validator_services::{
     duties_service::{self, DutiesService, DutiesServiceBuilder},
     inclusion_list_service::InclusionListService,
     latency_service,
+    payload_attestation_service::PayloadAttestationService,
     preparation_service::{PreparationService, PreparationServiceBuilder},
     sync_committee_service::SyncCommitteeService,
 };
@@ -72,6 +75,8 @@ pub const AGGREGATION_PRE_COMPUTE_EPOCHS: u64 = 2;
 /// Number of slots in advance to compute sync selection proofs when in `distributed` mode.
 pub const AGGREGATION_PRE_COMPUTE_SLOTS_DISTRIBUTED: u64 = 1;
 
+const MAX_HEAD_EVENT_QUEUE_LEN: usize = 1_024;
+
 type ValidatorStore<E> = LighthouseValidatorStore<SystemTimeSlotClock, E>;
 
 #[derive(Clone)]
@@ -82,6 +87,7 @@ pub struct ProductionValidatorClient<E: EthSpec> {
     attestation_service: AttestationService<ValidatorStore<E>, SystemTimeSlotClock>,
     sync_committee_service: SyncCommitteeService<ValidatorStore<E>, SystemTimeSlotClock>,
     inclusion_list_service: InclusionListService<ValidatorStore<E>, SystemTimeSlotClock>,
+    payload_attestation_service: PayloadAttestationService<ValidatorStore<E>, SystemTimeSlotClock>,
     doppelganger_service: Option<Arc<DoppelgangerService>>,
     preparation_service: PreparationService<ValidatorStore<E>, SystemTimeSlotClock>,
     validator_store: Arc<ValidatorStore<E>>,
@@ -186,6 +192,9 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
             info!(new_validators, "Completed validator discovery");
         }
 
+        // Check for all validators' fee recipient
+        validator_defs.check_all_fee_recipients(config.validator_store.fee_recipient)?;
+
         let validators = InitializedValidators::from_definitions(
             validator_defs,
             config.validator_dir.clone(),
@@ -271,7 +280,7 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
         let beacon_node_setup = |x: (usize, &SensitiveUrl)| {
             let i = x.0;
             let url = x.1;
-            let slot_duration = Duration::from_secs(context.eth2_config.spec.seconds_per_slot);
+            let slot_duration = context.eth2_config.spec.get_slot_duration();
 
             let mut beacon_node_http_client_builder = ClientBuilder::new();
 
@@ -367,11 +376,22 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
             context.eth2_config.spec.clone(),
         );
 
-        // Perform some potentially long-running initialization tasks.
-        let (genesis_time, genesis_validators_root) = tokio::select! {
-            tuple = init_from_beacon_node::<E>(&beacon_nodes, &proposer_nodes) => tuple?,
-            () = context.executor.exit() => return Err("Shutting down".to_string())
-        };
+        let (genesis_time, genesis_validators_root) =
+            if let Some(eth2_network_config) = context.eth2_network_config.as_ref() {
+                let time = eth2_network_config
+                    .genesis_time::<E>()?
+                    .ok_or("no genesis time")?;
+                let root = eth2_network_config
+                    .genesis_validators_root::<E>()?
+                    .ok_or("no genesis validators root")?;
+                (time, root)
+            } else {
+                // Perform some potentially long-running initialization tasks.
+                tokio::select! {
+                    tuple = init_from_beacon_node::<E>(&beacon_nodes, &proposer_nodes) => tuple?,
+                    () = context.executor.exit() => return Err("Shutting down".to_string()),
+                }
+            };
 
         // Update the metrics server.
         if let Some(ctx) = &validator_metrics_ctx {
@@ -381,12 +401,23 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
         let slot_clock = SystemTimeSlotClock::new(
             context.eth2_config.spec.genesis_slot,
             Duration::from_secs(genesis_time),
-            Duration::from_secs(context.eth2_config.spec.seconds_per_slot),
+            context.eth2_config.spec.get_slot_duration(),
         );
 
         beacon_nodes.set_slot_clock(slot_clock.clone());
         proposer_nodes.set_slot_clock(slot_clock.clone());
 
+        // Only the beacon_nodes are used for attestation duties and thus biconditionally
+        // proposer_nodes do not need head_send ref.
+        let head_monitor_rx = if config.enable_beacon_head_monitor {
+            let (head_monitor_tx, head_receiver) =
+                mpsc::channel::<HeadEvent>(MAX_HEAD_EVENT_QUEUE_LEN);
+            beacon_nodes.set_head_send(Arc::new(head_monitor_tx));
+            Some(Mutex::new(head_receiver))
+        } else {
+            None
+        };
+
         let beacon_nodes = Arc::new(beacon_nodes);
         start_fallback_updater_service::<_, E>(context.executor.clone(), beacon_nodes.clone())?;
 
@@ -497,15 +528,17 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
 
         let block_service = block_service_builder.build()?;
 
-        let attestation_service = AttestationServiceBuilder::new()
+        let attestation_builder = AttestationServiceBuilder::new()
             .duties_service(duties_service.clone())
             .slot_clock(slot_clock.clone())
             .validator_store(validator_store.clone())
             .beacon_nodes(beacon_nodes.clone())
             .executor(context.executor.clone())
+            .head_monitor_rx(head_monitor_rx)
             .chain_spec(context.eth2_config.spec.clone())
-            .disable(config.disable_attesting)
-            .build()?;
+            .disable(config.disable_attesting);
+
+        let attestation_service = attestation_builder.build()?;
 
         let preparation_service = PreparationServiceBuilder::new()
             .slot_clock(slot_clock.clone())
@@ -535,6 +568,15 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
             .disable(false)
             .build()?;
 
+        let payload_attestation_service = PayloadAttestationService::new(
+            duties_service.clone(),
+            validator_store.clone(),
+            slot_clock.clone(),
+            beacon_nodes.clone(),
+            context.executor.clone(),
+            context.eth2_config.spec.clone(),
+        );
+
         Ok(Self {
             context,
             duties_service,
@@ -542,6 +584,7 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
             attestation_service,
             sync_committee_service,
             inclusion_list_service,
+            payload_attestation_service,
             doppelganger_service,
             preparation_service,
             validator_store,
@@ -618,6 +661,13 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
             .start_update_service(&self.context.eth2_config.spec)
             .map_err(|e| format!("Unable to start inclusion list service: {}", e))?;
 
+        if self.context.eth2_config.spec.is_gloas_scheduled() {
+            self.payload_attestation_service
+                .clone()
+                .start_update_service()
+                .map_err(|e| format!("Unable to start payload attestation service: {}", e))?;
+        }
+
         self.preparation_service
             .clone()
             .start_update_service(&self.context.eth2_config.spec)

validator_client/validator_metrics/src/lib.rs

@@ -22,7 +22,12 @@ pub const UPDATE_ATTESTERS_CURRENT_EPOCH: &str = "update_attesters_current_epoch
 pub const UPDATE_ATTESTERS_NEXT_EPOCH: &str = "update_attesters_next_epoch";
 pub const UPDATE_ATTESTERS_FETCH: &str = "update_attesters_fetch";
 pub const UPDATE_ATTESTERS_STORE: &str = "update_attesters_store";
+pub const UPDATE_PTC_CURRENT_EPOCH: &str = "update_ptc_current_epoch";
+pub const UPDATE_PTC_NEXT_EPOCH: &str = "update_ptc_next_epoch";
+pub const UPDATE_PTC_FETCH: &str = "update_ptc_fetch";
+pub const UPDATE_PTC_STORE: &str = "update_ptc_store";
 pub const ATTESTER_DUTIES_HTTP_POST: &str = "attester_duties_http_post";
+pub const PTC_DUTIES_HTTP_POST: &str = "ptc_duties_http_post";
 pub const PROPOSER_DUTIES_HTTP_GET: &str = "proposer_duties_http_get";
 pub const VALIDATOR_DUTIES_SYNC_HTTP_POST: &str = "validator_duties_sync_http_post";
 pub const VALIDATOR_ID_HTTP_GET: &str = "validator_id_http_get";
@@ -162,6 +167,13 @@ pub static ATTESTER_COUNT: LazyLock<Result<IntGaugeVec>> = LazyLock::new(|| {
         &["task"],
     )
 });
+pub static PTC_COUNT: LazyLock<Result<IntGaugeVec>> = LazyLock::new(|| {
+    try_create_int_gauge_vec(
+        "vc_beacon_ptc_count",
+        "Number of PTC (Payload Timeliness Committee) validators on this host",
+        &["task"],
+    )
+});
 pub static PROPOSAL_CHANGED: LazyLock<Result<IntCounter>> = LazyLock::new(|| {
     try_create_int_counter(
         "vc_beacon_block_proposal_changed",

validator_client/validator_services/Cargo.toml

@@ -13,6 +13,7 @@ futures = { workspace = true }
 graffiti_file = { workspace = true }
 logging = { workspace = true }
 parking_lot = { workspace = true }
+reqwest = { workspace = true }
 safe_arith = { workspace = true }
 slot_clock = { workspace = true }
 task_executor = { workspace = true }

validator_client/validator_services/src/attestation_service.rs

@@ -1,17 +1,19 @@
 use crate::duties_service::{DutiesService, DutyAndProof};
-use beacon_node_fallback::{ApiTopic, BeaconNodeFallback};
-use futures::future::join_all;
+use beacon_node_fallback::{ApiTopic, BeaconNodeFallback, beacon_head_monitor::HeadEvent};
+use futures::StreamExt;
 use logging::crit;
 use slot_clock::SlotClock;
 use std::collections::HashMap;
 use std::ops::Deref;
 use std::sync::Arc;
 use task_executor::TaskExecutor;
+use tokio::sync::Mutex;
+use tokio::sync::mpsc;
 use tokio::time::{Duration, Instant, sleep, sleep_until};
-use tracing::{Instrument, Span, debug, error, info, info_span, instrument, trace, warn};
+use tracing::{Instrument, debug, error, info, info_span, instrument, warn};
 use tree_hash::TreeHash;
-use types::{Attestation, AttestationData, ChainSpec, CommitteeIndex, EthSpec, Slot};
-use validator_store::{Error as ValidatorStoreError, ValidatorStore};
+use types::{Attestation, AttestationData, ChainSpec, CommitteeIndex, EthSpec, Hash256, Slot};
+use validator_store::{AggregateToSign, AttestationToSign, ValidatorStore};
 
 /// Builds an `AttestationService`.
 #[derive(Default)]
@@ -22,6 +24,7 @@ pub struct AttestationServiceBuilder<S: ValidatorStore, T: SlotClock + 'static>
     beacon_nodes: Option<Arc<BeaconNodeFallback<T>>>,
     executor: Option<TaskExecutor>,
     chain_spec: Option<Arc<ChainSpec>>,
+    head_monitor_rx: Option<Mutex<mpsc::Receiver<HeadEvent>>>,
     disable: bool,
 }
 
@@ -34,6 +37,7 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> AttestationServiceBuil
             beacon_nodes: None,
             executor: None,
             chain_spec: None,
+            head_monitor_rx: None,
             disable: false,
         }
     }
@@ -73,6 +77,13 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> AttestationServiceBuil
         self
     }
 
+    pub fn head_monitor_rx(
+        mut self,
+        head_monitor_rx: Option<Mutex<mpsc::Receiver<HeadEvent>>>,
+    ) -> Self {
+        self.head_monitor_rx = head_monitor_rx;
+        self
+    }
     pub fn build(self) -> Result<AttestationService<S, T>, String> {
         Ok(AttestationService {
             inner: Arc::new(Inner {
@@ -94,7 +105,9 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> AttestationServiceBuil
                 chain_spec: self
                     .chain_spec
                     .ok_or("Cannot build AttestationService without chain_spec")?,
+                head_monitor_rx: self.head_monitor_rx,
                 disable: self.disable,
+                latest_attested_slot: Mutex::new(Slot::default()),
             }),
         })
     }
@@ -108,10 +121,13 @@ pub struct Inner<S, T> {
     beacon_nodes: Arc<BeaconNodeFallback<T>>,
     executor: TaskExecutor,
     chain_spec: Arc<ChainSpec>,
+    head_monitor_rx: Option<Mutex<mpsc::Receiver<HeadEvent>>>,
     disable: bool,
+    latest_attested_slot: Mutex<Slot>,
 }
 
-/// Attempts to produce attestations for all known validators 1/3rd of the way through each slot.
+/// Attempts to produce attestations for all known validators 1/3rd of the way through each slot
+/// or when a head event is received from the BNs.
 ///
 /// If any validators are on the same committee, a single attestation will be downloaded and
 /// returned to the beacon node. This attestation will have a signature from each of the
@@ -144,7 +160,7 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> AttestationService<S,
             return Ok(());
         }
 
-        let slot_duration = Duration::from_secs(spec.seconds_per_slot);
+        let slot_duration = spec.get_slot_duration();
         let duration_to_next_slot = self
             .slot_clock
             .duration_to_next_slot()
@@ -157,21 +173,46 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> AttestationService<S,
 
         let executor = self.executor.clone();
 
+        let unaggregated_attestation_due = self.chain_spec.get_unaggregated_attestation_due();
+
         let interval_fut = async move {
             loop {
-                if let Some(duration_to_next_slot) = self.slot_clock.duration_to_next_slot() {
-                    sleep(duration_to_next_slot + slot_duration / 3).await;
-
-                    if let Err(e) = self.spawn_attestation_tasks(slot_duration) {
-                        crit!(error = e, "Failed to spawn attestation tasks")
-                    } else {
-                        trace!("Spawned attestation tasks");
-                    }
-                } else {
+                let Some(duration) = self.slot_clock.duration_to_next_slot() else {
                     error!("Failed to read slot clock");
-                    // If we can't read the slot clock, just wait another slot.
                     sleep(slot_duration).await;
                     continue;
+                };
+
+                let beacon_node_data = if self.head_monitor_rx.is_some() {
+                    tokio::select! {
+                        _ = sleep(duration + unaggregated_attestation_due) => None,
+                        event = self.poll_for_head_events() =>
+                            event.map(|event| (event.beacon_node_index, event.beacon_block_root)),
+                    }
+                } else {
+                    sleep(duration + unaggregated_attestation_due).await;
+                    None
+                };
+
+                let Some(current_slot) = self.slot_clock.now() else {
+                    error!("Failed to read slot clock after trigger");
+                    continue;
+                };
+
+                let mut last_slot = self.latest_attested_slot.lock().await;
+
+                if current_slot <= *last_slot {
+                    debug!(%current_slot, "Attestation already initiated for the slot");
+                    continue;
+                }
+
+                match self.spawn_attestation_tasks(beacon_node_data).await {
+                    Ok(_) => {
+                        *last_slot = current_slot;
+                    }
+                    Err(e) => {
+                        crit!(error = e, "Failed to spawn attestation tasks")
+                    }
                 }
             }
         };
@@ -180,15 +221,38 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> AttestationService<S,
         Ok(())
     }
 
+    async fn poll_for_head_events(&self) -> Option<HeadEvent> {
+        let Some(receiver) = &self.head_monitor_rx else {
+            return None;
+        };
+        let mut receiver = receiver.lock().await;
+        loop {
+            match receiver.recv().await {
+                Some(head_event) => {
+                    // Only return head events for the current slot - this ensures the
+                    // block for this slot has been produced before triggering attestation
+                    let current_slot = self.slot_clock.now()?;
+                    if head_event.slot == current_slot {
+                        return Some(head_event);
+                    }
+                    // Head event is for a previous slot, keep waiting
+                }
+                None => {
+                    warn!("Head monitor channel closed unexpectedly");
+                    return None;
+                }
+            }
+        }
+    }
+
     /// Spawn only one new task for attestation post-Electra
     /// For each required aggregates, spawn a new task that downloads, signs and uploads the
     /// aggregates to the beacon node.
-    fn spawn_attestation_tasks(&self, slot_duration: Duration) -> Result<(), String> {
+    async fn spawn_attestation_tasks(
+        &self,
+        beacon_node_data: Option<(usize, Hash256)>,
+    ) -> Result<(), String> {
         let slot = self.slot_clock.now().ok_or("Failed to read slot clock")?;
-        let duration_to_next_slot = self
-            .slot_clock
-            .duration_to_next_slot()
-            .ok_or("Unable to determine duration to next slot")?;
 
         // Create and publish an `Attestation` for all validators only once
         // as the committee_index is not included in AttestationData post-Electra
@@ -199,29 +263,89 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> AttestationService<S,
             return Ok(());
         }
 
+        debug!(
+            %slot,
+            from_head_monitor = beacon_node_data.is_some(),
+            "Starting attestation production"
+        );
+
         let attestation_service = self.clone();
 
-        let attestation_data_handle = self
+        let mut attestation_data_from_head_event = None;
+
+        if let Some((beacon_node_index, expected_block_root)) = beacon_node_data {
+            match attestation_service
+                .beacon_nodes
+                .run_on_candidate_index(beacon_node_index, |beacon_node| async move {
+                    let _timer = validator_metrics::start_timer_vec(
+                        &validator_metrics::ATTESTATION_SERVICE_TIMES,
+                        &[validator_metrics::ATTESTATIONS_HTTP_GET],
+                    );
+                    let data = beacon_node
+                        .get_validator_attestation_data(slot, 0)
+                        .await
+                        .map_err(|e| format!("Failed to produce attestation data: {:?}", e))?
+                        .data;
+
+                    if data.beacon_block_root != expected_block_root {
+                        return Err(format!(
+                            "Attestation block root mismatch: expected {:?}, got {:?}",
+                            expected_block_root, data.beacon_block_root
+                        ));
+                    }
+                    Ok(data)
+                })
+                .await
+            {
+                Ok(data) => attestation_data_from_head_event = Some(data),
+                Err(error) => {
+                    warn!(?error, "Failed to attest based on head event");
+                }
+            }
+        }
+
+        // If the beacon node that sent us the head failed to attest, wait until the attestation
+        // deadline then try all BNs.
+        let attestation_data = if let Some(attestation_data) = attestation_data_from_head_event {
+            attestation_data
+        } else {
+            let duration_to_deadline = self
+                .slot_clock
+                .duration_to_slot(slot + 1)
+                .and_then(|duration_to_next_slot| {
+                    duration_to_next_slot
+                        .checked_add(self.chain_spec.get_unaggregated_attestation_due())
+                })
+                .map(|next_slot_deadline| {
+                    next_slot_deadline.saturating_sub(self.chain_spec.get_slot_duration())
+                })
+                .unwrap_or(Duration::from_secs(0));
+            sleep(duration_to_deadline).await;
+
+            attestation_service
+                .beacon_nodes
+                .first_success(|beacon_node| async move {
+                    let _timer = validator_metrics::start_timer_vec(
+                        &validator_metrics::ATTESTATION_SERVICE_TIMES,
+                        &[validator_metrics::ATTESTATIONS_HTTP_GET],
+                    );
+                    let data = beacon_node
+                        .get_validator_attestation_data(slot, 0)
+                        .await
+                        .map_err(|e| format!("Failed to produce attestation data: {:?}", e))?
+                        .data;
+                    Ok::<AttestationData, String>(data)
+                })
+                .await
+                .map_err(|e| e.to_string())?
+        };
+
+        // Sign and publish attestations.
+        let publication_handle = self
             .inner
             .executor
             .spawn_handle(
                 async move {
-                    let attestation_data = attestation_service
-                        .beacon_nodes
-                        .first_success(|beacon_node| async move {
-                            let _timer = validator_metrics::start_timer_vec(
-                                &validator_metrics::ATTESTATION_SERVICE_TIMES,
-                                &[validator_metrics::ATTESTATIONS_HTTP_GET],
-                            );
-                            beacon_node
-                                .get_validator_attestation_data(slot, 0)
-                                .await
-                                .map_err(|e| format!("Failed to produce attestation data: {:?}", e))
-                                .map(|result| result.data)
-                        })
-                        .await
-                        .map_err(|e| e.to_string())?;
-
                     attestation_service
                         .sign_and_publish_attestations(
                             slot,
@@ -231,7 +355,7 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> AttestationService<S,
                         .await
                         .map_err(|e| {
                             crit!(
-                                error = format!("{:?}", e),
+                                error = e,
                                 slot = slot.as_u64(),
                                 "Error during attestation routine"
                             );
@@ -239,15 +363,20 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> AttestationService<S,
                         })?;
                     Ok::<AttestationData, String>(attestation_data)
                 },
-                "unaggregated attestation production",
+                "unaggregated attestation publication",
             )
             .ok_or("Failed to spawn attestation data task")?;
 
         // If a validator needs to publish an aggregate attestation, they must do so at 2/3
         // through the slot. This delay triggers at this time
+        let duration_to_next_slot = self
+            .slot_clock
+            .duration_to_slot(slot + 1)
+            .ok_or("Unable to determine duration to next slot")?;
         let aggregate_production_instant = Instant::now()
             + duration_to_next_slot
-                .checked_sub(slot_duration / 3)
+                .checked_add(self.chain_spec.get_aggregate_attestation_due())
+                .and_then(|offset| offset.checked_sub(self.chain_spec.get_slot_duration()))
                 .unwrap_or_else(|| Duration::from_secs(0));
 
         let aggregate_duties_by_committee_index: HashMap<CommitteeIndex, Vec<DutyAndProof>> = self
@@ -267,7 +396,7 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> AttestationService<S,
         self.inner.executor.spawn(
             async move {
                 // Log an error if the handle fails and return, skipping aggregates
-                let attestation_data = match attestation_data_handle.await {
+                let attestation_data = match publication_handle.await {
                     Ok(Some(Ok(data))) => data,
                     Ok(Some(Err(err))) => {
                         error!(?err, "Attestation production failed");
@@ -310,7 +439,7 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> AttestationService<S,
     }
 
     #[instrument(
-        name = "handle_aggregates",
+        name = "lh_handle_aggregates",
         skip_all,
         fields(%slot, %committee_index)
     )]
@@ -365,7 +494,7 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> AttestationService<S,
     ///
     /// The given `validator_duties` should already be filtered to only contain those that match
     /// `slot`. Critical errors will be logged if this is not the case.
-    #[instrument(skip_all, fields(%slot, %attestation_data.beacon_block_root))]
+    #[instrument(name = "lh_sign_and_publish_attestations", skip_all, fields(%slot, %attestation_data.beacon_block_root))]
     async fn sign_and_publish_attestations(
         &self,
         slot: Slot,
@@ -383,160 +512,157 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> AttestationService<S,
             .ok_or("Unable to determine current slot from clock")?
             .epoch(S::E::slots_per_epoch());
 
-        // Create futures to produce signed `Attestation` objects.
-        let attestation_data_ref = &attestation_data;
-        let signing_futures = validator_duties.iter().map(|duty_and_proof| {
-            async move {
-                let duty = &duty_and_proof.duty;
-                let attestation_data = attestation_data_ref;
+        // Make sure the target epoch is not higher than the current epoch to avoid potential attacks.
+        if attestation_data.target.epoch > current_epoch {
+            return Err(format!(
+                "Attestation target epoch {} is higher than current epoch {}",
+                attestation_data.target.epoch, current_epoch
+            ));
+        }
 
-                // Ensure that the attestation matches the duties.
-                if !duty.match_attestation_data::<S::E>(attestation_data, &self.chain_spec) {
+        // Create attestations for each validator duty.
+        let mut attestations_to_sign = Vec::with_capacity(validator_duties.len());
+
+        for duty_and_proof in validator_duties {
+            let duty = &duty_and_proof.duty;
+
+            // Ensure that the attestation matches the duties.
+            if !duty.match_attestation_data::<S::E>(&attestation_data, &self.chain_spec) {
+                crit!(
+                    validator = ?duty.pubkey,
+                    duty_slot = %duty.slot,
+                    attestation_slot = %attestation_data.slot,
+                    duty_index = duty.committee_index,
+                    attestation_index = attestation_data.index,
+                    "Inconsistent validator duties during signing"
+                );
+                continue;
+            }
+
+            let attestation = match Attestation::empty_for_signing(
+                duty.committee_index,
+                duty.committee_length as usize,
+                attestation_data.slot,
+                attestation_data.beacon_block_root,
+                attestation_data.source,
+                attestation_data.target,
+                attestation_data.index != 0,
+                &self.chain_spec,
+            ) {
+                Ok(attestation) => attestation,
+                Err(err) => {
                     crit!(
                         validator = ?duty.pubkey,
-                        duty_slot = %duty.slot,
-                        attestation_slot = %attestation_data.slot,
-                        duty_index = duty.committee_index,
-                        attestation_index = attestation_data.index,
-                        "Inconsistent validator duties during signing"
+                        ?duty,
+                        ?err,
+                        "Invalid validator duties during signing"
                     );
-                    return None;
+                    continue;
                 }
+            };
 
-                let mut attestation = match Attestation::empty_for_signing(
-                    duty.committee_index,
-                    duty.committee_length as usize,
-                    attestation_data.slot,
-                    attestation_data.beacon_block_root,
-                    attestation_data.source,
-                    attestation_data.target,
-                    &self.chain_spec,
-                ) {
-                    Ok(attestation) => attestation,
-                    Err(err) => {
-                        crit!(
-                            validator = ?duty.pubkey,
-                            ?duty,
-                            ?err,
-                            "Invalid validator duties during signing"
-                        );
-                        return None;
-                    }
-                };
+            attestations_to_sign.push(AttestationToSign {
+                validator_index: duty.validator_index,
+                pubkey: duty.pubkey,
+                validator_committee_index: duty.validator_committee_index as usize,
+                attestation,
+            });
+        }
 
-                match self
-                    .validator_store
-                    .sign_attestation(
-                        duty.pubkey,
-                        duty.validator_committee_index as usize,
-                        &mut attestation,
-                        current_epoch,
-                    )
-                    .await
-                {
-                    Ok(()) => Some((attestation, duty.validator_index)),
-                    Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
-                        // A pubkey can be missing when a validator was recently
-                        // removed via the API.
-                        warn!(
-                            info = "a validator may have recently been removed from this VC",
-                            pubkey = ?pubkey,
-                            validator = ?duty.pubkey,
-                            slot = slot.as_u64(),
-                            "Missing pubkey for attestation"
-                        );
-                        None
-                    }
-                    Err(e) => {
-                        crit!(
-                            error = ?e,
-                            validator = ?duty.pubkey,
-                            slot = slot.as_u64(),
-                            "Failed to sign attestation"
-                        );
-                        None
-                    }
-                }
-            }
-            .instrument(Span::current())
-        });
-
-        // Execute all the futures in parallel, collecting any successful results.
-        let (ref attestations, ref validator_indices): (Vec<_>, Vec<_>) = join_all(signing_futures)
-            .instrument(info_span!(
-                "sign_attestations",
-                count = validator_duties.len()
-            ))
-            .await
-            .into_iter()
-            .flatten()
-            .unzip();
-
-        if attestations.is_empty() {
-            warn!("No attestations were published");
+        if attestations_to_sign.is_empty() {
+            warn!("No valid attestations to sign");
             return Ok(());
         }
+
+        let attestation_stream = self.validator_store.sign_attestations(attestations_to_sign);
+        tokio::pin!(attestation_stream);
+
         let fork_name = self
             .chain_spec
             .fork_name_at_slot::<S::E>(attestation_data.slot);
 
-        // Post the attestations to the BN.
-        match self
-            .beacon_nodes
-            .request(ApiTopic::Attestations, |beacon_node| async move {
-                let _timer = validator_metrics::start_timer_vec(
-                    &validator_metrics::ATTESTATION_SERVICE_TIMES,
-                    &[validator_metrics::ATTESTATIONS_HTTP_POST],
-                );
+        // Publish each batch as it arrives from the stream.
+        let mut received_non_empty_batch = false;
+        while let Some(result) = attestation_stream.next().await {
+            match result {
+                Ok(batch) if !batch.is_empty() => {
+                    received_non_empty_batch = true;
 
-                let single_attestations = attestations
-                    .iter()
-                    .zip(validator_indices)
-                    .filter_map(|(a, i)| {
-                        match a.to_single_attestation_with_attester_index(*i) {
-                            Ok(a) => Some(a),
-                            Err(e) => {
-                                // This shouldn't happen unless BN and VC are out of sync with
-                                // respect to the Electra fork.
-                                error!(
-                                    error = ?e,
-                                    committee_index = attestation_data.index,
-                                    slot = slot.as_u64(),
-                                    "type" = "unaggregated",
-                                    "Unable to convert to SingleAttestation"
-                                );
-                                None
+                    let single_attestations = batch
+                        .iter()
+                        .filter_map(|(attester_index, attestation)| {
+                            match attestation
+                                .to_single_attestation_with_attester_index(*attester_index)
+                            {
+                                Ok(single_attestation) => Some(single_attestation),
+                                Err(e) => {
+                                    // This shouldn't happen unless BN and VC are out of sync with
+                                    // respect to the Electra fork.
+                                    error!(
+                                        error = ?e,
+                                        committee_index = attestation_data.index,
+                                        slot = slot.as_u64(),
+                                        "type" = "unaggregated",
+                                        "Unable to convert to SingleAttestation"
+                                    );
+                                    None
+                                }
                             }
-                        }
-                    })
-                    .collect::<Vec<_>>();
+                        })
+                        .collect::<Vec<_>>();
+                    let single_attestations = &single_attestations;
+                    let validator_indices = single_attestations
+                        .iter()
+                        .map(|att| att.attester_index)
+                        .collect::<Vec<_>>();
+                    let published_count = single_attestations.len();
 
-                beacon_node
-                    .post_beacon_pool_attestations_v2::<S::E>(single_attestations, fork_name)
-                    .await
-            })
-            .instrument(info_span!(
-                "publish_attestations",
-                count = attestations.len()
-            ))
-            .await
-        {
-            Ok(()) => info!(
-                count = attestations.len(),
-                validator_indices = ?validator_indices,
-                head_block = ?attestation_data.beacon_block_root,
-                committee_index = attestation_data.index,
-                slot = attestation_data.slot.as_u64(),
-                "type" = "unaggregated",
-                "Successfully published attestations"
-            ),
-            Err(e) => error!(
-                error = %e,
-                committee_index = attestation_data.index,
-                slot = slot.as_u64(),
-                "type" = "unaggregated",
-                "Unable to publish attestations"
-            ),
+                    // Post the attestations to the BN.
+                    match self
+                        .beacon_nodes
+                        .request(ApiTopic::Attestations, |beacon_node| async move {
+                            let _timer = validator_metrics::start_timer_vec(
+                                &validator_metrics::ATTESTATION_SERVICE_TIMES,
+                                &[validator_metrics::ATTESTATIONS_HTTP_POST],
+                            );
+
+                            beacon_node
+                                .post_beacon_pool_attestations_v2::<S::E>(
+                                    single_attestations.clone(),
+                                    fork_name,
+                                )
+                                .await
+                        })
+                        .instrument(info_span!("publish_attestations", count = published_count))
+                        .await
+                    {
+                        Ok(()) => info!(
+                            count = published_count,
+                            validator_indices = ?validator_indices,
+                            head_block = ?attestation_data.beacon_block_root,
+                            committee_index = attestation_data.index,
+                            slot = attestation_data.slot.as_u64(),
+                            "type" = "unaggregated",
+                            "Successfully published attestations"
+                        ),
+                        Err(e) => error!(
+                            error = %e,
+                            committee_index = attestation_data.index,
+                            slot = slot.as_u64(),
+                            "type" = "unaggregated",
+                            "Unable to publish attestations"
+                        ),
+                    }
+                }
+                Err(e) => {
+                    crit!(error = ?e, "Failed to sign attestations");
+                }
+                _ => {}
+            }
+        }
+
+        if !received_non_empty_batch {
+            warn!("No attestations were published");
         }
 
         Ok(())
@@ -612,113 +738,103 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> AttestationService<S,
             .await
             .map_err(|e| e.to_string())?;
 
-        // Create futures to produce the signed aggregated attestations.
-        let signing_futures = validator_duties.iter().map(|duty_and_proof| async move {
-            let duty = &duty_and_proof.duty;
-            let selection_proof = duty_and_proof.selection_proof.as_ref()?;
-
-            if !duty.match_attestation_data::<S::E>(attestation_data, &self.chain_spec) {
-                crit!("Inconsistent validator duties during signing");
-                return None;
-            }
-
-            match self
-                .validator_store
-                .produce_signed_aggregate_and_proof(
-                    duty.pubkey,
-                    duty.validator_index,
-                    aggregated_attestation.clone(),
-                    selection_proof.clone(),
-                )
-                .await
-            {
-                Ok(aggregate) => Some(aggregate),
-                Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
-                    // A pubkey can be missing when a validator was recently
-                    // removed via the API.
-                    debug!(?pubkey, "Missing pubkey for aggregate");
-                    None
-                }
-                Err(e) => {
-                    crit!(
-                        error = ?e,
-                        pubkey = ?duty.pubkey,
-                        "Failed to sign aggregate"
-                    );
-                    None
-                }
-            }
-        });
-
-        // Execute all the futures in parallel, collecting any successful results.
-        let aggregator_count = validator_duties
+        // Build the batch of aggregates to sign.
+        let aggregates_to_sign: Vec<_> = validator_duties
             .iter()
-            .filter(|d| d.selection_proof.is_some())
-            .count();
-        let signed_aggregate_and_proofs = join_all(signing_futures)
-            .instrument(info_span!("sign_aggregates", count = aggregator_count))
-            .await
-            .into_iter()
-            .flatten()
-            .collect::<Vec<_>>();
+            .filter_map(|duty_and_proof| {
+                let duty = &duty_and_proof.duty;
+                let selection_proof = duty_and_proof.selection_proof.as_ref()?;
 
-        if !signed_aggregate_and_proofs.is_empty() {
-            let signed_aggregate_and_proofs_slice = signed_aggregate_and_proofs.as_slice();
-            match self
-                .beacon_nodes
-                .first_success(|beacon_node| async move {
-                    let _timer = validator_metrics::start_timer_vec(
-                        &validator_metrics::ATTESTATION_SERVICE_TIMES,
-                        &[validator_metrics::AGGREGATES_HTTP_POST],
-                    );
-                    if fork_name.electra_enabled() {
-                        beacon_node
-                            .post_validator_aggregate_and_proof_v2(
-                                signed_aggregate_and_proofs_slice,
-                                fork_name,
-                            )
-                            .await
-                    } else {
-                        beacon_node
-                            .post_validator_aggregate_and_proof_v1(
-                                signed_aggregate_and_proofs_slice,
-                            )
-                            .await
-                    }
+                if !duty.match_attestation_data::<S::E>(attestation_data, &self.chain_spec) {
+                    crit!("Inconsistent validator duties during signing");
+                    return None;
+                }
+
+                Some(AggregateToSign {
+                    pubkey: duty.pubkey,
+                    aggregator_index: duty.validator_index,
+                    aggregate: aggregated_attestation.clone(),
+                    selection_proof: selection_proof.clone(),
                 })
-                .instrument(info_span!(
-                    "publish_aggregates",
-                    count = signed_aggregate_and_proofs.len()
-                ))
-                .await
-            {
-                Ok(()) => {
-                    for signed_aggregate_and_proof in signed_aggregate_and_proofs {
-                        let attestation = signed_aggregate_and_proof.message().aggregate();
-                        info!(
-                            aggregator = signed_aggregate_and_proof.message().aggregator_index(),
-                            signatures = attestation.num_set_aggregation_bits(),
-                            head_block = format!("{:?}", attestation.data().beacon_block_root),
-                            committee_index = attestation.committee_index(),
-                            slot = attestation.data().slot.as_u64(),
-                            "type" = "aggregated",
-                            "Successfully published attestation"
-                        );
+            })
+            .collect();
+
+        // Sign aggregates. Returns a stream of batches.
+        let aggregate_stream = self
+            .validator_store
+            .sign_aggregate_and_proofs(aggregates_to_sign);
+        tokio::pin!(aggregate_stream);
+
+        // Publish each batch as it arrives from the stream.
+        while let Some(result) = aggregate_stream.next().await {
+            match result {
+                Ok(batch) if !batch.is_empty() => {
+                    let signed_aggregate_and_proofs = batch.as_slice();
+                    match self
+                        .beacon_nodes
+                        .first_success(|beacon_node| async move {
+                            let _timer = validator_metrics::start_timer_vec(
+                                &validator_metrics::ATTESTATION_SERVICE_TIMES,
+                                &[validator_metrics::AGGREGATES_HTTP_POST],
+                            );
+                            if fork_name.electra_enabled() {
+                                beacon_node
+                                    .post_validator_aggregate_and_proof_v2(
+                                        signed_aggregate_and_proofs,
+                                        fork_name,
+                                    )
+                                    .await
+                            } else {
+                                beacon_node
+                                    .post_validator_aggregate_and_proof_v1(
+                                        signed_aggregate_and_proofs,
+                                    )
+                                    .await
+                            }
+                        })
+                        .instrument(info_span!(
+                            "publish_aggregates",
+                            count = signed_aggregate_and_proofs.len()
+                        ))
+                        .await
+                    {
+                        Ok(()) => {
+                            for signed_aggregate_and_proof in signed_aggregate_and_proofs {
+                                let attestation = signed_aggregate_and_proof.message().aggregate();
+                                info!(
+                                    aggregator =
+                                        signed_aggregate_and_proof.message().aggregator_index(),
+                                    signatures = attestation.num_set_aggregation_bits(),
+                                    head_block =
+                                        format!("{:?}", attestation.data().beacon_block_root),
+                                    committee_index = attestation.committee_index(),
+                                    slot = attestation.data().slot.as_u64(),
+                                    "type" = "aggregated",
+                                    "Successfully published attestation"
+                                );
+                            }
+                        }
+                        Err(e) => {
+                            for signed_aggregate_and_proof in signed_aggregate_and_proofs {
+                                let attestation = &signed_aggregate_and_proof.message().aggregate();
+                                crit!(
+                                    error = %e,
+                                    aggregator = signed_aggregate_and_proof
+                                        .message()
+                                        .aggregator_index(),
+                                    committee_index = attestation.committee_index(),
+                                    slot = attestation.data().slot.as_u64(),
+                                    "type" = "aggregated",
+                                    "Failed to publish attestation"
+                                );
+                            }
+                        }
                     }
                 }
                 Err(e) => {
-                    for signed_aggregate_and_proof in signed_aggregate_and_proofs {
-                        let attestation = &signed_aggregate_and_proof.message().aggregate();
-                        crit!(
-                            error = %e,
-                            aggregator = signed_aggregate_and_proof.message().aggregator_index(),
-                            committee_index = attestation.committee_index(),
-                            slot = attestation.data().slot.as_u64(),
-                            "type" = "aggregated",
-                            "Failed to publish attestation"
-                        );
-                    }
+                    crit!(error = ?e, "Failed to sign aggregates");
                 }
+                _ => {}
             }
         }
 

validator_client/validator_services/src/block_service.rs

@@ -1,9 +1,10 @@
 use beacon_node_fallback::{ApiTopic, BeaconNodeFallback, Error as FallbackError, Errors};
 use bls::PublicKeyBytes;
+use eth2::BeaconNodeHttpClient;
 use eth2::types::GraffitiPolicy;
-use eth2::{BeaconNodeHttpClient, StatusCode};
 use graffiti_file::{GraffitiFile, determine_graffiti};
 use logging::crit;
+use reqwest::StatusCode;
 use slot_clock::SlotClock;
 use std::fmt::Debug;
 use std::future::Future;
@@ -13,6 +14,7 @@ use std::time::Duration;
 use task_executor::TaskExecutor;
 use tokio::sync::mpsc;
 use tracing::{Instrument, debug, error, info, info_span, instrument, trace, warn};
+use types::consts::gloas::BUILDER_INDEX_SELF_BUILD;
 use types::{BlockType, ChainSpec, EthSpec, Graffiti, Slot};
 use validator_store::{Error as ValidatorStoreError, SignedBlock, UnsignedBlock, ValidatorStore};
 
@@ -333,7 +335,7 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> BlockService<S, T> {
     #[instrument(skip_all, fields(%slot, ?validator_pubkey))]
     async fn sign_and_publish_block(
         &self,
-        proposer_fallback: ProposerFallback<T>,
+        proposer_fallback: &ProposerFallback<T>,
         slot: Slot,
         graffiti: Option<Graffiti>,
         validator_pubkey: &PublicKeyBytes,
@@ -402,7 +404,7 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> BlockService<S, T> {
     }
 
     #[instrument(
-        name = "block_proposal_duty_cycle",
+        name = "lh_block_proposal_duty_cycle",
         skip_all,
         fields(%slot, ?validator_pubkey)
     )]
@@ -459,73 +461,145 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> BlockService<S, T> {
 
         info!(slot = slot.as_u64(), "Requesting unsigned block");
 
-        // Request an SSZ block from all beacon nodes in order, returning on the first successful response.
-        // If all nodes fail, run a second pass falling back to JSON.
-        //
-        // Proposer nodes will always be tried last during each pass since it's likely that they don't have a
-        // great view of attestations on the network.
-        let ssz_block_response = proposer_fallback
-            .request_proposers_last(|beacon_node| async move {
-                let _get_timer = validator_metrics::start_timer_vec(
-                    &validator_metrics::BLOCK_SERVICE_TIMES,
-                    &[validator_metrics::BEACON_BLOCK_HTTP_GET],
-                );
-                beacon_node
-                    .get_validator_blocks_v3_ssz::<S::E>(
-                        slot,
-                        randao_reveal_ref,
-                        graffiti.as_ref(),
-                        builder_boost_factor,
-                        self_ref.graffiti_policy,
-                    )
-                    .await
-            })
-            .await;
+        // Check if Gloas fork is active at this slot
+        let fork_name = self_ref.chain_spec.fork_name_at_slot::<S::E>(slot);
 
-        let block_response = match ssz_block_response {
-            Ok((ssz_block_response, _metadata)) => ssz_block_response,
-            Err(e) => {
-                warn!(
-                    slot = slot.as_u64(),
-                    error = %e,
-                    "SSZ block production failed, falling back to JSON"
-                );
+        let (block_proposer, unsigned_block) = if fork_name.gloas_enabled() {
+            // Use V4 block production for Gloas
+            // Request an SSZ block from all beacon nodes in order, returning on the first successful response.
+            // If all nodes fail, run a second pass falling back to JSON.
+            let ssz_block_response = proposer_fallback
+                .request_proposers_last(|beacon_node| async move {
+                    let _get_timer = validator_metrics::start_timer_vec(
+                        &validator_metrics::BLOCK_SERVICE_TIMES,
+                        &[validator_metrics::BEACON_BLOCK_HTTP_GET],
+                    );
+                    beacon_node
+                        .get_validator_blocks_v4_ssz::<S::E>(
+                            slot,
+                            randao_reveal_ref,
+                            graffiti.as_ref(),
+                            builder_boost_factor,
+                            self_ref.graffiti_policy,
+                        )
+                        .await
+                })
+                .await;
 
-                proposer_fallback
-                    .request_proposers_last(|beacon_node| async move {
-                        let _get_timer = validator_metrics::start_timer_vec(
-                            &validator_metrics::BLOCK_SERVICE_TIMES,
-                            &[validator_metrics::BEACON_BLOCK_HTTP_GET],
-                        );
-                        let (json_block_response, _metadata) = beacon_node
-                            .get_validator_blocks_v3::<S::E>(
-                                slot,
-                                randao_reveal_ref,
-                                graffiti.as_ref(),
-                                builder_boost_factor,
-                                self_ref.graffiti_policy,
-                            )
-                            .await
-                            .map_err(|e| {
-                                BlockError::Recoverable(format!(
-                                    "Error from beacon node when producing block: {:?}",
-                                    e
-                                ))
-                            })?;
+            let block_response = match ssz_block_response {
+                Ok((ssz_block_response, _metadata)) => ssz_block_response,
+                Err(e) => {
+                    warn!(
+                        slot = slot.as_u64(),
+                        error = %e,
+                        "SSZ V4 block production failed, falling back to JSON"
+                    );
 
-                        Ok(json_block_response.data)
-                    })
-                    .await
-                    .map_err(BlockError::from)?
-            }
-        };
+                    proposer_fallback
+                        .request_proposers_last(|beacon_node| async move {
+                            let _get_timer = validator_metrics::start_timer_vec(
+                                &validator_metrics::BLOCK_SERVICE_TIMES,
+                                &[validator_metrics::BEACON_BLOCK_HTTP_GET],
+                            );
+                            let (json_block_response, _metadata) = beacon_node
+                                .get_validator_blocks_v4::<S::E>(
+                                    slot,
+                                    randao_reveal_ref,
+                                    graffiti.as_ref(),
+                                    builder_boost_factor,
+                                    self_ref.graffiti_policy,
+                                )
+                                .await
+                                .map_err(|e| {
+                                    BlockError::Recoverable(format!(
+                                        "Error from beacon node when producing block: {:?}",
+                                        e
+                                    ))
+                                })?;
 
-        let (block_proposer, unsigned_block) = match block_response {
-            eth2::types::ProduceBlockV3Response::Full(block) => {
-                (block.block().proposer_index(), UnsignedBlock::Full(block))
-            }
-            eth2::types::ProduceBlockV3Response::Blinded(block) => {
-                (block.proposer_index(), UnsignedBlock::Blinded(block))
+                            Ok(json_block_response.data)
+                        })
+                        .await
+                        .map_err(BlockError::from)?
+                }
+            };
+
+            // Gloas blocks don't have blobs (they're in the execution layer)
+            let block_contents = eth2::types::FullBlockContents::Block(block_response);
+            (
+                block_contents.block().proposer_index(),
+                UnsignedBlock::Full(block_contents),
+            )
+        } else {
+            // Use V3 block production for pre-Gloas forks
+            // Request an SSZ block from all beacon nodes in order, returning on the first successful response.
+            // If all nodes fail, run a second pass falling back to JSON.
+            //
+            // Proposer nodes will always be tried last during each pass since it's likely that they don't have a
+            // great view of attestations on the network.
+            let ssz_block_response = proposer_fallback
+                .request_proposers_last(|beacon_node| async move {
+                    let _get_timer = validator_metrics::start_timer_vec(
+                        &validator_metrics::BLOCK_SERVICE_TIMES,
+                        &[validator_metrics::BEACON_BLOCK_HTTP_GET],
+                    );
+                    beacon_node
+                        .get_validator_blocks_v3_ssz::<S::E>(
+                            slot,
+                            randao_reveal_ref,
+                            graffiti.as_ref(),
+                            builder_boost_factor,
+                            self_ref.graffiti_policy,
+                        )
+                        .await
+                })
+                .await;
+
+            let block_response = match ssz_block_response {
+                Ok((ssz_block_response, _metadata)) => ssz_block_response,
+                Err(e) => {
+                    warn!(
+                        slot = slot.as_u64(),
+                        error = %e,
+                        "SSZ block production failed, falling back to JSON"
+                    );
+
+                    proposer_fallback
+                        .request_proposers_last(|beacon_node| async move {
+                            let _get_timer = validator_metrics::start_timer_vec(
+                                &validator_metrics::BLOCK_SERVICE_TIMES,
+                                &[validator_metrics::BEACON_BLOCK_HTTP_GET],
+                            );
+                            let (json_block_response, _metadata) = beacon_node
+                                .get_validator_blocks_v3::<S::E>(
+                                    slot,
+                                    randao_reveal_ref,
+                                    graffiti.as_ref(),
+                                    builder_boost_factor,
+                                    self_ref.graffiti_policy,
+                                )
+                                .await
+                                .map_err(|e| {
+                                    BlockError::Recoverable(format!(
+                                        "Error from beacon node when producing block: {:?}",
+                                        e
+                                    ))
+                                })?;
+
+                            Ok(json_block_response.data)
+                        })
+                        .await
+                        .map_err(BlockError::from)?
+                }
+            };
+
+            match block_response {
+                eth2::types::ProduceBlockV3Response::Full(block) => {
+                    (block.block().proposer_index(), UnsignedBlock::Full(block))
+                }
+                eth2::types::ProduceBlockV3Response::Blinded(block) => {
+                    (block.proposer_index(), UnsignedBlock::Blinded(block))
+                }
             }
         };
 
@@ -538,7 +612,7 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> BlockService<S, T> {
 
         self_ref
             .sign_and_publish_block(
-                proposer_fallback,
+                &proposer_fallback,
                 slot,
                 graffiti,
                 &validator_pubkey,
@@ -546,6 +620,108 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> BlockService<S, T> {
             )
             .await?;
 
+        // TODO(gloas) we only need to fetch, sign and publish the envelope in the local building case.
+        // Right now we always default to local building. Once we implement trustless/trusted builder logic
+        // we should check the bid for index == BUILDER_INDEX_SELF_BUILD
+        if fork_name.gloas_enabled() {
+            self_ref
+                .fetch_sign_and_publish_payload_envelope(
+                    &proposer_fallback,
+                    slot,
+                    &validator_pubkey,
+                )
+                .await?;
+        }
+
+        Ok(())
+    }
+
+    /// Fetch, sign, and publish the execution payload envelope for Gloas.
+    /// This should be called after the block has been published.
+    ///
+    /// TODO(gloas): For multi-BN setups, we need to track which beacon node produced the block
+    /// and fetch the envelope from that same node. The envelope is cached per-BN,
+    /// so fetching from a different BN than the one that built the block will fail.
+    /// See: https://github.com/sigp/lighthouse/pull/8313
+    #[instrument(skip_all)]
+    async fn fetch_sign_and_publish_payload_envelope(
+        &self,
+        _proposer_fallback: &ProposerFallback<T>,
+        slot: Slot,
+        validator_pubkey: &PublicKeyBytes,
+    ) -> Result<(), BlockError> {
+        info!(slot = slot.as_u64(), "Fetching execution payload envelope");
+
+        // Fetch the envelope from the beacon node. Use builder_index=BUILDER_INDEX_SELF_BUILD for local building.
+        // TODO(gloas): Use proposer_fallback once multi-BN is supported.
+        let envelope = self
+            .beacon_nodes
+            .first_success(|beacon_node| async move {
+                beacon_node
+                    .get_validator_execution_payload_envelope_ssz::<S::E>(
+                        slot,
+                        BUILDER_INDEX_SELF_BUILD,
+                    )
+                    .await
+                    .map_err(|e| {
+                        BlockError::Recoverable(format!(
+                            "Error fetching execution payload envelope: {:?}",
+                            e
+                        ))
+                    })
+            })
+            .await?;
+
+        info!(
+            slot = slot.as_u64(),
+            beacon_block_root = %envelope.beacon_block_root,
+            "Received execution payload envelope, signing"
+        );
+
+        // Sign the envelope
+        let signed_envelope = self
+            .validator_store
+            .sign_execution_payload_envelope(*validator_pubkey, envelope)
+            .await
+            .map_err(|e| {
+                BlockError::Recoverable(format!(
+                    "Error signing execution payload envelope: {:?}",
+                    e
+                ))
+            })?;
+
+        info!(
+            slot = slot.as_u64(),
+            "Signed execution payload envelope, publishing"
+        );
+
+        let fork_name = self.chain_spec.fork_name_at_slot::<S::E>(slot);
+
+        // Publish the signed envelope
+        // TODO(gloas): Use proposer_fallback once multi-BN is supported.
+        self.beacon_nodes
+            .first_success(|beacon_node| {
+                let signed_envelope = signed_envelope.clone();
+                async move {
+                    beacon_node
+                        .post_beacon_execution_payload_envelope_ssz(&signed_envelope, fork_name)
+                        .await
+                        .map_err(|e| {
+                            BlockError::Recoverable(format!(
+                                "Error publishing execution payload envelope: {:?}",
+                                e
+                            ))
+                        })
+                }
+            })
+            .await?;
+
+        info!(
+            slot = slot.as_u64(),
+            beacon_block_root = %signed_envelope.message.beacon_block_root,
+            "Successfully published signed execution payload envelope"
+        );
+
         Ok(())
     }
 

validator_client/validator_services/src/duties_service.rs

@@ -13,11 +13,11 @@ use beacon_node_fallback::{ApiTopic, BeaconNodeFallback};
 use bls::PublicKeyBytes;
 use eth2::types::{
     AttesterData, BeaconCommitteeSelection, BeaconCommitteeSubscription, DutiesResponse,
-    InclusionListDuty, ProposerData, StateId, ValidatorId,
+    InclusionListDuty, ProposerData, PtcDuty, StateId, ValidatorId,
 };
 use futures::{
     StreamExt,
-    stream::{self, FuturesUnordered},
+    stream::{self, FuturesUnordered, TryStreamExt},
 };
 use parking_lot::{RwLock, RwLockWriteGuard};
 use safe_arith::{ArithError, SafeArith};
@@ -46,6 +46,7 @@ const VALIDATOR_METRICS_MIN_COUNT: usize = 64;
 /// The initial request is used to determine if further requests are required, so that it
 /// reduces the amount of data that needs to be transferred.
 const INITIAL_DUTIES_QUERY_SIZE: usize = 1;
+const INITIAL_PTC_DUTIES_QUERY_SIZE: usize = 1;
 
 /// Offsets from the attestation duty slot at which a subscription should be sent.
 const ATTESTATION_SUBSCRIPTION_OFFSETS: [u64; 8] = [3, 4, 5, 6, 7, 8, 16, 32];
@@ -83,6 +84,7 @@ const _: () = assert!(ATTESTATION_SUBSCRIPTION_OFFSETS[0] > MIN_ATTESTATION_SUBS
 pub enum Error<T> {
     UnableToReadSlotClock,
     FailedToDownloadAttesters(#[allow(dead_code)] String),
+    FailedToDownloadPtc(#[allow(dead_code)] String),
     FailedToProduceSelectionProof(#[allow(dead_code)] ValidatorStoreError<T>),
     InvalidModulo(#[allow(dead_code)] ArithError),
     Arith(#[allow(dead_code)] ArithError),
@@ -142,71 +144,15 @@ impl Default for SelectionProofConfig {
 /// Create a selection proof for `duty`.
 ///
 /// Return `Ok(None)` if the attesting validator is not an aggregator.
-async fn make_selection_proof<S: ValidatorStore + 'static, T: SlotClock>(
+async fn make_selection_proof<S: ValidatorStore>(
     duty: &AttesterData,
     validator_store: &S,
     spec: &ChainSpec,
-    beacon_nodes: &Arc<BeaconNodeFallback<T>>,
-    config: &SelectionProofConfig,
 ) -> Result<Option<SelectionProof>, Error<S::Error>> {
-    let selection_proof = if config.selections_endpoint {
-        let beacon_committee_selection = BeaconCommitteeSelection {
-            validator_index: duty.validator_index,
-            slot: duty.slot,
-            // This is partial selection proof
-            selection_proof: validator_store
-                .produce_selection_proof(duty.pubkey, duty.slot)
-                .await
-                .map_err(Error::FailedToProduceSelectionProof)?
-                .into(),
-        };
-        // Call the endpoint /eth/v1/validator/beacon_committee_selections
-        // by sending the BeaconCommitteeSelection that contains partial selection proof
-        // The middleware should return BeaconCommitteeSelection that contains full selection proof
-        let middleware_response = beacon_nodes
-            .first_success(|beacon_node| {
-                let selection_data = beacon_committee_selection.clone();
-                debug!(
-                    "validator_index" = duty.validator_index,
-                    "slot" = %duty.slot,
-                    "partial selection proof" = ?beacon_committee_selection.selection_proof,
-                    "Sending selection to middleware"
-                );
-                async move {
-                    beacon_node
-                        .post_validator_beacon_committee_selections(&[selection_data])
-                        .await
-                }
-            })
-            .await;
-
-        let response_data = middleware_response
-            .map_err(|e| {
-                Error::FailedToProduceSelectionProof(ValidatorStoreError::Middleware(e.to_string()))
-            })?
-            .data
-            .pop()
-            .ok_or_else(|| {
-                Error::FailedToProduceSelectionProof(ValidatorStoreError::Middleware(format!(
-                    "attestation selection proof - empty response for validator {}",
-                    duty.validator_index
-                )))
-            })?;
-
-        debug!(
-            "validator_index" = response_data.validator_index,
-            "slot" = %response_data.slot,
-            // The selection proof from middleware response will be a full selection proof
-            "full selection proof" = ?response_data.selection_proof,
-            "Received selection from middleware"
-        );
-        SelectionProof::from(response_data.selection_proof)
-    } else {
-        validator_store
-            .produce_selection_proof(duty.pubkey, duty.slot)
-            .await
-            .map_err(Error::FailedToProduceSelectionProof)?
-    };
+    let selection_proof = validator_store
+        .produce_selection_proof(duty.pubkey, duty.slot)
+        .await
+        .map_err(Error::FailedToProduceSelectionProof)?;
 
     selection_proof
         .is_aggregator(duty.committee_length as usize, spec)
@@ -222,6 +168,69 @@ async fn make_selection_proof<S: ValidatorStore + 'static, T: SlotClock>(
         })
 }
 
+/// Create a Vec<BeaconCommitteeSelection> for every epoch
+/// so that when calling the selections_endpoint later, it calls once per epoch with duties of all slots in that epoch
+async fn make_beacon_committee_selection<S: ValidatorStore, T: SlotClock>(
+    duties_service: &Arc<DutiesService<S, T>>,
+    duties: &[AttesterData],
+) -> Result<Vec<BeaconCommitteeSelection>, Error<S::Error>> {
+    // collect the BeaconCommitteeSelection in duties
+    let beacon_committee_selections = duties
+        .iter()
+        .map(|duty| {
+            let validator_store = &duties_service.validator_store;
+            async move {
+                let partial_selection_proof = validator_store
+                    .produce_selection_proof(duty.pubkey, duty.slot)
+                    .await
+                    .map_err(Error::FailedToProduceSelectionProof)?;
+                Ok::<BeaconCommitteeSelection, Error<S::Error>>(BeaconCommitteeSelection {
+                    validator_index: duty.validator_index,
+                    slot: duty.slot,
+                    selection_proof: partial_selection_proof.into(),
+                })
+            }
+        })
+        .collect::<FuturesUnordered<_>>()
+        .try_collect::<Vec<_>>()
+        .await?;
+
+    let epoch = duties
+        .first()
+        .map(|attester_data| attester_data.slot.epoch(S::E::slots_per_epoch()))
+        .unwrap_or_default();
+
+    debug!(
+        %epoch,
+        count = beacon_committee_selections.len(),
+        "Sending selections to middleware"
+    );
+
+    let selections = duties_service
+        .beacon_nodes
+        .first_success(|beacon_node| {
+            let selections = beacon_committee_selections.clone();
+            async move {
+                beacon_node
+                    .post_validator_beacon_committee_selections(&selections)
+                    .await
+            }
+        })
+        .await
+        .map_err(|e| {
+            Error::FailedToProduceSelectionProof(ValidatorStoreError::Middleware(e.to_string()))
+        })?
+        .data;
+
+    debug!(
+        %epoch,
+        count = beacon_committee_selections.len(),
+        "Received selections from middleware"
+    );
+
+    Ok(selections)
+}
+
 impl DutyAndProof {
     /// Create a new `DutyAndProof` with the selection proof waiting to be filled in.
     pub fn new_without_selection_proof(duty: AttesterData, current_slot: Slot) -> Self {
@@ -279,6 +288,7 @@ type AttesterMap = HashMap<PublicKeyBytes, HashMap<Epoch, (DependentRoot, DutyAn
 type ProposerMap = HashMap<Epoch, (DependentRoot, Vec<ProposerData>)>;
 type InclusionListDutiesMap =
     HashMap<PublicKeyBytes, HashMap<Epoch, (DependentRoot, InclusionListDuty)>>;
+type PtcMap = HashMap<Epoch, (DependentRoot, Vec<PtcDuty>)>;
 
 pub struct DutiesServiceBuilder<S, T> {
     /// Provides the canonical list of locally-managed validators.
@@ -381,6 +391,7 @@ impl<S, T> DutiesServiceBuilder<S, T> {
             proposers: Default::default(),
             sync_duties: SyncDutiesMap::new(self.sync_selection_proof_config),
             inclusion_list_duties: Default::default(),
+            ptc_duties: Default::default(),
             validator_store: self
                 .validator_store
                 .ok_or("Cannot build DutiesService without validator_store")?,
@@ -413,6 +424,8 @@ pub struct DutiesService<S, T> {
     /// Maps a validator public key to their inclusion list committee duties for each epoch.
     pub inclusion_list_duties: RwLock<InclusionListDutiesMap>,
     pub sync_duties: SyncDutiesMap,
+    /// Maps an epoch to PTC duties for locally-managed validators.
+    pub ptc_duties: RwLock<PtcMap>,
     /// Provides the canonical list of locally-managed validators.
     pub validator_store: Arc<S>,
     /// Maps unknown validator pubkeys to the next slot time when a poll should be conducted again.
@@ -464,13 +477,22 @@ impl<S: ValidatorStore, T: SlotClock + 'static> DutiesService<S, T> {
             .voting_pubkeys(DoppelgangerStatus::only_safe);
         self.attesters
             .read()
-            .iter()
-            .filter_map(|(_, map)| map.get(&epoch))
+            .values()
+            .filter_map(|map| map.get(&epoch))
             .map(|(_, duty_and_proof)| duty_and_proof)
             .filter(|duty_and_proof| signing_pubkeys.contains(&duty_and_proof.duty.pubkey))
             .count()
     }
 
+    /// Returns the total number of validators that have PTC duties in the given epoch.
+    pub fn ptc_count(&self, epoch: Epoch) -> usize {
+        self.ptc_duties
+            .read()
+            .get(&epoch)
+            .map(|(_, duties)| duties.len())
+            .unwrap_or(0)
+    }
+
     /// Returns the total number of validators that are in a doppelganger detection period.
     pub fn doppelganger_detecting_count(&self) -> usize {
         self.validator_store
@@ -517,8 +539,8 @@ impl<S: ValidatorStore, T: SlotClock + 'static> DutiesService<S, T> {
 
         self.attesters
             .read()
-            .iter()
-            .filter_map(|(_, map)| map.get(&epoch))
+            .values()
+            .filter_map(|map| map.get(&epoch))
             .map(|(_, duty_and_proof)| duty_and_proof)
             .filter(|duty_and_proof| {
                 duty_and_proof.duty.slot == slot
@@ -556,6 +578,25 @@ impl<S: ValidatorStore, T: SlotClock + 'static> DutiesService<S, T> {
         self.enable_high_validator_count_metrics
             || self.total_validator_count() <= VALIDATOR_METRICS_MIN_COUNT
     }
+
+    /// Get PTC duties for a specific slot.
+    ///
+    /// Returns duties for local validators who have PTC assignments at the given slot.
+    pub fn get_ptc_duties_for_slot(&self, slot: Slot) -> Vec<PtcDuty> {
+        let epoch = slot.epoch(S::E::slots_per_epoch());
+
+        self.ptc_duties
+            .read()
+            .get(&epoch)
+            .map(|(_, ptc_duties)| {
+                ptc_duties
+                    .iter()
+                    .filter(|ptc_duty| ptc_duty.slot == slot)
+                    .cloned()
+                    .collect()
+            })
+            .unwrap_or_default()
+    }
 }
 
 /// Start the service that periodically polls the beacon node for validator duties. This will start
@@ -711,6 +752,61 @@ pub fn start_update_service<S: ValidatorStore + 'static, T: SlotClock + 'static>
         },
         "duties_service_inclusion_list_committee",
     );
+
+    // Spawn the task which keeps track of local PTC duties.
+    // Only start PTC duties service if Gloas fork is scheduled.
+    if core_duties_service.spec.is_gloas_scheduled() {
+        let duties_service = core_duties_service.clone();
+        core_duties_service.executor.spawn(
+            async move {
+                loop {
+                    // Check if we've reached the Gloas fork epoch before polling
+                    let Some(current_slot) = duties_service.slot_clock.now() else {
+                        // Unable to read slot clock, sleep and try again
+                        sleep(duties_service.slot_clock.slot_duration()).await;
+                        continue;
+                    };
+
+                    let current_epoch = current_slot.epoch(S::E::slots_per_epoch());
+                    let Some(gloas_fork_epoch) = duties_service.spec.gloas_fork_epoch else {
+                        // Gloas fork epoch not configured, should not reach here
+                        break;
+                    };
+
+                    if current_epoch + 1 < gloas_fork_epoch {
+                        // Wait until the next slot and check again
+                        if let Some(duration) = duties_service.slot_clock.duration_to_next_slot() {
+                            sleep(duration).await;
+                        } else {
+                            sleep(duties_service.slot_clock.slot_duration()).await;
+                        }
+                        continue;
+                    }
+
+                    if let Err(e) = poll_beacon_ptc_attesters(&duties_service).await {
+                        error!(
+                            error = ?e,
+                           "Failed to poll PTC duties"
+                        );
+                    }
+
+                    // Wait until the next slot before polling again.
+                    // This doesn't mean that the beacon node will get polled every slot
+                    // as the PTC duties service will return early if it deems it already has
+                    // enough information.
+                    if let Some(duration) = duties_service.slot_clock.duration_to_next_slot() {
+                        sleep(duration).await;
+                    } else {
+                        // Just sleep for one slot if we are unable to read the system clock, this gives
+                        // us an opportunity for the clock to eventually come good.
+                        sleep(duties_service.slot_clock.slot_duration()).await;
+                        continue;
+                    }
+                }
+            },
+            "duties_service_ptc",
+        );
+    }
 }
 
 /// Iterate through all the voting pubkeys in the `ValidatorStore` and attempt to learn any unknown
@@ -943,8 +1039,8 @@ async fn poll_beacon_attesters<S: ValidatorStore + 'static, T: SlotClock + 'stat
         duties_service
             .attesters
             .read()
-            .iter()
-            .filter_map(|(_, map)| map.get(epoch))
+            .values()
+            .filter_map(|map| map.get(epoch))
             .filter(|(_, duty_and_proof)| {
                 duty_and_proof
                     .subscription_slots
@@ -1331,6 +1427,26 @@ fn process_duty_and_proof<S: ValidatorStore>(
     }
 }
 
+async fn post_validator_duties_ptc<S: ValidatorStore, T: SlotClock + 'static>(
+    duties_service: &Arc<DutiesService<S, T>>,
+    epoch: Epoch,
+    validator_indices: &[u64],
+) -> Result<DutiesResponse<Vec<PtcDuty>>, Error<S::Error>> {
+    duties_service
+        .beacon_nodes
+        .first_success(|beacon_node| async move {
+            let _timer = validator_metrics::start_timer_vec(
+                &validator_metrics::DUTIES_SERVICE_TIMES,
+                &[validator_metrics::PTC_DUTIES_HTTP_POST],
+            );
+            beacon_node
+                .post_validator_duties_ptc(epoch, validator_indices)
+                .await
+        })
+        .await
+        .map_err(|e| Error::FailedToDownloadPtc(e.to_string()))
+}
+
 /// Compute the attestation selection proofs for the `duties` and add them to the `attesters` map.
 ///
 /// Duties are computed in batches each slot. If a re-org is detected then the process will
@@ -1343,14 +1459,21 @@ async fn fill_in_selection_proofs<S: ValidatorStore + 'static, T: SlotClock + 's
     // Sort duties by slot in a BTreeMap.
     let mut duties_by_slot: BTreeMap<Slot, Vec<_>> = BTreeMap::new();
 
-    for duty in duties {
-        duties_by_slot.entry(duty.slot).or_default().push(duty);
+    for duty in &duties {
+        duties_by_slot
+            .entry(duty.slot)
+            .or_default()
+            .push(duty.clone());
     }
 
     // At halfway through each slot when nothing else is likely to be getting signed, sign a batch
     // of selection proofs and insert them into the duties service `attesters` map.
     let slot_clock = &duties_service.slot_clock;
 
+    // Create a HashMap for BeaconCommitteeSelection to match the duty later for distributed case involving middleware
+    let mut selection_hashmap = HashMap::new();
+    let mut call_selection_endpoint = false;
+
     while !duties_by_slot.is_empty() {
         if let Some(duration) = slot_clock.duration_to_next_slot() {
             sleep(
@@ -1389,10 +1512,77 @@ async fn fill_in_selection_proofs<S: ValidatorStore + 'static, T: SlotClock + 's
                 &[validator_metrics::ATTESTATION_SELECTION_PROOFS],
             );
 
-            // In distributed case, we want to send all partial selection proofs to the middleware to determine aggregation duties,
-            // as the middleware will need to have a threshold of partial selection proofs to be able to return the full selection proof
-            // Thus, sign selection proofs in parallel in distributed case; Otherwise, sign them serially in non-distributed (normal) case
-            if duties_service.selection_proof_config.parallel_sign {
+            // for distributed case that uses the selections_endpoint
+            if duties_service.selection_proof_config.selections_endpoint {
+                // Using lookahead_slot to determine if it is the first slot of an epoch
+                let is_lookahead_slot_epoch_start = lookahead_slot % S::E::slots_per_epoch() == 0;
+
+                // Call the selection endpoint only at the first slot of an epoch or when it errors
+                if is_lookahead_slot_epoch_start || call_selection_endpoint {
+                    let beacon_committee_selections =
+                        make_beacon_committee_selection(&duties_service, &duties).await;
+
+                    let selections = match beacon_committee_selections {
+                        Ok(selections) => selections,
+                        Err(e) => {
+                            error!(
+                                error = ?e,
+                                "Failed to fetch selection proofs"
+                            );
+                            // If calling the endpoint fails, change to true so that it will retry the next slot
+                            call_selection_endpoint = true;
+                            continue;
+                        }
+                    };
+
+                    for selection in &selections {
+                        // This is a full_selection_proof returned by middleware
+                        let selection_proof =
+                            SelectionProof::from(selection.selection_proof.clone());
+                        selection_hashmap
+                            .insert((selection.validator_index, selection.slot), selection_proof);
+                    }
+                    // Once we have the selection_proof, we don't call the selections_endpoint again
+                    call_selection_endpoint = false;
+                }
+
+                for duty in relevant_duties.into_values().flatten() {
+                    let key = (duty.validator_index, duty.slot);
+
+                    let result = if let Some(selection_proof) = selection_hashmap.remove(&key) {
+                        match selection_proof
+                            .is_aggregator(duty.committee_length as usize, &duties_service.spec)
+                            .map_err(Error::<S::Error>::InvalidModulo)
+                        {
+                            // Aggregator, return the result
+                            Ok(true) => Ok((duty, Some(selection_proof))),
+                            // Not an aggregator, do nothing and continue
+                            Ok(false) => continue,
+                            Err(_) => return,
+                        }
+                    } else {
+                        Err(Error::FailedToProduceSelectionProof(
+                            ValidatorStoreError::Middleware(format!(
+                                "Missing selection proof for validator {} slot {}",
+                                duty.validator_index, duty.slot
+                            )),
+                        ))
+                    };
+
+                    let mut attesters = duties_service.attesters.write();
+                    // if process_duty_and_proof returns false, exit the loop
+                    if !process_duty_and_proof::<S>(
+                        &mut attesters,
+                        result,
+                        dependent_root,
+                        current_slot,
+                    ) {
+                        return;
+                    }
+                }
+            }
+            // For distributed case that uses parallel_sign
+            else if duties_service.selection_proof_config.parallel_sign {
                 let mut duty_and_proof_results = relevant_duties
                     .into_values()
                     .flatten()
@@ -1401,8 +1591,6 @@ async fn fill_in_selection_proofs<S: ValidatorStore + 'static, T: SlotClock + 's
                             &duty,
                             duties_service.validator_store.as_ref(),
                             &duties_service.spec,
-                            &duties_service.beacon_nodes,
-                            &duties_service.selection_proof_config,
                         )
                         .await?;
                         Ok((duty, opt_selection_proof))
@@ -1429,8 +1617,6 @@ async fn fill_in_selection_proofs<S: ValidatorStore + 'static, T: SlotClock + 's
                             &duty,
                             duties_service.validator_store.as_ref(),
                             &duties_service.spec,
-                            &duties_service.beacon_nodes,
-                            &duties_service.selection_proof_config,
                         )
                         .await?;
                         Ok((duty, opt_selection_proof))
@@ -1857,6 +2043,209 @@ async fn poll_beacon_proposers<S: ValidatorStore, T: SlotClock + 'static>(
     Ok(())
 }
 
+/// Query the beacon node for ptc duties for any known validators.
+async fn poll_beacon_ptc_attesters<S: ValidatorStore + 'static, T: SlotClock + 'static>(
+    duties_service: &Arc<DutiesService<S, T>>,
+) -> Result<(), Error<S::Error>> {
+    let current_epoch_timer = validator_metrics::start_timer_vec(
+        &validator_metrics::DUTIES_SERVICE_TIMES,
+        &[validator_metrics::UPDATE_PTC_CURRENT_EPOCH],
+    );
+
+    let current_slot = duties_service
+        .slot_clock
+        .now()
+        .ok_or(Error::UnableToReadSlotClock)?;
+    let current_epoch = current_slot.epoch(S::E::slots_per_epoch());
+
+    // Collect *all* pubkeys, even those undergoing doppelganger protection.
+    let local_pubkeys: HashSet<_> = duties_service
+        .validator_store
+        .voting_pubkeys(DoppelgangerStatus::ignored);
+
+    let local_indices = {
+        let mut local_indices = Vec::with_capacity(local_pubkeys.len());
+
+        for &pubkey in &local_pubkeys {
+            if let Some(validator_index) = duties_service.validator_store.validator_index(&pubkey) {
+                local_indices.push(validator_index)
+            }
+        }
+        local_indices
+    };
+
+    // Poll for current epoch
+    if let Err(e) = poll_beacon_ptc_attesters_for_epoch(
+        duties_service,
+        current_epoch,
+        &local_indices,
+        &local_pubkeys,
+    )
+    .await
+    {
+        error!(
+            %current_epoch,
+            request_epoch = %current_epoch,
+            err = ?e,
+            "Failed to download PTC duties"
+        );
+    }
+    drop(current_epoch_timer);
+    let next_epoch_timer = validator_metrics::start_timer_vec(
+        &validator_metrics::DUTIES_SERVICE_TIMES,
+        &[validator_metrics::UPDATE_PTC_NEXT_EPOCH],
+    );
+
+    // Poll for next epoch
+    let next_epoch = current_epoch + 1;
+    if let Err(e) = poll_beacon_ptc_attesters_for_epoch(
+        duties_service,
+        next_epoch,
+        &local_indices,
+        &local_pubkeys,
+    )
+    .await
+    {
+        error!(
+            %current_epoch,
+            request_epoch = %next_epoch,
+            err = ?e,
+            "Failed to download PTC duties"
+        );
+    }
+    drop(next_epoch_timer);
+
+    // Prune old duties.
+    duties_service
+        .ptc_duties
+        .write()
+        .retain(|&epoch, _| epoch + HISTORICAL_DUTIES_EPOCHS >= current_epoch);
+
+    Ok(())
+}
+
+/// For the given `local_indices` and `local_pubkeys`, download the PTC duties for the given `epoch` and
+/// store them in `duties_service.ptc_duties` using bandwidth optimization.
+async fn poll_beacon_ptc_attesters_for_epoch<
+    S: ValidatorStore + 'static,
+    T: SlotClock + 'static,
+>(
+    duties_service: &Arc<DutiesService<S, T>>,
+    epoch: Epoch,
+    local_indices: &[u64],
+    local_pubkeys: &HashSet<PublicKeyBytes>,
+) -> Result<(), Error<S::Error>> {
+    // No need to bother the BN if we don't have any validators.
+    if local_indices.is_empty() {
+        debug!(
+            %epoch,
+            "No validators, not downloading PTC duties"
+        );
+        return Ok(());
+    }
+
+    let fetch_timer = validator_metrics::start_timer_vec(
+        &validator_metrics::DUTIES_SERVICE_TIMES,
+        &[validator_metrics::UPDATE_PTC_FETCH],
+    );
+
+    // TODO(gloas) Unlike attester duties which use `get_uninitialized_validators` to detect
+    // newly-added validators, PTC duties only check dependent_root changes. Validators added
+    // mid-epoch won't get PTC duties until the next epoch boundary. We should probably fix this.
+    let initial_indices_to_request =
+        &local_indices[0..min(INITIAL_PTC_DUTIES_QUERY_SIZE, local_indices.len())];
+
+    let response =
+        post_validator_duties_ptc(duties_service, epoch, initial_indices_to_request).await?;
+    let dependent_root = response.dependent_root;
+
+    // Check if we need to update duties for this epoch and collect validators to update.
+    // We update if we have no epoch data OR if the dependent_root changed.
+    let validators_to_update = {
+        // Avoid holding the read-lock for any longer than required.
+        let ptc_duties = duties_service.ptc_duties.read();
+        let needs_update = ptc_duties.get(&epoch).is_none_or(|(prior_root, _duties)| {
+            // Update if dependent_root changed
+            *prior_root != dependent_root
+        });
+
+        if needs_update {
+            local_pubkeys.iter().collect::<Vec<_>>()
+        } else {
+            Vec::new()
+        }
+    };
+
+    if validators_to_update.is_empty() {
+        // No validators have conflicting (epoch, dependent_root) values for this epoch.
+        return Ok(());
+    }
+
+    // Make a request for all indices that require updating which we have not already made a request for.
+    let indices_to_request = validators_to_update
+        .iter()
+        .filter_map(|pubkey| duties_service.validator_store.validator_index(pubkey))
+        .filter(|validator_index| !initial_indices_to_request.contains(validator_index))
+        .collect::<Vec<_>>();
+
+    // Filter the initial duties by their relevance so that we don't hit warnings about
+    // overwriting duties.
+    let new_initial_duties = response
+        .data
+        .into_iter()
+        .filter(|duty| validators_to_update.contains(&&duty.pubkey));
+
+    let mut new_duties = if !indices_to_request.is_empty() {
+        post_validator_duties_ptc(duties_service, epoch, indices_to_request.as_slice())
+            .await?
+            .data
+    } else {
+        vec![]
+    };
+    new_duties.extend(new_initial_duties);
+
+    drop(fetch_timer);
+
+    let _store_timer = validator_metrics::start_timer_vec(
+        &validator_metrics::DUTIES_SERVICE_TIMES,
+        &[validator_metrics::UPDATE_PTC_STORE],
+    );
+
+    debug!(
+        %dependent_root,
+        num_new_duties = new_duties.len(),
+        "Downloaded PTC duties"
+    );
+
+    // Update duties - we only reach here if dependent_root changed or epoch is missing
+    let mut ptc_duties = duties_service.ptc_duties.write();
+
+    match ptc_duties.entry(epoch) {
+        hash_map::Entry::Occupied(mut entry) => {
+            // Dependent root must have changed, so we do complete replacement.
+            // We cannot support partial updates for the same dependent_root.
+            // The beacon node may return incomplete duty lists and we cannot distinguish between "no duties" and
+            // "duties not included in this response". We could query all local validators in each
+            // `post_validator_duties_ptc` call regardless of dependent_root changes, but the bandwidth
+            // cost is likely not justified since PTC assignments are sparse.
+            let (existing_root, _existing_duties) = entry.get();
+            debug!(
+                old_root = %existing_root,
+                new_root = %dependent_root,
+                "PTC dependent root changed, replacing all duties"
+            );
+
+            *entry.get_mut() = (dependent_root, new_duties);
+        }
+        hash_map::Entry::Vacant(entry) => {
+            // No existing duties for this epoch
+            entry.insert((dependent_root, new_duties));
+        }
+    }
+
+    Ok(())
+}
+
 /// Notify the block service if it should produce a block.
 async fn notify_block_production_service<S: ValidatorStore>(
     current_slot: Slot,

validator_client/validator_services/src/inclusion_list_service.rs

@@ -136,7 +136,7 @@ impl<S, T> Deref for InclusionListService<S, T> {
 impl<S: ValidatorStore + 'static, T: SlotClock + 'static> InclusionListService<S, T> {
     /// Starts the service which periodically produces inclusion lists.
     pub fn start_update_service(self, spec: &ChainSpec) -> Result<(), String> {
-        let slot_duration = Duration::from_secs(spec.seconds_per_slot);
+        let slot_duration = spec.get_slot_duration();
 
         let duration_to_next_slot = self
             .slot_clock

validator_client/validator_services/src/lib.rs

@@ -4,6 +4,7 @@ pub mod duties_service;
 pub mod inclusion_list_service;
 pub mod latency_service;
 pub mod notifier_service;
+pub mod payload_attestation_service;
 pub mod preparation_service;
 pub mod sync;
 pub mod sync_committee_service;

validator_client/validator_services/src/notifier_service.rs

@@ -2,7 +2,7 @@ use crate::duties_service::DutiesService;
 use slot_clock::SlotClock;
 use std::sync::Arc;
 use task_executor::TaskExecutor;
-use tokio::time::{Duration, sleep};
+use tokio::time::sleep;
 use tracing::{debug, error, info};
 use types::{ChainSpec, EthSpec};
 use validator_metrics::set_gauge;
@@ -14,7 +14,7 @@ pub fn spawn_notifier<S: ValidatorStore + 'static, T: SlotClock + 'static>(
     executor: TaskExecutor,
     spec: &ChainSpec,
 ) -> Result<(), String> {
-    let slot_duration = Duration::from_secs(spec.seconds_per_slot);
+    let slot_duration = spec.get_slot_duration();
 
     let interval_fut = async move {
         loop {
@@ -109,6 +109,7 @@ pub async fn notify<S: ValidatorStore, T: SlotClock + 'static>(
         let total_validators = duties_service.total_validator_count();
         let proposing_validators = duties_service.proposer_count(epoch);
         let attesting_validators = duties_service.attester_count(epoch);
+        let ptc_validators = duties_service.ptc_count(epoch);
         let doppelganger_detecting_validators = duties_service.doppelganger_detecting_count();
 
         if doppelganger_detecting_validators > 0 {
@@ -126,6 +127,7 @@ pub async fn notify<S: ValidatorStore, T: SlotClock + 'static>(
         } else if total_validators == attesting_validators {
             info!(
                 current_epoch_proposers = proposing_validators,
+                current_epoch_ptc = ptc_validators,
                 active_validators = attesting_validators,
                 total_validators = total_validators,
                 %epoch,
@@ -135,6 +137,7 @@ pub async fn notify<S: ValidatorStore, T: SlotClock + 'static>(
         } else if attesting_validators > 0 {
             info!(
                 current_epoch_proposers = proposing_validators,
+                current_epoch_ptc = ptc_validators,
                 active_validators = attesting_validators,
                 total_validators = total_validators,
                 %epoch,
@@ -146,7 +149,7 @@ pub async fn notify<S: ValidatorStore, T: SlotClock + 'static>(
                 validators = total_validators,
                 %epoch,
                 %slot,
-                "Awaiting activation"
+                "All validators inactive"
             );
         }
     } else {

validator_client/validator_services/src/payload_attestation_service.rs

@@ -0,0 +1,243 @@
+use crate::duties_service::DutiesService;
+use beacon_node_fallback::BeaconNodeFallback;
+use logging::crit;
+use slot_clock::SlotClock;
+use std::ops::Deref;
+use std::sync::Arc;
+use task_executor::TaskExecutor;
+use tokio::time::sleep;
+use tracing::{debug, error, info};
+use types::{ChainSpec, EthSpec};
+use validator_store::ValidatorStore;
+
+pub struct Inner<S, T> {
+    duties_service: Arc<DutiesService<S, T>>,
+    validator_store: Arc<S>,
+    slot_clock: T,
+    beacon_nodes: Arc<BeaconNodeFallback<T>>,
+    executor: TaskExecutor,
+    chain_spec: Arc<ChainSpec>,
+}
+
+pub struct PayloadAttestationService<S, T> {
+    inner: Arc<Inner<S, T>>,
+}
+
+impl<S, T> Clone for PayloadAttestationService<S, T> {
+    fn clone(&self) -> Self {
+        Self {
+            inner: self.inner.clone(),
+        }
+    }
+}
+
+impl<S, T> Deref for PayloadAttestationService<S, T> {
+    type Target = Inner<S, T>;
+
+    fn deref(&self) -> &Self::Target {
+        self.inner.deref()
+    }
+}
+
+impl<S: ValidatorStore + 'static, T: SlotClock + 'static> PayloadAttestationService<S, T> {
+    pub fn new(
+        duties_service: Arc<DutiesService<S, T>>,
+        validator_store: Arc<S>,
+        slot_clock: T,
+        beacon_nodes: Arc<BeaconNodeFallback<T>>,
+        executor: TaskExecutor,
+        chain_spec: Arc<ChainSpec>,
+    ) -> Self {
+        Self {
+            inner: Arc::new(Inner {
+                duties_service,
+                validator_store,
+                slot_clock,
+                beacon_nodes,
+                executor,
+                chain_spec,
+            }),
+        }
+    }
+
+    pub fn start_update_service(self) -> Result<(), String> {
+        let slot_duration = self.chain_spec.get_slot_duration();
+        let payload_attestation_due = self.chain_spec.get_payload_attestation_due();
+
+        info!(
+            payload_attestation_due_ms = payload_attestation_due.as_millis(),
+            "Payload attestation service started"
+        );
+
+        let executor = self.executor.clone();
+
+        let interval_fut = async move {
+            loop {
+                let Some(duration_to_next_slot) = self.slot_clock.duration_to_next_slot() else {
+                    error!("Failed to read slot clock");
+                    sleep(slot_duration).await;
+                    continue;
+                };
+
+                let Some(current_slot) = self.slot_clock.now() else {
+                    error!("Failed to read slot clock after trigger");
+                    continue;
+                };
+
+                if !self
+                    .chain_spec
+                    .fork_name_at_slot::<S::E>(current_slot)
+                    .gloas_enabled()
+                {
+                    let duration_to_next_epoch = self
+                        .slot_clock
+                        .duration_to_next_epoch(S::E::slots_per_epoch())
+                        .unwrap_or_else(|| {
+                            self.chain_spec.get_slot_duration() * S::E::slots_per_epoch() as u32
+                        });
+                    sleep(duration_to_next_epoch).await;
+                    continue;
+                }
+
+                sleep(duration_to_next_slot + payload_attestation_due).await;
+
+                let Some(attestation_slot) = self.slot_clock.now() else {
+                    error!("Failed to read slot clock after sleep");
+                    continue;
+                };
+
+                let service = self.clone();
+                self.executor.spawn(
+                    async move {
+                        service.produce_and_publish(attestation_slot).await;
+                    },
+                    "payload_attestation_producer",
+                );
+            }
+        };
+
+        executor.spawn(interval_fut, "payload_attestation_service");
+        Ok(())
+    }
+
+    async fn produce_and_publish(&self, slot: types::Slot) {
+        let duties = self.duties_service.get_ptc_duties_for_slot(slot);
+
+        if duties.is_empty() {
+            return;
+        }
+
+        debug!(
+            %slot,
+            duty_count = duties.len(),
+            "Producing payload attestations"
+        );
+
+        let attestation_data = match self
+            .beacon_nodes
+            .first_success(|beacon_node| async move {
+                beacon_node
+                    .get_validator_payload_attestation_data(slot)
+                    .await
+                    .map_err(|e| format!("Failed to get payload attestation data: {e:?}"))
+                    .map(|resp| resp.into_data())
+            })
+            .await
+        {
+            Ok(data) => data,
+            Err(e) => {
+                crit!(
+                    error = %e,
+                    %slot,
+                    "Failed to produce payload attestation data"
+                );
+                return;
+            }
+        };
+
+        debug!(
+            %slot,
+            beacon_block_root = ?attestation_data.beacon_block_root,
+            payload_present = attestation_data.payload_present,
+            "Received payload attestation data"
+        );
+
+        let mut messages = Vec::with_capacity(duties.len());
+
+        for duty in &duties {
+            match self
+                .validator_store
+                .sign_payload_attestation(duty.pubkey, attestation_data.clone())
+                .await
+            {
+                Ok(message) => {
+                    messages.push(message);
+                }
+                Err(e) => {
+                    crit!(
+                        error = ?e,
+                        validator = ?duty.pubkey,
+                        %slot,
+                        "Failed to sign payload attestation"
+                    );
+                }
+            }
+        }
+
+        if messages.is_empty() {
+            return;
+        }
+
+        let count = messages.len();
+        let fork_name = self.chain_spec.fork_name_at_slot::<S::E>(slot);
+        let result = self
+            .beacon_nodes
+            .first_success(|beacon_node| {
+                let messages = messages.clone();
+                async move {
+                    beacon_node
+                        .post_beacon_pool_payload_attestations_ssz(&messages, fork_name)
+                        .await
+                        .map_err(|e| format!("Failed to publish payload attestations (SSZ): {e:?}"))
+                }
+            })
+            .await;
+
+        let result = match result {
+            Ok(()) => Ok(()),
+            Err(_) => {
+                debug!(%slot, "SSZ publish failed, falling back to JSON");
+                self.beacon_nodes
+                    .first_success(|beacon_node| {
+                        let messages = messages.clone();
+                        async move {
+                            beacon_node
+                                .post_beacon_pool_payload_attestations(&messages, fork_name)
+                                .await
+                                .map_err(|e| {
+                                    format!("Failed to publish payload attestations (JSON): {e:?}")
+                                })
+                        }
+                    })
+                    .await
+            }
+        };
+
+        match result {
+            Ok(()) => {
+                info!(
+                    %slot,
+                    %count,
+                    "Successfully published payload attestations"
+                );
+            }
+            Err(e) => {
+                crit!(
+                    error = %e,
+                    %slot,
+                    "Failed to publish payload attestations"
+                );
+            }
+        }
+    }
+}

validator_client/validator_services/src/preparation_service.rs

@@ -8,7 +8,7 @@ use std::ops::Deref;
 use std::sync::Arc;
 use std::time::{SystemTime, UNIX_EPOCH};
 use task_executor::TaskExecutor;
-use tokio::time::{Duration, sleep};
+use tokio::time::sleep;
 use tracing::{debug, error, info, warn};
 use types::{
     Address, ChainSpec, EthSpec, ProposerPreparationData, SignedValidatorRegistrationData,
@@ -174,7 +174,7 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> PreparationService<S,
 
     /// Starts the service which periodically produces proposer preparations.
     pub fn start_proposer_prepare_service(self, spec: &ChainSpec) -> Result<(), String> {
-        let slot_duration = Duration::from_secs(spec.seconds_per_slot);
+        let slot_duration = spec.get_slot_duration();
         info!("Proposer preparation service started");
 
         let executor = self.executor.clone();
@@ -214,7 +214,7 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> PreparationService<S,
         info!("Validator registration service started");
 
         let spec = spec.clone();
-        let slot_duration = Duration::from_secs(spec.seconds_per_slot);
+        let slot_duration = spec.get_slot_duration();
 
         let executor = self.executor.clone();
 

validator_client/validator_services/src/sync_committee_service.rs

@@ -2,8 +2,8 @@ use crate::duties_service::DutiesService;
 use beacon_node_fallback::{ApiTopic, BeaconNodeFallback};
 use bls::PublicKeyBytes;
 use eth2::types::BlockId;
+use futures::StreamExt;
 use futures::future::FutureExt;
-use futures::future::join_all;
 use logging::crit;
 use slot_clock::SlotClock;
 use std::collections::HashMap;
@@ -17,7 +17,7 @@ use types::{
     ChainSpec, EthSpec, Hash256, Slot, SyncCommitteeSubscription, SyncContributionData, SyncDuty,
     SyncSelectionProof, SyncSubnetId,
 };
-use validator_store::{Error as ValidatorStoreError, ValidatorStore};
+use validator_store::{ContributionToSign, SyncMessageToSign, ValidatorStore};
 
 pub const SUBSCRIPTION_LOOKAHEAD_EPOCHS: u64 = 4;
 
@@ -93,7 +93,7 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> SyncCommitteeService<S
             return Ok(());
         }
 
-        let slot_duration = Duration::from_secs(spec.seconds_per_slot);
+        let slot_duration = spec.get_slot_duration();
         let duration_to_next_slot = self
             .slot_clock
             .duration_to_next_slot()
@@ -106,18 +106,20 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> SyncCommitteeService<S
 
         let executor = self.executor.clone();
 
+        let sync_message_slot_component = spec.get_sync_message_due();
+
         let interval_fut = async move {
             loop {
                 if let Some(duration_to_next_slot) = self.slot_clock.duration_to_next_slot() {
                     // Wait for contribution broadcast interval 1/3 of the way through the slot.
-                    sleep(duration_to_next_slot + slot_duration / 3).await;
+                    sleep(duration_to_next_slot + sync_message_slot_component).await;
 
                     // Do nothing if the Altair fork has not yet occurred.
                     if !self.altair_fork_activated() {
                         continue;
                     }
 
-                    if let Err(e) = self.spawn_contribution_tasks(slot_duration).await {
+                    if let Err(e) = self.spawn_contribution_tasks().await {
                         crit!(
                             error = ?e,
                             "Failed to spawn sync contribution tasks"
@@ -140,7 +142,8 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> SyncCommitteeService<S
         Ok(())
     }
 
-    async fn spawn_contribution_tasks(&self, slot_duration: Duration) -> Result<(), String> {
+    async fn spawn_contribution_tasks(&self) -> Result<(), String> {
+        let spec = &self.duties_service.spec;
         let slot = self.slot_clock.now().ok_or("Failed to read slot clock")?;
         let duration_to_next_slot = self
             .slot_clock
@@ -151,7 +154,8 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> SyncCommitteeService<S
         // through the slot. This delay triggers at this time
         let aggregate_production_instant = Instant::now()
             + duration_to_next_slot
-                .checked_sub(slot_duration / 3)
+                .checked_add(spec.get_contribution_message_due())
+                .and_then(|offset| offset.checked_sub(spec.get_slot_duration()))
                 .unwrap_or_else(|| Duration::from_secs(0));
 
         let Some(slot_duties) = self
@@ -210,7 +214,7 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> SyncCommitteeService<S
                     .map(|_| ())
                     .await
             }
-            .instrument(info_span!("sync_committee_signature_publish", %slot)),
+            .instrument(info_span!("lh_sync_committee_signature_publish", %slot)),
             "sync_committee_signature_publish",
         );
 
@@ -228,7 +232,7 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> SyncCommitteeService<S
                     .map(|_| ())
                     .await
             }
-            .instrument(info_span!("sync_committee_aggregate_publish", %slot)),
+            .instrument(info_span!("lh_sync_committee_aggregate_publish", %slot)),
             "sync_committee_aggregate_publish",
         );
 
@@ -243,78 +247,57 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> SyncCommitteeService<S
         beacon_block_root: Hash256,
         validator_duties: Vec<SyncDuty>,
     ) -> Result<(), ()> {
-        // Create futures to produce sync committee signatures.
-        let signature_futures = validator_duties.iter().map(|duty| async move {
-            match self
-                .validator_store
-                .produce_sync_committee_signature(
-                    slot,
-                    beacon_block_root,
-                    duty.validator_index,
-                    &duty.pubkey,
-                )
-                .await
-            {
-                Ok(signature) => Some(signature),
-                Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
-                    // A pubkey can be missing when a validator was recently
-                    // removed via the API.
-                    debug!(
-                        ?pubkey,
-                        validator_index = duty.validator_index,
-                        %slot,
-                        "Missing pubkey for sync committee signature"
-                    );
-                    None
+        let messages_to_sign: Vec<_> = validator_duties
+            .iter()
+            .map(|duty| SyncMessageToSign {
+                slot,
+                beacon_block_root,
+                validator_index: duty.validator_index,
+                pubkey: duty.pubkey,
+            })
+            .collect();
+
+        let signature_stream = self
+            .validator_store
+            .sign_sync_committee_signatures(messages_to_sign);
+        tokio::pin!(signature_stream);
+
+        while let Some(result) = signature_stream.next().await {
+            match result {
+                Ok(committee_signatures) if !committee_signatures.is_empty() => {
+                    let committee_signatures = &committee_signatures;
+                    match self
+                        .beacon_nodes
+                        .request(ApiTopic::SyncCommittee, |beacon_node| async move {
+                            beacon_node
+                                .post_beacon_pool_sync_committee_signatures(committee_signatures)
+                                .await
+                        })
+                        .instrument(info_span!(
+                            "publish_sync_signatures",
+                            count = committee_signatures.len()
+                        ))
+                        .await
+                    {
+                        Ok(()) => info!(
+                            count = committee_signatures.len(),
+                            head_block = ?beacon_block_root,
+                            %slot,
+                            "Successfully published sync committee messages"
+                        ),
+                        Err(e) => error!(
+                            %slot,
+                            error = %e,
+                            "Unable to publish sync committee messages"
+                        ),
+                    }
                 }
                 Err(e) => {
-                    crit!(
-                        validator_index = duty.validator_index,
-                        %slot,
-                        error = ?e,
-                        "Failed to sign sync committee signature"
-                    );
-                    None
+                    crit!(%slot, error = ?e, "Failed to sign sync committee signatures");
                 }
+                _ => {}
             }
-        });
-
-        // Execute all the futures in parallel, collecting any successful results.
-        let committee_signatures = &join_all(signature_futures)
-            .instrument(info_span!(
-                "sign_sync_signatures",
-                count = validator_duties.len()
-            ))
-            .await
-            .into_iter()
-            .flatten()
-            .collect::<Vec<_>>();
-
-        self.beacon_nodes
-            .request(ApiTopic::SyncCommittee, |beacon_node| async move {
-                beacon_node
-                    .post_beacon_pool_sync_committee_signatures(committee_signatures)
-                    .await
-            })
-            .instrument(info_span!(
-                "publish_sync_signatures",
-                count = committee_signatures.len()
-            ))
-            .await
-            .map_err(|e| {
-                error!(
-                    %slot,
-                    error = %e,
-                    "Unable to publish sync committee messages"
-                );
-            })?;
-
-        info!(
-            count = committee_signatures.len(),
-            head_block = ?beacon_block_root,
-            %slot,
-            "Successfully published sync committee messages"
-        );
+        }
 
         Ok(())
     }
@@ -341,7 +324,7 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> SyncCommitteeService<S
                         .map(|_| ())
                         .await
                 }
-                .instrument(info_span!("publish_sync_committee_aggregate_for_subnet", %slot, ?beacon_block_root, %subnet_id)),
+                .instrument(info_span!("lh_publish_sync_committee_aggregate_for_subnet", %slot, ?beacon_block_root, %subnet_id)),
                 "sync_committee_aggregate_publish_subnet",
             );
         }
@@ -385,77 +368,61 @@ impl<S: ValidatorStore + 'static, T: SlotClock + 'static> SyncCommitteeService<S
             })?
             .data;
 
-        // Create futures to produce signed contributions.
-        let aggregator_count = subnet_aggregators.len();
-        let signature_futures = subnet_aggregators.into_iter().map(
-            |(aggregator_index, aggregator_pk, selection_proof)| async move {
-                match self
-                    .validator_store
-                    .produce_signed_contribution_and_proof(
-                        aggregator_index,
-                        aggregator_pk,
-                        contribution.clone(),
-                        selection_proof,
-                    )
-                    .await
-                {
-                    Ok(signed_contribution) => Some(signed_contribution),
-                    Err(ValidatorStoreError::UnknownPubkey(pubkey)) => {
-                        // A pubkey can be missing when a validator was recently
-                        // removed via the API.
-                        debug!(?pubkey, %slot, "Missing pubkey for sync contribution");
-                        None
-                    }
-                    Err(e) => {
-                        crit!(
+        let contributions_to_sign: Vec<_> = subnet_aggregators
+            .into_iter()
+            .map(
+                |(aggregator_index, aggregator_pk, selection_proof)| ContributionToSign {
+                    aggregator_index,
+                    aggregator_pubkey: aggregator_pk,
+                    contribution: contribution.clone(),
+                    selection_proof,
+                },
+            )
+            .collect();
+
+        let contribution_stream = self
+            .validator_store
+            .sign_sync_committee_contributions(contributions_to_sign);
+        tokio::pin!(contribution_stream);
+
+        while let Some(result) = contribution_stream.next().await {
+            match result {
+                Ok(signed_contributions) if !signed_contributions.is_empty() => {
+                    let signed_contributions = &signed_contributions;
+                    // Publish to the beacon node.
+                    match self
+                        .beacon_nodes
+                        .first_success(|beacon_node| async move {
+                            beacon_node
+                                .post_validator_contribution_and_proofs(signed_contributions)
+                                .await
+                        })
+                        .instrument(info_span!(
+                            "publish_sync_contributions",
+                            count = signed_contributions.len()
+                        ))
+                        .await
+                    {
+                        Ok(()) => info!(
+                            subnet = %subnet_id,
+                            beacon_block_root = %beacon_block_root,
+                            num_signers = contribution.aggregation_bits.num_set_bits(),
                             %slot,
-                            error = ?e,
-                            "Unable to sign sync committee contribution"
-                        );
-                        None
+                            "Successfully published sync contributions"
+                        ),
+                        Err(e) => error!(
+                            %slot,
+                            error = %e,
+                            "Unable to publish signed contributions and proofs"
+                        ),
                     }
                 }
-            },
-        );
-
-        // Execute all the futures in parallel, collecting any successful results.
-        let signed_contributions = &join_all(signature_futures)
-            .instrument(info_span!(
-                "sign_sync_contributions",
-                count = aggregator_count
-            ))
-            .await
-            .into_iter()
-            .flatten()
-            .collect::<Vec<_>>();
-
-        // Publish to the beacon node.
-        self.beacon_nodes
-            .first_success(|beacon_node| async move {
-                beacon_node
-                    .post_validator_contribution_and_proofs(signed_contributions)
-                    .await
-            })
-            .instrument(info_span!(
-                "publish_sync_contributions",
-                count = signed_contributions.len()
-            ))
-            .await
-            .map_err(|e| {
-                error!(
-                    %slot,
-                    error = %e,
-                    "Unable to publish signed contributions and proofs"
-                );
-            })?;
-
-        info!(
-            subnet = %subnet_id,
-            beacon_block_root = %beacon_block_root,
-            num_signers = contribution.aggregation_bits.num_set_bits(),
-            %slot,
-            "Successfully published sync contributions"
-        );
+                Err(e) => {
+                    crit!(%slot, error = ?e, "Failed to sign sync committee contributions");
+                }
+                _ => {}
+            }
+        }
 
         Ok(())
     }

validator_client/validator_store/Cargo.toml

@@ -7,5 +7,6 @@ authors = ["Sigma Prime <contact@sigmaprime.io>"]
 [dependencies]
 bls = { workspace = true }
 eth2 = { workspace = true }
+futures = { workspace = true }
 slashing_protection = { workspace = true }
 types = { workspace = true }

validator_client/validator_store/src/lib.rs

@@ -1,13 +1,16 @@
 use bls::{PublicKeyBytes, Signature};
 use eth2::types::{FullBlockContents, PublishBlockRequest};
+use futures::Stream;
 use slashing_protection::NotSafe;
 use std::fmt::Debug;
 use std::future::Future;
 use std::sync::Arc;
 use types::{
-    Address, Attestation, AttestationError, BlindedBeaconBlock, Epoch, EthSpec, Graffiti, Hash256,
-    InclusionList, SelectionProof, SignedAggregateAndProof, SignedBlindedBeaconBlock,
-    SignedContributionAndProof, SignedInclusionList, SignedValidatorRegistrationData, Slot,
+    Address, Attestation, AttestationError, BlindedBeaconBlock, Epoch, EthSpec,
+    ExecutionPayloadEnvelope, Graffiti, Hash256, InclusionList, PayloadAttestationData,
+    PayloadAttestationMessage, SelectionProof, SignedAggregateAndProof, SignedBlindedBeaconBlock,
+    SignedContributionAndProof, SignedExecutionPayloadEnvelope, SignedInclusionList,
+    SignedValidatorRegistrationData, Slot,
     SyncCommitteeContribution, SyncCommitteeMessage, SyncSelectionProof, SyncSubnetId,
     ValidatorRegistrationData,
 };
@@ -20,9 +23,9 @@ pub enum Error<T> {
     Slashable(NotSafe),
     SameData,
     GreaterThanCurrentSlot { slot: Slot, current_slot: Slot },
-    GreaterThanCurrentEpoch { epoch: Epoch, current_epoch: Epoch },
     UnableToSignAttestation(AttestationError),
     SpecificError(T),
+    ExecutorError,
     Middleware(String),
 }
 
@@ -32,6 +35,38 @@ impl<T> From<T> for Error<T> {
     }
 }
 
+/// Input for batch attestation signing
+pub struct AttestationToSign<E: EthSpec> {
+    pub validator_index: u64,
+    pub pubkey: PublicKeyBytes,
+    pub validator_committee_index: usize,
+    pub attestation: Attestation<E>,
+}
+
+/// Input for batch aggregate signing
+pub struct AggregateToSign<E: EthSpec> {
+    pub pubkey: PublicKeyBytes,
+    pub aggregator_index: u64,
+    pub aggregate: Attestation<E>,
+    pub selection_proof: SelectionProof,
+}
+
+/// Input for batch sync committee message signing
+pub struct SyncMessageToSign {
+    pub slot: Slot,
+    pub beacon_block_root: Hash256,
+    pub validator_index: u64,
+    pub pubkey: PublicKeyBytes,
+}
+
+/// Input for batch sync committee contribution signing
+pub struct ContributionToSign<E: EthSpec> {
+    pub aggregator_index: u64,
+    pub aggregator_pubkey: PublicKeyBytes,
+    pub contribution: SyncCommitteeContribution<E>,
+    pub selection_proof: SyncSelectionProof,
+}
+
 /// A helper struct, used for passing data from the validator store to services.
 pub struct ProposalData {
     pub validator_index: Option<u64>,
@@ -104,31 +139,26 @@ pub trait ValidatorStore: Send + Sync {
         current_slot: Slot,
     ) -> impl Future<Output = Result<SignedBlock<Self::E>, Error<Self::Error>>> + Send;
 
-    fn sign_attestation(
-        &self,
-        validator_pubkey: PublicKeyBytes,
-        validator_committee_position: usize,
-        attestation: &mut Attestation<Self::E>,
-        current_epoch: Epoch,
-    ) -> impl Future<Output = Result<(), Error<Self::Error>>> + Send;
+    /// Sign a batch of `attestations` and apply slashing protection to them.
+    ///
+    /// Returns a stream of batches of successfully signed attestations. Each batch contains
+    /// attestations that passed slashing protection, along with the validator index of the signer.
+    /// Eventually this will be replaced by `SingleAttestation` use.
+    ///
+    /// Output:
+    ///
+    /// * Vec of (validator_index, signed_attestation).
+    #[allow(clippy::type_complexity)]
+    fn sign_attestations(
+        self: &Arc<Self>,
+        attestations: Vec<AttestationToSign<Self::E>>,
+    ) -> impl Stream<Item = Result<Vec<(u64, Attestation<Self::E>)>, Error<Self::Error>>> + Send;
 
     fn sign_validator_registration_data(
         &self,
         validator_registration_data: ValidatorRegistrationData,
     ) -> impl Future<Output = Result<SignedValidatorRegistrationData, Error<Self::Error>>> + Send;
 
-    /// Signs an `AggregateAndProof` for a given validator.
-    ///
-    /// The resulting `SignedAggregateAndProof` is sent on the aggregation channel and cannot be
-    /// modified by actors other than the signing validator.
-    fn produce_signed_aggregate_and_proof(
-        &self,
-        validator_pubkey: PublicKeyBytes,
-        aggregator_index: u64,
-        aggregate: Attestation<Self::E>,
-        selection_proof: SelectionProof,
-    ) -> impl Future<Output = Result<SignedAggregateAndProof<Self::E>, Error<Self::Error>>> + Send;
-
     /// Produces a `SelectionProof` for the `slot`, signed by with corresponding secret key to
     /// `validator_pubkey`.
     fn produce_selection_proof(
@@ -145,21 +175,23 @@ pub trait ValidatorStore: Send + Sync {
         subnet_id: SyncSubnetId,
     ) -> impl Future<Output = Result<SyncSelectionProof, Error<Self::Error>>> + Send;
 
-    fn produce_sync_committee_signature(
-        &self,
-        slot: Slot,
-        beacon_block_root: Hash256,
-        validator_index: u64,
-        validator_pubkey: &PublicKeyBytes,
-    ) -> impl Future<Output = Result<SyncCommitteeMessage, Error<Self::Error>>> + Send;
+    /// Sign a batch of aggregate and proofs and return results as a stream of batches.
+    fn sign_aggregate_and_proofs(
+        self: &Arc<Self>,
+        aggregates: Vec<AggregateToSign<Self::E>>,
+    ) -> impl Stream<Item = Result<Vec<SignedAggregateAndProof<Self::E>>, Error<Self::Error>>> + Send;
 
-    fn produce_signed_contribution_and_proof(
-        &self,
-        aggregator_index: u64,
-        aggregator_pubkey: PublicKeyBytes,
-        contribution: SyncCommitteeContribution<Self::E>,
-        selection_proof: SyncSelectionProof,
-    ) -> impl Future<Output = Result<SignedContributionAndProof<Self::E>, Error<Self::Error>>> + Send;
+    /// Sign a batch of sync committee messages and return results as a stream of batches.
+    fn sign_sync_committee_signatures(
+        self: &Arc<Self>,
+        messages: Vec<SyncMessageToSign>,
+    ) -> impl Stream<Item = Result<Vec<SyncCommitteeMessage>, Error<Self::Error>>> + Send;
+
+    /// Sign a batch of sync committee contributions and return results as a stream of batches.
+    fn sign_sync_committee_contributions(
+        self: &Arc<Self>,
+        contributions: Vec<ContributionToSign<Self::E>>,
+    ) -> impl Stream<Item = Result<Vec<SignedContributionAndProof<Self::E>>, Error<Self::Error>>> + Send;
 
     fn sign_inclusion_list(
         &self,
@@ -174,6 +206,20 @@ pub trait ValidatorStore: Send + Sync {
     /// runs.
     fn prune_slashing_protection_db(&self, current_epoch: Epoch, first_run: bool);
 
+    /// Sign an `ExecutionPayloadEnvelope` for Gloas.
+    fn sign_execution_payload_envelope(
+        &self,
+        validator_pubkey: PublicKeyBytes,
+        envelope: ExecutionPayloadEnvelope<Self::E>,
+    ) -> impl Future<Output = Result<SignedExecutionPayloadEnvelope<Self::E>, Error<Self::Error>>> + Send;
+
+    /// Sign a `PayloadAttestationData` for the PTC.
+    fn sign_payload_attestation(
+        &self,
+        validator_pubkey: PublicKeyBytes,
+        data: PayloadAttestationData,
+    ) -> impl Future<Output = Result<PayloadAttestationMessage, Error<Self::Error>>> + Send;
+
     /// Returns `ProposalData` for the provided `pubkey` if it exists in `InitializedValidators`.
     /// `ProposalData` fields include defaulting logic described in `get_fee_recipient_defaulting`,
     /// `get_gas_limit_defaulting`, and `get_builder_proposals_defaulting`.