From ea458144eeca27870a90b3e83ac6634df518a87e Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Mon, 18 Aug 2025 13:40:25 +0530 Subject: [PATCH 01/55] using events api for eager start attestation tasks --- .../src/attestation_service.rs | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index da6e8f35886..5b6f2d13441 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -1,6 +1,8 @@ use crate::duties_service::{DutiesService, DutyAndProof}; use beacon_node_fallback::{ApiTopic, BeaconNodeFallback}; +use eth2::types::EventTopic; use futures::future::join_all; +use futures::StreamExt; use logging::crit; use slot_clock::SlotClock; use std::collections::HashMap; @@ -162,6 +164,11 @@ impl AttestationService {}, + _ = poll_head_event_on_all_beacon_nodes(&self.beacon_nodes) => {} + } + if let Err(e) = self.spawn_attestation_tasks(slot_duration) { crit!(error = e, "Failed to spawn attestation tasks") } else { @@ -709,6 +716,31 @@ impl AttestationService( + beacon_nodes: &Arc> +) -> Result<(), String> { + match beacon_nodes.first_success(|beacon_node| async move { + let mut event_stream = beacon_node.get_events::(&[EventTopic::Head]).await + .map_err(|e| format!("Failed to get event stream: {:?}", e))?; + + // Poll once for a head event to trigger early attestation processing + if let Some(event_result) = event_stream.next().await { + match event_result { + Ok(_event) => Ok(()), + Err(e) => Err(format!("Head event stream error: {:?}", e)) + } + } else { + Err("No head events received".to_string()) + } + }).await { + Ok(_) => Ok(()), + Err(e) => { + debug!("Failed to get head events from any beacon node: {}", e); + Ok(()) // Don't fail the entire process if head events aren't available + } + } +} + #[cfg(test)] mod tests { use super::*; From 46fe220f96574f00d4ecfeb2ca545f9bbf785bf2 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Mon, 18 Aug 2025 15:27:49 +0530 Subject: [PATCH 02/55] minor nits --- .../src/attestation_service.rs | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 5b6f2d13441..aff935d8059 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -166,7 +166,7 @@ impl AttestationService {}, - _ = poll_head_event_on_all_beacon_nodes(&self.beacon_nodes) => {} + _ = poll_head_event_on_all_beacon_nodes::(&self.beacon_nodes) => {} } if let Err(e) = self.spawn_attestation_tasks(slot_duration) { @@ -716,29 +716,32 @@ impl AttestationService( +async fn poll_head_event_on_all_beacon_nodes( beacon_nodes: &Arc> ) -> Result<(), String> { - match beacon_nodes.first_success(|beacon_node| async move { - let mut event_stream = beacon_node.get_events::(&[EventTopic::Head]).await + use eth2::types::EventKind; + + beacon_nodes.first_success(|beacon_node| async move { + let mut event_stream = beacon_node.get_events::(&[EventTopic::Head]).await .map_err(|e| format!("Failed to get event stream: {:?}", e))?; // Poll once for a head event to trigger early attestation processing if let Some(event_result) = event_stream.next().await { - match event_result { - Ok(_event) => Ok(()), - Err(e) => Err(format!("Head event stream error: {:?}", e)) + let event = event_result.map_err(|e| format!("Head event stream error: {:?}", e))?; + match event { + EventKind::Head(_) => { + trace!("Received head event, triggering early attestation processing"); + Ok(()) + }, + _ => Err("Received non-head event when expecting head event".to_string()) } } else { Err("No head events received".to_string()) } - }).await { - Ok(_) => Ok(()), - Err(e) => { - debug!("Failed to get head events from any beacon node: {}", e); - Ok(()) // Don't fail the entire process if head events aren't available - } - } + }).await.map_err(|e| { + debug!(error = %e, "Failed to get head events from any beacon node"); + e.to_string() + }) } #[cfg(test)] From de7226f1cc2a8668c199e40c2f7c53d2bd1c08ea Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Mon, 18 Aug 2025 16:06:07 +0530 Subject: [PATCH 03/55] clippy chill vibes --- .../validator_services/src/attestation_service.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index aff935d8059..f22872a1bb2 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -1,6 +1,6 @@ use crate::duties_service::{DutiesService, DutyAndProof}; use beacon_node_fallback::{ApiTopic, BeaconNodeFallback}; -use eth2::types::EventTopic; +use eth2::types::{EventTopic, EventKind}; use futures::future::join_all; use futures::StreamExt; use logging::crit; @@ -719,12 +719,11 @@ impl AttestationService( beacon_nodes: &Arc> ) -> Result<(), String> { - use eth2::types::EventKind; - + beacon_nodes.first_success(|beacon_node| async move { let mut event_stream = beacon_node.get_events::(&[EventTopic::Head]).await .map_err(|e| format!("Failed to get event stream: {:?}", e))?; - + // Poll once for a head event to trigger early attestation processing if let Some(event_result) = event_stream.next().await { let event = event_result.map_err(|e| format!("Head event stream error: {:?}", e))?; From 5f2a101ce05348f214816fc5633efd4826c5c7d2 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Mon, 18 Aug 2025 16:10:46 +0530 Subject: [PATCH 04/55] missed something --- validator_client/validator_services/src/attestation_service.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index f22872a1bb2..97c5cc31c40 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -162,8 +162,6 @@ impl AttestationService {}, _ = poll_head_event_on_all_beacon_nodes::(&self.beacon_nodes) => {} From ae1670529ada33ecfb7a7d9834c2144982e894c0 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Mon, 18 Aug 2025 19:49:06 +0530 Subject: [PATCH 05/55] linty --- .../src/attestation_service.rs | 53 ++++++++++--------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 97c5cc31c40..c747fc2d749 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -1,8 +1,8 @@ use crate::duties_service::{DutiesService, DutyAndProof}; use beacon_node_fallback::{ApiTopic, BeaconNodeFallback}; -use eth2::types::{EventTopic, EventKind}; -use futures::future::join_all; +use eth2::types::{EventKind, EventTopic}; use futures::StreamExt; +use futures::future::join_all; use logging::crit; use slot_clock::SlotClock; use std::collections::HashMap; @@ -715,30 +715,35 @@ impl AttestationService( - beacon_nodes: &Arc> + beacon_nodes: &Arc>, ) -> Result<(), String> { - - beacon_nodes.first_success(|beacon_node| async move { - let mut event_stream = beacon_node.get_events::(&[EventTopic::Head]).await - .map_err(|e| format!("Failed to get event stream: {:?}", e))?; - - // Poll once for a head event to trigger early attestation processing - if let Some(event_result) = event_stream.next().await { - let event = event_result.map_err(|e| format!("Head event stream error: {:?}", e))?; - match event { - EventKind::Head(_) => { - trace!("Received head event, triggering early attestation processing"); - Ok(()) - }, - _ => Err("Received non-head event when expecting head event".to_string()) + beacon_nodes + .first_success(|beacon_node| async move { + let mut event_stream = beacon_node + .get_events::(&[EventTopic::Head]) + .await + .map_err(|e| format!("Failed to get event stream: {:?}", e))?; + + // Poll once for a head event to trigger early attestation processing + if let Some(event_result) = event_stream.next().await { + let event = + event_result.map_err(|e| format!("Head event stream error: {:?}", e))?; + match event { + EventKind::Head(_) => { + trace!("Received head event, triggering early attestation processing"); + Ok(()) + } + _ => Err("Received non-head event when expecting head event".to_string()), + } + } else { + Err("No head events received".to_string()) } - } else { - Err("No head events received".to_string()) - } - }).await.map_err(|e| { - debug!(error = %e, "Failed to get head events from any beacon node"); - e.to_string() - }) + }) + .await + .map_err(|e| { + debug!(error = %e, "Failed to get head events from any beacon node"); + e.to_string() + }) } #[cfg(test)] From c13d0c311182632926d5c947ad4cea8525f682a4 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Fri, 19 Sep 2025 18:24:30 -0400 Subject: [PATCH 06/55] implemented head monitoring service --- .../beacon_node_fallback/src/lib.rs | 35 +++ validator_client/src/lib.rs | 20 ++ .../src/attestation_service.rs | 59 ++++- .../src/head_monitor_service.rs | 221 ++++++++++++++++++ .../validator_services/src/lib.rs | 1 + 5 files changed, 331 insertions(+), 5 deletions(-) create mode 100644 validator_client/validator_services/src/head_monitor_service.rs diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index a3f60d2de04..3519382eb72 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -646,6 +646,41 @@ impl BeaconNodeFallback { Err(Errors(errors)) } + /// Try `func` on a specific beacon node by index first, then fall back to the normal order. + /// Returns immediately if the preferred node succeeds, otherwise falls back to first_success. + /// This is an insurance against potential race conditions that may arise. + pub async fn first_success_from_index( + &self, + preferred_index: Option, + func: F, + ) -> Result> + where + F: Fn(BeaconNodeHttpClient) -> R + Clone, + R: Future>, + Err: Debug, + { + let candidates = self.candidates.read().await; + + // Try the preferred beacon node first if it exists + if let Some(preferred_idx) = preferred_index + && let Some(preferred_candidate) = candidates.iter().find(|c| c.index == preferred_idx) + { + let preferred_node = preferred_candidate.beacon_node.clone(); + drop(candidates); + + match Self::run_on_candidate(preferred_node, &func).await { + Ok(val) => return Ok(val), + Err(_) => { + return self.first_success(func).await; + } + } + } + + // Fall back to normal first_success behavior + drop(candidates); + self.first_success(func).await + } + /// Run the future `func` on `candidate` while reporting metrics. async fn run_on_candidate( candidate: BeaconNodeHttpClient, diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 71bdde10b02..51e6bcec5af 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -9,6 +9,7 @@ use metrics::set_gauge; use monitoring_api::{MonitoringHttpClient, ProcessType}; use sensitive_url::SensitiveUrl; use slashing_protection::{SLASHING_PROTECTION_FILENAME, SlashingDatabase}; +use tokio::sync::Mutex; use account_utils::validator_definitions::ValidatorDefinitions; use beacon_node_fallback::{ @@ -42,6 +43,7 @@ use validator_services::{ attestation_service::{AttestationService, AttestationServiceBuilder}, block_service::{BlockService, BlockServiceBuilder}, duties_service::{self, DutiesService, DutiesServiceBuilder}, + head_monitor_service::{HeadMonitorService, HeadMonitorServiceBuilder}, latency_service, preparation_service::{PreparationService, PreparationServiceBuilder}, sync_committee_service::SyncCommitteeService, @@ -79,6 +81,7 @@ pub struct ProductionValidatorClient { context: RuntimeContext, duties_service: Arc, SystemTimeSlotClock>>, block_service: BlockService, SystemTimeSlotClock>, + head_monitor_service: HeadMonitorService, SystemTimeSlotClock>, attestation_service: AttestationService, SystemTimeSlotClock>, sync_committee_service: SyncCommitteeService, SystemTimeSlotClock>, doppelganger_service: Option>, @@ -493,8 +496,18 @@ impl ProductionValidatorClient { block_service_builder = block_service_builder.proposer_nodes(proposer_nodes.clone()); } + let (head_sender, head_receiver) = mpsc::channel(1_024); + let block_service = block_service_builder.build()?; + let head_monitor_service = HeadMonitorServiceBuilder::new() + .slot_clock(slot_clock.clone()) + .executor(context.executor.clone()) + .validator_store(validator_store.clone()) + .beacon_nodes(beacon_nodes.clone()) + .head_monitor_tx(Arc::new(head_sender)) + .build()?; + let attestation_service = AttestationServiceBuilder::new() .duties_service(duties_service.clone()) .slot_clock(slot_clock.clone()) @@ -502,6 +515,7 @@ impl ProductionValidatorClient { .beacon_nodes(beacon_nodes.clone()) .executor(context.executor.clone()) .chain_spec(context.eth2_config.spec.clone()) + .head_monitor_rx(Arc::new(Mutex::new(head_receiver))) .disable(config.disable_attesting) .build()?; @@ -526,6 +540,7 @@ impl ProductionValidatorClient { context, duties_service, block_service, + head_monitor_service, attestation_service, sync_committee_service, doppelganger_service, @@ -604,6 +619,11 @@ impl ProductionValidatorClient { .start_update_service(&self.context.eth2_config.spec) .map_err(|e| format!("Unable to start preparation service: {}", e))?; + self.head_monitor_service + .clone() + .start_update_service() + .map_err(|e| format!("Unable to start head monitor service: {}", e))?; + if let Some(doppelganger_service) = self.doppelganger_service.clone() { DoppelgangerService::start_update_service( doppelganger_service, diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index da6e8f35886..5c50c06b1f4 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -1,4 +1,7 @@ use crate::duties_service::{DutiesService, DutyAndProof}; +use tokio::sync::Mutex; + +use crate::head_monitor_service::HeadEvent; use beacon_node_fallback::{ApiTopic, BeaconNodeFallback}; use futures::future::join_all; use logging::crit; @@ -7,6 +10,7 @@ use std::collections::HashMap; use std::ops::Deref; use std::sync::Arc; use task_executor::TaskExecutor; +use tokio::sync::mpsc; use tokio::time::{Duration, Instant, sleep, sleep_until}; use tracing::{debug, error, info, trace, warn}; use tree_hash::TreeHash; @@ -22,6 +26,7 @@ pub struct AttestationServiceBuilder beacon_nodes: Option>>, executor: Option, chain_spec: Option>, + head_monitor_rx: Option>>>, disable: bool, } @@ -34,6 +39,7 @@ impl AttestationServiceBuil beacon_nodes: None, executor: None, chain_spec: None, + head_monitor_rx: None, disable: false, } } @@ -73,6 +79,13 @@ impl AttestationServiceBuil self } + pub fn head_monitor_rx( + mut self, + head_monitor_rx: Arc>>, + ) -> Self { + self.head_monitor_rx = Some(head_monitor_rx); + self + } pub fn build(self) -> Result, String> { Ok(AttestationService { inner: Arc::new(Inner { @@ -94,6 +107,9 @@ impl AttestationServiceBuil chain_spec: self .chain_spec .ok_or("Cannot build AttestationService without chain_spec")?, + head_monitor_rx: self + .head_monitor_rx + .ok_or("Cannot build AttestationService without head_monitor_rx")?, disable: self.disable, }), }) @@ -108,6 +124,7 @@ pub struct Inner { beacon_nodes: Arc>, executor: TaskExecutor, chain_spec: Arc, + head_monitor_rx: Arc>>, disable: bool, } @@ -160,9 +177,17 @@ impl AttestationService = None; + tokio::select! { + _ = sleep(duration_to_next_slot + slot_duration / 3) => {}, + head_event = self.poll_for_head_events() => { + if let Ok(event) = head_event { + beacon_node_index = Some(event.beacon_node_index); + } + } + } - if let Err(e) = self.spawn_attestation_tasks(slot_duration) { + if let Err(e) = self.spawn_attestation_tasks(slot_duration, beacon_node_index) { crit!(error = e, "Failed to spawn attestation tasks") } else { trace!("Spawned attestation tasks"); @@ -180,9 +205,25 @@ impl AttestationService Result { + let mut receiver = self.head_monitor_rx.lock().await; + match receiver.recv().await { + Some(head_event) => Ok(head_event), + None => Err("Head monitor channel closed unexpectedly".to_string()), + } + } + /// For each each required attestation, spawn a new task that downloads, signs and uploads the /// attestation to the beacon node. - fn spawn_attestation_tasks(&self, slot_duration: Duration) -> Result<(), String> { + fn spawn_attestation_tasks( + &self, + slot_duration: Duration, + beacon_node_index: Option, + ) -> Result<(), String> { + info!( + "process attestation from beacon_node_index {:?}", + beacon_node_index + ); let slot = self.slot_clock.now().ok_or("Failed to read slot clock")?; let duration_to_next_slot = self .slot_clock @@ -221,6 +262,7 @@ impl AttestationService AttestationService, aggregate_production_instant: Instant, + candidate_beacon_node: Option, ) -> Result<(), ()> { let attestations_timer = validator_metrics::start_timer_vec( &validator_metrics::ATTESTATION_SERVICE_TIMES, @@ -265,7 +308,12 @@ impl AttestationService AttestationService, ) -> Result, String> { if validator_duties.is_empty() { return Ok(None); @@ -346,7 +395,7 @@ impl AttestationService; + +#[derive(Debug)] +pub enum Error { + BeaconNodeNotFound, +} + +#[derive(Default)] +pub struct HeadMonitorServiceBuilder { + validator_store: Option>, + slot_clock: Option, + beacon_nodes: Option>>, + executor: Option, + beacon_head_cache: Option>, + head_monitor_tx: Option>>, +} + +pub struct BeaconHeadCache { + cache: RwLock, +} + +impl BeaconHeadCache { + pub fn get(&self, beacon_node_index: usize) -> Option { + self.cache.read().get(&beacon_node_index).cloned() + } + + pub fn insert(&self, beacon_node_index: usize, head: SseHead) { + self.cache.write().insert(beacon_node_index, head); + } + + pub fn is_latest(&self, head: &SseHead) -> bool { + let cache = self.cache.read(); + cache + .values() + .all(|cache_head| head.slot >= cache_head.slot) + } +} + +#[derive(Debug)] +pub struct HeadEvent { + pub beacon_node_index: usize, +} + +impl HeadMonitorServiceBuilder { + pub fn new() -> Self { + Self { + validator_store: None, + slot_clock: None, + beacon_nodes: None, + executor: None, + beacon_head_cache: None, + head_monitor_tx: None, + } + } + + pub fn beacon_nodes(mut self, nodes: Arc>) -> Self { + self.beacon_nodes = Some(nodes); + self + } + + pub fn slot_clock(mut self, clock: T) -> Self { + self.slot_clock = Some(clock); + self + } + + pub fn executor(mut self, executor: TaskExecutor) -> Self { + self.executor = Some(executor); + self + } + + pub fn validator_store(mut self, validator_store: Arc) -> Self { + self.validator_store = Some(validator_store); + self + } + + pub fn beacon_head_cache(mut self, beacon_head_cache: Arc) -> Self { + self.beacon_head_cache = Some(beacon_head_cache); + self + } + + pub fn head_monitor_tx(mut self, head_monitor_tx: Arc>) -> Self { + self.head_monitor_tx = Some(head_monitor_tx); + self + } + pub fn build(self) -> Result, String> { + let beacon_head_cache = Arc::new(BeaconHeadCache { + cache: RwLock::new(HashMap::new()), + }); + Ok(HeadMonitorService { + inner: Arc::new(Inner { + _validator_store: self + .validator_store + .ok_or("Cannot build HeadMonitorService without validator_store")?, + + _slot_clock: self + .slot_clock + .ok_or("Cannot build HeadMonitorService without slot_clock")?, + beacon_nodes: self + .beacon_nodes + .ok_or("Cannot build HeadMonitorService without beacon_nodes")?, + executor: self + .executor + .ok_or("Cannot build HeadMonitorService without executor")?, + beacon_head_cache, + head_monitor_tx: self + .head_monitor_tx + .ok_or("Cannot build HeadMonitorService without head_monitor_rx")?, + }), + }) + } +} + +pub struct Inner { + _validator_store: Arc, + _slot_clock: T, + executor: TaskExecutor, + beacon_nodes: Arc>, + beacon_head_cache: Arc, + head_monitor_tx: Arc>, +} + +pub struct HeadMonitorService { + inner: Arc>, +} + +impl Clone for HeadMonitorService { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + } + } +} + +impl Deref for HeadMonitorService { + type Target = Inner; + + fn deref(&self) -> &Self::Target { + self.inner.deref() + } +} + +impl HeadMonitorService { + pub fn start_update_service(self) -> Result<(), String> { + let executor = self.executor.clone(); + let head_cache = self.beacon_head_cache.clone(); + + info!("Starting head monitoring service"); + let interval_fut = async move { + let candidates = { + let candidates_guard = self.beacon_nodes.candidates.read().await; + candidates_guard.clone() + }; + let mut tasks = vec![]; + + for candidate in candidates.iter() { + let head_event_stream = candidate + .beacon_node + .get_events::(&[EventTopic::Head]) + .await; + + let mut head_event_stream = match head_event_stream { + Ok(stream) => stream, + Err(e) => { + warn!("failed to get head event stream: {:?}", e); + continue; + } + }; + + let head_cache_clone = head_cache.clone(); + let sender_tx = self.head_monitor_tx.clone(); + + let stream_fut = async move { + while let Some(event_result) = head_event_stream.next().await { + if let Ok(EventKind::Head(head)) = event_result { + head_cache_clone.insert(candidate.index, head.clone()); + + if head_cache_clone.is_latest(&head) { + if let Ok(()) = sender_tx + .send(HeadEvent { + beacon_node_index: candidate.index, + }) + .await + { + } else { + warn!("Channel closed"); + break; + } + } + } + } + }; + + tasks.push(stream_fut); + } + + futures::future::join_all(tasks).await; + + drop(candidates); + }; + + executor.spawn(interval_fut, "head_monitor_service"); + + Ok(()) + } +} diff --git a/validator_client/validator_services/src/lib.rs b/validator_client/validator_services/src/lib.rs index 3b8bd9ae14b..1712f67d8e8 100644 --- a/validator_client/validator_services/src/lib.rs +++ b/validator_client/validator_services/src/lib.rs @@ -1,6 +1,7 @@ pub mod attestation_service; pub mod block_service; pub mod duties_service; +pub mod head_monitor_service; pub mod latency_service; pub mod notifier_service; pub mod preparation_service; From a1d36e6acac8a584e28e49d231bfaadd3d8fa836 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Fri, 19 Sep 2025 19:06:25 -0400 Subject: [PATCH 07/55] fixing some linting issues --- .../src/attestation_service.rs | 34 ------------------- 1 file changed, 34 deletions(-) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 5056ea2e1d7..5c50c06b1f4 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -3,8 +3,6 @@ use tokio::sync::Mutex; use crate::head_monitor_service::HeadEvent; use beacon_node_fallback::{ApiTopic, BeaconNodeFallback}; -use eth2::types::{EventKind, EventTopic}; -use futures::StreamExt; use futures::future::join_all; use logging::crit; use slot_clock::SlotClock; @@ -760,38 +758,6 @@ impl AttestationService( - beacon_nodes: &Arc>, -) -> Result<(), String> { - beacon_nodes - .first_success(|beacon_node| async move { - let mut event_stream = beacon_node - .get_events::(&[EventTopic::Head]) - .await - .map_err(|e| format!("Failed to get event stream: {:?}", e))?; - - // Poll once for a head event to trigger early attestation processing - if let Some(event_result) = event_stream.next().await { - let event = - event_result.map_err(|e| format!("Head event stream error: {:?}", e))?; - match event { - EventKind::Head(_) => { - trace!("Received head event, triggering early attestation processing"); - Ok(()) - } - _ => Err("Received non-head event when expecting head event".to_string()), - } - } else { - Err("No head events received".to_string()) - } - }) - .await - .map_err(|e| { - debug!(error = %e, "Failed to get head events from any beacon node"); - e.to_string() - }) -} - #[cfg(test)] mod tests { use super::*; From a489d324f21454f265a0d046f1e0704cdcb5f9bd Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Fri, 19 Sep 2025 21:46:01 -0400 Subject: [PATCH 08/55] comments and removing unwanted code --- .../src/head_monitor_service.rs | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/validator_client/validator_services/src/head_monitor_service.rs b/validator_client/validator_services/src/head_monitor_service.rs index f23c1ba5f56..c551dd436f4 100644 --- a/validator_client/validator_services/src/head_monitor_service.rs +++ b/validator_client/validator_services/src/head_monitor_service.rs @@ -16,21 +16,18 @@ use validator_store::ValidatorStore; type CacheHashMap = HashMap; -#[derive(Debug)] -pub enum Error { - BeaconNodeNotFound, -} - +// Builder for `HeadMonitorService` #[derive(Default)] pub struct HeadMonitorServiceBuilder { validator_store: Option>, - slot_clock: Option, beacon_nodes: Option>>, executor: Option, beacon_head_cache: Option>, head_monitor_tx: Option>>, } +// Cache to maintain the latest head received from each of the beacon nodes +// in the `BeaconNodeFallback` pub struct BeaconHeadCache { cache: RwLock, } @@ -50,8 +47,14 @@ impl BeaconHeadCache { .values() .all(|cache_head| head.slot >= cache_head.slot) } + + pub fn purge_cache(&self) { + self.cache.write().clear(); + } } +// This is used send the index derived from `CandidateBeaconNode` to the +// `AttestationService` for further processing #[derive(Debug)] pub struct HeadEvent { pub beacon_node_index: usize, @@ -61,7 +64,6 @@ impl HeadMonitorServiceBuil pub fn new() -> Self { Self { validator_store: None, - slot_clock: None, beacon_nodes: None, executor: None, beacon_head_cache: None, @@ -74,11 +76,6 @@ impl HeadMonitorServiceBuil self } - pub fn slot_clock(mut self, clock: T) -> Self { - self.slot_clock = Some(clock); - self - } - pub fn executor(mut self, executor: TaskExecutor) -> Self { self.executor = Some(executor); self @@ -107,10 +104,6 @@ impl HeadMonitorServiceBuil _validator_store: self .validator_store .ok_or("Cannot build HeadMonitorService without validator_store")?, - - _slot_clock: self - .slot_clock - .ok_or("Cannot build HeadMonitorService without slot_clock")?, beacon_nodes: self .beacon_nodes .ok_or("Cannot build HeadMonitorService without beacon_nodes")?, @@ -126,15 +119,22 @@ impl HeadMonitorServiceBuil } } +// Helper to minimise `Arc` usage pub struct Inner { _validator_store: Arc, - _slot_clock: T, executor: TaskExecutor, beacon_nodes: Arc>, beacon_head_cache: Arc, head_monitor_tx: Arc>, } +// Runs a non-terminating loop to update the `BeaconHeadCache` with the latest head received +// from the candidate beacon_nodes. This is an attempt to stream events to beacon nodes and +// potential start attestion duties earlier as soon as latest head is receive from any of the +// beacon node in contrast to attest at the 1/3rd mark in the slot. +// +// The cache and the candidate BNs list are periodically refresh/purged to avoid race condition that may +// be arise due to change in ranking of beacon_nodes thus affecting the indices of beacon_nodes pub struct HeadMonitorService { inner: Arc>, } @@ -156,6 +156,7 @@ impl Deref for HeadMonitorService { } impl HeadMonitorService { + // Starts the service to perpetually stream head events from connected beacon_nodes pub fn start_update_service(self) -> Result<(), String> { let executor = self.executor.clone(); let head_cache = self.beacon_head_cache.clone(); @@ -198,7 +199,7 @@ impl HeadMonitorService HeadMonitorService Date: Fri, 19 Sep 2025 21:48:41 -0400 Subject: [PATCH 09/55] clippy change --- validator_client/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 51e6bcec5af..c5efb569a4d 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -501,7 +501,6 @@ impl ProductionValidatorClient { let block_service = block_service_builder.build()?; let head_monitor_service = HeadMonitorServiceBuilder::new() - .slot_clock(slot_clock.clone()) .executor(context.executor.clone()) .validator_store(validator_store.clone()) .beacon_nodes(beacon_nodes.clone()) From b65fc3079bb2604fdd0dd1c304ee3c0d7ec55e9f Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Wed, 15 Oct 2025 20:03:46 -0400 Subject: [PATCH 10/55] same data attestation bug solved --- .../src/attestation_service.rs | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 5c50c06b1f4..3067a339c87 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -111,6 +111,7 @@ impl AttestationServiceBuil .head_monitor_rx .ok_or("Cannot build AttestationService without head_monitor_rx")?, disable: self.disable, + latest_attested_slot: Mutex::new(Slot::default()), }), }) } @@ -126,9 +127,11 @@ pub struct Inner { chain_spec: Arc, head_monitor_rx: Arc>>, disable: bool, + latest_attested_slot: Mutex, } -/// Attempts to produce attestations for all known validators 1/3rd of the way through each slot. +/// Attempts to produce attestations for all known validators 1/3rd of the way through each slot +/// or when a head event is received from the BNs. /// /// If any validators are on the same committee, a single attestation will be downloaded and /// returned to the beacon node. This attestation will have a signature from each of the @@ -178,6 +181,7 @@ impl AttestationService = None; + tokio::select! { _ = sleep(duration_to_next_slot + slot_duration / 3) => {}, head_event = self.poll_for_head_events() => { @@ -187,14 +191,26 @@ impl AttestationService *last_slot { + if let Err(e) = + self.spawn_attestation_tasks(slot_duration, beacon_node_index) + { + crit!(error = e, "Failed to spawn attestation tasks") + } else { + *last_slot = current_slot; + trace!(?current_slot, "Spawned attestation tasks"); + } + } else { + debug!(?current_slot, ?last_slot, "Already attested for this slot"); + } } else { - trace!("Spawned attestation tasks"); + error!("Failed to read slot clock after trigger"); } } else { error!("Failed to read slot clock"); - // If we can't read the slot clock, just wait another slot. sleep(slot_duration).await; continue; } @@ -221,9 +237,10 @@ impl AttestationService, ) -> Result<(), String> { info!( - "process attestation from beacon_node_index {:?}", - beacon_node_index + ?beacon_node_index, + "process attestation from beacon_node_index", ); + let slot = self.slot_clock.now().ok_or("Failed to read slot clock")?; let duration_to_next_slot = self .slot_clock From b25703fef6338f3cad3f543c583c7295cd12b8d3 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Mon, 20 Oct 2025 13:44:38 -0400 Subject: [PATCH 11/55] fixing dangling conditions and amaking head_monitor_service optional --- .../beacon_node_fallback/src/lib.rs | 72 ++++++++- validator_client/src/lib.rs | 17 ++- .../src/head_monitor_service.rs | 141 ++++++++---------- 3 files changed, 142 insertions(+), 88 deletions(-) diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index 3519382eb72..b729c4e0b14 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -8,12 +8,14 @@ use beacon_node_health::{ SyncDistanceTier, check_node_health, }; use clap::ValueEnum; +use eth2::types::SseHead; use eth2::{BeaconNodeHttpClient, Timeouts}; use futures::future; use sensitive_url::SensitiveUrl; use serde::{Deserialize, Serialize, Serializer, ser::SerializeStruct}; use slot_clock::SlotClock; use std::cmp::Ordering; +use std::collections::HashMap; use std::fmt; use std::fmt::Debug; use std::future::Future; @@ -49,6 +51,48 @@ pub struct Config { pub sync_tolerances: BeaconNodeSyncDistanceTiers, } +type CacheHashMap = HashMap; + +/// Cache to maintain the latest head received from each of the beacon nodes +/// in the `BeaconNodeFallback`. +#[derive(Debug)] +pub struct BeaconHeadCache { + cache: RwLock, +} + +impl BeaconHeadCache { + pub fn new() -> Self { + Self { + cache: RwLock::new(HashMap::new()), + } + } + + pub async fn get(&self, beacon_node_index: usize) -> Option { + self.cache.read().await.get(&beacon_node_index).cloned() + } + + pub async fn insert(&self, beacon_node_index: usize, head: SseHead) { + self.cache.write().await.insert(beacon_node_index, head); + } + + pub async fn is_latest(&self, head: &SseHead) -> bool { + let cache = self.cache.read().await; + cache + .values() + .all(|cache_head| head.slot >= cache_head.slot) + } + + pub async fn purge_cache(&self) { + self.cache.write().await.clear(); + } +} + +impl Default for BeaconHeadCache { + fn default() -> Self { + Self::new() + } +} + /// Indicates a measurement of latency between the VC and a BN. pub struct LatencyMeasurement { /// An identifier for the beacon node (e.g. the URL). @@ -378,6 +422,7 @@ impl CandidateBeaconNode { #[derive(Clone, Debug)] pub struct BeaconNodeFallback { pub candidates: Arc>>, + beacon_head_cache: Option>, distance_tiers: BeaconNodeSyncDistanceTiers, slot_clock: Option, broadcast_topics: Vec, @@ -387,6 +432,7 @@ pub struct BeaconNodeFallback { impl BeaconNodeFallback { pub fn new( candidates: Vec, + beacon_head_cache: Arc, config: Config, broadcast_topics: Vec, spec: Arc, @@ -394,6 +440,7 @@ impl BeaconNodeFallback { let distance_tiers = config.sync_tolerances; Self { candidates: Arc::new(RwLock::new(candidates)), + beacon_head_cache: Some(beacon_head_cache), distance_tiers, slot_clock: None, broadcast_topics, @@ -410,6 +457,14 @@ impl BeaconNodeFallback { self.slot_clock = Some(slot_clock); } + /// Set the beacon head cache reference. + /// + /// This is used to refresh beacon_head_cache in the `HeadMonitorService` to avoid dangling + /// reference to BNs when the candidate is change in the `update_candidates_list` + pub fn set_beacon_head_cache(&mut self, cache: Arc) { + self.beacon_head_cache = Some(cache); + } + /// The count of candidates, regardless of their state. pub async fn num_total(&self) -> usize { self.candidates.read().await.len() @@ -493,6 +548,10 @@ impl BeaconNodeFallback { let mut candidates = self.candidates.write().await; *candidates = new_candidates; + if let Some(cache) = &self.beacon_head_cache { + cache.purge_cache().await; + } + Ok(new_list) } @@ -962,8 +1021,17 @@ mod tests { topics: Vec, spec: Arc, ) -> BeaconNodeFallback { - let mut beacon_node_fallback = - BeaconNodeFallback::new(candidates, Config::default(), topics, spec); + let beacon_head_cache = Arc::new(BeaconHeadCache { + cache: RwLock::new(HashMap::new()), + }); + + let mut beacon_node_fallback = BeaconNodeFallback::new( + candidates, + beacon_head_cache, + Config::default(), + topics, + spec, + ); beacon_node_fallback.set_slot_clock(TestingSlotClock::new( Slot::new(1), diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index c5efb569a4d..520c9cec775 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -13,7 +13,7 @@ use tokio::sync::Mutex; use account_utils::validator_definitions::ValidatorDefinitions; use beacon_node_fallback::{ - BeaconNodeFallback, CandidateBeaconNode, start_fallback_updater_service, + BeaconHeadCache, BeaconNodeFallback, CandidateBeaconNode, start_fallback_updater_service, }; use clap::ArgMatches; use doppelganger_service::DoppelgangerService; @@ -355,8 +355,11 @@ impl ProductionValidatorClient { // Initialize the number of connected, avaliable beacon nodes to 0. set_gauge(&validator_metrics::AVAILABLE_BEACON_NODES_COUNT, 0); + let beacon_head_cache = Arc::new(BeaconHeadCache::new()); + let mut beacon_nodes: BeaconNodeFallback<_> = BeaconNodeFallback::new( candidates, + beacon_head_cache.clone(), config.beacon_node_fallback, config.broadcast_topics.clone(), context.eth2_config.spec.clone(), @@ -364,6 +367,7 @@ impl ProductionValidatorClient { let mut proposer_nodes: BeaconNodeFallback<_> = BeaconNodeFallback::new( proposer_candidates, + beacon_head_cache.clone(), config.beacon_node_fallback, config.broadcast_topics.clone(), context.eth2_config.spec.clone(), @@ -504,6 +508,7 @@ impl ProductionValidatorClient { .executor(context.executor.clone()) .validator_store(validator_store.clone()) .beacon_nodes(beacon_nodes.clone()) + .beacon_head_cache(beacon_head_cache.clone()) .head_monitor_tx(Arc::new(head_sender)) .build()?; @@ -618,10 +623,12 @@ impl ProductionValidatorClient { .start_update_service(&self.context.eth2_config.spec) .map_err(|e| format!("Unable to start preparation service: {}", e))?; - self.head_monitor_service - .clone() - .start_update_service() - .map_err(|e| format!("Unable to start head monitor service: {}", e))?; + if let Err(e) = self.head_monitor_service.clone().start_update_service() { + warn!( + error = %e, + "Unable to start head monitor service, validator client running compromised performance" + ); + } if let Some(doppelganger_service) = self.doppelganger_service.clone() { DoppelgangerService::start_update_service( diff --git a/validator_client/validator_services/src/head_monitor_service.rs b/validator_client/validator_services/src/head_monitor_service.rs index c551dd436f4..be624e58403 100644 --- a/validator_client/validator_services/src/head_monitor_service.rs +++ b/validator_client/validator_services/src/head_monitor_service.rs @@ -1,21 +1,17 @@ -use beacon_node_fallback::BeaconNodeFallback; +use beacon_node_fallback::{BeaconHeadCache, BeaconNodeFallback}; use eth2::types::EventTopic; -use eth2::types::SseHead; use tokio::sync::mpsc; +use tokio::time::{Duration, sleep}; use tracing::{info, warn}; use eth2::types::EventKind; use futures::StreamExt; -use parking_lot::RwLock; use slot_clock::SlotClock; -use std::collections::HashMap; use std::ops::Deref; use std::sync::Arc; use task_executor::TaskExecutor; use validator_store::ValidatorStore; -type CacheHashMap = HashMap; - // Builder for `HeadMonitorService` #[derive(Default)] pub struct HeadMonitorServiceBuilder { @@ -26,33 +22,6 @@ pub struct HeadMonitorServiceBuilder head_monitor_tx: Option>>, } -// Cache to maintain the latest head received from each of the beacon nodes -// in the `BeaconNodeFallback` -pub struct BeaconHeadCache { - cache: RwLock, -} - -impl BeaconHeadCache { - pub fn get(&self, beacon_node_index: usize) -> Option { - self.cache.read().get(&beacon_node_index).cloned() - } - - pub fn insert(&self, beacon_node_index: usize, head: SseHead) { - self.cache.write().insert(beacon_node_index, head); - } - - pub fn is_latest(&self, head: &SseHead) -> bool { - let cache = self.cache.read(); - cache - .values() - .all(|cache_head| head.slot >= cache_head.slot) - } - - pub fn purge_cache(&self) { - self.cache.write().clear(); - } -} - // This is used send the index derived from `CandidateBeaconNode` to the // `AttestationService` for further processing #[derive(Debug)] @@ -96,9 +65,6 @@ impl HeadMonitorServiceBuil self } pub fn build(self) -> Result, String> { - let beacon_head_cache = Arc::new(BeaconHeadCache { - cache: RwLock::new(HashMap::new()), - }); Ok(HeadMonitorService { inner: Arc::new(Inner { _validator_store: self @@ -110,7 +76,9 @@ impl HeadMonitorServiceBuil executor: self .executor .ok_or("Cannot build HeadMonitorService without executor")?, - beacon_head_cache, + beacon_head_cache: self + .beacon_head_cache + .ok_or("Cannot build HeadMonitorService without beacon_head_cache")?, head_monitor_tx: self .head_monitor_tx .ok_or("Cannot build HeadMonitorService without head_monitor_rx")?, @@ -163,57 +131,68 @@ impl HeadMonitorService(&[EventTopic::Head]) - .await; - - let mut head_event_stream = match head_event_stream { - Ok(stream) => stream, - Err(e) => { - warn!("failed to get head event stream: {:?}", e); - continue; - } + loop { + let candidates = { + let candidates_guard = self.beacon_nodes.candidates.read().await; + candidates_guard.clone() }; - - let head_cache_clone = head_cache.clone(); - let sender_tx = self.head_monitor_tx.clone(); - - let stream_fut = async move { - while let Some(event_result) = head_event_stream.next().await { - if let Ok(EventKind::Head(head)) = event_result { - head_cache_clone.insert(candidate.index, head.clone()); - - if head_cache_clone.is_latest(&head) { - if let Ok(()) = sender_tx - .send(HeadEvent { - beacon_node_index: candidate.index, - }) - .await - { - } else { - warn!("Head monitoring service channel closed"); - break; + let mut tasks = vec![]; + + for candidate in candidates.iter() { + let head_event_stream = candidate + .beacon_node + .get_events::(&[EventTopic::Head]) + .await; + + let mut head_event_stream = match head_event_stream { + Ok(stream) => stream, + Err(e) => { + warn!("failed to get head event stream: {:?}", e); + continue; + } + }; + + let head_cache_clone = head_cache.clone(); + let sender_tx = self.head_monitor_tx.clone(); + + let stream_fut = async move { + while let Some(event_result) = head_event_stream.next().await { + if let Ok(EventKind::Head(head)) = event_result { + head_cache_clone.insert(candidate.index, head.clone()).await; + + if head_cache_clone.is_latest(&head).await { + if let Ok(()) = sender_tx + .send(HeadEvent { + beacon_node_index: candidate.index, + }) + .await + { + } else { + warn!("Head monitoring service channel closed"); + break; + } } } } - } - }; + }; - tasks.push(stream_fut); - } + tasks.push(stream_fut); + } + + if tasks.is_empty() { + warn!("No beacon nodes available for head event streaming, retrying in 5 seconds"); + sleep(Duration::from_secs(5)).await; + continue; + } - futures::future::join_all(tasks).await; + futures::future::join_all(tasks).await; - drop(candidates); - self.beacon_head_cache.purge_cache(); + drop(candidates); + self.beacon_head_cache.purge_cache().await; + + // Add a small delay before reconnecting to avoid hammering beacon nodes + sleep(Duration::from_secs(1)).await; + } }; executor.spawn(interval_fut, "head_monitor_service"); From c4d851c699d858e901333195a7c9d8b46940c938 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Mon, 20 Oct 2025 13:48:40 -0400 Subject: [PATCH 12/55] fmt --- .../validator_services/src/head_monitor_service.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/validator_client/validator_services/src/head_monitor_service.rs b/validator_client/validator_services/src/head_monitor_service.rs index be624e58403..0a42ee4f170 100644 --- a/validator_client/validator_services/src/head_monitor_service.rs +++ b/validator_client/validator_services/src/head_monitor_service.rs @@ -180,7 +180,9 @@ impl HeadMonitorService Date: Mon, 20 Oct 2025 17:24:45 -0400 Subject: [PATCH 13/55] update comment --- .../validator_services/src/head_monitor_service.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/validator_client/validator_services/src/head_monitor_service.rs b/validator_client/validator_services/src/head_monitor_service.rs index 0a42ee4f170..2b76f68e1a3 100644 --- a/validator_client/validator_services/src/head_monitor_service.rs +++ b/validator_client/validator_services/src/head_monitor_service.rs @@ -101,8 +101,10 @@ pub struct Inner { // potential start attestion duties earlier as soon as latest head is receive from any of the // beacon node in contrast to attest at the 1/3rd mark in the slot. // -// The cache and the candidate BNs list are periodically refresh/purged to avoid race condition that may -// be arise due to change in ranking of beacon_nodes thus affecting the indices of beacon_nodes +// +// The cache and the candidate BNs list are refresh/purged to avoid dangling reference conditions +// that arise due to `update_candidates_list`. The cache is refresh use the shared reference between +// `HeadMonitorService` and `BeaconNodeFallback`. pub struct HeadMonitorService { inner: Arc>, } From 0e35ee5c9372fd4d9c2afbe4093cfde3996a6921 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Tue, 28 Oct 2025 18:58:02 -0400 Subject: [PATCH 14/55] massive refact --- .../src/beacon_head_monitor.rs | 142 ++++++++++++ .../beacon_node_fallback/src/lib.rs | 103 +++------ validator_client/src/lib.rs | 26 +-- .../src/attestation_service.rs | 3 +- .../src/head_monitor_service.rs | 206 ------------------ .../validator_services/src/lib.rs | 1 - 6 files changed, 181 insertions(+), 300 deletions(-) create mode 100644 validator_client/beacon_node_fallback/src/beacon_head_monitor.rs delete mode 100644 validator_client/validator_services/src/head_monitor_service.rs diff --git a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs new file mode 100644 index 00000000000..0c921da471f --- /dev/null +++ b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs @@ -0,0 +1,142 @@ +use crate::BeaconNodeFallback; +use eth2::types::EthSpec; +use eth2::types::SseHead; +use std::collections::HashMap; +use tokio::sync::RwLock; + +use eth2::types::EventTopic; +use tracing::{info, warn}; + +use eth2::types::EventKind; +use futures::StreamExt; +use slot_clock::SlotClock; +use std::sync::Arc; + +type CacheHashMap = HashMap; + +// This is used send the index derived from `CandidateBeaconNode` to the +// `AttestationService` for further processing +#[derive(Debug)] +pub struct HeadEvent { + pub beacon_node_index: usize, +} + +/// Cache to maintain the latest head received from each of the beacon nodes +/// in the `BeaconNodeFallback`. +#[derive(Debug)] +pub struct BeaconHeadCache { + cache: RwLock, +} + +impl BeaconHeadCache { + pub fn new() -> Self { + Self { + cache: RwLock::new(HashMap::new()), + } + } + + pub async fn get(&self, beacon_node_index: usize) -> Option { + self.cache.read().await.get(&beacon_node_index).cloned() + } + + pub async fn insert(&self, beacon_node_index: usize, head: SseHead) { + self.cache.write().await.insert(beacon_node_index, head); + } + + pub async fn is_latest(&self, head: &SseHead) -> bool { + let cache = self.cache.read().await; + cache + .values() + .all(|cache_head| head.slot >= cache_head.slot) + } + + pub async fn purge_cache(&self) { + self.cache.write().await.clear(); + } +} + +impl Default for BeaconHeadCache { + fn default() -> Self { + Self::new() + } +} + +// Runs a non-terminating loop to update the `BeaconHeadCache` with the latest head received +// from the candidate beacon_nodes. This is an attempt to stream events to beacon nodes and +// potential start attestion duties earlier as soon as latest head is receive from any of the +// beacon node in contrast to attest at the 1/3rd mark in the slot. +// +// +// The cache and the candidate BNs list are refresh/purged to avoid dangling reference conditions +// that arise due to `update_candidates_list`. The cache is refresh use the shared reference between +// `HeadMonitorService` and `BeaconNodeFallback`. + +// Starts the service to perpetually stream head events from connected beacon_nodes +pub async fn poll_head_event_from_beacon_nodes( + beacon_nodes: Arc>, +) -> Result<(), String> { + let head_cache = beacon_nodes.beacon_head_cache.clone().expect("asdf"); + let head_monitor_send = beacon_nodes.head_monitor_send.clone().expect("asdf"); + + info!("Starting head monitoring service"); + let candidates = { + let candidates_guard = beacon_nodes.candidates.read().await; + candidates_guard.clone() + }; + let mut tasks = vec![]; + + for candidate in candidates.iter() { + let head_event_stream = candidate + .beacon_node + .get_events::(&[EventTopic::Head]) + .await; + + let mut head_event_stream = match head_event_stream { + Ok(stream) => stream, + Err(e) => { + warn!("failed to get head event stream: {:?}", e); + continue; + } + }; + + let sender_tx = head_monitor_send.clone(); + let head_cache_ref = head_cache.clone(); + + let stream_fut = async move { + while let Some(event_result) = head_event_stream.next().await { + if let Ok(EventKind::Head(head)) = event_result { + head_cache_ref.insert(candidate.index, head.clone()).await; + + if head_cache_ref.is_latest(&head).await { + if let Ok(()) = sender_tx + .send(HeadEvent { + beacon_node_index: candidate.index, + }) + .await + { + } else { + warn!("Head monitoring service channel closed"); + break; + } + } + } + } + }; + + tasks.push(stream_fut); + } + + if tasks.is_empty() { + head_cache.purge_cache().await; + return Err( + "No beacon nodes available for head event streaming, retry in sometime".to_string(), + ); + } + + futures::future::join_all(tasks).await; + + drop(candidates); + head_cache.purge_cache().await; + + Ok(()) +} diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index b729c4e0b14..4c23426abf5 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -2,20 +2,21 @@ //! "fallback" behaviour; it will try a request on all of the nodes until one or none of them //! succeed. +pub mod beacon_head_monitor; pub mod beacon_node_health; + +use beacon_head_monitor::{BeaconHeadCache, HeadEvent, poll_head_event_from_beacon_nodes}; use beacon_node_health::{ BeaconNodeHealth, BeaconNodeSyncDistanceTiers, ExecutionEngineHealth, IsOptimistic, SyncDistanceTier, check_node_health, }; use clap::ValueEnum; -use eth2::types::SseHead; use eth2::{BeaconNodeHttpClient, Timeouts}; use futures::future; use sensitive_url::SensitiveUrl; use serde::{Deserialize, Serialize, Serializer, ser::SerializeStruct}; use slot_clock::SlotClock; use std::cmp::Ordering; -use std::collections::HashMap; use std::fmt; use std::fmt::Debug; use std::future::Future; @@ -24,7 +25,11 @@ use std::time::{Duration, Instant}; use std::vec::Vec; use strum::EnumVariantNames; use task_executor::TaskExecutor; -use tokio::{sync::RwLock, time::sleep}; + +use tokio::{ + sync::{RwLock, mpsc}, + time::sleep, +}; use tracing::{debug, error, warn}; use types::{ChainSpec, Config as ConfigSpec, EthSpec, Slot}; use validator_metrics::{ENDPOINT_ERRORS, ENDPOINT_REQUESTS, inc_counter_vec}; @@ -51,48 +56,6 @@ pub struct Config { pub sync_tolerances: BeaconNodeSyncDistanceTiers, } -type CacheHashMap = HashMap; - -/// Cache to maintain the latest head received from each of the beacon nodes -/// in the `BeaconNodeFallback`. -#[derive(Debug)] -pub struct BeaconHeadCache { - cache: RwLock, -} - -impl BeaconHeadCache { - pub fn new() -> Self { - Self { - cache: RwLock::new(HashMap::new()), - } - } - - pub async fn get(&self, beacon_node_index: usize) -> Option { - self.cache.read().await.get(&beacon_node_index).cloned() - } - - pub async fn insert(&self, beacon_node_index: usize, head: SseHead) { - self.cache.write().await.insert(beacon_node_index, head); - } - - pub async fn is_latest(&self, head: &SseHead) -> bool { - let cache = self.cache.read().await; - cache - .values() - .all(|cache_head| head.slot >= cache_head.slot) - } - - pub async fn purge_cache(&self) { - self.cache.write().await.clear(); - } -} - -impl Default for BeaconHeadCache { - fn default() -> Self { - Self::new() - } -} - /// Indicates a measurement of latency between the VC and a BN. pub struct LatencyMeasurement { /// An identifier for the beacon node (e.g. the URL). @@ -112,6 +75,20 @@ pub fn start_fallback_updater_service( return Err("Cannot start fallback updater without slot clock"); } + let beacon_nodes_ref = beacon_nodes.clone(); + + let head_monitor_future = async move { + loop { + if let Err(err) = + poll_head_event_from_beacon_nodes::(beacon_nodes_ref.clone()).await + { + warn!(error=?err, "Head service failed"); + } + } + }; + + executor.spawn(head_monitor_future, "head_monitoring"); + let future = async move { loop { beacon_nodes.update_all_candidates::().await; @@ -422,9 +399,10 @@ impl CandidateBeaconNode { #[derive(Clone, Debug)] pub struct BeaconNodeFallback { pub candidates: Arc>>, - beacon_head_cache: Option>, distance_tiers: BeaconNodeSyncDistanceTiers, slot_clock: Option, + beacon_head_cache: Option>, + head_monitor_send: Option>>, broadcast_topics: Vec, spec: Arc, } @@ -432,7 +410,6 @@ pub struct BeaconNodeFallback { impl BeaconNodeFallback { pub fn new( candidates: Vec, - beacon_head_cache: Arc, config: Config, broadcast_topics: Vec, spec: Arc, @@ -440,9 +417,10 @@ impl BeaconNodeFallback { let distance_tiers = config.sync_tolerances; Self { candidates: Arc::new(RwLock::new(candidates)), - beacon_head_cache: Some(beacon_head_cache), distance_tiers, slot_clock: None, + beacon_head_cache: Some(Arc::new(BeaconHeadCache::new())), + head_monitor_send: None, broadcast_topics, spec, } @@ -457,19 +435,15 @@ impl BeaconNodeFallback { self.slot_clock = Some(slot_clock); } - /// Set the beacon head cache reference. - /// - /// This is used to refresh beacon_head_cache in the `HeadMonitorService` to avoid dangling - /// reference to BNs when the candidate is change in the `update_candidates_list` - pub fn set_beacon_head_cache(&mut self, cache: Arc) { - self.beacon_head_cache = Some(cache); - } - /// The count of candidates, regardless of their state. pub async fn num_total(&self) -> usize { self.candidates.read().await.len() } + pub fn set_head_send(&mut self, head_monitor_send: Arc>) { + self.head_monitor_send = Some(head_monitor_send); + } + /// The count of candidates that are online and compatible, but not necessarily synced. pub async fn num_available(&self) -> usize { let mut n = 0; @@ -482,6 +456,10 @@ impl BeaconNodeFallback { n } + pub fn spawn_head_monitor_process(&self, _executor: TaskExecutor) { + let _future = async move {}; + } + // Returns all data required by the VC notifier. pub async fn get_notifier_info(&self) -> (Vec, usize, usize) { let candidates = self.candidates.read().await; @@ -1021,17 +999,8 @@ mod tests { topics: Vec, spec: Arc, ) -> BeaconNodeFallback { - let beacon_head_cache = Arc::new(BeaconHeadCache { - cache: RwLock::new(HashMap::new()), - }); - - let mut beacon_node_fallback = BeaconNodeFallback::new( - candidates, - beacon_head_cache, - Config::default(), - topics, - spec, - ); + let mut beacon_node_fallback = + BeaconNodeFallback::new(candidates, Config::default(), topics, spec); beacon_node_fallback.set_slot_clock(TestingSlotClock::new( Slot::new(1), diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 520c9cec775..1dca8fc9e85 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -13,7 +13,7 @@ use tokio::sync::Mutex; use account_utils::validator_definitions::ValidatorDefinitions; use beacon_node_fallback::{ - BeaconHeadCache, BeaconNodeFallback, CandidateBeaconNode, start_fallback_updater_service, + BeaconNodeFallback, CandidateBeaconNode, start_fallback_updater_service, }; use clap::ArgMatches; use doppelganger_service::DoppelgangerService; @@ -43,7 +43,6 @@ use validator_services::{ attestation_service::{AttestationService, AttestationServiceBuilder}, block_service::{BlockService, BlockServiceBuilder}, duties_service::{self, DutiesService, DutiesServiceBuilder}, - head_monitor_service::{HeadMonitorService, HeadMonitorServiceBuilder}, latency_service, preparation_service::{PreparationService, PreparationServiceBuilder}, sync_committee_service::SyncCommitteeService, @@ -81,7 +80,6 @@ pub struct ProductionValidatorClient { context: RuntimeContext, duties_service: Arc, SystemTimeSlotClock>>, block_service: BlockService, SystemTimeSlotClock>, - head_monitor_service: HeadMonitorService, SystemTimeSlotClock>, attestation_service: AttestationService, SystemTimeSlotClock>, sync_committee_service: SyncCommitteeService, SystemTimeSlotClock>, doppelganger_service: Option>, @@ -355,11 +353,8 @@ impl ProductionValidatorClient { // Initialize the number of connected, avaliable beacon nodes to 0. set_gauge(&validator_metrics::AVAILABLE_BEACON_NODES_COUNT, 0); - let beacon_head_cache = Arc::new(BeaconHeadCache::new()); - let mut beacon_nodes: BeaconNodeFallback<_> = BeaconNodeFallback::new( candidates, - beacon_head_cache.clone(), config.beacon_node_fallback, config.broadcast_topics.clone(), context.eth2_config.spec.clone(), @@ -367,7 +362,6 @@ impl ProductionValidatorClient { let mut proposer_nodes: BeaconNodeFallback<_> = BeaconNodeFallback::new( proposer_candidates, - beacon_head_cache.clone(), config.beacon_node_fallback, config.broadcast_topics.clone(), context.eth2_config.spec.clone(), @@ -500,18 +494,10 @@ impl ProductionValidatorClient { block_service_builder = block_service_builder.proposer_nodes(proposer_nodes.clone()); } - let (head_sender, head_receiver) = mpsc::channel(1_024); + let (_, head_receiver) = mpsc::channel(1_024); let block_service = block_service_builder.build()?; - let head_monitor_service = HeadMonitorServiceBuilder::new() - .executor(context.executor.clone()) - .validator_store(validator_store.clone()) - .beacon_nodes(beacon_nodes.clone()) - .beacon_head_cache(beacon_head_cache.clone()) - .head_monitor_tx(Arc::new(head_sender)) - .build()?; - let attestation_service = AttestationServiceBuilder::new() .duties_service(duties_service.clone()) .slot_clock(slot_clock.clone()) @@ -544,7 +530,6 @@ impl ProductionValidatorClient { context, duties_service, block_service, - head_monitor_service, attestation_service, sync_committee_service, doppelganger_service, @@ -623,13 +608,6 @@ impl ProductionValidatorClient { .start_update_service(&self.context.eth2_config.spec) .map_err(|e| format!("Unable to start preparation service: {}", e))?; - if let Err(e) = self.head_monitor_service.clone().start_update_service() { - warn!( - error = %e, - "Unable to start head monitor service, validator client running compromised performance" - ); - } - if let Some(doppelganger_service) = self.doppelganger_service.clone() { DoppelgangerService::start_update_service( doppelganger_service, diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 3067a339c87..a3644ca5949 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -1,8 +1,7 @@ use crate::duties_service::{DutiesService, DutyAndProof}; use tokio::sync::Mutex; -use crate::head_monitor_service::HeadEvent; -use beacon_node_fallback::{ApiTopic, BeaconNodeFallback}; +use beacon_node_fallback::{ApiTopic, BeaconNodeFallback, beacon_head_monitor::HeadEvent}; use futures::future::join_all; use logging::crit; use slot_clock::SlotClock; diff --git a/validator_client/validator_services/src/head_monitor_service.rs b/validator_client/validator_services/src/head_monitor_service.rs deleted file mode 100644 index 2b76f68e1a3..00000000000 --- a/validator_client/validator_services/src/head_monitor_service.rs +++ /dev/null @@ -1,206 +0,0 @@ -use beacon_node_fallback::{BeaconHeadCache, BeaconNodeFallback}; -use eth2::types::EventTopic; -use tokio::sync::mpsc; -use tokio::time::{Duration, sleep}; -use tracing::{info, warn}; - -use eth2::types::EventKind; -use futures::StreamExt; -use slot_clock::SlotClock; -use std::ops::Deref; -use std::sync::Arc; -use task_executor::TaskExecutor; -use validator_store::ValidatorStore; - -// Builder for `HeadMonitorService` -#[derive(Default)] -pub struct HeadMonitorServiceBuilder { - validator_store: Option>, - beacon_nodes: Option>>, - executor: Option, - beacon_head_cache: Option>, - head_monitor_tx: Option>>, -} - -// This is used send the index derived from `CandidateBeaconNode` to the -// `AttestationService` for further processing -#[derive(Debug)] -pub struct HeadEvent { - pub beacon_node_index: usize, -} - -impl HeadMonitorServiceBuilder { - pub fn new() -> Self { - Self { - validator_store: None, - beacon_nodes: None, - executor: None, - beacon_head_cache: None, - head_monitor_tx: None, - } - } - - pub fn beacon_nodes(mut self, nodes: Arc>) -> Self { - self.beacon_nodes = Some(nodes); - self - } - - pub fn executor(mut self, executor: TaskExecutor) -> Self { - self.executor = Some(executor); - self - } - - pub fn validator_store(mut self, validator_store: Arc) -> Self { - self.validator_store = Some(validator_store); - self - } - - pub fn beacon_head_cache(mut self, beacon_head_cache: Arc) -> Self { - self.beacon_head_cache = Some(beacon_head_cache); - self - } - - pub fn head_monitor_tx(mut self, head_monitor_tx: Arc>) -> Self { - self.head_monitor_tx = Some(head_monitor_tx); - self - } - pub fn build(self) -> Result, String> { - Ok(HeadMonitorService { - inner: Arc::new(Inner { - _validator_store: self - .validator_store - .ok_or("Cannot build HeadMonitorService without validator_store")?, - beacon_nodes: self - .beacon_nodes - .ok_or("Cannot build HeadMonitorService without beacon_nodes")?, - executor: self - .executor - .ok_or("Cannot build HeadMonitorService without executor")?, - beacon_head_cache: self - .beacon_head_cache - .ok_or("Cannot build HeadMonitorService without beacon_head_cache")?, - head_monitor_tx: self - .head_monitor_tx - .ok_or("Cannot build HeadMonitorService without head_monitor_rx")?, - }), - }) - } -} - -// Helper to minimise `Arc` usage -pub struct Inner { - _validator_store: Arc, - executor: TaskExecutor, - beacon_nodes: Arc>, - beacon_head_cache: Arc, - head_monitor_tx: Arc>, -} - -// Runs a non-terminating loop to update the `BeaconHeadCache` with the latest head received -// from the candidate beacon_nodes. This is an attempt to stream events to beacon nodes and -// potential start attestion duties earlier as soon as latest head is receive from any of the -// beacon node in contrast to attest at the 1/3rd mark in the slot. -// -// -// The cache and the candidate BNs list are refresh/purged to avoid dangling reference conditions -// that arise due to `update_candidates_list`. The cache is refresh use the shared reference between -// `HeadMonitorService` and `BeaconNodeFallback`. -pub struct HeadMonitorService { - inner: Arc>, -} - -impl Clone for HeadMonitorService { - fn clone(&self) -> Self { - Self { - inner: self.inner.clone(), - } - } -} - -impl Deref for HeadMonitorService { - type Target = Inner; - - fn deref(&self) -> &Self::Target { - self.inner.deref() - } -} - -impl HeadMonitorService { - // Starts the service to perpetually stream head events from connected beacon_nodes - pub fn start_update_service(self) -> Result<(), String> { - let executor = self.executor.clone(); - let head_cache = self.beacon_head_cache.clone(); - - info!("Starting head monitoring service"); - let interval_fut = async move { - loop { - let candidates = { - let candidates_guard = self.beacon_nodes.candidates.read().await; - candidates_guard.clone() - }; - let mut tasks = vec![]; - - for candidate in candidates.iter() { - let head_event_stream = candidate - .beacon_node - .get_events::(&[EventTopic::Head]) - .await; - - let mut head_event_stream = match head_event_stream { - Ok(stream) => stream, - Err(e) => { - warn!("failed to get head event stream: {:?}", e); - continue; - } - }; - - let head_cache_clone = head_cache.clone(); - let sender_tx = self.head_monitor_tx.clone(); - - let stream_fut = async move { - while let Some(event_result) = head_event_stream.next().await { - if let Ok(EventKind::Head(head)) = event_result { - head_cache_clone.insert(candidate.index, head.clone()).await; - - if head_cache_clone.is_latest(&head).await { - if let Ok(()) = sender_tx - .send(HeadEvent { - beacon_node_index: candidate.index, - }) - .await - { - } else { - warn!("Head monitoring service channel closed"); - break; - } - } - } - } - }; - - tasks.push(stream_fut); - } - - if tasks.is_empty() { - warn!( - "No beacon nodes available for head event streaming, retrying in 5 seconds" - ); - sleep(Duration::from_secs(5)).await; - continue; - } - - futures::future::join_all(tasks).await; - - drop(candidates); - self.beacon_head_cache.purge_cache().await; - - // Add a small delay before reconnecting to avoid hammering beacon nodes - sleep(Duration::from_secs(1)).await; - } - }; - - executor.spawn(interval_fut, "head_monitor_service"); - - Ok(()) - } -} diff --git a/validator_client/validator_services/src/lib.rs b/validator_client/validator_services/src/lib.rs index 1712f67d8e8..3b8bd9ae14b 100644 --- a/validator_client/validator_services/src/lib.rs +++ b/validator_client/validator_services/src/lib.rs @@ -1,7 +1,6 @@ pub mod attestation_service; pub mod block_service; pub mod duties_service; -pub mod head_monitor_service; pub mod latency_service; pub mod notifier_service; pub mod preparation_service; From 29867d2c4466ad1909a4a0840434da2243126660 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Wed, 29 Oct 2025 02:03:40 -0400 Subject: [PATCH 15/55] fixes and linting --- .../src/beacon_head_monitor.rs | 15 ++++++++++----- validator_client/src/lib.rs | 11 +++++++++-- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs index 0c921da471f..5f09073e554 100644 --- a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs +++ b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs @@ -68,15 +68,20 @@ impl Default for BeaconHeadCache { // // // The cache and the candidate BNs list are refresh/purged to avoid dangling reference conditions -// that arise due to `update_candidates_list`. The cache is refresh use the shared reference between -// `HeadMonitorService` and `BeaconNodeFallback`. - +// that arise due to `update_candidates_list`. +// // Starts the service to perpetually stream head events from connected beacon_nodes pub async fn poll_head_event_from_beacon_nodes( beacon_nodes: Arc>, ) -> Result<(), String> { - let head_cache = beacon_nodes.beacon_head_cache.clone().expect("asdf"); - let head_monitor_send = beacon_nodes.head_monitor_send.clone().expect("asdf"); + let head_cache = beacon_nodes + .beacon_head_cache + .clone() + .expect("Unable to start head monitor without beacon_head_cache"); + let head_monitor_send = beacon_nodes + .head_monitor_send + .clone() + .expect("Unable to start head monitor without head_monitor_send"); info!("Starting head monitoring service"); let candidates = { diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 1dca8fc9e85..34f26a1ac2e 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -73,6 +73,8 @@ pub const AGGREGATION_PRE_COMPUTE_EPOCHS: u64 = 2; /// Number of slots in advance to compute sync selection proofs when in `distributed` mode. pub const AGGREGATION_PRE_COMPUTE_SLOTS_DISTRIBUTED: u64 = 1; +const MAX_HEAD_EVENT_QUEUE_LEN: usize = 1_024; + type ValidatorStore = LighthouseValidatorStore; #[derive(Clone)] @@ -384,9 +386,16 @@ impl ProductionValidatorClient { Duration::from_secs(context.eth2_config.spec.seconds_per_slot), ); + let (head_send, head_receiver) = mpsc::channel(MAX_HEAD_EVENT_QUEUE_LEN); + + let head_send_ref = Arc::new(head_send); + beacon_nodes.set_slot_clock(slot_clock.clone()); proposer_nodes.set_slot_clock(slot_clock.clone()); + beacon_nodes.set_head_send(head_send_ref.clone()); + proposer_nodes.set_head_send(head_send_ref.clone()); + let beacon_nodes = Arc::new(beacon_nodes); start_fallback_updater_service::<_, E>(context.executor.clone(), beacon_nodes.clone())?; @@ -494,8 +503,6 @@ impl ProductionValidatorClient { block_service_builder = block_service_builder.proposer_nodes(proposer_nodes.clone()); } - let (_, head_receiver) = mpsc::channel(1_024); - let block_service = block_service_builder.build()?; let attestation_service = AttestationServiceBuilder::new() From fd4387603f51abc10a43ad5b1ac0525254bf982a Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Fri, 31 Oct 2025 04:10:08 -0400 Subject: [PATCH 16/55] remove unused code --- validator_client/beacon_node_fallback/src/lib.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index 4c23426abf5..7c2df5fab30 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -456,10 +456,6 @@ impl BeaconNodeFallback { n } - pub fn spawn_head_monitor_process(&self, _executor: TaskExecutor) { - let _future = async move {}; - } - // Returns all data required by the VC notifier. pub async fn get_notifier_info(&self) -> (Vec, usize, usize) { let candidates = self.candidates.read().await; From b054a1032a0c7cd1352695b20ff31202eb8f4f72 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Fri, 31 Oct 2025 20:27:45 -0400 Subject: [PATCH 17/55] changes --- .../src/beacon_head_monitor.rs | 23 +++---- .../src/attestation_service.rs | 60 +++++++++---------- 2 files changed, 41 insertions(+), 42 deletions(-) diff --git a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs index 5f09073e554..16749bee873 100644 --- a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs +++ b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs @@ -112,17 +112,18 @@ pub async fn poll_head_event_from_beacon_nodes AttestationService = None; - - tokio::select! { - _ = sleep(duration_to_next_slot + slot_duration / 3) => {}, - head_event = self.poll_for_head_events() => { - if let Ok(event) = head_event { - beacon_node_index = Some(event.beacon_node_index); - } - } - } - - if let Some(current_slot) = self.slot_clock.now() { - let mut last_slot = self.latest_attested_slot.lock().await; - - if current_slot > *last_slot { - if let Err(e) = - self.spawn_attestation_tasks(slot_duration, beacon_node_index) - { - crit!(error = e, "Failed to spawn attestation tasks") - } else { - *last_slot = current_slot; - trace!(?current_slot, "Spawned attestation tasks"); - } - } else { - debug!(?current_slot, ?last_slot, "Already attested for this slot"); - } - } else { - error!("Failed to read slot clock after trigger"); - } - } else { + let Some(duration) = self.slot_clock.duration_to_next_slot() else { error!("Failed to read slot clock"); sleep(slot_duration).await; continue; + }; + + let beacon_node_index = tokio::select! { + _ = sleep(duration + slot_duration /3 ) => None, + Ok(event) = self.poll_for_head_events() => Some(event.beacon_node_index), + else => None + }; + + let Some(current_slot) = self.slot_clock.now() else { + error!("Failed to read slot clock after trigger"); + continue; + }; + + let mut last_slot = self.latest_attested_slot.lock().await; + + if current_slot <= *last_slot { + debug!(?current_slot, "Attestation already initiated for the slot"); + continue; + } + + match self.spawn_attestation_tasks(slot_duration, beacon_node_index) { + Ok(_) => { + *last_slot = current_slot; + trace!(?current_slot, "Spawned attestation tasks"); + } + Err(e) => { + crit!(error = e, "Failed to spawn attestation tasks") + } } } }; From 32eed9abef3c5d5dfcbe3c6cf9b5783eed8adbb2 Mon Sep 17 00:00:00 2001 From: Hopinheimer Date: Mon, 10 Nov 2025 17:25:57 -0500 Subject: [PATCH 18/55] addressing comments --- .../beacon_node_fallback/src/beacon_head_monitor.rs | 7 +++---- validator_client/beacon_node_fallback/src/lib.rs | 12 +++++------- .../validator_services/src/attestation_service.rs | 2 +- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs index 16749bee873..ab28c229430 100644 --- a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs +++ b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs @@ -1,13 +1,12 @@ use crate::BeaconNodeFallback; -use eth2::types::EthSpec; -use eth2::types::SseHead; use std::collections::HashMap; use tokio::sync::RwLock; -use eth2::types::EventTopic; +use types::EthSpec; + +use eth2::types::{EventKind, EventTopic, SseHead}; use tracing::{info, warn}; -use eth2::types::EventKind; use futures::StreamExt; use slot_clock::SlotClock; use std::sync::Arc; diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index baf4e3a7c0b..939029747bd 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -435,15 +435,15 @@ impl BeaconNodeFallback { self.slot_clock = Some(slot_clock); } + pub fn set_head_send(&mut self, head_monitor_send: Arc>) { + self.head_monitor_send = Some(head_monitor_send); + } + /// The count of candidates, regardless of their state. pub async fn num_total(&self) -> usize { self.candidates.read().await.len() } - pub fn set_head_send(&mut self, head_monitor_send: Arc>) { - self.head_monitor_send = Some(head_monitor_send); - } - /// The count of candidates that are online and compatible, but not necessarily synced. pub async fn num_available(&self) -> usize { let mut n = 0; @@ -692,10 +692,9 @@ impl BeaconNodeFallback { R: Future>, Err: Debug, { - let candidates = self.candidates.read().await; - // Try the preferred beacon node first if it exists if let Some(preferred_idx) = preferred_index + && let candidates = self.candidates.read().await && let Some(preferred_candidate) = candidates.iter().find(|c| c.index == preferred_idx) { let preferred_node = preferred_candidate.beacon_node.clone(); @@ -710,7 +709,6 @@ impl BeaconNodeFallback { } // Fall back to normal first_success behavior - drop(candidates); self.first_success(func).await } diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 429621332e8..531ea7e5445 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -233,7 +233,7 @@ impl AttestationService, ) -> Result<(), String> { - info!( + debug!( ?beacon_node_index, "process attestation from beacon_node_index", ); From dac9f007765aad91d69468cf73ce54a019496781 Mon Sep 17 00:00:00 2001 From: Hopinheimer Date: Tue, 11 Nov 2025 15:06:19 -0500 Subject: [PATCH 19/55] removing unwanted logs --- .../validator_services/src/attestation_service.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 531ea7e5445..46e4174bedc 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -203,10 +203,7 @@ impl AttestationService { - *last_slot = current_slot; - trace!(?current_slot, "Spawned attestation tasks"); - } + Ok(_) => *last_slot = current_slot, Err(e) => { crit!(error = e, "Failed to spawn attestation tasks") } From bf1471c05dee3dc5f1865c05e283e06337b8ae57 Mon Sep 17 00:00:00 2001 From: Hopinheimer Date: Tue, 11 Nov 2025 15:13:02 -0500 Subject: [PATCH 20/55] fmt --- validator_client/validator_services/src/attestation_service.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 46e4174bedc..7a985017d7a 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -11,7 +11,7 @@ use std::sync::Arc; use task_executor::TaskExecutor; use tokio::sync::mpsc; use tokio::time::{Duration, Instant, sleep, sleep_until}; -use tracing::{debug, error, info, trace, warn}; +use tracing::{debug, error, info, warn}; use tree_hash::TreeHash; use types::{Attestation, AttestationData, ChainSpec, CommitteeIndex, EthSpec, Slot}; use validator_store::{Error as ValidatorStoreError, ValidatorStore}; From 39b9a584c6f69ea43c7d1ac480d40c13b6579da7 Mon Sep 17 00:00:00 2001 From: Hopinheimer Date: Fri, 14 Nov 2025 03:29:04 -0500 Subject: [PATCH 21/55] fixing a unwanted service starting bug --- .../beacon_node_fallback/src/lib.rs | 22 +++++++++++-------- validator_client/src/lib.rs | 3 ++- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index 939029747bd..fa72d3bb3c3 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -77,17 +77,21 @@ pub fn start_fallback_updater_service( let beacon_nodes_ref = beacon_nodes.clone(); - let head_monitor_future = async move { - loop { - if let Err(err) = - poll_head_event_from_beacon_nodes::(beacon_nodes_ref.clone()).await - { - warn!(error=?err, "Head service failed"); + // the existence of head_monitor_send is overloaded with the predicate of + // requirement of starting the head monitoring service or not. + if beacon_nodes_ref.head_monitor_send.is_some() { + let head_monitor_future = async move { + loop { + if let Err(err) = + poll_head_event_from_beacon_nodes::(beacon_nodes_ref.clone()).await + { + warn!(error=?err, "Head service failed"); + } } - } - }; + }; - executor.spawn(head_monitor_future, "head_monitoring"); + executor.spawn(head_monitor_future, "head_monitoring"); + } let future = async move { loop { diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 34f26a1ac2e..0c6999791e1 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -393,8 +393,9 @@ impl ProductionValidatorClient { beacon_nodes.set_slot_clock(slot_clock.clone()); proposer_nodes.set_slot_clock(slot_clock.clone()); + // Only the beacon_nodes are used for attestation duties and thus biconditionally + // proposer_nodes do not need head_send ref. beacon_nodes.set_head_send(head_send_ref.clone()); - proposer_nodes.set_head_send(head_send_ref.clone()); let beacon_nodes = Arc::new(beacon_nodes); start_fallback_updater_service::<_, E>(context.executor.clone(), beacon_nodes.clone())?; From 07d9a126f4ec3a76809ca73e0cb1663d76f81662 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Wed, 26 Nov 2025 13:23:37 -0500 Subject: [PATCH 22/55] making the service feature flagged --- validator_client/src/cli.rs | 11 +++++++ validator_client/src/config.rs | 5 +++ validator_client/src/lib.rs | 28 ++++++++++------ .../src/attestation_service.rs | 33 ++++++++++++------- 4 files changed, 55 insertions(+), 22 deletions(-) diff --git a/validator_client/src/cli.rs b/validator_client/src/cli.rs index 477781d3e88..2fe1ca9e9e7 100644 --- a/validator_client/src/cli.rs +++ b/validator_client/src/cli.rs @@ -466,6 +466,17 @@ pub struct ValidatorClient { )] pub beacon_nodes_sync_tolerances: Vec, + #[clap( + long, + help = "Enable the beacon head monitor so fallback head updates trigger duties when a lagging primary is detected. \ + This keeps the attestation service responsive when using multiple beacon nodes, but it relies on the \ + fallback service streaming head events which may increase network usage. \ + When not enabled (default), duties are only triggered on slot boundaries and ignore fallback head changes.", + display_order = 0, + help_heading = FLAG_HEADER + )] + pub enable_beacon_head_monitor: bool, + #[clap( long, help = "Disable Lighthouse's slashing protection for all web3signer keys. This can \ diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index 04d69dc9dc1..3483eeac6f4 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -80,6 +80,9 @@ pub struct Config { pub broadcast_topics: Vec, /// Enables a service which attempts to measure latency between the VC and BNs. pub enable_latency_measurement_service: bool, + /// Enables the beacon head monitor that reacts to fallback head updates. + #[serde(default)] + pub enable_beacon_head_monitor: bool, /// Defines the number of validators per `validator/register_validator` request sent to the BN. pub validator_registration_batch_size: usize, /// Whether we are running with distributed network support. @@ -129,6 +132,7 @@ impl Default for Config { builder_registration_timestamp_override: None, broadcast_topics: vec![ApiTopic::Subscriptions], enable_latency_measurement_service: true, + enable_beacon_head_monitor: false, validator_registration_batch_size: 500, distributed: false, initialized_validators: <_>::default(), @@ -368,6 +372,7 @@ impl Config { config.validator_store.builder_boost_factor = validator_client_config.builder_boost_factor; config.enable_latency_measurement_service = !validator_client_config.disable_latency_measurement_service; + config.enable_beacon_head_monitor = validator_client_config.enable_beacon_head_monitor; config.validator_registration_batch_size = validator_client_config.validator_registration_batch_size; diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 0c6999791e1..44ea96ecc93 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -13,7 +13,8 @@ use tokio::sync::Mutex; use account_utils::validator_definitions::ValidatorDefinitions; use beacon_node_fallback::{ - BeaconNodeFallback, CandidateBeaconNode, start_fallback_updater_service, + BeaconNodeFallback, CandidateBeaconNode, beacon_head_monitor::HeadEvent, + start_fallback_updater_service, }; use clap::ArgMatches; use doppelganger_service::DoppelgangerService; @@ -386,16 +387,19 @@ impl ProductionValidatorClient { Duration::from_secs(context.eth2_config.spec.seconds_per_slot), ); - let (head_send, head_receiver) = mpsc::channel(MAX_HEAD_EVENT_QUEUE_LEN); - - let head_send_ref = Arc::new(head_send); - beacon_nodes.set_slot_clock(slot_clock.clone()); proposer_nodes.set_slot_clock(slot_clock.clone()); // Only the beacon_nodes are used for attestation duties and thus biconditionally // proposer_nodes do not need head_send ref. - beacon_nodes.set_head_send(head_send_ref.clone()); + let head_monitor_rx = if config.enable_beacon_head_monitor { + let (head_monitor_tx, head_receiver) = + mpsc::channel::(MAX_HEAD_EVENT_QUEUE_LEN); + beacon_nodes.set_head_send(Arc::new(head_monitor_tx)); + Some(Arc::new(Mutex::new(head_receiver))) + } else { + None + }; let beacon_nodes = Arc::new(beacon_nodes); start_fallback_updater_service::<_, E>(context.executor.clone(), beacon_nodes.clone())?; @@ -506,16 +510,20 @@ impl ProductionValidatorClient { let block_service = block_service_builder.build()?; - let attestation_service = AttestationServiceBuilder::new() + let mut attestation_builder = AttestationServiceBuilder::new() .duties_service(duties_service.clone()) .slot_clock(slot_clock.clone()) .validator_store(validator_store.clone()) .beacon_nodes(beacon_nodes.clone()) .executor(context.executor.clone()) .chain_spec(context.eth2_config.spec.clone()) - .head_monitor_rx(Arc::new(Mutex::new(head_receiver))) - .disable(config.disable_attesting) - .build()?; + .disable(config.disable_attesting); + + if let Some(head_monitor_rx) = head_monitor_rx { + attestation_builder = attestation_builder.head_monitor_rx(head_monitor_rx); + } + + let attestation_service = attestation_builder.build()?; let preparation_service = PreparationServiceBuilder::new() .slot_clock(slot_clock.clone()) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 7a985017d7a..c60c32275c6 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -106,9 +106,7 @@ impl AttestationServiceBuil chain_spec: self .chain_spec .ok_or("Cannot build AttestationService without chain_spec")?, - head_monitor_rx: self - .head_monitor_rx - .ok_or("Cannot build AttestationService without head_monitor_rx")?, + head_monitor_rx: self.head_monitor_rx, disable: self.disable, latest_attested_slot: Mutex::new(Slot::default()), }), @@ -124,7 +122,7 @@ pub struct Inner { beacon_nodes: Arc>, executor: TaskExecutor, chain_spec: Arc, - head_monitor_rx: Arc>>, + head_monitor_rx: Option>>>, disable: bool, latest_attested_slot: Mutex, } @@ -184,10 +182,15 @@ impl AttestationService None, - Ok(event) = self.poll_for_head_events() => Some(event.beacon_node_index), - else => None + let slot_trigger_delay = slot_duration / 3; + let beacon_node_index = if self.head_monitor_rx.is_some() { + tokio::select! { + _ = sleep(duration + slot_trigger_delay) => None, + event = self.poll_for_head_events() => event.map(|event| event.beacon_node_index), + } + } else { + sleep(duration + slot_trigger_delay).await; + None }; let Some(current_slot) = self.slot_clock.now() else { @@ -215,11 +218,17 @@ impl AttestationService Result { - let mut receiver = self.head_monitor_rx.lock().await; + async fn poll_for_head_events(&self) -> Option { + let Some(receiver) = &self.head_monitor_rx else { + return None; + }; + let mut receiver = receiver.lock().await; match receiver.recv().await { - Some(head_event) => Ok(head_event), - None => Err("Head monitor channel closed unexpectedly".to_string()), + Some(head_event) => Some(head_event), + None => { + warn!("Head monitor channel closed unexpectedly"); + None + } } } From a82960be449cd50643c6590e4a831f768a074aaa Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Wed, 26 Nov 2025 13:41:49 -0500 Subject: [PATCH 23/55] cleaned up some logic --- validator_client/beacon_node_fallback/src/lib.rs | 8 +++++++- validator_client/src/lib.rs | 5 +---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index fa72d3bb3c3..fc647a03937 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -423,7 +423,7 @@ impl BeaconNodeFallback { candidates: Arc::new(RwLock::new(candidates)), distance_tiers, slot_clock: None, - beacon_head_cache: Some(Arc::new(BeaconHeadCache::new())), + beacon_head_cache: None, head_monitor_send: None, broadcast_topics, spec, @@ -439,8 +439,14 @@ impl BeaconNodeFallback { self.slot_clock = Some(slot_clock); } + + /// This the head monitor channel that streams events from all the beacon node that the + /// validator client is connected in the `BeaconNodeFallback`. This is also initialize the + /// beacon_head_cache under the assumption the beacon_head_cache will always be needed when + /// head_monitor_send is set. pub fn set_head_send(&mut self, head_monitor_send: Arc>) { self.head_monitor_send = Some(head_monitor_send); + self.beacon_head_cache = Some(Arc::new(BeaconHeadCache::new())); } /// The count of candidates, regardless of their state. diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 44ea96ecc93..d165c848d59 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -516,13 +516,10 @@ impl ProductionValidatorClient { .validator_store(validator_store.clone()) .beacon_nodes(beacon_nodes.clone()) .executor(context.executor.clone()) + .head_monitor_rx(head_monitor_rx) .chain_spec(context.eth2_config.spec.clone()) .disable(config.disable_attesting); - if let Some(head_monitor_rx) = head_monitor_rx { - attestation_builder = attestation_builder.head_monitor_rx(head_monitor_rx); - } - let attestation_service = attestation_builder.build()?; let preparation_service = PreparationServiceBuilder::new() From 6bddeebb54268dfb0b9a8e754ba676891897ec6c Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Wed, 26 Nov 2025 15:31:05 -0500 Subject: [PATCH 24/55] fix --- validator_client/beacon_node_fallback/src/lib.rs | 1 - validator_client/src/lib.rs | 2 +- .../validator_services/src/attestation_service.rs | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index fc647a03937..fbc3798879a 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -439,7 +439,6 @@ impl BeaconNodeFallback { self.slot_clock = Some(slot_clock); } - /// This the head monitor channel that streams events from all the beacon node that the /// validator client is connected in the `BeaconNodeFallback`. This is also initialize the /// beacon_head_cache under the assumption the beacon_head_cache will always be needed when diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index d165c848d59..71f637f2873 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -510,7 +510,7 @@ impl ProductionValidatorClient { let block_service = block_service_builder.build()?; - let mut attestation_builder = AttestationServiceBuilder::new() + let attestation_builder = AttestationServiceBuilder::new() .duties_service(duties_service.clone()) .slot_clock(slot_clock.clone()) .validator_store(validator_store.clone()) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index c60c32275c6..ed650b5e624 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -80,9 +80,9 @@ impl AttestationServiceBuil pub fn head_monitor_rx( mut self, - head_monitor_rx: Arc>>, + head_monitor_rx: Option>>>, ) -> Self { - self.head_monitor_rx = Some(head_monitor_rx); + self.head_monitor_rx = head_monitor_rx; self } pub fn build(self) -> Result, String> { From 409d937ed2e9eea224da9688299b834c4f674544 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Wed, 26 Nov 2025 17:06:38 -0500 Subject: [PATCH 25/55] updating docs --- book/src/help_vc.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/book/src/help_vc.md b/book/src/help_vc.md index b19ff0ba388..f61917a0dc4 100644 --- a/book/src/help_vc.md +++ b/book/src/help_vc.md @@ -205,6 +205,13 @@ Flags: --distributed Enables functionality required for running the validator in a distributed validator cluster. + --enable-beacon-head-monitor + Enable the beacon head monitor so fallback head updates trigger duties + when a lagging primary is detected. This keeps the attestation + service responsive when using multiple beacon nodes, but it relies on + the fallback service streaming head events which may increase network + usage. When not enabled (default), duties are only triggered on slot + boundaries and ignore fallback head changes. --enable-doppelganger-protection If this flag is set, Lighthouse will delay startup for three epochs and monitor for messages on the network by any of the validators From 0d8160f46840c3c11e5ccc0806ee090140d8533f Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Thu, 27 Nov 2025 16:49:58 -0500 Subject: [PATCH 26/55] addressing comments --- .../beacon_node_fallback/src/beacon_head_monitor.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs index ab28c229430..136a6af6607 100644 --- a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs +++ b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs @@ -1,15 +1,12 @@ use crate::BeaconNodeFallback; -use std::collections::HashMap; -use tokio::sync::RwLock; - -use types::EthSpec; - use eth2::types::{EventKind, EventTopic, SseHead}; -use tracing::{info, warn}; - use futures::StreamExt; use slot_clock::SlotClock; +use std::collections::HashMap; use std::sync::Arc; +use tokio::sync::RwLock; +use tracing::{info, warn}; +use types::EthSpec; type CacheHashMap = HashMap; From 70e6d186db0db0ec5bed732feb3f03cdfb8e4db4 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Mon, 1 Dec 2025 16:38:13 -0500 Subject: [PATCH 27/55] testsss --- .../src/beacon_head_monitor.rs | 225 ++++++++++++++++++ .../beacon_node_fallback/src/lib.rs | 129 ++++++++++ 2 files changed, 354 insertions(+) diff --git a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs index 136a6af6607..e838b286e5c 100644 --- a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs +++ b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs @@ -142,3 +142,228 @@ pub async fn poll_head_event_from_beacon_nodes SseHead { + SseHead { + slot: types::Slot::new(slot), + block: Hash256::from_low_u64_be(block_root as u64), + state: Hash256::from_low_u64_be(block_root as u64), + epoch_transition: false, + previous_duty_dependent_root: Hash256::from_low_u64_be(block_root as u64), + current_duty_dependent_root: Hash256::from_low_u64_be(block_root as u64), + execution_optimistic: false, + } + } + + #[tokio::test] + async fn test_beacon_head_cache_insertion_and_retrieval() { + let cache = BeaconHeadCache::new(); + let head_1 = create_sse_head(1, 1); + let head_2 = create_sse_head(2, 2); + + cache.insert(0, head_1.clone()).await; + cache.insert(1, head_2.clone()).await; + + assert_eq!(cache.get(0).await, Some(head_1)); + assert_eq!(cache.get(1).await, Some(head_2)); + assert_eq!(cache.get(2).await, None); + } + + #[tokio::test] + async fn test_beacon_head_cache_update() { + let cache = BeaconHeadCache::new(); + let head_old = create_sse_head(1, 1); + let head_new = create_sse_head(2, 2); + + cache.insert(0, head_old).await; + cache.insert(0, head_new.clone()).await; + + assert_eq!(cache.get(0).await, Some(head_new)); + } + + #[tokio::test] + async fn test_is_latest_with_higher_slot() { + let cache = BeaconHeadCache::new(); + let head_1 = create_sse_head(1, 1); + let head_2 = create_sse_head(2, 2); + let head_3 = create_sse_head(3, 3); + + cache.insert(0, head_1).await; + cache.insert(1, head_2).await; + + assert!(cache.is_latest(&head_3).await); + } + + #[tokio::test] + async fn test_is_latest_with_lower_slot() { + let cache = BeaconHeadCache::new(); + let head_1 = create_sse_head(1, 1); + let head_2 = create_sse_head(2, 2); + let head_older = create_sse_head(1, 99); + + cache.insert(0, head_1).await; + cache.insert(1, head_2).await; + + assert!(!cache.is_latest(&head_older).await); + } + + #[tokio::test] + async fn test_is_latest_with_equal_slot() { + let cache = BeaconHeadCache::new(); + let head_1 = create_sse_head(5, 1); + let head_2 = create_sse_head(5, 2); + let head_equal = create_sse_head(5, 3); + + cache.insert(0, head_1).await; + cache.insert(1, head_2).await; + + assert!(cache.is_latest(&head_equal).await); + } + + #[tokio::test] + async fn test_is_latest_empty_cache() { + let cache = BeaconHeadCache::new(); + let head = create_sse_head(1, 1); + + assert!(cache.is_latest(&head).await); + } + + #[tokio::test] + async fn test_purge_cache_clears_all_entries() { + let cache = BeaconHeadCache::new(); + let head_1 = create_sse_head(1, 1); + let head_2 = create_sse_head(2, 2); + + cache.insert(0, head_1).await; + cache.insert(1, head_2).await; + + assert!(cache.get(0).await.is_some()); + assert!(cache.get(1).await.is_some()); + + cache.purge_cache().await; + + assert!(cache.get(0).await.is_none()); + assert!(cache.get(1).await.is_none()); + } + + #[tokio::test] + async fn test_head_event_creation() { + let event = HeadEvent { + beacon_node_index: 42, + }; + assert_eq!(event.beacon_node_index, 42); + } + + #[tokio::test] + async fn test_cache_caches_multiple_heads_from_different_nodes() { + let cache = BeaconHeadCache::new(); + let head_1 = create_sse_head(10, 1); + let head_2 = create_sse_head(5, 2); + let head_3 = create_sse_head(8, 3); + + cache.insert(0, head_1.clone()).await; + cache.insert(1, head_2.clone()).await; + cache.insert(2, head_3.clone()).await; + + // Verify all are stored + assert_eq!(cache.get(0).await, Some(head_1)); + assert_eq!(cache.get(1).await, Some(head_2)); + assert_eq!(cache.get(2).await, Some(head_3)); + + // The latest should be slot 10 + let head_10 = create_sse_head(10, 99); + assert!(cache.is_latest(&head_10).await); + + // Anything with slot > 10 should be latest + let head_11 = create_sse_head(11, 99); + assert!(cache.is_latest(&head_11).await); + + // Anything with slot < 10 should not be latest + let head_9 = create_sse_head(9, 99); + assert!(!cache.is_latest(&head_9).await); + } + + #[tokio::test] + async fn test_cache_handles_concurrent_operations() { + let cache = Arc::new(BeaconHeadCache::new()); + let mut handles = vec![]; + + // Spawn multiple tasks that insert heads concurrently + for i in 0..10 { + let cache_clone = cache.clone(); + let handle = tokio::spawn(async move { + let head = create_sse_head(i as u64, (i % 256) as u8); + cache_clone.insert(i, head).await; + }); + handles.push(handle); + } + + // Wait for all tasks to complete + for handle in handles { + handle.await.unwrap(); + } + + // Verify all heads are cached + for i in 0..10 { + assert!(cache.get(i).await.is_some()); + } + } + + #[tokio::test] + async fn test_is_latest_after_cache_updates() { + let cache = BeaconHeadCache::new(); + + // Start with head at slot 5 + let head_5 = create_sse_head(5, 1); + cache.insert(0, head_5.clone()).await; + assert!(cache.is_latest(&head_5).await); + + // Add a higher slot + let head_10 = create_sse_head(10, 2); + cache.insert(1, head_10.clone()).await; + + // head_5 should no longer be latest + assert!(!cache.is_latest(&head_5).await); + // head_10 should be latest + assert!(cache.is_latest(&head_10).await); + + // Add an even higher slot + let head_15 = create_sse_head(15, 3); + cache.insert(2, head_15.clone()).await; + + // head_10 should no longer be latest + assert!(!cache.is_latest(&head_10).await); + // head_15 should be latest + assert!(cache.is_latest(&head_15).await); + } + + #[tokio::test] + async fn test_cache_default_is_empty() { + let cache = BeaconHeadCache::default(); + assert!(cache.get(0).await.is_none()); + assert!(cache.get(999).await.is_none()); + } + + #[tokio::test] + async fn test_is_latest_with_multiple_same_slot_heads() { + let cache = BeaconHeadCache::new(); + let head_slot_5_node1 = create_sse_head(5, 1); + let head_slot_5_node2 = create_sse_head(5, 2); + let head_slot_5_node3 = create_sse_head(5, 3); + + cache.insert(0, head_slot_5_node1).await; + cache.insert(1, head_slot_5_node2).await; + + // All heads with slot 5 should be considered latest + assert!(cache.is_latest(&head_slot_5_node3).await); + + // But heads with slot 4 should not be latest + let head_slot_4 = create_sse_head(4, 4); + assert!(!cache.is_latest(&head_slot_4).await); + } +} diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index fbc3798879a..e24750cc7c2 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -1147,4 +1147,133 @@ mod tests { mock1.expect(3).assert(); mock2.expect(3).assert(); } + + #[tokio::test] + async fn first_success_from_index_tries_preferred_node_first() { + let spec = Arc::new(MainnetEthSpec::default_spec()); + let (mut mock_beacon_node_1, beacon_node_1) = new_mock_beacon_node(0, &spec).await; + let (mut mock_beacon_node_2, beacon_node_2) = new_mock_beacon_node(1, &spec).await; + let (mut mock_beacon_node_3, beacon_node_3) = new_mock_beacon_node(2, &spec).await; + + let beacon_node_fallback = create_beacon_node_fallback( + vec![beacon_node_1, beacon_node_2, beacon_node_3], + vec![], + spec.clone(), + ); + + let mock1 = mock_beacon_node_1.mock_offline_node(); + let _mock2 = mock_beacon_node_2.mock_online_node(); + let mock3 = mock_beacon_node_3.mock_online_node(); + + // Request with preferred_index=1 (beacon_node_2) + let result = beacon_node_fallback + .first_success_from_index(Some(1), |client| async move { + client.get_node_version().await + }) + .await; + + // Should succeed since beacon_node_2 is online + assert!(result.is_ok()); + + // mock1 should not be called since preferred node succeeds + mock1.expect(0).assert(); + mock3.expect(0).assert(); + } + + #[tokio::test] + async fn first_success_from_index_falls_back_when_preferred_fails() { + let spec = Arc::new(MainnetEthSpec::default_spec()); + let (mut mock_beacon_node_1, beacon_node_1) = new_mock_beacon_node(0, &spec).await; + let (mut mock_beacon_node_2, beacon_node_2) = new_mock_beacon_node(1, &spec).await; + let (mut mock_beacon_node_3, beacon_node_3) = new_mock_beacon_node(2, &spec).await; + + let beacon_node_fallback = create_beacon_node_fallback( + vec![beacon_node_1, beacon_node_2, beacon_node_3], + vec![], + spec.clone(), + ); + + let _mock1 = mock_beacon_node_1.mock_online_node(); + let mock2 = mock_beacon_node_2.mock_offline_node(); + let _mock3 = mock_beacon_node_3.mock_offline_node(); + + // Request with preferred_index=1 (beacon_node_2), but it's offline + let result = beacon_node_fallback + .first_success_from_index(Some(1), |client| async move { + client.get_node_version().await + }) + .await; + + // Should succeed by falling back to beacon_node_1 + assert!(result.is_ok()); + + // mock2 should be called at least once (the preferred attempt) + mock2.expect(1).assert(); + // since the result was ok we can safely assume that the fallback first_success + // behaviour succeeded instead of checking if either of mock1/mock3 received hits + } + + #[tokio::test] + async fn first_success_from_index_with_none_falls_back_to_first_success() { + let spec = Arc::new(MainnetEthSpec::default_spec()); + let (mut mock_beacon_node_1, beacon_node_1) = new_mock_beacon_node(0, &spec).await; + let (mut mock_beacon_node_2, beacon_node_2) = new_mock_beacon_node(1, &spec).await; + let (mut mock_beacon_node_3, beacon_node_3) = new_mock_beacon_node(2, &spec).await; + + let beacon_node_fallback = create_beacon_node_fallback( + vec![beacon_node_1, beacon_node_2, beacon_node_3], + vec![], + spec.clone(), + ); + + let _mock1 = mock_beacon_node_1.mock_offline_node(); + let _mock2 = mock_beacon_node_2.mock_offline_node(); + let mock3 = mock_beacon_node_3.mock_online_node(); + + // Request with preferred_index=None + let result = beacon_node_fallback + .first_success_from_index(None, |client| async move { + client.get_node_version().await + }) + .await; + + // Should succeed with beacon_node_3 in the first pass + assert!(result.is_ok()); + + // mock3 should be called once in the first pass + mock3.expect(1).assert(); + } + + #[tokio::test] + async fn first_success_from_index_all_offline() { + let spec = Arc::new(MainnetEthSpec::default_spec()); + let (mut mock_beacon_node_1, beacon_node_1) = new_mock_beacon_node(0, &spec).await; + let (mut mock_beacon_node_2, beacon_node_2) = new_mock_beacon_node(1, &spec).await; + let (mut mock_beacon_node_3, beacon_node_3) = new_mock_beacon_node(2, &spec).await; + + let beacon_node_fallback = create_beacon_node_fallback( + vec![beacon_node_1, beacon_node_2, beacon_node_3], + vec![], + spec.clone(), + ); + + let _mock1 = mock_beacon_node_1.mock_offline_node(); + let mock2 = mock_beacon_node_2.mock_offline_node(); + let _mock3 = mock_beacon_node_3.mock_offline_node(); + + // Request with preferred_index=1, but all nodes are offline + let result = beacon_node_fallback + .first_success_from_index(Some(1), |client| async move { + client.get_node_version().await + }) + .await; + + // Should fail since all nodes are offline + assert!(result.is_err()); + + // Preferred node (mock2) should be called 3 times: + // - 1 time for the preferred attempt + // - 2 more times from the fallback to first_success (first and second pass) + mock2.expect(3).assert(); + } } From 84c8291c0c27028d59444cab806804fe654a30d4 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Mon, 1 Dec 2025 16:38:40 -0500 Subject: [PATCH 28/55] fmt --- .../src/beacon_head_monitor.rs | 2 +- .../beacon_node_fallback/src/lib.rs | 30 +++++++++++-------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs index e838b286e5c..4fd8ca17c1c 100644 --- a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs +++ b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs @@ -146,7 +146,7 @@ pub async fn poll_head_event_from_beacon_nodes SseHead { SseHead { diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index e24750cc7c2..33fdc9cf338 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -1167,9 +1167,10 @@ mod tests { // Request with preferred_index=1 (beacon_node_2) let result = beacon_node_fallback - .first_success_from_index(Some(1), |client| async move { - client.get_node_version().await - }) + .first_success_from_index( + Some(1), + |client| async move { client.get_node_version().await }, + ) .await; // Should succeed since beacon_node_2 is online @@ -1199,9 +1200,10 @@ mod tests { // Request with preferred_index=1 (beacon_node_2), but it's offline let result = beacon_node_fallback - .first_success_from_index(Some(1), |client| async move { - client.get_node_version().await - }) + .first_success_from_index( + Some(1), + |client| async move { client.get_node_version().await }, + ) .await; // Should succeed by falling back to beacon_node_1 @@ -1211,7 +1213,7 @@ mod tests { mock2.expect(1).assert(); // since the result was ok we can safely assume that the fallback first_success // behaviour succeeded instead of checking if either of mock1/mock3 received hits - } + } #[tokio::test] async fn first_success_from_index_with_none_falls_back_to_first_success() { @@ -1232,9 +1234,10 @@ mod tests { // Request with preferred_index=None let result = beacon_node_fallback - .first_success_from_index(None, |client| async move { - client.get_node_version().await - }) + .first_success_from_index( + None, + |client| async move { client.get_node_version().await }, + ) .await; // Should succeed with beacon_node_3 in the first pass @@ -1263,9 +1266,10 @@ mod tests { // Request with preferred_index=1, but all nodes are offline let result = beacon_node_fallback - .first_success_from_index(Some(1), |client| async move { - client.get_node_version().await - }) + .first_success_from_index( + Some(1), + |client| async move { client.get_node_version().await }, + ) .await; // Should fail since all nodes are offline From 8b45d57983378afabf08e0cbeff29f2177d85519 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Wed, 17 Dec 2025 03:57:22 -0500 Subject: [PATCH 29/55] addressing comments --- book/src/help_vc.md | 2 +- .../src/beacon_head_monitor.rs | 13 +++++++++++-- .../beacon_node_fallback/src/lib.rs | 15 +++++++++++++-- validator_client/src/cli.rs | 3 ++- validator_client/src/config.rs | 2 +- validator_client/src/lib.rs | 2 +- .../src/attestation_service.rs | 17 ++++++++++------- 7 files changed, 39 insertions(+), 15 deletions(-) diff --git a/book/src/help_vc.md b/book/src/help_vc.md index f61917a0dc4..f28487c9460 100644 --- a/book/src/help_vc.md +++ b/book/src/help_vc.md @@ -207,7 +207,7 @@ Flags: distributed validator cluster. --enable-beacon-head-monitor Enable the beacon head monitor so fallback head updates trigger duties - when a lagging primary is detected. This keeps the attestation + when a lagging primary is detected. This keeps the attestation service responsive when using multiple beacon nodes, but it relies on the fallback service streaming head events which may increase network usage. When not enabled (default), duties are only triggered on slot diff --git a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs index 4fd8ca17c1c..cd5f1865d33 100644 --- a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs +++ b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs @@ -25,20 +25,27 @@ pub struct BeaconHeadCache { } impl BeaconHeadCache { + /// Creates a new empty beacon head cache. pub fn new() -> Self { Self { cache: RwLock::new(HashMap::new()), } } + /// Retrieves the cached head for a specific beacon node. + /// Returns `None` if no head has been cached for that node yet. pub async fn get(&self, beacon_node_index: usize) -> Option { self.cache.read().await.get(&beacon_node_index).cloned() } + /// Stores or updates the head event for a specific beacon node. + /// Replaces any previously cached head for the given node. pub async fn insert(&self, beacon_node_index: usize, head: SseHead) { self.cache.write().await.insert(beacon_node_index, head); } + /// Checks if the given head is the latest among all cached heads. + /// Returns `true` if the head's slot is >= all cached heads' slots. pub async fn is_latest(&self, head: &SseHead) -> bool { let cache = self.cache.read().await; cache @@ -46,6 +53,8 @@ impl BeaconHeadCache { .all(|cache_head| head.slot >= cache_head.slot) } + /// Clears all cached heads, removing entries for all beacon nodes. + /// Useful when beacon node candidates are refreshed to avoid stale references. pub async fn purge_cache(&self) { self.cache.write().await.clear(); } @@ -73,11 +82,11 @@ pub async fn poll_head_event_from_beacon_nodes( if let Err(err) = poll_head_event_from_beacon_nodes::(beacon_nodes_ref.clone()).await { - warn!(error=?err, "Head service failed"); + warn!(error=?err, "Head service failed, retrying starting next slot"); + let sleep_time = beacon_nodes_ref + .slot_clock + .as_ref() + .and_then(|slot_clock| { + let slot = slot_clock.now()?; + let till_next_slot = slot_clock.duration_to_slot(slot + 1)?; + + till_next_slot.checked_sub(SLOT_LOOKAHEAD) + }) + .unwrap_or_else(|| Duration::from_secs(1)); + + sleep(sleep_time).await } } }; diff --git a/validator_client/src/cli.rs b/validator_client/src/cli.rs index 2fe1ca9e9e7..3ccb4b3802f 100644 --- a/validator_client/src/cli.rs +++ b/validator_client/src/cli.rs @@ -468,7 +468,8 @@ pub struct ValidatorClient { #[clap( long, - help = "Enable the beacon head monitor so fallback head updates trigger duties when a lagging primary is detected. \ + help = "Enable the beacon head monitor so fallback head updates trigger duties when a lagging \ + primary is detected. \ This keeps the attestation service responsive when using multiple beacon nodes, but it relies on the \ fallback service streaming head events which may increase network usage. \ When not enabled (default), duties are only triggered on slot boundaries and ignore fallback head changes.", diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index 3483eeac6f4..b5e86e13458 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -132,7 +132,7 @@ impl Default for Config { builder_registration_timestamp_override: None, broadcast_topics: vec![ApiTopic::Subscriptions], enable_latency_measurement_service: true, - enable_beacon_head_monitor: false, + enable_beacon_head_monitor: true, validator_registration_batch_size: 500, distributed: false, initialized_validators: <_>::default(), diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 71f637f2873..1a575ede282 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -396,7 +396,7 @@ impl ProductionValidatorClient { let (head_monitor_tx, head_receiver) = mpsc::channel::(MAX_HEAD_EVENT_QUEUE_LEN); beacon_nodes.set_head_send(Arc::new(head_monitor_tx)); - Some(Arc::new(Mutex::new(head_receiver))) + Some(Mutex::new(head_receiver)) } else { None }; diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 83e20df9f4c..d92d468759f 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -1,6 +1,4 @@ use crate::duties_service::{DutiesService, DutyAndProof}; -use tokio::sync::Mutex; - use beacon_node_fallback::{ApiTopic, BeaconNodeFallback, beacon_head_monitor::HeadEvent}; use futures::future::join_all; use logging::crit; @@ -9,6 +7,7 @@ use std::collections::HashMap; use std::ops::Deref; use std::sync::Arc; use task_executor::TaskExecutor; +use tokio::sync::Mutex; use tokio::sync::mpsc; use tokio::time::{Duration, Instant, sleep, sleep_until}; use tracing::{Instrument, Span, debug, error, info, info_span, instrument, warn}; @@ -25,7 +24,7 @@ pub struct AttestationServiceBuilder beacon_nodes: Option>>, executor: Option, chain_spec: Option>, - head_monitor_rx: Option>>>, + head_monitor_rx: Option>>, disable: bool, } @@ -80,7 +79,7 @@ impl AttestationServiceBuil pub fn head_monitor_rx( mut self, - head_monitor_rx: Option>>>, + head_monitor_rx: Option>>, ) -> Self { self.head_monitor_rx = head_monitor_rx; self @@ -122,7 +121,7 @@ pub struct Inner { beacon_nodes: Arc>, executor: TaskExecutor, chain_spec: Arc, - head_monitor_rx: Option>>>, + head_monitor_rx: Option>>, disable: bool, latest_attested_slot: Mutex, } @@ -235,7 +234,11 @@ impl AttestationService,) -> Result<(), String> { + fn spawn_attestation_tasks( + &self, + slot_duration: Duration, + beacon_node_index: Option, + ) -> Result<(), String> { let slot = self.slot_clock.now().ok_or("Failed to read slot clock")?; let duration_to_next_slot = self .slot_clock @@ -432,7 +435,7 @@ impl AttestationService Date: Wed, 17 Dec 2025 04:34:30 -0500 Subject: [PATCH 30/55] resolving unstable changes --- .../beacon_node_fallback/src/beacon_head_monitor.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs index cd5f1865d33..30ae6a35f1b 100644 --- a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs +++ b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs @@ -155,7 +155,8 @@ pub async fn poll_head_event_from_beacon_nodes SseHead { SseHead { From dfe2935a84a61f3c1df62c7f7204625c6c1fda10 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Wed, 17 Dec 2025 04:55:25 -0500 Subject: [PATCH 31/55] clippy --- book/src/help_vc.md | 6 +++--- .../beacon_node_fallback/src/beacon_head_monitor.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/book/src/help_vc.md b/book/src/help_vc.md index ecc1ee74f23..e44cf3f106a 100644 --- a/book/src/help_vc.md +++ b/book/src/help_vc.md @@ -207,9 +207,9 @@ Flags: distributed validator cluster. --enable-beacon-head-monitor Enable the beacon head monitor so fallback head updates trigger duties - when a lagging primary is detected. This keeps the attestation - service responsive when using multiple beacon nodes, but it relies on - the fallback service streaming head events which may increase network + when a lagging primary is detected. This keeps the attestation service + responsive when using multiple beacon nodes, but it relies on the + fallback service streaming head events which may increase network usage. When not enabled (default), duties are only triggered on slot boundaries and ignore fallback head changes. --enable-doppelganger-protection diff --git a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs index 30ae6a35f1b..08298adddea 100644 --- a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs +++ b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs @@ -155,8 +155,8 @@ pub async fn poll_head_event_from_beacon_nodes SseHead { SseHead { From 7b7ab0957656dd4714c579dfa86006a6f004ed8a Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Wed, 17 Dec 2025 15:19:19 -0500 Subject: [PATCH 32/55] update cli docs for default behaviour --- book/src/help_vc.md | 5 +++-- validator_client/src/cli.rs | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/book/src/help_vc.md b/book/src/help_vc.md index e44cf3f106a..e4a5df07539 100644 --- a/book/src/help_vc.md +++ b/book/src/help_vc.md @@ -210,8 +210,9 @@ Flags: when a lagging primary is detected. This keeps the attestation service responsive when using multiple beacon nodes, but it relies on the fallback service streaming head events which may increase network - usage. When not enabled (default), duties are only triggered on slot - boundaries and ignore fallback head changes. + usage. When is not enabled, duties are only triggered on slot + boundaries and ignore fallback head changes. The current default + behaviour is to have this feature enabled. --enable-doppelganger-protection If this flag is set, Lighthouse will delay startup for three epochs and monitor for messages on the network by any of the validators diff --git a/validator_client/src/cli.rs b/validator_client/src/cli.rs index b87c1af1e2d..e00769911eb 100644 --- a/validator_client/src/cli.rs +++ b/validator_client/src/cli.rs @@ -482,7 +482,8 @@ pub struct ValidatorClient { primary is detected. \ This keeps the attestation service responsive when using multiple beacon nodes, but it relies on the \ fallback service streaming head events which may increase network usage. \ - When not enabled (default), duties are only triggered on slot boundaries and ignore fallback head changes.", + When is not enabled, duties are only triggered on slot boundaries and ignore fallback head changes. \ + The current default behaviour is to have this feature enabled.", display_order = 0, help_heading = FLAG_HEADER )] From c2092ebf3c75e3ef6a8244daf903a2c32643ac28 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 15 Jan 2026 16:32:41 +1100 Subject: [PATCH 33/55] Small tweaks --- validator_client/beacon_node_fallback/src/lib.rs | 13 ++++--------- validator_client/src/config.rs | 1 - 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index eeaef4fb028..7fd83e7c3e5 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -88,13 +88,8 @@ pub fn start_fallback_updater_service( let sleep_time = beacon_nodes_ref .slot_clock .as_ref() - .and_then(|slot_clock| { - let slot = slot_clock.now()?; - let till_next_slot = slot_clock.duration_to_slot(slot + 1)?; - - till_next_slot.checked_sub(SLOT_LOOKAHEAD) - }) - .unwrap_or_else(|| Duration::from_secs(1)); + .and_then(|slot_clock| slot_clock.duration_to_next_slot()) + .unwrap_or_else(|| Duration::from_secs(12)); sleep(sleep_time).await } @@ -450,8 +445,8 @@ impl BeaconNodeFallback { self.slot_clock = Some(slot_clock); } - /// This the head monitor channel that streams events from all the beacon node that the - /// validator client is connected in the `BeaconNodeFallback`. This is also initialize the + /// This the head monitor channel that streams events from all the beacon nodes that the + /// validator client is connected in the `BeaconNodeFallback`. This also initializes the /// beacon_head_cache under the assumption the beacon_head_cache will always be needed when /// head_monitor_send is set. pub fn set_head_send(&mut self, head_monitor_send: Arc>) { diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index 92a30ae180f..83e9c24bde4 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -83,7 +83,6 @@ pub struct Config { /// Enables a service which attempts to measure latency between the VC and BNs. pub enable_latency_measurement_service: bool, /// Enables the beacon head monitor that reacts to fallback head updates. - #[serde(default)] pub enable_beacon_head_monitor: bool, /// Defines the number of validators per `validator/register_validator` request sent to the BN. pub validator_registration_batch_size: usize, From b388efed59c7f53d010b8f169c7501ac31d7d655 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 15 Jan 2026 16:41:20 +1100 Subject: [PATCH 34/55] Flip flag polarity and make it work --- lighthouse/tests/validator_client.rs | 18 ++++++++++++++++++ validator_client/src/cli.rs | 12 +++++------- validator_client/src/config.rs | 2 +- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/lighthouse/tests/validator_client.rs b/lighthouse/tests/validator_client.rs index ee3e910b369..6fd5a6538ce 100644 --- a/lighthouse/tests/validator_client.rs +++ b/lighthouse/tests/validator_client.rs @@ -758,3 +758,21 @@ fn validator_proposer_nodes() { ); }); } + +// Head monitor is enabled by default. +#[test] +fn head_monitor_default() { + CommandLineTest::new().run().with_config(|config| { + assert!(config.enable_beacon_head_monitor); + }); +} + +#[test] +fn head_monitor_disabled() { + CommandLineTest::new() + .flag("disable-beacon-head-monitor", None) + .run() + .with_config(|config| { + assert!(!config.enable_beacon_head_monitor); + }); +} diff --git a/validator_client/src/cli.rs b/validator_client/src/cli.rs index e00769911eb..0eb0e9e5dda 100644 --- a/validator_client/src/cli.rs +++ b/validator_client/src/cli.rs @@ -478,16 +478,14 @@ pub struct ValidatorClient { #[clap( long, - help = "Enable the beacon head monitor so fallback head updates trigger duties when a lagging \ - primary is detected. \ - This keeps the attestation service responsive when using multiple beacon nodes, but it relies on the \ - fallback service streaming head events which may increase network usage. \ - When is not enabled, duties are only triggered on slot boundaries and ignore fallback head changes. \ - The current default behaviour is to have this feature enabled.", + help = "Disable the beacon head monitor which tries to attest as soon as any of the \ + configured beacon nodes sends a head event. Leaving the service enabled is \ + recommended, but disabling it can lead to reduced bandwidth and more predictable \ + usage of the primary beacon node (rather than the fastest BN).", display_order = 0, help_heading = FLAG_HEADER )] - pub enable_beacon_head_monitor: bool, + pub disable_beacon_head_monitor: bool, #[clap( long, diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index 83e9c24bde4..fd9985363fe 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -380,7 +380,7 @@ impl Config { config.validator_store.builder_boost_factor = validator_client_config.builder_boost_factor; config.enable_latency_measurement_service = !validator_client_config.disable_latency_measurement_service; - config.enable_beacon_head_monitor = validator_client_config.enable_beacon_head_monitor; + config.enable_beacon_head_monitor = !validator_client_config.disable_beacon_head_monitor; config.validator_registration_batch_size = validator_client_config.validator_registration_batch_size; From 4bfafc1830e5e0207133f6f0cc9b3b480cbb33ef Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 15 Jan 2026 21:55:23 +1100 Subject: [PATCH 35/55] Update book --- book/src/help_vc.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/book/src/help_vc.md b/book/src/help_vc.md index e4a5df07539..4647780ea8c 100644 --- a/book/src/help_vc.md +++ b/book/src/help_vc.md @@ -185,6 +185,12 @@ Flags: If present, do not attempt to discover new validators in the validators-dir. Validators will need to be manually added to the validator_definitions.yml file. + --disable-beacon-head-monitor + Disable the beacon head monitor which tries to attest as soon as any + of the configured beacon nodes sends a head event. Leaving the service + enabled is recommended, but disabling it can lead to reduced bandwidth + and more predictable usage of the primary beacon node (rather than the + fastest BN). --disable-latency-measurement-service Disables the service that periodically attempts to measure latency to BNs. @@ -205,14 +211,6 @@ Flags: --distributed Enables functionality required for running the validator in a distributed validator cluster. - --enable-beacon-head-monitor - Enable the beacon head monitor so fallback head updates trigger duties - when a lagging primary is detected. This keeps the attestation service - responsive when using multiple beacon nodes, but it relies on the - fallback service streaming head events which may increase network - usage. When is not enabled, duties are only triggered on slot - boundaries and ignore fallback head changes. The current default - behaviour is to have this feature enabled. --enable-doppelganger-protection If this flag is set, Lighthouse will delay startup for three epochs and monitor for messages on the network by any of the validators From face04fc8a9444f1dd70a14721c7b1d3fb4efd0d Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 15 Jan 2026 21:58:52 +1100 Subject: [PATCH 36/55] Add eth2 events feature --- validator_client/beacon_node_fallback/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator_client/beacon_node_fallback/Cargo.toml b/validator_client/beacon_node_fallback/Cargo.toml index 481aece48b2..bc1ac20d44c 100644 --- a/validator_client/beacon_node_fallback/Cargo.toml +++ b/validator_client/beacon_node_fallback/Cargo.toml @@ -11,7 +11,7 @@ path = "src/lib.rs" [dependencies] bls = { workspace = true } clap = { workspace = true } -eth2 = { workspace = true } +eth2 = { workspace = true, features = ["events"] } futures = { workspace = true } itertools = { workspace = true } sensitive_url = { workspace = true } From b30d710f22357a0760661d90d21e657c8f7d8577 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Thu, 15 Jan 2026 14:33:35 -0500 Subject: [PATCH 37/55] addressing comments --- .../src/attestation_service.rs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 78a5a2ffc6d..2ea89e3b172 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -205,7 +205,17 @@ impl AttestationService *last_slot = current_slot, + Ok(_) => { + *last_slot = current_slot; + let duration = if let Some(duration) = self.slot_clock.duration_to_next_slot() { + duration + } else { + error!("Failed to read slot clock"); + slot_duration + }; + + sleep(duration).await; + }, Err(e) => { crit!(error = e, "Failed to spawn attestation tasks") } @@ -254,6 +264,12 @@ impl AttestationService Date: Thu, 15 Jan 2026 14:42:30 -0500 Subject: [PATCH 38/55] linting --- .../validator_services/src/attestation_service.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 2ea89e3b172..f3c347f3627 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -207,15 +207,16 @@ impl AttestationService { *last_slot = current_slot; - let duration = if let Some(duration) = self.slot_clock.duration_to_next_slot() { - duration - } else { - error!("Failed to read slot clock"); - slot_duration - }; + let duration = + if let Some(duration) = self.slot_clock.duration_to_next_slot() { + duration + } else { + error!("Failed to read slot clock"); + slot_duration + }; sleep(duration).await; - }, + } Err(e) => { crit!(error = e, "Failed to spawn attestation tasks") } From 516ccd7e2c1813c94eb566277b5f7763e61f0c74 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Sun, 18 Jan 2026 21:11:53 -0500 Subject: [PATCH 39/55] fixing unfavorable condition --- .../src/beacon_head_monitor.rs | 12 +++++++- .../src/attestation_service.rs | 29 ++++++++++--------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs index 08298adddea..7a41251ceb7 100644 --- a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs +++ b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs @@ -15,6 +15,7 @@ type CacheHashMap = HashMap; #[derive(Debug)] pub struct HeadEvent { pub beacon_node_index: usize, + pub slot: types::Slot, } /// Cache to maintain the latest head received from each of the beacon nodes @@ -121,9 +122,16 @@ pub async fn poll_head_event_from_beacon_nodes SseHead { SseHead { @@ -265,8 +273,10 @@ mod tests { async fn test_head_event_creation() { let event = HeadEvent { beacon_node_index: 42, + slot: Slot::new(123), }; assert_eq!(event.beacon_node_index, 42); + assert_eq!(event.slot, Slot::new(123)); } #[tokio::test] diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index f3c347f3627..23ee8dfbfef 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -207,15 +207,6 @@ impl AttestationService { *last_slot = current_slot; - let duration = - if let Some(duration) = self.slot_clock.duration_to_next_slot() { - duration - } else { - error!("Failed to read slot clock"); - slot_duration - }; - - sleep(duration).await; } Err(e) => { crit!(error = e, "Failed to spawn attestation tasks") @@ -233,11 +224,21 @@ impl AttestationService Some(head_event), - None => { - warn!("Head monitor channel closed unexpectedly"); - None + loop { + match receiver.recv().await { + Some(head_event) => { + // Only return head events for the current slot - this ensures the + // block for this slot has been produced before triggering attestation + let current_slot = self.slot_clock.now()?; + if head_event.slot == current_slot { + return Some(head_event); + } + // Head event is for a previous slot, keep waiting + } + None => { + warn!("Head monitor channel closed unexpectedly"); + return None; + } } } } From 6ee9c1d224317756a92932e9cbb2648b07487f4c Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Wed, 28 Jan 2026 12:18:15 +1100 Subject: [PATCH 40/55] Use Display for Slot --- validator_client/validator_services/src/attestation_service.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 734e2a87f04..fee0b5935a5 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -200,7 +200,7 @@ impl AttestationService Date: Wed, 28 Jan 2026 12:37:05 +1100 Subject: [PATCH 41/55] Add more debug logs and ignore optimistic heads fully --- .../src/beacon_head_monitor.rs | 31 +++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs index 7a41251ceb7..cf979adfe72 100644 --- a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs +++ b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs @@ -5,7 +5,7 @@ use slot_clock::SlotClock; use std::collections::HashMap; use std::sync::Arc; use tokio::sync::RwLock; -use tracing::{info, warn}; +use tracing::{debug, info, warn}; use types::EthSpec; type CacheHashMap = HashMap; @@ -116,15 +116,34 @@ pub async fn poll_head_event_from_beacon_nodes Date: Thu, 29 Jan 2026 11:47:24 +1100 Subject: [PATCH 42/55] Use stream::select_all instead of join_all and other refactors --- .../src/beacon_head_monitor.rs | 130 ++++++++++-------- .../beacon_node_fallback/src/lib.rs | 4 +- 2 files changed, 71 insertions(+), 63 deletions(-) diff --git a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs index cf979adfe72..e9897b22175 100644 --- a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs +++ b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs @@ -94,89 +94,97 @@ pub async fn poll_head_event_from_beacon_nodes(&[EventTopic::Head]) .await; - let mut head_event_stream = match head_event_stream { + let head_event_stream = match head_event_stream { Ok(stream) => stream, Err(e) => { - warn!("failed to get head event stream: {:?}", e); + warn!(error = ?e, node_index = candidate.index, "Failed to get head event stream"); continue; } }; - let sender_tx = head_monitor_send.clone(); - let head_cache_ref = head_cache.clone(); + streams.push(head_event_stream.map(|event| (candidate.index, event))); + } - let stream_fut = async move { - while let Some(event_result) = head_event_stream.next().await { - if let Ok(EventKind::Head(head)) = event_result { + if streams.is_empty() { + return Err("No beacon nodes available for head event streaming".to_string()); + } + + // Combine streams into a single stream and poll events from any of them. + let mut combined_stream = futures::stream::select_all(streams); + + while let Some((candidate_index, event_result)) = combined_stream.next().await { + match event_result { + Ok(EventKind::Head(head)) => { + debug!( + candidate_index, + block_root = ?head.block, + slot = %head.slot, + "New head from beacon node" + ); + + // Skip optimistic heads - the beacon node can't produce valid + // attestation data when its execution layer is not verified + if head.execution_optimistic { debug!( - node_index = candidate.index, + candidate_index, block_root = ?head.block, slot = %head.slot, - "New head from beacon node" + "Skipping optimistic head" ); - - // Skip optimistic heads - the beacon node can't produce valid - // attestation data when its execution layer is not verified - if head.execution_optimistic { - debug!( - node_index = candidate.index, - block_root = ?head.block, - slot = %head.slot, - "Skipping optimistic head" - ); - continue; - } - - head_cache_ref.insert(candidate.index, head.clone()).await; - - if !head_cache_ref.is_latest(&head).await { - debug!( - node_index = candidate.index, - block_root = ?head.block, - slot = %head.slot, - "Skipping stale head" - ); - continue; - } - - if sender_tx - .send(HeadEvent { - beacon_node_index: candidate.index, - slot: head.slot, - }) - .await - .is_err() - { - warn!("Head monitoring service channel closed"); - } + continue; } - } - }; - - tasks.push(stream_fut); - } - if tasks.is_empty() { - head_cache.purge_cache().await; - return Err( - "No beacon nodes available for head event streaming, retry in sometime".to_string(), - ); - } + head_cache.insert(candidate_index, head.clone()).await; - futures::future::join_all(tasks).await; + if !head_cache.is_latest(&head).await { + debug!( + candidate_index, + block_root = ?head.block, + slot = %head.slot, + "Skipping stale head" + ); + continue; + } - drop(candidates); - head_cache.purge_cache().await; + if head_monitor_send + .send(HeadEvent { + beacon_node_index: candidate_index, + slot: head.slot, + }) + .await + .is_err() + { + return Err("Head monitoring service channel closed".into()); + } + } + Ok(event) => { + warn!( + event_kind = event.topic_name(), + candidate_index, "Received unexpected event from BN" + ); + continue; + } + Err(e) => { + return Err(format!("Head monitoring stream error {e:?}")); + } + } + } - Ok(()) + Err("Stream ended unexpectedly".into()) } #[cfg(test)] diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index 7fd83e7c3e5..3ce242c9090 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -81,10 +81,10 @@ pub fn start_fallback_updater_service( if beacon_nodes_ref.head_monitor_send.is_some() { let head_monitor_future = async move { loop { - if let Err(err) = + if let Err(error) = poll_head_event_from_beacon_nodes::(beacon_nodes_ref.clone()).await { - warn!(error=?err, "Head service failed, retrying starting next slot"); + warn!(error, "Head service failed retrying starting next slot"); let sleep_time = beacon_nodes_ref .slot_clock .as_ref() From 585f95854a55036f2517bf0d51d1c0b31a6395e0 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Fri, 30 Jan 2026 13:50:02 -0500 Subject: [PATCH 43/55] using configured client instead of `EventSource::default` --- common/eth2/src/error.rs | 3 +++ common/eth2/src/lib.rs | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/common/eth2/src/error.rs b/common/eth2/src/error.rs index 1f21220b798..671a617c9ec 100644 --- a/common/eth2/src/error.rs +++ b/common/eth2/src/error.rs @@ -17,6 +17,8 @@ pub enum Error { #[cfg(feature = "events")] /// The `reqwest_eventsource` client raised an error. SseClient(Box), + #[cfg(feature = "events")] + SseEventSource(reqwest_eventsource::CannotCloneRequestError), /// The server returned an error message where the body was able to be parsed. ServerMessage(ErrorMessage), /// The server returned an error message with an array of errors. @@ -100,6 +102,7 @@ impl Error { None } } + Error::SseEventSource(_) => None, Error::ServerMessage(msg) => StatusCode::try_from(msg.code).ok(), Error::ServerIndexedMessage(msg) => StatusCode::try_from(msg.code).ok(), Error::StatusCode(status) => Some(*status), diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 8746e3c063c..d029bdfd0bf 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -40,7 +40,7 @@ use reqwest::{ header::{HeaderMap, HeaderValue}, }; #[cfg(feature = "events")] -use reqwest_eventsource::{Event, EventSource}; +use reqwest_eventsource::{Event, RequestBuilderExt}; use serde::{Serialize, de::DeserializeOwned}; use ssz::Encode; use std::fmt; @@ -2800,7 +2800,11 @@ impl BeaconNodeHttpClient { .join(","); path.query_pairs_mut().append_pair("topics", &topic_string); - let mut es = EventSource::get(path); + let mut es = self + .client + .get(path) + .eventsource() + .map_err(Error::SseEventSource)?; // If we don't await `Event::Open` here, then the consumer // will not get any Message events until they start awaiting the stream. // This is a way to register the stream with the sse server before From c6ac1a6dd431a65f2c573b9984054289f77a12f1 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 2 Feb 2026 12:19:14 +1100 Subject: [PATCH 44/55] Lengthen SSE timeout beyond 12s --- common/eth2/src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index d029bdfd0bf..589f20f257d 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -2800,9 +2800,12 @@ impl BeaconNodeHttpClient { .join(","); path.query_pairs_mut().append_pair("topics", &topic_string); + // Long timeout for events, some events are quite infrequent. + let event_timeout = Duration::from_secs(5 * 60); let mut es = self .client .get(path) + .timeout(event_timeout) .eventsource() .map_err(Error::SseEventSource)?; // If we don't await `Event::Open` here, then the consumer From b3804f390d2f628f3c182941f619b3d8746d5008 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 2 Feb 2026 12:19:29 +1100 Subject: [PATCH 45/55] Better error message --- .../beacon_node_fallback/src/beacon_head_monitor.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs index e9897b22175..5e199954c63 100644 --- a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs +++ b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs @@ -179,7 +179,9 @@ pub async fn poll_head_event_from_beacon_nodes { - return Err(format!("Head monitoring stream error {e:?}")); + return Err(format!( + "Head monitoring stream error, node: {candidate_index}, error: {e:?}" + )); } } } From a985826ff13641e878dc9ce98bfd1ac7667d1dfe Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 2 Feb 2026 13:17:43 +1100 Subject: [PATCH 46/55] Try `sseer` --- Cargo.lock | 69 ++++++++++++++++++++-------------------- common/eth2/Cargo.toml | 5 +-- common/eth2/src/error.rs | 15 ++------- common/eth2/src/lib.rs | 44 +++++++++---------------- 4 files changed, 55 insertions(+), 78 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 622ff881837..4197d7cb29b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1615,6 +1615,16 @@ dependencies = [ "serde", ] +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "c-kzg" version = "2.1.5" @@ -3098,7 +3108,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3106,6 +3116,7 @@ name = "eth2" version = "0.1.0" dependencies = [ "bls", + "bytes", "context_deserialize", "educe", "eip_3076", @@ -3120,10 +3131,10 @@ dependencies = [ "proto_array", "rand 0.9.2", "reqwest", - "reqwest-eventsource", "sensitive_url", "serde", "serde_json", + "sseer", "ssz_types", "superstruct", "test_random_derive", @@ -3317,17 +3328,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "eventsource-stream" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74fef4569247a5f429d9156b9d0a2599914385dd189c539334c625d8099d90ab" -dependencies = [ - "futures-core", - "nom", - "pin-project-lite", -] - [[package]] name = "execution_engine_integration" version = "0.1.0" @@ -4729,7 +4729,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6291,7 +6291,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7199,7 +7199,7 @@ dependencies = [ "once_cell", "socket2 0.6.1", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -7489,22 +7489,6 @@ dependencies = [ "webpki-roots", ] -[[package]] -name = "reqwest-eventsource" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "632c55746dbb44275691640e7b40c907c16a2dc1a5842aa98aaec90da6ec6bde" -dependencies = [ - "eventsource-stream", - "futures-core", - "futures-timer", - "mime", - "nom", - "pin-project-lite", - "reqwest", - "thiserror 1.0.69", -] - [[package]] name = "resolv-conf" version = "0.7.6" @@ -7716,7 +7700,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -8461,6 +8445,21 @@ dependencies = [ "der", ] +[[package]] +name = "sseer" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d741f202dad1ff5d3dd94c2a1c82bf8ccdcc2a7ed5a3ceb8c84f53efd2e705ac" +dependencies = [ + "bytes", + "bytes-utils", + "futures-core", + "futures-timer", + "http-body-util", + "memchr", + "pin-project-lite", +] + [[package]] name = "ssz_types" version = "0.14.0" @@ -8781,7 +8780,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.2", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -10132,7 +10131,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/common/eth2/Cargo.toml b/common/eth2/Cargo.toml index da8aba5ded9..325f4657a64 100644 --- a/common/eth2/Cargo.toml +++ b/common/eth2/Cargo.toml @@ -7,10 +7,11 @@ edition = { workspace = true } [features] default = [] lighthouse = ["proto_array", "eth2_keystore", "eip_3076", "zeroize"] -events = ["reqwest-eventsource", "futures", "futures-util"] +events = ["sseer", "futures", "futures-util"] [dependencies] bls = { workspace = true } +bytes = { workspace = true } context_deserialize = { workspace = true } educe = { workspace = true } eip_3076 = { workspace = true, optional = true } @@ -24,10 +25,10 @@ mediatype = "0.19.13" pretty_reqwest_error = { workspace = true } proto_array = { workspace = true, optional = true } reqwest = { workspace = true } -reqwest-eventsource = { version = "0.6.0", optional = true } sensitive_url = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +sseer = { version = "0.1", optional = true } ssz_types = { workspace = true } superstruct = { workspace = true } types = { workspace = true } diff --git a/common/eth2/src/error.rs b/common/eth2/src/error.rs index 671a617c9ec..15277e8f05d 100644 --- a/common/eth2/src/error.rs +++ b/common/eth2/src/error.rs @@ -15,10 +15,8 @@ pub enum Error { /// The `reqwest` client raised an error. HttpClient(PrettyReqwestError), #[cfg(feature = "events")] - /// The `reqwest_eventsource` client raised an error. - SseClient(Box), - #[cfg(feature = "events")] - SseEventSource(reqwest_eventsource::CannotCloneRequestError), + /// Error from SSE stream processing. + SseStream(String), /// The server returned an error message where the body was able to be parsed. ServerMessage(ErrorMessage), /// The server returned an error message with an array of errors. @@ -95,14 +93,7 @@ impl Error { match self { Error::HttpClient(error) => error.inner().status(), #[cfg(feature = "events")] - Error::SseClient(error) => { - if let reqwest_eventsource::Error::InvalidStatusCode(status, _) = error.as_ref() { - Some(*status) - } else { - None - } - } - Error::SseEventSource(_) => None, + Error::SseStream(_) => None, Error::ServerMessage(msg) => StatusCode::try_from(msg.code).ok(), Error::ServerIndexedMessage(msg) => StatusCode::try_from(msg.code).ok(), Error::StatusCode(status) => Some(*status), diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 589f20f257d..d5538a8b94d 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -39,9 +39,9 @@ use reqwest::{ Body, IntoUrl, RequestBuilder, Response, header::{HeaderMap, HeaderValue}, }; -#[cfg(feature = "events")] -use reqwest_eventsource::{Event, RequestBuilderExt}; use serde::{Serialize, de::DeserializeOwned}; +#[cfg(feature = "events")] +use sseer::EventStream; use ssz::Encode; use std::fmt; use std::future::Future; @@ -2800,34 +2800,20 @@ impl BeaconNodeHttpClient { .join(","); path.query_pairs_mut().append_pair("topics", &topic_string); - // Long timeout for events, some events are quite infrequent. - let event_timeout = Duration::from_secs(5 * 60); - let mut es = self - .client - .get(path) - .timeout(event_timeout) - .eventsource() - .map_err(Error::SseEventSource)?; - // If we don't await `Event::Open` here, then the consumer - // will not get any Message events until they start awaiting the stream. - // This is a way to register the stream with the sse server before - // message events start getting emitted. - while let Some(event) = es.next().await { - match event { - Ok(Event::Open) => break, - Err(err) => return Err(Error::SseClient(err.into())), - // This should never happen as we are guaranteed to get the - // Open event before any message starts coming through. - Ok(Event::Message(_)) => continue, - } - } - Ok(Box::pin(es.filter_map(|event| async move { + let response = self.client.get(path).timeout(Duration::MAX).send().await?; + + let byte_stream = response.bytes_stream().map(|result| { + result + .map(|bytes| bytes::Bytes::copy_from_slice(&bytes)) + .map_err(std::io::Error::other) + }); + + let event_stream = EventStream::new(byte_stream); + + Ok(Box::pin(event_stream.filter_map(|event| async move { match event { - Ok(Event::Open) => None, - Ok(Event::Message(message)) => { - Some(EventKind::from_sse_bytes(&message.event, &message.data)) - } - Err(err) => Some(Err(Error::SseClient(err.into()))), + Ok(sse_event) => Some(EventKind::from_sse_bytes(&sse_event.event, &sse_event.data)), + Err(err) => Some(Err(Error::SseStream(err.to_string()))), } }))) } From 4197d0f50b5aac48d01913cd20c38adbad92d73d Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 2 Feb 2026 13:25:27 +1100 Subject: [PATCH 47/55] Disable nginx cache/buffering --- beacon_node/http_api/src/lib.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 58cd2a3bdbc..4451b52c2d6 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -3205,7 +3205,15 @@ pub fn serve( let s = futures::stream::select_all(receivers); - Ok(warp::sse::reply(warp::sse::keep_alive().stream(s))) + Ok(warp::reply::with_header( + warp::reply::with_header( + warp::sse::reply(warp::sse::keep_alive().stream(s)), + "X-Accel-Buffering", + "no", + ), + "Cache-Control", + "no-cache, no-store, must-revalidate", + )) }) }, ); From 80c5d735d3f52f12d2179235f6ce0ae08899a2fd Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 2 Feb 2026 16:59:52 +1100 Subject: [PATCH 48/55] Revert "Try `sseer`" This reverts commit a985826ff13641e878dc9ce98bfd1ac7667d1dfe. --- Cargo.lock | 69 ++++++++++++++++++++-------------------- common/eth2/Cargo.toml | 5 ++- common/eth2/src/error.rs | 15 +++++++-- common/eth2/src/lib.rs | 44 ++++++++++++++++--------- 4 files changed, 78 insertions(+), 55 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4197d7cb29b..622ff881837 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1615,16 +1615,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bytes-utils" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" -dependencies = [ - "bytes", - "either", -] - [[package]] name = "c-kzg" version = "2.1.5" @@ -3108,7 +3098,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3116,7 +3106,6 @@ name = "eth2" version = "0.1.0" dependencies = [ "bls", - "bytes", "context_deserialize", "educe", "eip_3076", @@ -3131,10 +3120,10 @@ dependencies = [ "proto_array", "rand 0.9.2", "reqwest", + "reqwest-eventsource", "sensitive_url", "serde", "serde_json", - "sseer", "ssz_types", "superstruct", "test_random_derive", @@ -3328,6 +3317,17 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "eventsource-stream" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74fef4569247a5f429d9156b9d0a2599914385dd189c539334c625d8099d90ab" +dependencies = [ + "futures-core", + "nom", + "pin-project-lite", +] + [[package]] name = "execution_engine_integration" version = "0.1.0" @@ -4729,7 +4729,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6291,7 +6291,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7199,7 +7199,7 @@ dependencies = [ "once_cell", "socket2 0.6.1", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -7489,6 +7489,22 @@ dependencies = [ "webpki-roots", ] +[[package]] +name = "reqwest-eventsource" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "632c55746dbb44275691640e7b40c907c16a2dc1a5842aa98aaec90da6ec6bde" +dependencies = [ + "eventsource-stream", + "futures-core", + "futures-timer", + "mime", + "nom", + "pin-project-lite", + "reqwest", + "thiserror 1.0.69", +] + [[package]] name = "resolv-conf" version = "0.7.6" @@ -7700,7 +7716,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -8445,21 +8461,6 @@ dependencies = [ "der", ] -[[package]] -name = "sseer" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d741f202dad1ff5d3dd94c2a1c82bf8ccdcc2a7ed5a3ceb8c84f53efd2e705ac" -dependencies = [ - "bytes", - "bytes-utils", - "futures-core", - "futures-timer", - "http-body-util", - "memchr", - "pin-project-lite", -] - [[package]] name = "ssz_types" version = "0.14.0" @@ -8780,7 +8781,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.2", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -10131,7 +10132,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] diff --git a/common/eth2/Cargo.toml b/common/eth2/Cargo.toml index 325f4657a64..da8aba5ded9 100644 --- a/common/eth2/Cargo.toml +++ b/common/eth2/Cargo.toml @@ -7,11 +7,10 @@ edition = { workspace = true } [features] default = [] lighthouse = ["proto_array", "eth2_keystore", "eip_3076", "zeroize"] -events = ["sseer", "futures", "futures-util"] +events = ["reqwest-eventsource", "futures", "futures-util"] [dependencies] bls = { workspace = true } -bytes = { workspace = true } context_deserialize = { workspace = true } educe = { workspace = true } eip_3076 = { workspace = true, optional = true } @@ -25,10 +24,10 @@ mediatype = "0.19.13" pretty_reqwest_error = { workspace = true } proto_array = { workspace = true, optional = true } reqwest = { workspace = true } +reqwest-eventsource = { version = "0.6.0", optional = true } sensitive_url = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -sseer = { version = "0.1", optional = true } ssz_types = { workspace = true } superstruct = { workspace = true } types = { workspace = true } diff --git a/common/eth2/src/error.rs b/common/eth2/src/error.rs index 15277e8f05d..671a617c9ec 100644 --- a/common/eth2/src/error.rs +++ b/common/eth2/src/error.rs @@ -15,8 +15,10 @@ pub enum Error { /// The `reqwest` client raised an error. HttpClient(PrettyReqwestError), #[cfg(feature = "events")] - /// Error from SSE stream processing. - SseStream(String), + /// The `reqwest_eventsource` client raised an error. + SseClient(Box), + #[cfg(feature = "events")] + SseEventSource(reqwest_eventsource::CannotCloneRequestError), /// The server returned an error message where the body was able to be parsed. ServerMessage(ErrorMessage), /// The server returned an error message with an array of errors. @@ -93,7 +95,14 @@ impl Error { match self { Error::HttpClient(error) => error.inner().status(), #[cfg(feature = "events")] - Error::SseStream(_) => None, + Error::SseClient(error) => { + if let reqwest_eventsource::Error::InvalidStatusCode(status, _) = error.as_ref() { + Some(*status) + } else { + None + } + } + Error::SseEventSource(_) => None, Error::ServerMessage(msg) => StatusCode::try_from(msg.code).ok(), Error::ServerIndexedMessage(msg) => StatusCode::try_from(msg.code).ok(), Error::StatusCode(status) => Some(*status), diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index d5538a8b94d..589f20f257d 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -39,9 +39,9 @@ use reqwest::{ Body, IntoUrl, RequestBuilder, Response, header::{HeaderMap, HeaderValue}, }; -use serde::{Serialize, de::DeserializeOwned}; #[cfg(feature = "events")] -use sseer::EventStream; +use reqwest_eventsource::{Event, RequestBuilderExt}; +use serde::{Serialize, de::DeserializeOwned}; use ssz::Encode; use std::fmt; use std::future::Future; @@ -2800,20 +2800,34 @@ impl BeaconNodeHttpClient { .join(","); path.query_pairs_mut().append_pair("topics", &topic_string); - let response = self.client.get(path).timeout(Duration::MAX).send().await?; - - let byte_stream = response.bytes_stream().map(|result| { - result - .map(|bytes| bytes::Bytes::copy_from_slice(&bytes)) - .map_err(std::io::Error::other) - }); - - let event_stream = EventStream::new(byte_stream); - - Ok(Box::pin(event_stream.filter_map(|event| async move { + // Long timeout for events, some events are quite infrequent. + let event_timeout = Duration::from_secs(5 * 60); + let mut es = self + .client + .get(path) + .timeout(event_timeout) + .eventsource() + .map_err(Error::SseEventSource)?; + // If we don't await `Event::Open` here, then the consumer + // will not get any Message events until they start awaiting the stream. + // This is a way to register the stream with the sse server before + // message events start getting emitted. + while let Some(event) = es.next().await { + match event { + Ok(Event::Open) => break, + Err(err) => return Err(Error::SseClient(err.into())), + // This should never happen as we are guaranteed to get the + // Open event before any message starts coming through. + Ok(Event::Message(_)) => continue, + } + } + Ok(Box::pin(es.filter_map(|event| async move { match event { - Ok(sse_event) => Some(EventKind::from_sse_bytes(&sse_event.event, &sse_event.data)), - Err(err) => Some(Err(Error::SseStream(err.to_string()))), + Ok(Event::Open) => None, + Ok(Event::Message(message)) => { + Some(EventKind::from_sse_bytes(&message.event, &message.data)) + } + Err(err) => Some(Err(Error::SseClient(err.into()))), } }))) } From 3ee6f7acee96ff00561155421a59ef167503ce60 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 2 Feb 2026 17:09:33 +1100 Subject: [PATCH 49/55] More nginx headers (X-Accel-Expires) --- beacon_node/http_api/src/lib.rs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 4451b52c2d6..79e01cf7131 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -3205,15 +3205,19 @@ pub fn serve( let s = futures::stream::select_all(receivers); - Ok(warp::reply::with_header( - warp::reply::with_header( - warp::sse::reply(warp::sse::keep_alive().stream(s)), - "X-Accel-Buffering", - "no", - ), + let response = warp::sse::reply(warp::sse::keep_alive().stream(s)); + + // Set headers to bypass nginx caching and buffering, which breaks realtime + // delivery. + let response = warp::reply::with_header(response, "X-Accel-Buffering", "no"); + let response = warp::reply::with_header(response, "X-Accel-Expires", "0"); + let response = warp::reply::with_header( + response, "Cache-Control", "no-cache, no-store, must-revalidate", - )) + ); + + Ok(response) }) }, ); From 8547f213db6a48b60c1b7b966dee868f8c8c825c Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Mon, 2 Feb 2026 21:10:36 -0500 Subject: [PATCH 50/55] addressing comments --- .../beacon_node_fallback/src/beacon_head_monitor.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs index 5e199954c63..02aea277767 100644 --- a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs +++ b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs @@ -10,7 +10,7 @@ use types::EthSpec; type CacheHashMap = HashMap; -// This is used send the index derived from `CandidateBeaconNode` to the +// This is used to send the index derived from `CandidateBeaconNode` to the // `AttestationService` for further processing #[derive(Debug)] pub struct HeadEvent { @@ -69,7 +69,7 @@ impl Default for BeaconHeadCache { // Runs a non-terminating loop to update the `BeaconHeadCache` with the latest head received // from the candidate beacon_nodes. This is an attempt to stream events to beacon nodes and -// potential start attestion duties earlier as soon as latest head is receive from any of the +// potential start attestation duties earlier as soon as latest head is receive from any of the // beacon node in contrast to attest at the 1/3rd mark in the slot. // // From b2cc70339115c97af1f47a11627c032dc6127a53 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 3 Feb 2026 12:40:56 +1100 Subject: [PATCH 51/55] Make events timeout configurable --- common/eth2/src/lib.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 589f20f257d..10382b028a8 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -76,6 +76,8 @@ const HTTP_GET_BEACON_BLOCK_SSZ_TIMEOUT_QUOTIENT: u32 = 4; const HTTP_GET_DEBUG_BEACON_STATE_QUOTIENT: u32 = 4; const HTTP_GET_DEPOSIT_SNAPSHOT_QUOTIENT: u32 = 4; const HTTP_GET_VALIDATOR_BLOCK_TIMEOUT_QUOTIENT: u32 = 4; +// Generally the timeout for events should be longer than a slot. +const HTTP_GET_EVENTS_TIMEOUT_MULTIPLIER: u32 = 50; const HTTP_DEFAULT_TIMEOUT_QUOTIENT: u32 = 4; /// A struct to define a variety of different timeouts for different validator tasks to ensure @@ -96,6 +98,7 @@ pub struct Timeouts { pub get_debug_beacon_states: Duration, pub get_deposit_snapshot: Duration, pub get_validator_block: Duration, + pub events: Duration, pub default: Duration, } @@ -116,6 +119,7 @@ impl Timeouts { get_debug_beacon_states: timeout, get_deposit_snapshot: timeout, get_validator_block: timeout, + events: HTTP_GET_EVENTS_TIMEOUT_MULTIPLIER * timeout, default: timeout, } } @@ -138,6 +142,7 @@ impl Timeouts { get_debug_beacon_states: base_timeout / HTTP_GET_DEBUG_BEACON_STATE_QUOTIENT, get_deposit_snapshot: base_timeout / HTTP_GET_DEPOSIT_SNAPSHOT_QUOTIENT, get_validator_block: base_timeout / HTTP_GET_VALIDATOR_BLOCK_TIMEOUT_QUOTIENT, + events: HTTP_GET_EVENTS_TIMEOUT_MULTIPLIER * base_timeout, default: base_timeout / HTTP_DEFAULT_TIMEOUT_QUOTIENT, } } @@ -2800,12 +2805,10 @@ impl BeaconNodeHttpClient { .join(","); path.query_pairs_mut().append_pair("topics", &topic_string); - // Long timeout for events, some events are quite infrequent. - let event_timeout = Duration::from_secs(5 * 60); let mut es = self .client .get(path) - .timeout(event_timeout) + .timeout(self.timeouts.events) .eventsource() .map_err(Error::SseEventSource)?; // If we don't await `Event::Open` here, then the consumer From 1752ef514b3c35193e309b7b4fbd1179c5c52fe5 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Mon, 2 Feb 2026 23:03:00 -0500 Subject: [PATCH 52/55] checking block_root before starting attestation --- .../src/beacon_head_monitor.rs | 7 ++++- .../beacon_node_fallback/src/lib.rs | 6 +--- validator_client/src/config.rs | 2 +- .../src/attestation_service.rs | 31 ++++++++++++++----- 4 files changed, 32 insertions(+), 14 deletions(-) diff --git a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs index 02aea277767..bed107d856d 100644 --- a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs +++ b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs @@ -1,5 +1,5 @@ use crate::BeaconNodeFallback; -use eth2::types::{EventKind, EventTopic, SseHead}; +use eth2::types::{EventKind, EventTopic, Hash256, SseHead}; use futures::StreamExt; use slot_clock::SlotClock; use std::collections::HashMap; @@ -16,6 +16,7 @@ type CacheHashMap = HashMap; pub struct HeadEvent { pub beacon_node_index: usize, pub slot: types::Slot, + pub beacon_block_root: Hash256, } /// Cache to maintain the latest head received from each of the beacon nodes @@ -164,6 +165,7 @@ pub async fn poll_head_event_from_beacon_nodes( poll_head_event_from_beacon_nodes::(beacon_nodes_ref.clone()).await { warn!(error, "Head service failed retrying starting next slot"); - let sleep_time = beacon_nodes_ref - .slot_clock - .as_ref() - .and_then(|slot_clock| slot_clock.duration_to_next_slot()) - .unwrap_or_else(|| Duration::from_secs(12)); + let sleep_time = beacon_nodes_ref.spec.get_slot_duration(); sleep(sleep_time).await } } diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index fd9985363fe..d68a78b705f 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -82,7 +82,7 @@ pub struct Config { pub broadcast_topics: Vec, /// Enables a service which attempts to measure latency between the VC and BNs. pub enable_latency_measurement_service: bool, - /// Enables the beacon head monitor that reacts to fallback head updates. + /// Enables the beacon head monitor that reacts to head updates from connected beacon nodes. pub enable_beacon_head_monitor: bool, /// Defines the number of validators per `validator/register_validator` request sent to the BN. pub validator_registration_batch_size: usize, diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index d5f133fa17e..40b72359fd3 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -12,7 +12,7 @@ use tokio::sync::mpsc; use tokio::time::{Duration, Instant, sleep, sleep_until}; use tracing::{Instrument, debug, error, info, info_span, instrument, warn}; use tree_hash::TreeHash; -use types::{Attestation, AttestationData, ChainSpec, CommitteeIndex, EthSpec, Slot}; +use types::{Attestation, AttestationData, ChainSpec, CommitteeIndex, EthSpec, Hash256, Slot}; use validator_store::{Error as ValidatorStoreError, ValidatorStore}; /// Builds an `AttestationService`. @@ -186,7 +186,8 @@ impl AttestationService None, - event = self.poll_for_head_events() => event.map(|event| event.beacon_node_index), + event = self.poll_for_head_events() => + event.map(|event| (event.beacon_node_index, event.beacon_block_root)), } } else { sleep(duration + unaggregated_attestation_due).await; @@ -247,7 +248,10 @@ impl AttestationService) -> Result<(), String> { + fn spawn_attestation_tasks( + &self, + beacon_node_data: Option<(usize, Hash256)>, + ) -> Result<(), String> { let slot = self.slot_clock.now().ok_or("Failed to read slot clock")?; let duration_to_next_slot = self .slot_clock @@ -265,7 +269,7 @@ impl AttestationService AttestationService AttestationService Date: Tue, 3 Feb 2026 15:09:53 +1100 Subject: [PATCH 53/55] Restore duration to next slot logic --- validator_client/beacon_node_fallback/src/lib.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index 3bb023f8cc8..ad5fb11f164 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -86,7 +86,11 @@ pub fn start_fallback_updater_service( { warn!(error, "Head service failed retrying starting next slot"); - let sleep_time = beacon_nodes_ref.spec.get_slot_duration(); + let sleep_time = beacon_nodes_ref + .slot_clock + .as_ref() + .and_then(|slot_clock| slot_clock.duration_to_next_slot()) + .unwrap_or_else(|| beacon_nodes_ref.spec.get_slot_duration()); sleep(sleep_time).await } } From 43fafe79589c5a90349806f0a3e7aab67e0a4fe5 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 3 Feb 2026 15:12:35 +1100 Subject: [PATCH 54/55] Minor formatting tweaks --- .../validator_services/src/attestation_service.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 40b72359fd3..0d13faa1829 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -183,11 +183,11 @@ impl AttestationService None, event = self.poll_for_head_events() => - event.map(|event| (event.beacon_node_index, event.beacon_block_root)), + event.map(|event| (event.beacon_node_index, event.beacon_block_root)), } } else { sleep(duration + unaggregated_attestation_due).await; @@ -206,7 +206,7 @@ impl AttestationService { *last_slot = current_slot; } From 0e9dab3878c8f594ac4e3b0094b71514424b3e54 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 3 Feb 2026 16:55:20 +1100 Subject: [PATCH 55/55] Fall back to all BNs at 4s if first head-event BN fails --- .../beacon_node_fallback/src/lib.rs | 131 ++++-------------- .../src/attestation_service.rs | 117 +++++++++++----- 2 files changed, 104 insertions(+), 144 deletions(-) diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index ad5fb11f164..b36ec70aa3a 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -127,12 +127,15 @@ pub fn start_fallback_updater_service( pub enum Error { /// We attempted to contact the node but it failed. RequestFailed(T), + /// The beacon node with the requested index was not available. + CandidateIndexUnknown(usize), } impl Error { pub fn request_failure(&self) -> Option<&T> { match self { Error::RequestFailed(e) => Some(e), + Error::CandidateIndexUnknown(_) => None, } } } @@ -694,37 +697,30 @@ impl BeaconNodeFallback { Err(Errors(errors)) } - /// Try `func` on a specific beacon node by index first, then fall back to the normal order. - /// Returns immediately if the preferred node succeeds, otherwise falls back to first_success. - /// This is an insurance against potential race conditions that may arise. - pub async fn first_success_from_index( + /// Try `func` on a specific beacon node by index. + /// + /// Returns immediately if the preferred node succeeds, otherwise return an error. + pub async fn run_on_candidate_index( &self, - preferred_index: Option, + candidate_index: usize, func: F, - ) -> Result> + ) -> Result> where F: Fn(BeaconNodeHttpClient) -> R + Clone, R: Future>, Err: Debug, { - // Try the preferred beacon node first if it exists - if let Some(preferred_idx) = preferred_index - && let candidates = self.candidates.read().await - && let Some(preferred_candidate) = candidates.iter().find(|c| c.index == preferred_idx) - { - let preferred_node = preferred_candidate.beacon_node.clone(); - drop(candidates); - - match Self::run_on_candidate(preferred_node, &func).await { - Ok(val) => return Ok(val), - Err(_) => { - return self.first_success(func).await; - } - } - } + // Find the requested beacon node or return an error. + let candidates = self.candidates.read().await; + let Some(candidate) = candidates.iter().find(|c| c.index == candidate_index) else { + return Err(Error::CandidateIndexUnknown(candidate_index)); + }; + let candidate_node = candidate.beacon_node.clone(); + drop(candidates); - // Fall back to normal first_success behavior - self.first_success(func).await + Self::run_on_candidate(candidate_node, &func) + .await + .map_err(|(_, err)| err) } /// Run the future `func` on `candidate` while reporting metrics. @@ -1156,7 +1152,7 @@ mod tests { } #[tokio::test] - async fn first_success_from_index_tries_preferred_node_first() { + async fn run_on_candidate_index_success() { let spec = Arc::new(MainnetEthSpec::default_spec()); let (mut mock_beacon_node_1, beacon_node_1) = new_mock_beacon_node(0, &spec).await; let (mut mock_beacon_node_2, beacon_node_2) = new_mock_beacon_node(1, &spec).await; @@ -1174,10 +1170,7 @@ mod tests { // Request with preferred_index=1 (beacon_node_2) let result = beacon_node_fallback - .first_success_from_index( - Some(1), - |client| async move { client.get_node_version().await }, - ) + .run_on_candidate_index(1, |client| async move { client.get_node_version().await }) .await; // Should succeed since beacon_node_2 is online @@ -1189,7 +1182,7 @@ mod tests { } #[tokio::test] - async fn first_success_from_index_falls_back_when_preferred_fails() { + async fn run_on_candidate_index_error() { let spec = Arc::new(MainnetEthSpec::default_spec()); let (mut mock_beacon_node_1, beacon_node_1) = new_mock_beacon_node(0, &spec).await; let (mut mock_beacon_node_2, beacon_node_2) = new_mock_beacon_node(1, &spec).await; @@ -1202,89 +1195,15 @@ mod tests { ); let _mock1 = mock_beacon_node_1.mock_online_node(); - let mock2 = mock_beacon_node_2.mock_offline_node(); - let _mock3 = mock_beacon_node_3.mock_offline_node(); - - // Request with preferred_index=1 (beacon_node_2), but it's offline - let result = beacon_node_fallback - .first_success_from_index( - Some(1), - |client| async move { client.get_node_version().await }, - ) - .await; - - // Should succeed by falling back to beacon_node_1 - assert!(result.is_ok()); - - // mock2 should be called at least once (the preferred attempt) - mock2.expect(1).assert(); - // since the result was ok we can safely assume that the fallback first_success - // behaviour succeeded instead of checking if either of mock1/mock3 received hits - } - - #[tokio::test] - async fn first_success_from_index_with_none_falls_back_to_first_success() { - let spec = Arc::new(MainnetEthSpec::default_spec()); - let (mut mock_beacon_node_1, beacon_node_1) = new_mock_beacon_node(0, &spec).await; - let (mut mock_beacon_node_2, beacon_node_2) = new_mock_beacon_node(1, &spec).await; - let (mut mock_beacon_node_3, beacon_node_3) = new_mock_beacon_node(2, &spec).await; - - let beacon_node_fallback = create_beacon_node_fallback( - vec![beacon_node_1, beacon_node_2, beacon_node_3], - vec![], - spec.clone(), - ); - - let _mock1 = mock_beacon_node_1.mock_offline_node(); let _mock2 = mock_beacon_node_2.mock_offline_node(); - let mock3 = mock_beacon_node_3.mock_online_node(); - - // Request with preferred_index=None - let result = beacon_node_fallback - .first_success_from_index( - None, - |client| async move { client.get_node_version().await }, - ) - .await; - - // Should succeed with beacon_node_3 in the first pass - assert!(result.is_ok()); - - // mock3 should be called once in the first pass - mock3.expect(1).assert(); - } - - #[tokio::test] - async fn first_success_from_index_all_offline() { - let spec = Arc::new(MainnetEthSpec::default_spec()); - let (mut mock_beacon_node_1, beacon_node_1) = new_mock_beacon_node(0, &spec).await; - let (mut mock_beacon_node_2, beacon_node_2) = new_mock_beacon_node(1, &spec).await; - let (mut mock_beacon_node_3, beacon_node_3) = new_mock_beacon_node(2, &spec).await; - - let beacon_node_fallback = create_beacon_node_fallback( - vec![beacon_node_1, beacon_node_2, beacon_node_3], - vec![], - spec.clone(), - ); - - let _mock1 = mock_beacon_node_1.mock_offline_node(); - let mock2 = mock_beacon_node_2.mock_offline_node(); let _mock3 = mock_beacon_node_3.mock_offline_node(); - // Request with preferred_index=1, but all nodes are offline + // Request with preferred_index=1 (beacon_node_2), but it's offline let result = beacon_node_fallback - .first_success_from_index( - Some(1), - |client| async move { client.get_node_version().await }, - ) + .run_on_candidate_index(1, |client| async move { client.get_node_version().await }) .await; - // Should fail since all nodes are offline + // Should fail. assert!(result.is_err()); - - // Preferred node (mock2) should be called 3 times: - // - 1 time for the preferred attempt - // - 2 more times from the fallback to first_success (first and second pass) - mock2.expect(3).assert(); } } diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 0d13faa1829..a9d52833127 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -206,7 +206,7 @@ impl AttestationService { *last_slot = current_slot; } @@ -248,15 +248,11 @@ impl AttestationService, ) -> Result<(), String> { let slot = self.slot_clock.now().ok_or("Failed to read slot clock")?; - let duration_to_next_slot = self - .slot_clock - .duration_to_next_slot() - .ok_or("Unable to determine duration to next slot")?; // Create and publish an `Attestation` for all validators only once // as the committee_index is not included in AttestationData post-Electra @@ -275,40 +271,81 @@ impl AttestationService attestation_data_from_head_event = Some(data), + Err(error) => { + warn!(?error, "Failed to attest based on head event"); + } + } + } + + // If the beacon node that sent us the head failed to attest, wait until the attestation + // deadline then try all BNs. + let attestation_data = if let Some(attestation_data) = attestation_data_from_head_event { + attestation_data + } else { + let duration_to_deadline = self + .slot_clock + .duration_to_slot(slot + 1) + .and_then(|duration_to_next_slot| { + duration_to_next_slot + .checked_add(self.chain_spec.get_unaggregated_attestation_due()) + }) + .map(|next_slot_deadline| { + next_slot_deadline.saturating_sub(self.chain_spec.get_slot_duration()) + }) + .unwrap_or(Duration::from_secs(0)); + sleep(duration_to_deadline).await; + + attestation_service + .beacon_nodes + .first_success(|beacon_node| async move { + let _timer = validator_metrics::start_timer_vec( + &validator_metrics::ATTESTATION_SERVICE_TIMES, + &[validator_metrics::ATTESTATIONS_HTTP_GET], + ); + let data = beacon_node + .get_validator_attestation_data(slot, 0) + .await + .map_err(|e| format!("Failed to produce attestation data: {:?}", e))? + .data; + Ok::(data) + }) + .await + .map_err(|e| e.to_string())? + }; + + // Sign and publish attestations. + let publication_handle = self .inner .executor .spawn_handle( async move { - let (beacon_node_index, expected_block_root) = beacon_node_data.unzip(); - let attestation_data = attestation_service - .beacon_nodes - .first_success_from_index(beacon_node_index, |beacon_node| async move { - let _timer = validator_metrics::start_timer_vec( - &validator_metrics::ATTESTATION_SERVICE_TIMES, - &[validator_metrics::ATTESTATIONS_HTTP_GET], - ); - let data = beacon_node - .get_validator_attestation_data(slot, 0) - .await - .map_err(|e| { - format!("Failed to produce attestation data: {:?}", e) - })? - .data; - - if let Some(root) = expected_block_root - && data.beacon_block_root != root - { - return Err(format!( - "Attestation block root mismatch: expected {:?}, got {:?}", - root, data.beacon_block_root - )); - } - Ok(data) - }) - .await - .map_err(|e| e.to_string())?; - attestation_service .sign_and_publish_attestations( slot, @@ -326,12 +363,16 @@ impl AttestationService(attestation_data) }, - "unaggregated attestation production", + "unaggregated attestation publication", ) .ok_or("Failed to spawn attestation data task")?; // If a validator needs to publish an aggregate attestation, they must do so at 2/3 // through the slot. This delay triggers at this time + let duration_to_next_slot = self + .slot_clock + .duration_to_slot(slot + 1) + .ok_or("Unable to determine duration to next slot")?; let aggregate_production_instant = Instant::now() + duration_to_next_slot .checked_add(self.chain_spec.get_aggregate_attestation_due()) @@ -355,7 +396,7 @@ impl AttestationService data, Ok(Some(Err(err))) => { error!(?err, "Attestation production failed");