From ea458144eeca27870a90b3e83ac6634df518a87e Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Mon, 18 Aug 2025 13:40:25 +0530 Subject: [PATCH 01/41] using events api for eager start attestation tasks --- .../src/attestation_service.rs | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index da6e8f35886..5b6f2d13441 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -1,6 +1,8 @@ use crate::duties_service::{DutiesService, DutyAndProof}; use beacon_node_fallback::{ApiTopic, BeaconNodeFallback}; +use eth2::types::EventTopic; use futures::future::join_all; +use futures::StreamExt; use logging::crit; use slot_clock::SlotClock; use std::collections::HashMap; @@ -162,6 +164,11 @@ impl AttestationService {}, + _ = poll_head_event_on_all_beacon_nodes(&self.beacon_nodes) => {} + } + if let Err(e) = self.spawn_attestation_tasks(slot_duration) { crit!(error = e, "Failed to spawn attestation tasks") } else { @@ -709,6 +716,31 @@ impl AttestationService( + beacon_nodes: &Arc> +) -> Result<(), String> { + match beacon_nodes.first_success(|beacon_node| async move { + let mut event_stream = beacon_node.get_events::(&[EventTopic::Head]).await + .map_err(|e| format!("Failed to get event stream: {:?}", e))?; + + // Poll once for a head event to trigger early attestation processing + if let Some(event_result) = event_stream.next().await { + match event_result { + Ok(_event) => Ok(()), + Err(e) => Err(format!("Head event stream error: {:?}", e)) + } + } else { + Err("No head events received".to_string()) + } + }).await { + Ok(_) => Ok(()), + Err(e) => { + debug!("Failed to get head events from any beacon node: {}", e); + Ok(()) // Don't fail the entire process if head events aren't available + } + } +} + #[cfg(test)] mod tests { use super::*; From 46fe220f96574f00d4ecfeb2ca545f9bbf785bf2 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Mon, 18 Aug 2025 15:27:49 +0530 Subject: [PATCH 02/41] minor nits --- .../src/attestation_service.rs | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 5b6f2d13441..aff935d8059 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -166,7 +166,7 @@ impl AttestationService {}, - _ = poll_head_event_on_all_beacon_nodes(&self.beacon_nodes) => {} + _ = poll_head_event_on_all_beacon_nodes::(&self.beacon_nodes) => {} } if let Err(e) = self.spawn_attestation_tasks(slot_duration) { @@ -716,29 +716,32 @@ impl AttestationService( +async fn poll_head_event_on_all_beacon_nodes( beacon_nodes: &Arc> ) -> Result<(), String> { - match beacon_nodes.first_success(|beacon_node| async move { - let mut event_stream = beacon_node.get_events::(&[EventTopic::Head]).await + use eth2::types::EventKind; + + beacon_nodes.first_success(|beacon_node| async move { + let mut event_stream = beacon_node.get_events::(&[EventTopic::Head]).await .map_err(|e| format!("Failed to get event stream: {:?}", e))?; // Poll once for a head event to trigger early attestation processing if let Some(event_result) = event_stream.next().await { - match event_result { - Ok(_event) => Ok(()), - Err(e) => Err(format!("Head event stream error: {:?}", e)) + let event = event_result.map_err(|e| format!("Head event stream error: {:?}", e))?; + match event { + EventKind::Head(_) => { + trace!("Received head event, triggering early attestation processing"); + Ok(()) + }, + _ => Err("Received non-head event when expecting head event".to_string()) } } else { Err("No head events received".to_string()) } - }).await { - Ok(_) => Ok(()), - Err(e) => { - debug!("Failed to get head events from any beacon node: {}", e); - Ok(()) // Don't fail the entire process if head events aren't available - } - } + }).await.map_err(|e| { + debug!(error = %e, "Failed to get head events from any beacon node"); + e.to_string() + }) } #[cfg(test)] From de7226f1cc2a8668c199e40c2f7c53d2bd1c08ea Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Mon, 18 Aug 2025 16:06:07 +0530 Subject: [PATCH 03/41] clippy chill vibes --- .../validator_services/src/attestation_service.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index aff935d8059..f22872a1bb2 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -1,6 +1,6 @@ use crate::duties_service::{DutiesService, DutyAndProof}; use beacon_node_fallback::{ApiTopic, BeaconNodeFallback}; -use eth2::types::EventTopic; +use eth2::types::{EventTopic, EventKind}; use futures::future::join_all; use futures::StreamExt; use logging::crit; @@ -719,12 +719,11 @@ impl AttestationService( beacon_nodes: &Arc> ) -> Result<(), String> { - use eth2::types::EventKind; - + beacon_nodes.first_success(|beacon_node| async move { let mut event_stream = beacon_node.get_events::(&[EventTopic::Head]).await .map_err(|e| format!("Failed to get event stream: {:?}", e))?; - + // Poll once for a head event to trigger early attestation processing if let Some(event_result) = event_stream.next().await { let event = event_result.map_err(|e| format!("Head event stream error: {:?}", e))?; From 5f2a101ce05348f214816fc5633efd4826c5c7d2 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Mon, 18 Aug 2025 16:10:46 +0530 Subject: [PATCH 04/41] missed something --- validator_client/validator_services/src/attestation_service.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index f22872a1bb2..97c5cc31c40 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -162,8 +162,6 @@ impl AttestationService {}, _ = poll_head_event_on_all_beacon_nodes::(&self.beacon_nodes) => {} From ae1670529ada33ecfb7a7d9834c2144982e894c0 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Mon, 18 Aug 2025 19:49:06 +0530 Subject: [PATCH 05/41] linty --- .../src/attestation_service.rs | 53 ++++++++++--------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 97c5cc31c40..c747fc2d749 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -1,8 +1,8 @@ use crate::duties_service::{DutiesService, DutyAndProof}; use beacon_node_fallback::{ApiTopic, BeaconNodeFallback}; -use eth2::types::{EventTopic, EventKind}; -use futures::future::join_all; +use eth2::types::{EventKind, EventTopic}; use futures::StreamExt; +use futures::future::join_all; use logging::crit; use slot_clock::SlotClock; use std::collections::HashMap; @@ -715,30 +715,35 @@ impl AttestationService( - beacon_nodes: &Arc> + beacon_nodes: &Arc>, ) -> Result<(), String> { - - beacon_nodes.first_success(|beacon_node| async move { - let mut event_stream = beacon_node.get_events::(&[EventTopic::Head]).await - .map_err(|e| format!("Failed to get event stream: {:?}", e))?; - - // Poll once for a head event to trigger early attestation processing - if let Some(event_result) = event_stream.next().await { - let event = event_result.map_err(|e| format!("Head event stream error: {:?}", e))?; - match event { - EventKind::Head(_) => { - trace!("Received head event, triggering early attestation processing"); - Ok(()) - }, - _ => Err("Received non-head event when expecting head event".to_string()) + beacon_nodes + .first_success(|beacon_node| async move { + let mut event_stream = beacon_node + .get_events::(&[EventTopic::Head]) + .await + .map_err(|e| format!("Failed to get event stream: {:?}", e))?; + + // Poll once for a head event to trigger early attestation processing + if let Some(event_result) = event_stream.next().await { + let event = + event_result.map_err(|e| format!("Head event stream error: {:?}", e))?; + match event { + EventKind::Head(_) => { + trace!("Received head event, triggering early attestation processing"); + Ok(()) + } + _ => Err("Received non-head event when expecting head event".to_string()), + } + } else { + Err("No head events received".to_string()) } - } else { - Err("No head events received".to_string()) - } - }).await.map_err(|e| { - debug!(error = %e, "Failed to get head events from any beacon node"); - e.to_string() - }) + }) + .await + .map_err(|e| { + debug!(error = %e, "Failed to get head events from any beacon node"); + e.to_string() + }) } #[cfg(test)] From c13d0c311182632926d5c947ad4cea8525f682a4 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Fri, 19 Sep 2025 18:24:30 -0400 Subject: [PATCH 06/41] implemented head monitoring service --- .../beacon_node_fallback/src/lib.rs | 35 +++ validator_client/src/lib.rs | 20 ++ .../src/attestation_service.rs | 59 ++++- .../src/head_monitor_service.rs | 221 ++++++++++++++++++ .../validator_services/src/lib.rs | 1 + 5 files changed, 331 insertions(+), 5 deletions(-) create mode 100644 validator_client/validator_services/src/head_monitor_service.rs diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index a3f60d2de04..3519382eb72 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -646,6 +646,41 @@ impl BeaconNodeFallback { Err(Errors(errors)) } + /// Try `func` on a specific beacon node by index first, then fall back to the normal order. + /// Returns immediately if the preferred node succeeds, otherwise falls back to first_success. + /// This is an insurance against potential race conditions that may arise. + pub async fn first_success_from_index( + &self, + preferred_index: Option, + func: F, + ) -> Result> + where + F: Fn(BeaconNodeHttpClient) -> R + Clone, + R: Future>, + Err: Debug, + { + let candidates = self.candidates.read().await; + + // Try the preferred beacon node first if it exists + if let Some(preferred_idx) = preferred_index + && let Some(preferred_candidate) = candidates.iter().find(|c| c.index == preferred_idx) + { + let preferred_node = preferred_candidate.beacon_node.clone(); + drop(candidates); + + match Self::run_on_candidate(preferred_node, &func).await { + Ok(val) => return Ok(val), + Err(_) => { + return self.first_success(func).await; + } + } + } + + // Fall back to normal first_success behavior + drop(candidates); + self.first_success(func).await + } + /// Run the future `func` on `candidate` while reporting metrics. async fn run_on_candidate( candidate: BeaconNodeHttpClient, diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 71bdde10b02..51e6bcec5af 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -9,6 +9,7 @@ use metrics::set_gauge; use monitoring_api::{MonitoringHttpClient, ProcessType}; use sensitive_url::SensitiveUrl; use slashing_protection::{SLASHING_PROTECTION_FILENAME, SlashingDatabase}; +use tokio::sync::Mutex; use account_utils::validator_definitions::ValidatorDefinitions; use beacon_node_fallback::{ @@ -42,6 +43,7 @@ use validator_services::{ attestation_service::{AttestationService, AttestationServiceBuilder}, block_service::{BlockService, BlockServiceBuilder}, duties_service::{self, DutiesService, DutiesServiceBuilder}, + head_monitor_service::{HeadMonitorService, HeadMonitorServiceBuilder}, latency_service, preparation_service::{PreparationService, PreparationServiceBuilder}, sync_committee_service::SyncCommitteeService, @@ -79,6 +81,7 @@ pub struct ProductionValidatorClient { context: RuntimeContext, duties_service: Arc, SystemTimeSlotClock>>, block_service: BlockService, SystemTimeSlotClock>, + head_monitor_service: HeadMonitorService, SystemTimeSlotClock>, attestation_service: AttestationService, SystemTimeSlotClock>, sync_committee_service: SyncCommitteeService, SystemTimeSlotClock>, doppelganger_service: Option>, @@ -493,8 +496,18 @@ impl ProductionValidatorClient { block_service_builder = block_service_builder.proposer_nodes(proposer_nodes.clone()); } + let (head_sender, head_receiver) = mpsc::channel(1_024); + let block_service = block_service_builder.build()?; + let head_monitor_service = HeadMonitorServiceBuilder::new() + .slot_clock(slot_clock.clone()) + .executor(context.executor.clone()) + .validator_store(validator_store.clone()) + .beacon_nodes(beacon_nodes.clone()) + .head_monitor_tx(Arc::new(head_sender)) + .build()?; + let attestation_service = AttestationServiceBuilder::new() .duties_service(duties_service.clone()) .slot_clock(slot_clock.clone()) @@ -502,6 +515,7 @@ impl ProductionValidatorClient { .beacon_nodes(beacon_nodes.clone()) .executor(context.executor.clone()) .chain_spec(context.eth2_config.spec.clone()) + .head_monitor_rx(Arc::new(Mutex::new(head_receiver))) .disable(config.disable_attesting) .build()?; @@ -526,6 +540,7 @@ impl ProductionValidatorClient { context, duties_service, block_service, + head_monitor_service, attestation_service, sync_committee_service, doppelganger_service, @@ -604,6 +619,11 @@ impl ProductionValidatorClient { .start_update_service(&self.context.eth2_config.spec) .map_err(|e| format!("Unable to start preparation service: {}", e))?; + self.head_monitor_service + .clone() + .start_update_service() + .map_err(|e| format!("Unable to start head monitor service: {}", e))?; + if let Some(doppelganger_service) = self.doppelganger_service.clone() { DoppelgangerService::start_update_service( doppelganger_service, diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index da6e8f35886..5c50c06b1f4 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -1,4 +1,7 @@ use crate::duties_service::{DutiesService, DutyAndProof}; +use tokio::sync::Mutex; + +use crate::head_monitor_service::HeadEvent; use beacon_node_fallback::{ApiTopic, BeaconNodeFallback}; use futures::future::join_all; use logging::crit; @@ -7,6 +10,7 @@ use std::collections::HashMap; use std::ops::Deref; use std::sync::Arc; use task_executor::TaskExecutor; +use tokio::sync::mpsc; use tokio::time::{Duration, Instant, sleep, sleep_until}; use tracing::{debug, error, info, trace, warn}; use tree_hash::TreeHash; @@ -22,6 +26,7 @@ pub struct AttestationServiceBuilder beacon_nodes: Option>>, executor: Option, chain_spec: Option>, + head_monitor_rx: Option>>>, disable: bool, } @@ -34,6 +39,7 @@ impl AttestationServiceBuil beacon_nodes: None, executor: None, chain_spec: None, + head_monitor_rx: None, disable: false, } } @@ -73,6 +79,13 @@ impl AttestationServiceBuil self } + pub fn head_monitor_rx( + mut self, + head_monitor_rx: Arc>>, + ) -> Self { + self.head_monitor_rx = Some(head_monitor_rx); + self + } pub fn build(self) -> Result, String> { Ok(AttestationService { inner: Arc::new(Inner { @@ -94,6 +107,9 @@ impl AttestationServiceBuil chain_spec: self .chain_spec .ok_or("Cannot build AttestationService without chain_spec")?, + head_monitor_rx: self + .head_monitor_rx + .ok_or("Cannot build AttestationService without head_monitor_rx")?, disable: self.disable, }), }) @@ -108,6 +124,7 @@ pub struct Inner { beacon_nodes: Arc>, executor: TaskExecutor, chain_spec: Arc, + head_monitor_rx: Arc>>, disable: bool, } @@ -160,9 +177,17 @@ impl AttestationService = None; + tokio::select! { + _ = sleep(duration_to_next_slot + slot_duration / 3) => {}, + head_event = self.poll_for_head_events() => { + if let Ok(event) = head_event { + beacon_node_index = Some(event.beacon_node_index); + } + } + } - if let Err(e) = self.spawn_attestation_tasks(slot_duration) { + if let Err(e) = self.spawn_attestation_tasks(slot_duration, beacon_node_index) { crit!(error = e, "Failed to spawn attestation tasks") } else { trace!("Spawned attestation tasks"); @@ -180,9 +205,25 @@ impl AttestationService Result { + let mut receiver = self.head_monitor_rx.lock().await; + match receiver.recv().await { + Some(head_event) => Ok(head_event), + None => Err("Head monitor channel closed unexpectedly".to_string()), + } + } + /// For each each required attestation, spawn a new task that downloads, signs and uploads the /// attestation to the beacon node. - fn spawn_attestation_tasks(&self, slot_duration: Duration) -> Result<(), String> { + fn spawn_attestation_tasks( + &self, + slot_duration: Duration, + beacon_node_index: Option, + ) -> Result<(), String> { + info!( + "process attestation from beacon_node_index {:?}", + beacon_node_index + ); let slot = self.slot_clock.now().ok_or("Failed to read slot clock")?; let duration_to_next_slot = self .slot_clock @@ -221,6 +262,7 @@ impl AttestationService AttestationService, aggregate_production_instant: Instant, + candidate_beacon_node: Option, ) -> Result<(), ()> { let attestations_timer = validator_metrics::start_timer_vec( &validator_metrics::ATTESTATION_SERVICE_TIMES, @@ -265,7 +308,12 @@ impl AttestationService AttestationService, ) -> Result, String> { if validator_duties.is_empty() { return Ok(None); @@ -346,7 +395,7 @@ impl AttestationService; + +#[derive(Debug)] +pub enum Error { + BeaconNodeNotFound, +} + +#[derive(Default)] +pub struct HeadMonitorServiceBuilder { + validator_store: Option>, + slot_clock: Option, + beacon_nodes: Option>>, + executor: Option, + beacon_head_cache: Option>, + head_monitor_tx: Option>>, +} + +pub struct BeaconHeadCache { + cache: RwLock, +} + +impl BeaconHeadCache { + pub fn get(&self, beacon_node_index: usize) -> Option { + self.cache.read().get(&beacon_node_index).cloned() + } + + pub fn insert(&self, beacon_node_index: usize, head: SseHead) { + self.cache.write().insert(beacon_node_index, head); + } + + pub fn is_latest(&self, head: &SseHead) -> bool { + let cache = self.cache.read(); + cache + .values() + .all(|cache_head| head.slot >= cache_head.slot) + } +} + +#[derive(Debug)] +pub struct HeadEvent { + pub beacon_node_index: usize, +} + +impl HeadMonitorServiceBuilder { + pub fn new() -> Self { + Self { + validator_store: None, + slot_clock: None, + beacon_nodes: None, + executor: None, + beacon_head_cache: None, + head_monitor_tx: None, + } + } + + pub fn beacon_nodes(mut self, nodes: Arc>) -> Self { + self.beacon_nodes = Some(nodes); + self + } + + pub fn slot_clock(mut self, clock: T) -> Self { + self.slot_clock = Some(clock); + self + } + + pub fn executor(mut self, executor: TaskExecutor) -> Self { + self.executor = Some(executor); + self + } + + pub fn validator_store(mut self, validator_store: Arc) -> Self { + self.validator_store = Some(validator_store); + self + } + + pub fn beacon_head_cache(mut self, beacon_head_cache: Arc) -> Self { + self.beacon_head_cache = Some(beacon_head_cache); + self + } + + pub fn head_monitor_tx(mut self, head_monitor_tx: Arc>) -> Self { + self.head_monitor_tx = Some(head_monitor_tx); + self + } + pub fn build(self) -> Result, String> { + let beacon_head_cache = Arc::new(BeaconHeadCache { + cache: RwLock::new(HashMap::new()), + }); + Ok(HeadMonitorService { + inner: Arc::new(Inner { + _validator_store: self + .validator_store + .ok_or("Cannot build HeadMonitorService without validator_store")?, + + _slot_clock: self + .slot_clock + .ok_or("Cannot build HeadMonitorService without slot_clock")?, + beacon_nodes: self + .beacon_nodes + .ok_or("Cannot build HeadMonitorService without beacon_nodes")?, + executor: self + .executor + .ok_or("Cannot build HeadMonitorService without executor")?, + beacon_head_cache, + head_monitor_tx: self + .head_monitor_tx + .ok_or("Cannot build HeadMonitorService without head_monitor_rx")?, + }), + }) + } +} + +pub struct Inner { + _validator_store: Arc, + _slot_clock: T, + executor: TaskExecutor, + beacon_nodes: Arc>, + beacon_head_cache: Arc, + head_monitor_tx: Arc>, +} + +pub struct HeadMonitorService { + inner: Arc>, +} + +impl Clone for HeadMonitorService { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + } + } +} + +impl Deref for HeadMonitorService { + type Target = Inner; + + fn deref(&self) -> &Self::Target { + self.inner.deref() + } +} + +impl HeadMonitorService { + pub fn start_update_service(self) -> Result<(), String> { + let executor = self.executor.clone(); + let head_cache = self.beacon_head_cache.clone(); + + info!("Starting head monitoring service"); + let interval_fut = async move { + let candidates = { + let candidates_guard = self.beacon_nodes.candidates.read().await; + candidates_guard.clone() + }; + let mut tasks = vec![]; + + for candidate in candidates.iter() { + let head_event_stream = candidate + .beacon_node + .get_events::(&[EventTopic::Head]) + .await; + + let mut head_event_stream = match head_event_stream { + Ok(stream) => stream, + Err(e) => { + warn!("failed to get head event stream: {:?}", e); + continue; + } + }; + + let head_cache_clone = head_cache.clone(); + let sender_tx = self.head_monitor_tx.clone(); + + let stream_fut = async move { + while let Some(event_result) = head_event_stream.next().await { + if let Ok(EventKind::Head(head)) = event_result { + head_cache_clone.insert(candidate.index, head.clone()); + + if head_cache_clone.is_latest(&head) { + if let Ok(()) = sender_tx + .send(HeadEvent { + beacon_node_index: candidate.index, + }) + .await + { + } else { + warn!("Channel closed"); + break; + } + } + } + } + }; + + tasks.push(stream_fut); + } + + futures::future::join_all(tasks).await; + + drop(candidates); + }; + + executor.spawn(interval_fut, "head_monitor_service"); + + Ok(()) + } +} diff --git a/validator_client/validator_services/src/lib.rs b/validator_client/validator_services/src/lib.rs index 3b8bd9ae14b..1712f67d8e8 100644 --- a/validator_client/validator_services/src/lib.rs +++ b/validator_client/validator_services/src/lib.rs @@ -1,6 +1,7 @@ pub mod attestation_service; pub mod block_service; pub mod duties_service; +pub mod head_monitor_service; pub mod latency_service; pub mod notifier_service; pub mod preparation_service; From a1d36e6acac8a584e28e49d231bfaadd3d8fa836 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Fri, 19 Sep 2025 19:06:25 -0400 Subject: [PATCH 07/41] fixing some linting issues --- .../src/attestation_service.rs | 34 ------------------- 1 file changed, 34 deletions(-) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 5056ea2e1d7..5c50c06b1f4 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -3,8 +3,6 @@ use tokio::sync::Mutex; use crate::head_monitor_service::HeadEvent; use beacon_node_fallback::{ApiTopic, BeaconNodeFallback}; -use eth2::types::{EventKind, EventTopic}; -use futures::StreamExt; use futures::future::join_all; use logging::crit; use slot_clock::SlotClock; @@ -760,38 +758,6 @@ impl AttestationService( - beacon_nodes: &Arc>, -) -> Result<(), String> { - beacon_nodes - .first_success(|beacon_node| async move { - let mut event_stream = beacon_node - .get_events::(&[EventTopic::Head]) - .await - .map_err(|e| format!("Failed to get event stream: {:?}", e))?; - - // Poll once for a head event to trigger early attestation processing - if let Some(event_result) = event_stream.next().await { - let event = - event_result.map_err(|e| format!("Head event stream error: {:?}", e))?; - match event { - EventKind::Head(_) => { - trace!("Received head event, triggering early attestation processing"); - Ok(()) - } - _ => Err("Received non-head event when expecting head event".to_string()), - } - } else { - Err("No head events received".to_string()) - } - }) - .await - .map_err(|e| { - debug!(error = %e, "Failed to get head events from any beacon node"); - e.to_string() - }) -} - #[cfg(test)] mod tests { use super::*; From a489d324f21454f265a0d046f1e0704cdcb5f9bd Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Fri, 19 Sep 2025 21:46:01 -0400 Subject: [PATCH 08/41] comments and removing unwanted code --- .../src/head_monitor_service.rs | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/validator_client/validator_services/src/head_monitor_service.rs b/validator_client/validator_services/src/head_monitor_service.rs index f23c1ba5f56..c551dd436f4 100644 --- a/validator_client/validator_services/src/head_monitor_service.rs +++ b/validator_client/validator_services/src/head_monitor_service.rs @@ -16,21 +16,18 @@ use validator_store::ValidatorStore; type CacheHashMap = HashMap; -#[derive(Debug)] -pub enum Error { - BeaconNodeNotFound, -} - +// Builder for `HeadMonitorService` #[derive(Default)] pub struct HeadMonitorServiceBuilder { validator_store: Option>, - slot_clock: Option, beacon_nodes: Option>>, executor: Option, beacon_head_cache: Option>, head_monitor_tx: Option>>, } +// Cache to maintain the latest head received from each of the beacon nodes +// in the `BeaconNodeFallback` pub struct BeaconHeadCache { cache: RwLock, } @@ -50,8 +47,14 @@ impl BeaconHeadCache { .values() .all(|cache_head| head.slot >= cache_head.slot) } + + pub fn purge_cache(&self) { + self.cache.write().clear(); + } } +// This is used send the index derived from `CandidateBeaconNode` to the +// `AttestationService` for further processing #[derive(Debug)] pub struct HeadEvent { pub beacon_node_index: usize, @@ -61,7 +64,6 @@ impl HeadMonitorServiceBuil pub fn new() -> Self { Self { validator_store: None, - slot_clock: None, beacon_nodes: None, executor: None, beacon_head_cache: None, @@ -74,11 +76,6 @@ impl HeadMonitorServiceBuil self } - pub fn slot_clock(mut self, clock: T) -> Self { - self.slot_clock = Some(clock); - self - } - pub fn executor(mut self, executor: TaskExecutor) -> Self { self.executor = Some(executor); self @@ -107,10 +104,6 @@ impl HeadMonitorServiceBuil _validator_store: self .validator_store .ok_or("Cannot build HeadMonitorService without validator_store")?, - - _slot_clock: self - .slot_clock - .ok_or("Cannot build HeadMonitorService without slot_clock")?, beacon_nodes: self .beacon_nodes .ok_or("Cannot build HeadMonitorService without beacon_nodes")?, @@ -126,15 +119,22 @@ impl HeadMonitorServiceBuil } } +// Helper to minimise `Arc` usage pub struct Inner { _validator_store: Arc, - _slot_clock: T, executor: TaskExecutor, beacon_nodes: Arc>, beacon_head_cache: Arc, head_monitor_tx: Arc>, } +// Runs a non-terminating loop to update the `BeaconHeadCache` with the latest head received +// from the candidate beacon_nodes. This is an attempt to stream events to beacon nodes and +// potential start attestion duties earlier as soon as latest head is receive from any of the +// beacon node in contrast to attest at the 1/3rd mark in the slot. +// +// The cache and the candidate BNs list are periodically refresh/purged to avoid race condition that may +// be arise due to change in ranking of beacon_nodes thus affecting the indices of beacon_nodes pub struct HeadMonitorService { inner: Arc>, } @@ -156,6 +156,7 @@ impl Deref for HeadMonitorService { } impl HeadMonitorService { + // Starts the service to perpetually stream head events from connected beacon_nodes pub fn start_update_service(self) -> Result<(), String> { let executor = self.executor.clone(); let head_cache = self.beacon_head_cache.clone(); @@ -198,7 +199,7 @@ impl HeadMonitorService HeadMonitorService Date: Fri, 19 Sep 2025 21:48:41 -0400 Subject: [PATCH 09/41] clippy change --- validator_client/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 51e6bcec5af..c5efb569a4d 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -501,7 +501,6 @@ impl ProductionValidatorClient { let block_service = block_service_builder.build()?; let head_monitor_service = HeadMonitorServiceBuilder::new() - .slot_clock(slot_clock.clone()) .executor(context.executor.clone()) .validator_store(validator_store.clone()) .beacon_nodes(beacon_nodes.clone()) From b65fc3079bb2604fdd0dd1c304ee3c0d7ec55e9f Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Wed, 15 Oct 2025 20:03:46 -0400 Subject: [PATCH 10/41] same data attestation bug solved --- .../src/attestation_service.rs | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 5c50c06b1f4..3067a339c87 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -111,6 +111,7 @@ impl AttestationServiceBuil .head_monitor_rx .ok_or("Cannot build AttestationService without head_monitor_rx")?, disable: self.disable, + latest_attested_slot: Mutex::new(Slot::default()), }), }) } @@ -126,9 +127,11 @@ pub struct Inner { chain_spec: Arc, head_monitor_rx: Arc>>, disable: bool, + latest_attested_slot: Mutex, } -/// Attempts to produce attestations for all known validators 1/3rd of the way through each slot. +/// Attempts to produce attestations for all known validators 1/3rd of the way through each slot +/// or when a head event is received from the BNs. /// /// If any validators are on the same committee, a single attestation will be downloaded and /// returned to the beacon node. This attestation will have a signature from each of the @@ -178,6 +181,7 @@ impl AttestationService = None; + tokio::select! { _ = sleep(duration_to_next_slot + slot_duration / 3) => {}, head_event = self.poll_for_head_events() => { @@ -187,14 +191,26 @@ impl AttestationService *last_slot { + if let Err(e) = + self.spawn_attestation_tasks(slot_duration, beacon_node_index) + { + crit!(error = e, "Failed to spawn attestation tasks") + } else { + *last_slot = current_slot; + trace!(?current_slot, "Spawned attestation tasks"); + } + } else { + debug!(?current_slot, ?last_slot, "Already attested for this slot"); + } } else { - trace!("Spawned attestation tasks"); + error!("Failed to read slot clock after trigger"); } } else { error!("Failed to read slot clock"); - // If we can't read the slot clock, just wait another slot. sleep(slot_duration).await; continue; } @@ -221,9 +237,10 @@ impl AttestationService, ) -> Result<(), String> { info!( - "process attestation from beacon_node_index {:?}", - beacon_node_index + ?beacon_node_index, + "process attestation from beacon_node_index", ); + let slot = self.slot_clock.now().ok_or("Failed to read slot clock")?; let duration_to_next_slot = self .slot_clock From b25703fef6338f3cad3f543c583c7295cd12b8d3 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Mon, 20 Oct 2025 13:44:38 -0400 Subject: [PATCH 11/41] fixing dangling conditions and amaking head_monitor_service optional --- .../beacon_node_fallback/src/lib.rs | 72 ++++++++- validator_client/src/lib.rs | 17 ++- .../src/head_monitor_service.rs | 141 ++++++++---------- 3 files changed, 142 insertions(+), 88 deletions(-) diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index 3519382eb72..b729c4e0b14 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -8,12 +8,14 @@ use beacon_node_health::{ SyncDistanceTier, check_node_health, }; use clap::ValueEnum; +use eth2::types::SseHead; use eth2::{BeaconNodeHttpClient, Timeouts}; use futures::future; use sensitive_url::SensitiveUrl; use serde::{Deserialize, Serialize, Serializer, ser::SerializeStruct}; use slot_clock::SlotClock; use std::cmp::Ordering; +use std::collections::HashMap; use std::fmt; use std::fmt::Debug; use std::future::Future; @@ -49,6 +51,48 @@ pub struct Config { pub sync_tolerances: BeaconNodeSyncDistanceTiers, } +type CacheHashMap = HashMap; + +/// Cache to maintain the latest head received from each of the beacon nodes +/// in the `BeaconNodeFallback`. +#[derive(Debug)] +pub struct BeaconHeadCache { + cache: RwLock, +} + +impl BeaconHeadCache { + pub fn new() -> Self { + Self { + cache: RwLock::new(HashMap::new()), + } + } + + pub async fn get(&self, beacon_node_index: usize) -> Option { + self.cache.read().await.get(&beacon_node_index).cloned() + } + + pub async fn insert(&self, beacon_node_index: usize, head: SseHead) { + self.cache.write().await.insert(beacon_node_index, head); + } + + pub async fn is_latest(&self, head: &SseHead) -> bool { + let cache = self.cache.read().await; + cache + .values() + .all(|cache_head| head.slot >= cache_head.slot) + } + + pub async fn purge_cache(&self) { + self.cache.write().await.clear(); + } +} + +impl Default for BeaconHeadCache { + fn default() -> Self { + Self::new() + } +} + /// Indicates a measurement of latency between the VC and a BN. pub struct LatencyMeasurement { /// An identifier for the beacon node (e.g. the URL). @@ -378,6 +422,7 @@ impl CandidateBeaconNode { #[derive(Clone, Debug)] pub struct BeaconNodeFallback { pub candidates: Arc>>, + beacon_head_cache: Option>, distance_tiers: BeaconNodeSyncDistanceTiers, slot_clock: Option, broadcast_topics: Vec, @@ -387,6 +432,7 @@ pub struct BeaconNodeFallback { impl BeaconNodeFallback { pub fn new( candidates: Vec, + beacon_head_cache: Arc, config: Config, broadcast_topics: Vec, spec: Arc, @@ -394,6 +440,7 @@ impl BeaconNodeFallback { let distance_tiers = config.sync_tolerances; Self { candidates: Arc::new(RwLock::new(candidates)), + beacon_head_cache: Some(beacon_head_cache), distance_tiers, slot_clock: None, broadcast_topics, @@ -410,6 +457,14 @@ impl BeaconNodeFallback { self.slot_clock = Some(slot_clock); } + /// Set the beacon head cache reference. + /// + /// This is used to refresh beacon_head_cache in the `HeadMonitorService` to avoid dangling + /// reference to BNs when the candidate is change in the `update_candidates_list` + pub fn set_beacon_head_cache(&mut self, cache: Arc) { + self.beacon_head_cache = Some(cache); + } + /// The count of candidates, regardless of their state. pub async fn num_total(&self) -> usize { self.candidates.read().await.len() @@ -493,6 +548,10 @@ impl BeaconNodeFallback { let mut candidates = self.candidates.write().await; *candidates = new_candidates; + if let Some(cache) = &self.beacon_head_cache { + cache.purge_cache().await; + } + Ok(new_list) } @@ -962,8 +1021,17 @@ mod tests { topics: Vec, spec: Arc, ) -> BeaconNodeFallback { - let mut beacon_node_fallback = - BeaconNodeFallback::new(candidates, Config::default(), topics, spec); + let beacon_head_cache = Arc::new(BeaconHeadCache { + cache: RwLock::new(HashMap::new()), + }); + + let mut beacon_node_fallback = BeaconNodeFallback::new( + candidates, + beacon_head_cache, + Config::default(), + topics, + spec, + ); beacon_node_fallback.set_slot_clock(TestingSlotClock::new( Slot::new(1), diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index c5efb569a4d..520c9cec775 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -13,7 +13,7 @@ use tokio::sync::Mutex; use account_utils::validator_definitions::ValidatorDefinitions; use beacon_node_fallback::{ - BeaconNodeFallback, CandidateBeaconNode, start_fallback_updater_service, + BeaconHeadCache, BeaconNodeFallback, CandidateBeaconNode, start_fallback_updater_service, }; use clap::ArgMatches; use doppelganger_service::DoppelgangerService; @@ -355,8 +355,11 @@ impl ProductionValidatorClient { // Initialize the number of connected, avaliable beacon nodes to 0. set_gauge(&validator_metrics::AVAILABLE_BEACON_NODES_COUNT, 0); + let beacon_head_cache = Arc::new(BeaconHeadCache::new()); + let mut beacon_nodes: BeaconNodeFallback<_> = BeaconNodeFallback::new( candidates, + beacon_head_cache.clone(), config.beacon_node_fallback, config.broadcast_topics.clone(), context.eth2_config.spec.clone(), @@ -364,6 +367,7 @@ impl ProductionValidatorClient { let mut proposer_nodes: BeaconNodeFallback<_> = BeaconNodeFallback::new( proposer_candidates, + beacon_head_cache.clone(), config.beacon_node_fallback, config.broadcast_topics.clone(), context.eth2_config.spec.clone(), @@ -504,6 +508,7 @@ impl ProductionValidatorClient { .executor(context.executor.clone()) .validator_store(validator_store.clone()) .beacon_nodes(beacon_nodes.clone()) + .beacon_head_cache(beacon_head_cache.clone()) .head_monitor_tx(Arc::new(head_sender)) .build()?; @@ -618,10 +623,12 @@ impl ProductionValidatorClient { .start_update_service(&self.context.eth2_config.spec) .map_err(|e| format!("Unable to start preparation service: {}", e))?; - self.head_monitor_service - .clone() - .start_update_service() - .map_err(|e| format!("Unable to start head monitor service: {}", e))?; + if let Err(e) = self.head_monitor_service.clone().start_update_service() { + warn!( + error = %e, + "Unable to start head monitor service, validator client running compromised performance" + ); + } if let Some(doppelganger_service) = self.doppelganger_service.clone() { DoppelgangerService::start_update_service( diff --git a/validator_client/validator_services/src/head_monitor_service.rs b/validator_client/validator_services/src/head_monitor_service.rs index c551dd436f4..be624e58403 100644 --- a/validator_client/validator_services/src/head_monitor_service.rs +++ b/validator_client/validator_services/src/head_monitor_service.rs @@ -1,21 +1,17 @@ -use beacon_node_fallback::BeaconNodeFallback; +use beacon_node_fallback::{BeaconHeadCache, BeaconNodeFallback}; use eth2::types::EventTopic; -use eth2::types::SseHead; use tokio::sync::mpsc; +use tokio::time::{Duration, sleep}; use tracing::{info, warn}; use eth2::types::EventKind; use futures::StreamExt; -use parking_lot::RwLock; use slot_clock::SlotClock; -use std::collections::HashMap; use std::ops::Deref; use std::sync::Arc; use task_executor::TaskExecutor; use validator_store::ValidatorStore; -type CacheHashMap = HashMap; - // Builder for `HeadMonitorService` #[derive(Default)] pub struct HeadMonitorServiceBuilder { @@ -26,33 +22,6 @@ pub struct HeadMonitorServiceBuilder head_monitor_tx: Option>>, } -// Cache to maintain the latest head received from each of the beacon nodes -// in the `BeaconNodeFallback` -pub struct BeaconHeadCache { - cache: RwLock, -} - -impl BeaconHeadCache { - pub fn get(&self, beacon_node_index: usize) -> Option { - self.cache.read().get(&beacon_node_index).cloned() - } - - pub fn insert(&self, beacon_node_index: usize, head: SseHead) { - self.cache.write().insert(beacon_node_index, head); - } - - pub fn is_latest(&self, head: &SseHead) -> bool { - let cache = self.cache.read(); - cache - .values() - .all(|cache_head| head.slot >= cache_head.slot) - } - - pub fn purge_cache(&self) { - self.cache.write().clear(); - } -} - // This is used send the index derived from `CandidateBeaconNode` to the // `AttestationService` for further processing #[derive(Debug)] @@ -96,9 +65,6 @@ impl HeadMonitorServiceBuil self } pub fn build(self) -> Result, String> { - let beacon_head_cache = Arc::new(BeaconHeadCache { - cache: RwLock::new(HashMap::new()), - }); Ok(HeadMonitorService { inner: Arc::new(Inner { _validator_store: self @@ -110,7 +76,9 @@ impl HeadMonitorServiceBuil executor: self .executor .ok_or("Cannot build HeadMonitorService without executor")?, - beacon_head_cache, + beacon_head_cache: self + .beacon_head_cache + .ok_or("Cannot build HeadMonitorService without beacon_head_cache")?, head_monitor_tx: self .head_monitor_tx .ok_or("Cannot build HeadMonitorService without head_monitor_rx")?, @@ -163,57 +131,68 @@ impl HeadMonitorService(&[EventTopic::Head]) - .await; - - let mut head_event_stream = match head_event_stream { - Ok(stream) => stream, - Err(e) => { - warn!("failed to get head event stream: {:?}", e); - continue; - } + loop { + let candidates = { + let candidates_guard = self.beacon_nodes.candidates.read().await; + candidates_guard.clone() }; - - let head_cache_clone = head_cache.clone(); - let sender_tx = self.head_monitor_tx.clone(); - - let stream_fut = async move { - while let Some(event_result) = head_event_stream.next().await { - if let Ok(EventKind::Head(head)) = event_result { - head_cache_clone.insert(candidate.index, head.clone()); - - if head_cache_clone.is_latest(&head) { - if let Ok(()) = sender_tx - .send(HeadEvent { - beacon_node_index: candidate.index, - }) - .await - { - } else { - warn!("Head monitoring service channel closed"); - break; + let mut tasks = vec![]; + + for candidate in candidates.iter() { + let head_event_stream = candidate + .beacon_node + .get_events::(&[EventTopic::Head]) + .await; + + let mut head_event_stream = match head_event_stream { + Ok(stream) => stream, + Err(e) => { + warn!("failed to get head event stream: {:?}", e); + continue; + } + }; + + let head_cache_clone = head_cache.clone(); + let sender_tx = self.head_monitor_tx.clone(); + + let stream_fut = async move { + while let Some(event_result) = head_event_stream.next().await { + if let Ok(EventKind::Head(head)) = event_result { + head_cache_clone.insert(candidate.index, head.clone()).await; + + if head_cache_clone.is_latest(&head).await { + if let Ok(()) = sender_tx + .send(HeadEvent { + beacon_node_index: candidate.index, + }) + .await + { + } else { + warn!("Head monitoring service channel closed"); + break; + } } } } - } - }; + }; - tasks.push(stream_fut); - } + tasks.push(stream_fut); + } + + if tasks.is_empty() { + warn!("No beacon nodes available for head event streaming, retrying in 5 seconds"); + sleep(Duration::from_secs(5)).await; + continue; + } - futures::future::join_all(tasks).await; + futures::future::join_all(tasks).await; - drop(candidates); - self.beacon_head_cache.purge_cache(); + drop(candidates); + self.beacon_head_cache.purge_cache().await; + + // Add a small delay before reconnecting to avoid hammering beacon nodes + sleep(Duration::from_secs(1)).await; + } }; executor.spawn(interval_fut, "head_monitor_service"); From c4d851c699d858e901333195a7c9d8b46940c938 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Mon, 20 Oct 2025 13:48:40 -0400 Subject: [PATCH 12/41] fmt --- .../validator_services/src/head_monitor_service.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/validator_client/validator_services/src/head_monitor_service.rs b/validator_client/validator_services/src/head_monitor_service.rs index be624e58403..0a42ee4f170 100644 --- a/validator_client/validator_services/src/head_monitor_service.rs +++ b/validator_client/validator_services/src/head_monitor_service.rs @@ -180,7 +180,9 @@ impl HeadMonitorService Date: Mon, 20 Oct 2025 17:24:45 -0400 Subject: [PATCH 13/41] update comment --- .../validator_services/src/head_monitor_service.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/validator_client/validator_services/src/head_monitor_service.rs b/validator_client/validator_services/src/head_monitor_service.rs index 0a42ee4f170..2b76f68e1a3 100644 --- a/validator_client/validator_services/src/head_monitor_service.rs +++ b/validator_client/validator_services/src/head_monitor_service.rs @@ -101,8 +101,10 @@ pub struct Inner { // potential start attestion duties earlier as soon as latest head is receive from any of the // beacon node in contrast to attest at the 1/3rd mark in the slot. // -// The cache and the candidate BNs list are periodically refresh/purged to avoid race condition that may -// be arise due to change in ranking of beacon_nodes thus affecting the indices of beacon_nodes +// +// The cache and the candidate BNs list are refresh/purged to avoid dangling reference conditions +// that arise due to `update_candidates_list`. The cache is refresh use the shared reference between +// `HeadMonitorService` and `BeaconNodeFallback`. pub struct HeadMonitorService { inner: Arc>, } From 0e35ee5c9372fd4d9c2afbe4093cfde3996a6921 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Tue, 28 Oct 2025 18:58:02 -0400 Subject: [PATCH 14/41] massive refact --- .../src/beacon_head_monitor.rs | 142 ++++++++++++ .../beacon_node_fallback/src/lib.rs | 103 +++------ validator_client/src/lib.rs | 26 +-- .../src/attestation_service.rs | 3 +- .../src/head_monitor_service.rs | 206 ------------------ .../validator_services/src/lib.rs | 1 - 6 files changed, 181 insertions(+), 300 deletions(-) create mode 100644 validator_client/beacon_node_fallback/src/beacon_head_monitor.rs delete mode 100644 validator_client/validator_services/src/head_monitor_service.rs diff --git a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs new file mode 100644 index 00000000000..0c921da471f --- /dev/null +++ b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs @@ -0,0 +1,142 @@ +use crate::BeaconNodeFallback; +use eth2::types::EthSpec; +use eth2::types::SseHead; +use std::collections::HashMap; +use tokio::sync::RwLock; + +use eth2::types::EventTopic; +use tracing::{info, warn}; + +use eth2::types::EventKind; +use futures::StreamExt; +use slot_clock::SlotClock; +use std::sync::Arc; + +type CacheHashMap = HashMap; + +// This is used send the index derived from `CandidateBeaconNode` to the +// `AttestationService` for further processing +#[derive(Debug)] +pub struct HeadEvent { + pub beacon_node_index: usize, +} + +/// Cache to maintain the latest head received from each of the beacon nodes +/// in the `BeaconNodeFallback`. +#[derive(Debug)] +pub struct BeaconHeadCache { + cache: RwLock, +} + +impl BeaconHeadCache { + pub fn new() -> Self { + Self { + cache: RwLock::new(HashMap::new()), + } + } + + pub async fn get(&self, beacon_node_index: usize) -> Option { + self.cache.read().await.get(&beacon_node_index).cloned() + } + + pub async fn insert(&self, beacon_node_index: usize, head: SseHead) { + self.cache.write().await.insert(beacon_node_index, head); + } + + pub async fn is_latest(&self, head: &SseHead) -> bool { + let cache = self.cache.read().await; + cache + .values() + .all(|cache_head| head.slot >= cache_head.slot) + } + + pub async fn purge_cache(&self) { + self.cache.write().await.clear(); + } +} + +impl Default for BeaconHeadCache { + fn default() -> Self { + Self::new() + } +} + +// Runs a non-terminating loop to update the `BeaconHeadCache` with the latest head received +// from the candidate beacon_nodes. This is an attempt to stream events to beacon nodes and +// potential start attestion duties earlier as soon as latest head is receive from any of the +// beacon node in contrast to attest at the 1/3rd mark in the slot. +// +// +// The cache and the candidate BNs list are refresh/purged to avoid dangling reference conditions +// that arise due to `update_candidates_list`. The cache is refresh use the shared reference between +// `HeadMonitorService` and `BeaconNodeFallback`. + +// Starts the service to perpetually stream head events from connected beacon_nodes +pub async fn poll_head_event_from_beacon_nodes( + beacon_nodes: Arc>, +) -> Result<(), String> { + let head_cache = beacon_nodes.beacon_head_cache.clone().expect("asdf"); + let head_monitor_send = beacon_nodes.head_monitor_send.clone().expect("asdf"); + + info!("Starting head monitoring service"); + let candidates = { + let candidates_guard = beacon_nodes.candidates.read().await; + candidates_guard.clone() + }; + let mut tasks = vec![]; + + for candidate in candidates.iter() { + let head_event_stream = candidate + .beacon_node + .get_events::(&[EventTopic::Head]) + .await; + + let mut head_event_stream = match head_event_stream { + Ok(stream) => stream, + Err(e) => { + warn!("failed to get head event stream: {:?}", e); + continue; + } + }; + + let sender_tx = head_monitor_send.clone(); + let head_cache_ref = head_cache.clone(); + + let stream_fut = async move { + while let Some(event_result) = head_event_stream.next().await { + if let Ok(EventKind::Head(head)) = event_result { + head_cache_ref.insert(candidate.index, head.clone()).await; + + if head_cache_ref.is_latest(&head).await { + if let Ok(()) = sender_tx + .send(HeadEvent { + beacon_node_index: candidate.index, + }) + .await + { + } else { + warn!("Head monitoring service channel closed"); + break; + } + } + } + } + }; + + tasks.push(stream_fut); + } + + if tasks.is_empty() { + head_cache.purge_cache().await; + return Err( + "No beacon nodes available for head event streaming, retry in sometime".to_string(), + ); + } + + futures::future::join_all(tasks).await; + + drop(candidates); + head_cache.purge_cache().await; + + Ok(()) +} diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index b729c4e0b14..4c23426abf5 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -2,20 +2,21 @@ //! "fallback" behaviour; it will try a request on all of the nodes until one or none of them //! succeed. +pub mod beacon_head_monitor; pub mod beacon_node_health; + +use beacon_head_monitor::{BeaconHeadCache, HeadEvent, poll_head_event_from_beacon_nodes}; use beacon_node_health::{ BeaconNodeHealth, BeaconNodeSyncDistanceTiers, ExecutionEngineHealth, IsOptimistic, SyncDistanceTier, check_node_health, }; use clap::ValueEnum; -use eth2::types::SseHead; use eth2::{BeaconNodeHttpClient, Timeouts}; use futures::future; use sensitive_url::SensitiveUrl; use serde::{Deserialize, Serialize, Serializer, ser::SerializeStruct}; use slot_clock::SlotClock; use std::cmp::Ordering; -use std::collections::HashMap; use std::fmt; use std::fmt::Debug; use std::future::Future; @@ -24,7 +25,11 @@ use std::time::{Duration, Instant}; use std::vec::Vec; use strum::EnumVariantNames; use task_executor::TaskExecutor; -use tokio::{sync::RwLock, time::sleep}; + +use tokio::{ + sync::{RwLock, mpsc}, + time::sleep, +}; use tracing::{debug, error, warn}; use types::{ChainSpec, Config as ConfigSpec, EthSpec, Slot}; use validator_metrics::{ENDPOINT_ERRORS, ENDPOINT_REQUESTS, inc_counter_vec}; @@ -51,48 +56,6 @@ pub struct Config { pub sync_tolerances: BeaconNodeSyncDistanceTiers, } -type CacheHashMap = HashMap; - -/// Cache to maintain the latest head received from each of the beacon nodes -/// in the `BeaconNodeFallback`. -#[derive(Debug)] -pub struct BeaconHeadCache { - cache: RwLock, -} - -impl BeaconHeadCache { - pub fn new() -> Self { - Self { - cache: RwLock::new(HashMap::new()), - } - } - - pub async fn get(&self, beacon_node_index: usize) -> Option { - self.cache.read().await.get(&beacon_node_index).cloned() - } - - pub async fn insert(&self, beacon_node_index: usize, head: SseHead) { - self.cache.write().await.insert(beacon_node_index, head); - } - - pub async fn is_latest(&self, head: &SseHead) -> bool { - let cache = self.cache.read().await; - cache - .values() - .all(|cache_head| head.slot >= cache_head.slot) - } - - pub async fn purge_cache(&self) { - self.cache.write().await.clear(); - } -} - -impl Default for BeaconHeadCache { - fn default() -> Self { - Self::new() - } -} - /// Indicates a measurement of latency between the VC and a BN. pub struct LatencyMeasurement { /// An identifier for the beacon node (e.g. the URL). @@ -112,6 +75,20 @@ pub fn start_fallback_updater_service( return Err("Cannot start fallback updater without slot clock"); } + let beacon_nodes_ref = beacon_nodes.clone(); + + let head_monitor_future = async move { + loop { + if let Err(err) = + poll_head_event_from_beacon_nodes::(beacon_nodes_ref.clone()).await + { + warn!(error=?err, "Head service failed"); + } + } + }; + + executor.spawn(head_monitor_future, "head_monitoring"); + let future = async move { loop { beacon_nodes.update_all_candidates::().await; @@ -422,9 +399,10 @@ impl CandidateBeaconNode { #[derive(Clone, Debug)] pub struct BeaconNodeFallback { pub candidates: Arc>>, - beacon_head_cache: Option>, distance_tiers: BeaconNodeSyncDistanceTiers, slot_clock: Option, + beacon_head_cache: Option>, + head_monitor_send: Option>>, broadcast_topics: Vec, spec: Arc, } @@ -432,7 +410,6 @@ pub struct BeaconNodeFallback { impl BeaconNodeFallback { pub fn new( candidates: Vec, - beacon_head_cache: Arc, config: Config, broadcast_topics: Vec, spec: Arc, @@ -440,9 +417,10 @@ impl BeaconNodeFallback { let distance_tiers = config.sync_tolerances; Self { candidates: Arc::new(RwLock::new(candidates)), - beacon_head_cache: Some(beacon_head_cache), distance_tiers, slot_clock: None, + beacon_head_cache: Some(Arc::new(BeaconHeadCache::new())), + head_monitor_send: None, broadcast_topics, spec, } @@ -457,19 +435,15 @@ impl BeaconNodeFallback { self.slot_clock = Some(slot_clock); } - /// Set the beacon head cache reference. - /// - /// This is used to refresh beacon_head_cache in the `HeadMonitorService` to avoid dangling - /// reference to BNs when the candidate is change in the `update_candidates_list` - pub fn set_beacon_head_cache(&mut self, cache: Arc) { - self.beacon_head_cache = Some(cache); - } - /// The count of candidates, regardless of their state. pub async fn num_total(&self) -> usize { self.candidates.read().await.len() } + pub fn set_head_send(&mut self, head_monitor_send: Arc>) { + self.head_monitor_send = Some(head_monitor_send); + } + /// The count of candidates that are online and compatible, but not necessarily synced. pub async fn num_available(&self) -> usize { let mut n = 0; @@ -482,6 +456,10 @@ impl BeaconNodeFallback { n } + pub fn spawn_head_monitor_process(&self, _executor: TaskExecutor) { + let _future = async move {}; + } + // Returns all data required by the VC notifier. pub async fn get_notifier_info(&self) -> (Vec, usize, usize) { let candidates = self.candidates.read().await; @@ -1021,17 +999,8 @@ mod tests { topics: Vec, spec: Arc, ) -> BeaconNodeFallback { - let beacon_head_cache = Arc::new(BeaconHeadCache { - cache: RwLock::new(HashMap::new()), - }); - - let mut beacon_node_fallback = BeaconNodeFallback::new( - candidates, - beacon_head_cache, - Config::default(), - topics, - spec, - ); + let mut beacon_node_fallback = + BeaconNodeFallback::new(candidates, Config::default(), topics, spec); beacon_node_fallback.set_slot_clock(TestingSlotClock::new( Slot::new(1), diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 520c9cec775..1dca8fc9e85 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -13,7 +13,7 @@ use tokio::sync::Mutex; use account_utils::validator_definitions::ValidatorDefinitions; use beacon_node_fallback::{ - BeaconHeadCache, BeaconNodeFallback, CandidateBeaconNode, start_fallback_updater_service, + BeaconNodeFallback, CandidateBeaconNode, start_fallback_updater_service, }; use clap::ArgMatches; use doppelganger_service::DoppelgangerService; @@ -43,7 +43,6 @@ use validator_services::{ attestation_service::{AttestationService, AttestationServiceBuilder}, block_service::{BlockService, BlockServiceBuilder}, duties_service::{self, DutiesService, DutiesServiceBuilder}, - head_monitor_service::{HeadMonitorService, HeadMonitorServiceBuilder}, latency_service, preparation_service::{PreparationService, PreparationServiceBuilder}, sync_committee_service::SyncCommitteeService, @@ -81,7 +80,6 @@ pub struct ProductionValidatorClient { context: RuntimeContext, duties_service: Arc, SystemTimeSlotClock>>, block_service: BlockService, SystemTimeSlotClock>, - head_monitor_service: HeadMonitorService, SystemTimeSlotClock>, attestation_service: AttestationService, SystemTimeSlotClock>, sync_committee_service: SyncCommitteeService, SystemTimeSlotClock>, doppelganger_service: Option>, @@ -355,11 +353,8 @@ impl ProductionValidatorClient { // Initialize the number of connected, avaliable beacon nodes to 0. set_gauge(&validator_metrics::AVAILABLE_BEACON_NODES_COUNT, 0); - let beacon_head_cache = Arc::new(BeaconHeadCache::new()); - let mut beacon_nodes: BeaconNodeFallback<_> = BeaconNodeFallback::new( candidates, - beacon_head_cache.clone(), config.beacon_node_fallback, config.broadcast_topics.clone(), context.eth2_config.spec.clone(), @@ -367,7 +362,6 @@ impl ProductionValidatorClient { let mut proposer_nodes: BeaconNodeFallback<_> = BeaconNodeFallback::new( proposer_candidates, - beacon_head_cache.clone(), config.beacon_node_fallback, config.broadcast_topics.clone(), context.eth2_config.spec.clone(), @@ -500,18 +494,10 @@ impl ProductionValidatorClient { block_service_builder = block_service_builder.proposer_nodes(proposer_nodes.clone()); } - let (head_sender, head_receiver) = mpsc::channel(1_024); + let (_, head_receiver) = mpsc::channel(1_024); let block_service = block_service_builder.build()?; - let head_monitor_service = HeadMonitorServiceBuilder::new() - .executor(context.executor.clone()) - .validator_store(validator_store.clone()) - .beacon_nodes(beacon_nodes.clone()) - .beacon_head_cache(beacon_head_cache.clone()) - .head_monitor_tx(Arc::new(head_sender)) - .build()?; - let attestation_service = AttestationServiceBuilder::new() .duties_service(duties_service.clone()) .slot_clock(slot_clock.clone()) @@ -544,7 +530,6 @@ impl ProductionValidatorClient { context, duties_service, block_service, - head_monitor_service, attestation_service, sync_committee_service, doppelganger_service, @@ -623,13 +608,6 @@ impl ProductionValidatorClient { .start_update_service(&self.context.eth2_config.spec) .map_err(|e| format!("Unable to start preparation service: {}", e))?; - if let Err(e) = self.head_monitor_service.clone().start_update_service() { - warn!( - error = %e, - "Unable to start head monitor service, validator client running compromised performance" - ); - } - if let Some(doppelganger_service) = self.doppelganger_service.clone() { DoppelgangerService::start_update_service( doppelganger_service, diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 3067a339c87..a3644ca5949 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -1,8 +1,7 @@ use crate::duties_service::{DutiesService, DutyAndProof}; use tokio::sync::Mutex; -use crate::head_monitor_service::HeadEvent; -use beacon_node_fallback::{ApiTopic, BeaconNodeFallback}; +use beacon_node_fallback::{ApiTopic, BeaconNodeFallback, beacon_head_monitor::HeadEvent}; use futures::future::join_all; use logging::crit; use slot_clock::SlotClock; diff --git a/validator_client/validator_services/src/head_monitor_service.rs b/validator_client/validator_services/src/head_monitor_service.rs deleted file mode 100644 index 2b76f68e1a3..00000000000 --- a/validator_client/validator_services/src/head_monitor_service.rs +++ /dev/null @@ -1,206 +0,0 @@ -use beacon_node_fallback::{BeaconHeadCache, BeaconNodeFallback}; -use eth2::types::EventTopic; -use tokio::sync::mpsc; -use tokio::time::{Duration, sleep}; -use tracing::{info, warn}; - -use eth2::types::EventKind; -use futures::StreamExt; -use slot_clock::SlotClock; -use std::ops::Deref; -use std::sync::Arc; -use task_executor::TaskExecutor; -use validator_store::ValidatorStore; - -// Builder for `HeadMonitorService` -#[derive(Default)] -pub struct HeadMonitorServiceBuilder { - validator_store: Option>, - beacon_nodes: Option>>, - executor: Option, - beacon_head_cache: Option>, - head_monitor_tx: Option>>, -} - -// This is used send the index derived from `CandidateBeaconNode` to the -// `AttestationService` for further processing -#[derive(Debug)] -pub struct HeadEvent { - pub beacon_node_index: usize, -} - -impl HeadMonitorServiceBuilder { - pub fn new() -> Self { - Self { - validator_store: None, - beacon_nodes: None, - executor: None, - beacon_head_cache: None, - head_monitor_tx: None, - } - } - - pub fn beacon_nodes(mut self, nodes: Arc>) -> Self { - self.beacon_nodes = Some(nodes); - self - } - - pub fn executor(mut self, executor: TaskExecutor) -> Self { - self.executor = Some(executor); - self - } - - pub fn validator_store(mut self, validator_store: Arc) -> Self { - self.validator_store = Some(validator_store); - self - } - - pub fn beacon_head_cache(mut self, beacon_head_cache: Arc) -> Self { - self.beacon_head_cache = Some(beacon_head_cache); - self - } - - pub fn head_monitor_tx(mut self, head_monitor_tx: Arc>) -> Self { - self.head_monitor_tx = Some(head_monitor_tx); - self - } - pub fn build(self) -> Result, String> { - Ok(HeadMonitorService { - inner: Arc::new(Inner { - _validator_store: self - .validator_store - .ok_or("Cannot build HeadMonitorService without validator_store")?, - beacon_nodes: self - .beacon_nodes - .ok_or("Cannot build HeadMonitorService without beacon_nodes")?, - executor: self - .executor - .ok_or("Cannot build HeadMonitorService without executor")?, - beacon_head_cache: self - .beacon_head_cache - .ok_or("Cannot build HeadMonitorService without beacon_head_cache")?, - head_monitor_tx: self - .head_monitor_tx - .ok_or("Cannot build HeadMonitorService without head_monitor_rx")?, - }), - }) - } -} - -// Helper to minimise `Arc` usage -pub struct Inner { - _validator_store: Arc, - executor: TaskExecutor, - beacon_nodes: Arc>, - beacon_head_cache: Arc, - head_monitor_tx: Arc>, -} - -// Runs a non-terminating loop to update the `BeaconHeadCache` with the latest head received -// from the candidate beacon_nodes. This is an attempt to stream events to beacon nodes and -// potential start attestion duties earlier as soon as latest head is receive from any of the -// beacon node in contrast to attest at the 1/3rd mark in the slot. -// -// -// The cache and the candidate BNs list are refresh/purged to avoid dangling reference conditions -// that arise due to `update_candidates_list`. The cache is refresh use the shared reference between -// `HeadMonitorService` and `BeaconNodeFallback`. -pub struct HeadMonitorService { - inner: Arc>, -} - -impl Clone for HeadMonitorService { - fn clone(&self) -> Self { - Self { - inner: self.inner.clone(), - } - } -} - -impl Deref for HeadMonitorService { - type Target = Inner; - - fn deref(&self) -> &Self::Target { - self.inner.deref() - } -} - -impl HeadMonitorService { - // Starts the service to perpetually stream head events from connected beacon_nodes - pub fn start_update_service(self) -> Result<(), String> { - let executor = self.executor.clone(); - let head_cache = self.beacon_head_cache.clone(); - - info!("Starting head monitoring service"); - let interval_fut = async move { - loop { - let candidates = { - let candidates_guard = self.beacon_nodes.candidates.read().await; - candidates_guard.clone() - }; - let mut tasks = vec![]; - - for candidate in candidates.iter() { - let head_event_stream = candidate - .beacon_node - .get_events::(&[EventTopic::Head]) - .await; - - let mut head_event_stream = match head_event_stream { - Ok(stream) => stream, - Err(e) => { - warn!("failed to get head event stream: {:?}", e); - continue; - } - }; - - let head_cache_clone = head_cache.clone(); - let sender_tx = self.head_monitor_tx.clone(); - - let stream_fut = async move { - while let Some(event_result) = head_event_stream.next().await { - if let Ok(EventKind::Head(head)) = event_result { - head_cache_clone.insert(candidate.index, head.clone()).await; - - if head_cache_clone.is_latest(&head).await { - if let Ok(()) = sender_tx - .send(HeadEvent { - beacon_node_index: candidate.index, - }) - .await - { - } else { - warn!("Head monitoring service channel closed"); - break; - } - } - } - } - }; - - tasks.push(stream_fut); - } - - if tasks.is_empty() { - warn!( - "No beacon nodes available for head event streaming, retrying in 5 seconds" - ); - sleep(Duration::from_secs(5)).await; - continue; - } - - futures::future::join_all(tasks).await; - - drop(candidates); - self.beacon_head_cache.purge_cache().await; - - // Add a small delay before reconnecting to avoid hammering beacon nodes - sleep(Duration::from_secs(1)).await; - } - }; - - executor.spawn(interval_fut, "head_monitor_service"); - - Ok(()) - } -} diff --git a/validator_client/validator_services/src/lib.rs b/validator_client/validator_services/src/lib.rs index 1712f67d8e8..3b8bd9ae14b 100644 --- a/validator_client/validator_services/src/lib.rs +++ b/validator_client/validator_services/src/lib.rs @@ -1,7 +1,6 @@ pub mod attestation_service; pub mod block_service; pub mod duties_service; -pub mod head_monitor_service; pub mod latency_service; pub mod notifier_service; pub mod preparation_service; From 29867d2c4466ad1909a4a0840434da2243126660 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Wed, 29 Oct 2025 02:03:40 -0400 Subject: [PATCH 15/41] fixes and linting --- .../src/beacon_head_monitor.rs | 15 ++++++++++----- validator_client/src/lib.rs | 11 +++++++++-- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs index 0c921da471f..5f09073e554 100644 --- a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs +++ b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs @@ -68,15 +68,20 @@ impl Default for BeaconHeadCache { // // // The cache and the candidate BNs list are refresh/purged to avoid dangling reference conditions -// that arise due to `update_candidates_list`. The cache is refresh use the shared reference between -// `HeadMonitorService` and `BeaconNodeFallback`. - +// that arise due to `update_candidates_list`. +// // Starts the service to perpetually stream head events from connected beacon_nodes pub async fn poll_head_event_from_beacon_nodes( beacon_nodes: Arc>, ) -> Result<(), String> { - let head_cache = beacon_nodes.beacon_head_cache.clone().expect("asdf"); - let head_monitor_send = beacon_nodes.head_monitor_send.clone().expect("asdf"); + let head_cache = beacon_nodes + .beacon_head_cache + .clone() + .expect("Unable to start head monitor without beacon_head_cache"); + let head_monitor_send = beacon_nodes + .head_monitor_send + .clone() + .expect("Unable to start head monitor without head_monitor_send"); info!("Starting head monitoring service"); let candidates = { diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 1dca8fc9e85..34f26a1ac2e 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -73,6 +73,8 @@ pub const AGGREGATION_PRE_COMPUTE_EPOCHS: u64 = 2; /// Number of slots in advance to compute sync selection proofs when in `distributed` mode. pub const AGGREGATION_PRE_COMPUTE_SLOTS_DISTRIBUTED: u64 = 1; +const MAX_HEAD_EVENT_QUEUE_LEN: usize = 1_024; + type ValidatorStore = LighthouseValidatorStore; #[derive(Clone)] @@ -384,9 +386,16 @@ impl ProductionValidatorClient { Duration::from_secs(context.eth2_config.spec.seconds_per_slot), ); + let (head_send, head_receiver) = mpsc::channel(MAX_HEAD_EVENT_QUEUE_LEN); + + let head_send_ref = Arc::new(head_send); + beacon_nodes.set_slot_clock(slot_clock.clone()); proposer_nodes.set_slot_clock(slot_clock.clone()); + beacon_nodes.set_head_send(head_send_ref.clone()); + proposer_nodes.set_head_send(head_send_ref.clone()); + let beacon_nodes = Arc::new(beacon_nodes); start_fallback_updater_service::<_, E>(context.executor.clone(), beacon_nodes.clone())?; @@ -494,8 +503,6 @@ impl ProductionValidatorClient { block_service_builder = block_service_builder.proposer_nodes(proposer_nodes.clone()); } - let (_, head_receiver) = mpsc::channel(1_024); - let block_service = block_service_builder.build()?; let attestation_service = AttestationServiceBuilder::new() From fd4387603f51abc10a43ad5b1ac0525254bf982a Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Fri, 31 Oct 2025 04:10:08 -0400 Subject: [PATCH 16/41] remove unused code --- validator_client/beacon_node_fallback/src/lib.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index 4c23426abf5..7c2df5fab30 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -456,10 +456,6 @@ impl BeaconNodeFallback { n } - pub fn spawn_head_monitor_process(&self, _executor: TaskExecutor) { - let _future = async move {}; - } - // Returns all data required by the VC notifier. pub async fn get_notifier_info(&self) -> (Vec, usize, usize) { let candidates = self.candidates.read().await; From b054a1032a0c7cd1352695b20ff31202eb8f4f72 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Fri, 31 Oct 2025 20:27:45 -0400 Subject: [PATCH 17/41] changes --- .../src/beacon_head_monitor.rs | 23 +++---- .../src/attestation_service.rs | 60 +++++++++---------- 2 files changed, 41 insertions(+), 42 deletions(-) diff --git a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs index 5f09073e554..16749bee873 100644 --- a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs +++ b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs @@ -112,17 +112,18 @@ pub async fn poll_head_event_from_beacon_nodes AttestationService = None; - - tokio::select! { - _ = sleep(duration_to_next_slot + slot_duration / 3) => {}, - head_event = self.poll_for_head_events() => { - if let Ok(event) = head_event { - beacon_node_index = Some(event.beacon_node_index); - } - } - } - - if let Some(current_slot) = self.slot_clock.now() { - let mut last_slot = self.latest_attested_slot.lock().await; - - if current_slot > *last_slot { - if let Err(e) = - self.spawn_attestation_tasks(slot_duration, beacon_node_index) - { - crit!(error = e, "Failed to spawn attestation tasks") - } else { - *last_slot = current_slot; - trace!(?current_slot, "Spawned attestation tasks"); - } - } else { - debug!(?current_slot, ?last_slot, "Already attested for this slot"); - } - } else { - error!("Failed to read slot clock after trigger"); - } - } else { + let Some(duration) = self.slot_clock.duration_to_next_slot() else { error!("Failed to read slot clock"); sleep(slot_duration).await; continue; + }; + + let beacon_node_index = tokio::select! { + _ = sleep(duration + slot_duration /3 ) => None, + Ok(event) = self.poll_for_head_events() => Some(event.beacon_node_index), + else => None + }; + + let Some(current_slot) = self.slot_clock.now() else { + error!("Failed to read slot clock after trigger"); + continue; + }; + + let mut last_slot = self.latest_attested_slot.lock().await; + + if current_slot <= *last_slot { + debug!(?current_slot, "Attestation already initiated for the slot"); + continue; + } + + match self.spawn_attestation_tasks(slot_duration, beacon_node_index) { + Ok(_) => { + *last_slot = current_slot; + trace!(?current_slot, "Spawned attestation tasks"); + } + Err(e) => { + crit!(error = e, "Failed to spawn attestation tasks") + } } } }; From 32eed9abef3c5d5dfcbe3c6cf9b5783eed8adbb2 Mon Sep 17 00:00:00 2001 From: Hopinheimer Date: Mon, 10 Nov 2025 17:25:57 -0500 Subject: [PATCH 18/41] addressing comments --- .../beacon_node_fallback/src/beacon_head_monitor.rs | 7 +++---- validator_client/beacon_node_fallback/src/lib.rs | 12 +++++------- .../validator_services/src/attestation_service.rs | 2 +- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs index 16749bee873..ab28c229430 100644 --- a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs +++ b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs @@ -1,13 +1,12 @@ use crate::BeaconNodeFallback; -use eth2::types::EthSpec; -use eth2::types::SseHead; use std::collections::HashMap; use tokio::sync::RwLock; -use eth2::types::EventTopic; +use types::EthSpec; + +use eth2::types::{EventKind, EventTopic, SseHead}; use tracing::{info, warn}; -use eth2::types::EventKind; use futures::StreamExt; use slot_clock::SlotClock; use std::sync::Arc; diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index baf4e3a7c0b..939029747bd 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -435,15 +435,15 @@ impl BeaconNodeFallback { self.slot_clock = Some(slot_clock); } + pub fn set_head_send(&mut self, head_monitor_send: Arc>) { + self.head_monitor_send = Some(head_monitor_send); + } + /// The count of candidates, regardless of their state. pub async fn num_total(&self) -> usize { self.candidates.read().await.len() } - pub fn set_head_send(&mut self, head_monitor_send: Arc>) { - self.head_monitor_send = Some(head_monitor_send); - } - /// The count of candidates that are online and compatible, but not necessarily synced. pub async fn num_available(&self) -> usize { let mut n = 0; @@ -692,10 +692,9 @@ impl BeaconNodeFallback { R: Future>, Err: Debug, { - let candidates = self.candidates.read().await; - // Try the preferred beacon node first if it exists if let Some(preferred_idx) = preferred_index + && let candidates = self.candidates.read().await && let Some(preferred_candidate) = candidates.iter().find(|c| c.index == preferred_idx) { let preferred_node = preferred_candidate.beacon_node.clone(); @@ -710,7 +709,6 @@ impl BeaconNodeFallback { } // Fall back to normal first_success behavior - drop(candidates); self.first_success(func).await } diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 429621332e8..531ea7e5445 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -233,7 +233,7 @@ impl AttestationService, ) -> Result<(), String> { - info!( + debug!( ?beacon_node_index, "process attestation from beacon_node_index", ); From dac9f007765aad91d69468cf73ce54a019496781 Mon Sep 17 00:00:00 2001 From: Hopinheimer Date: Tue, 11 Nov 2025 15:06:19 -0500 Subject: [PATCH 19/41] removing unwanted logs --- .../validator_services/src/attestation_service.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 531ea7e5445..46e4174bedc 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -203,10 +203,7 @@ impl AttestationService { - *last_slot = current_slot; - trace!(?current_slot, "Spawned attestation tasks"); - } + Ok(_) => *last_slot = current_slot, Err(e) => { crit!(error = e, "Failed to spawn attestation tasks") } From bf1471c05dee3dc5f1865c05e283e06337b8ae57 Mon Sep 17 00:00:00 2001 From: Hopinheimer Date: Tue, 11 Nov 2025 15:13:02 -0500 Subject: [PATCH 20/41] fmt --- validator_client/validator_services/src/attestation_service.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 46e4174bedc..7a985017d7a 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -11,7 +11,7 @@ use std::sync::Arc; use task_executor::TaskExecutor; use tokio::sync::mpsc; use tokio::time::{Duration, Instant, sleep, sleep_until}; -use tracing::{debug, error, info, trace, warn}; +use tracing::{debug, error, info, warn}; use tree_hash::TreeHash; use types::{Attestation, AttestationData, ChainSpec, CommitteeIndex, EthSpec, Slot}; use validator_store::{Error as ValidatorStoreError, ValidatorStore}; From 39b9a584c6f69ea43c7d1ac480d40c13b6579da7 Mon Sep 17 00:00:00 2001 From: Hopinheimer Date: Fri, 14 Nov 2025 03:29:04 -0500 Subject: [PATCH 21/41] fixing a unwanted service starting bug --- .../beacon_node_fallback/src/lib.rs | 22 +++++++++++-------- validator_client/src/lib.rs | 3 ++- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index 939029747bd..fa72d3bb3c3 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -77,17 +77,21 @@ pub fn start_fallback_updater_service( let beacon_nodes_ref = beacon_nodes.clone(); - let head_monitor_future = async move { - loop { - if let Err(err) = - poll_head_event_from_beacon_nodes::(beacon_nodes_ref.clone()).await - { - warn!(error=?err, "Head service failed"); + // the existence of head_monitor_send is overloaded with the predicate of + // requirement of starting the head monitoring service or not. + if beacon_nodes_ref.head_monitor_send.is_some() { + let head_monitor_future = async move { + loop { + if let Err(err) = + poll_head_event_from_beacon_nodes::(beacon_nodes_ref.clone()).await + { + warn!(error=?err, "Head service failed"); + } } - } - }; + }; - executor.spawn(head_monitor_future, "head_monitoring"); + executor.spawn(head_monitor_future, "head_monitoring"); + } let future = async move { loop { diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 34f26a1ac2e..0c6999791e1 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -393,8 +393,9 @@ impl ProductionValidatorClient { beacon_nodes.set_slot_clock(slot_clock.clone()); proposer_nodes.set_slot_clock(slot_clock.clone()); + // Only the beacon_nodes are used for attestation duties and thus biconditionally + // proposer_nodes do not need head_send ref. beacon_nodes.set_head_send(head_send_ref.clone()); - proposer_nodes.set_head_send(head_send_ref.clone()); let beacon_nodes = Arc::new(beacon_nodes); start_fallback_updater_service::<_, E>(context.executor.clone(), beacon_nodes.clone())?; From 07d9a126f4ec3a76809ca73e0cb1663d76f81662 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Wed, 26 Nov 2025 13:23:37 -0500 Subject: [PATCH 22/41] making the service feature flagged --- validator_client/src/cli.rs | 11 +++++++ validator_client/src/config.rs | 5 +++ validator_client/src/lib.rs | 28 ++++++++++------ .../src/attestation_service.rs | 33 ++++++++++++------- 4 files changed, 55 insertions(+), 22 deletions(-) diff --git a/validator_client/src/cli.rs b/validator_client/src/cli.rs index 477781d3e88..2fe1ca9e9e7 100644 --- a/validator_client/src/cli.rs +++ b/validator_client/src/cli.rs @@ -466,6 +466,17 @@ pub struct ValidatorClient { )] pub beacon_nodes_sync_tolerances: Vec, + #[clap( + long, + help = "Enable the beacon head monitor so fallback head updates trigger duties when a lagging primary is detected. \ + This keeps the attestation service responsive when using multiple beacon nodes, but it relies on the \ + fallback service streaming head events which may increase network usage. \ + When not enabled (default), duties are only triggered on slot boundaries and ignore fallback head changes.", + display_order = 0, + help_heading = FLAG_HEADER + )] + pub enable_beacon_head_monitor: bool, + #[clap( long, help = "Disable Lighthouse's slashing protection for all web3signer keys. This can \ diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index 04d69dc9dc1..3483eeac6f4 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -80,6 +80,9 @@ pub struct Config { pub broadcast_topics: Vec, /// Enables a service which attempts to measure latency between the VC and BNs. pub enable_latency_measurement_service: bool, + /// Enables the beacon head monitor that reacts to fallback head updates. + #[serde(default)] + pub enable_beacon_head_monitor: bool, /// Defines the number of validators per `validator/register_validator` request sent to the BN. pub validator_registration_batch_size: usize, /// Whether we are running with distributed network support. @@ -129,6 +132,7 @@ impl Default for Config { builder_registration_timestamp_override: None, broadcast_topics: vec![ApiTopic::Subscriptions], enable_latency_measurement_service: true, + enable_beacon_head_monitor: false, validator_registration_batch_size: 500, distributed: false, initialized_validators: <_>::default(), @@ -368,6 +372,7 @@ impl Config { config.validator_store.builder_boost_factor = validator_client_config.builder_boost_factor; config.enable_latency_measurement_service = !validator_client_config.disable_latency_measurement_service; + config.enable_beacon_head_monitor = validator_client_config.enable_beacon_head_monitor; config.validator_registration_batch_size = validator_client_config.validator_registration_batch_size; diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 0c6999791e1..44ea96ecc93 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -13,7 +13,8 @@ use tokio::sync::Mutex; use account_utils::validator_definitions::ValidatorDefinitions; use beacon_node_fallback::{ - BeaconNodeFallback, CandidateBeaconNode, start_fallback_updater_service, + BeaconNodeFallback, CandidateBeaconNode, beacon_head_monitor::HeadEvent, + start_fallback_updater_service, }; use clap::ArgMatches; use doppelganger_service::DoppelgangerService; @@ -386,16 +387,19 @@ impl ProductionValidatorClient { Duration::from_secs(context.eth2_config.spec.seconds_per_slot), ); - let (head_send, head_receiver) = mpsc::channel(MAX_HEAD_EVENT_QUEUE_LEN); - - let head_send_ref = Arc::new(head_send); - beacon_nodes.set_slot_clock(slot_clock.clone()); proposer_nodes.set_slot_clock(slot_clock.clone()); // Only the beacon_nodes are used for attestation duties and thus biconditionally // proposer_nodes do not need head_send ref. - beacon_nodes.set_head_send(head_send_ref.clone()); + let head_monitor_rx = if config.enable_beacon_head_monitor { + let (head_monitor_tx, head_receiver) = + mpsc::channel::(MAX_HEAD_EVENT_QUEUE_LEN); + beacon_nodes.set_head_send(Arc::new(head_monitor_tx)); + Some(Arc::new(Mutex::new(head_receiver))) + } else { + None + }; let beacon_nodes = Arc::new(beacon_nodes); start_fallback_updater_service::<_, E>(context.executor.clone(), beacon_nodes.clone())?; @@ -506,16 +510,20 @@ impl ProductionValidatorClient { let block_service = block_service_builder.build()?; - let attestation_service = AttestationServiceBuilder::new() + let mut attestation_builder = AttestationServiceBuilder::new() .duties_service(duties_service.clone()) .slot_clock(slot_clock.clone()) .validator_store(validator_store.clone()) .beacon_nodes(beacon_nodes.clone()) .executor(context.executor.clone()) .chain_spec(context.eth2_config.spec.clone()) - .head_monitor_rx(Arc::new(Mutex::new(head_receiver))) - .disable(config.disable_attesting) - .build()?; + .disable(config.disable_attesting); + + if let Some(head_monitor_rx) = head_monitor_rx { + attestation_builder = attestation_builder.head_monitor_rx(head_monitor_rx); + } + + let attestation_service = attestation_builder.build()?; let preparation_service = PreparationServiceBuilder::new() .slot_clock(slot_clock.clone()) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 7a985017d7a..c60c32275c6 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -106,9 +106,7 @@ impl AttestationServiceBuil chain_spec: self .chain_spec .ok_or("Cannot build AttestationService without chain_spec")?, - head_monitor_rx: self - .head_monitor_rx - .ok_or("Cannot build AttestationService without head_monitor_rx")?, + head_monitor_rx: self.head_monitor_rx, disable: self.disable, latest_attested_slot: Mutex::new(Slot::default()), }), @@ -124,7 +122,7 @@ pub struct Inner { beacon_nodes: Arc>, executor: TaskExecutor, chain_spec: Arc, - head_monitor_rx: Arc>>, + head_monitor_rx: Option>>>, disable: bool, latest_attested_slot: Mutex, } @@ -184,10 +182,15 @@ impl AttestationService None, - Ok(event) = self.poll_for_head_events() => Some(event.beacon_node_index), - else => None + let slot_trigger_delay = slot_duration / 3; + let beacon_node_index = if self.head_monitor_rx.is_some() { + tokio::select! { + _ = sleep(duration + slot_trigger_delay) => None, + event = self.poll_for_head_events() => event.map(|event| event.beacon_node_index), + } + } else { + sleep(duration + slot_trigger_delay).await; + None }; let Some(current_slot) = self.slot_clock.now() else { @@ -215,11 +218,17 @@ impl AttestationService Result { - let mut receiver = self.head_monitor_rx.lock().await; + async fn poll_for_head_events(&self) -> Option { + let Some(receiver) = &self.head_monitor_rx else { + return None; + }; + let mut receiver = receiver.lock().await; match receiver.recv().await { - Some(head_event) => Ok(head_event), - None => Err("Head monitor channel closed unexpectedly".to_string()), + Some(head_event) => Some(head_event), + None => { + warn!("Head monitor channel closed unexpectedly"); + None + } } } From a82960be449cd50643c6590e4a831f768a074aaa Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Wed, 26 Nov 2025 13:41:49 -0500 Subject: [PATCH 23/41] cleaned up some logic --- validator_client/beacon_node_fallback/src/lib.rs | 8 +++++++- validator_client/src/lib.rs | 5 +---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index fa72d3bb3c3..fc647a03937 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -423,7 +423,7 @@ impl BeaconNodeFallback { candidates: Arc::new(RwLock::new(candidates)), distance_tiers, slot_clock: None, - beacon_head_cache: Some(Arc::new(BeaconHeadCache::new())), + beacon_head_cache: None, head_monitor_send: None, broadcast_topics, spec, @@ -439,8 +439,14 @@ impl BeaconNodeFallback { self.slot_clock = Some(slot_clock); } + + /// This the head monitor channel that streams events from all the beacon node that the + /// validator client is connected in the `BeaconNodeFallback`. This is also initialize the + /// beacon_head_cache under the assumption the beacon_head_cache will always be needed when + /// head_monitor_send is set. pub fn set_head_send(&mut self, head_monitor_send: Arc>) { self.head_monitor_send = Some(head_monitor_send); + self.beacon_head_cache = Some(Arc::new(BeaconHeadCache::new())); } /// The count of candidates, regardless of their state. diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 44ea96ecc93..d165c848d59 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -516,13 +516,10 @@ impl ProductionValidatorClient { .validator_store(validator_store.clone()) .beacon_nodes(beacon_nodes.clone()) .executor(context.executor.clone()) + .head_monitor_rx(head_monitor_rx) .chain_spec(context.eth2_config.spec.clone()) .disable(config.disable_attesting); - if let Some(head_monitor_rx) = head_monitor_rx { - attestation_builder = attestation_builder.head_monitor_rx(head_monitor_rx); - } - let attestation_service = attestation_builder.build()?; let preparation_service = PreparationServiceBuilder::new() From 6bddeebb54268dfb0b9a8e754ba676891897ec6c Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Wed, 26 Nov 2025 15:31:05 -0500 Subject: [PATCH 24/41] fix --- validator_client/beacon_node_fallback/src/lib.rs | 1 - validator_client/src/lib.rs | 2 +- .../validator_services/src/attestation_service.rs | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index fc647a03937..fbc3798879a 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -439,7 +439,6 @@ impl BeaconNodeFallback { self.slot_clock = Some(slot_clock); } - /// This the head monitor channel that streams events from all the beacon node that the /// validator client is connected in the `BeaconNodeFallback`. This is also initialize the /// beacon_head_cache under the assumption the beacon_head_cache will always be needed when diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index d165c848d59..71f637f2873 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -510,7 +510,7 @@ impl ProductionValidatorClient { let block_service = block_service_builder.build()?; - let mut attestation_builder = AttestationServiceBuilder::new() + let attestation_builder = AttestationServiceBuilder::new() .duties_service(duties_service.clone()) .slot_clock(slot_clock.clone()) .validator_store(validator_store.clone()) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index c60c32275c6..ed650b5e624 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -80,9 +80,9 @@ impl AttestationServiceBuil pub fn head_monitor_rx( mut self, - head_monitor_rx: Arc>>, + head_monitor_rx: Option>>>, ) -> Self { - self.head_monitor_rx = Some(head_monitor_rx); + self.head_monitor_rx = head_monitor_rx; self } pub fn build(self) -> Result, String> { From 409d937ed2e9eea224da9688299b834c4f674544 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Wed, 26 Nov 2025 17:06:38 -0500 Subject: [PATCH 25/41] updating docs --- book/src/help_vc.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/book/src/help_vc.md b/book/src/help_vc.md index b19ff0ba388..f61917a0dc4 100644 --- a/book/src/help_vc.md +++ b/book/src/help_vc.md @@ -205,6 +205,13 @@ Flags: --distributed Enables functionality required for running the validator in a distributed validator cluster. + --enable-beacon-head-monitor + Enable the beacon head monitor so fallback head updates trigger duties + when a lagging primary is detected. This keeps the attestation + service responsive when using multiple beacon nodes, but it relies on + the fallback service streaming head events which may increase network + usage. When not enabled (default), duties are only triggered on slot + boundaries and ignore fallback head changes. --enable-doppelganger-protection If this flag is set, Lighthouse will delay startup for three epochs and monitor for messages on the network by any of the validators From 0d8160f46840c3c11e5ccc0806ee090140d8533f Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Thu, 27 Nov 2025 16:49:58 -0500 Subject: [PATCH 26/41] addressing comments --- .../beacon_node_fallback/src/beacon_head_monitor.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs index ab28c229430..136a6af6607 100644 --- a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs +++ b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs @@ -1,15 +1,12 @@ use crate::BeaconNodeFallback; -use std::collections::HashMap; -use tokio::sync::RwLock; - -use types::EthSpec; - use eth2::types::{EventKind, EventTopic, SseHead}; -use tracing::{info, warn}; - use futures::StreamExt; use slot_clock::SlotClock; +use std::collections::HashMap; use std::sync::Arc; +use tokio::sync::RwLock; +use tracing::{info, warn}; +use types::EthSpec; type CacheHashMap = HashMap; From 5e8c62b183a89beed7848a20efb081276af41bec Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Mon, 1 Dec 2025 17:26:40 -0300 Subject: [PATCH 27/41] Create attestation data service --- .github/workflows/docker-reproducible.yml | 176 ++++++++++ .vscode/settings.json | 5 + Cargo.lock | 87 ++--- Cargo.toml | 31 +- Dockerfile.reproducible | 32 +- Makefile | 61 +++- account_manager/src/validator/exit.rs | 3 + .../src/validator_pubkey_cache.rs | 111 ++++-- beacon_node/beacon_chain/tests/store_tests.rs | 29 +- beacon_node/client/src/builder.rs | 17 +- .../src/test_utils/mock_builder.rs | 2 +- beacon_node/http_api/src/test_utils.rs | 2 +- beacon_node/http_api/tests/tests.rs | 2 + beacon_node/lighthouse_network/Cargo.toml | 8 +- .../src/peer_manager/mod.rs | 326 ++++++++++-------- beacon_node/lighthouse_network/tests/main.rs | 2 + .../lighthouse_network/tests/rpc_tests.rs | 5 +- .../src/bls_to_execution_changes.rs | 2 +- beacon_node/operation_pool/src/lib.rs | 1 + book/src/advanced_blobs.md | 2 +- book/src/advanced_checkpoint_sync.md | 2 +- book/src/contributing_setup.md | 41 +++ book/src/ui_installation.md | 4 +- common/eth2/src/lib.rs | 6 +- common/eth2_interop_keypairs/Cargo.toml | 5 + common/eth2_interop_keypairs/tests/main.rs | 2 + common/malloc_utils/Cargo.toml | 2 + .../context_deserialize/Cargo.toml | 17 - .../context_deserialize/src/impls/core.rs | 103 ------ .../context_deserialize/src/impls/milhouse.rs | 45 --- .../context_deserialize/src/impls/mod.rs | 7 - .../context_deserialize/src/impls/ssz.rs | 51 --- .../context_deserialize/src/lib.rs | 13 - .../context_deserialize_derive/Cargo.toml | 16 - .../context_deserialize_derive/src/lib.rs | 118 ------- .../tests/context_deserialize_derive.rs | 93 ----- consensus/merkle_proof/Cargo.toml | 3 +- consensus/merkle_proof/src/lib.rs | 90 +++-- crypto/eth2_key_derivation/Cargo.toml | 5 + crypto/eth2_key_derivation/tests/main.rs | 2 + crypto/eth2_keystore/Cargo.toml | 5 + crypto/eth2_keystore/tests/main.rs | 4 + crypto/eth2_wallet/Cargo.toml | 5 + crypto/eth2_wallet/tests/main.rs | 3 + lcli/src/block_root.rs | 2 +- lcli/src/http_sync.rs | 4 +- lcli/src/skip_slots.rs | 2 +- lcli/src/state_root.rs | 2 +- lcli/src/transition_blocks.rs | 2 +- slasher/Cargo.toml | 5 + slasher/tests/main.rs | 5 + testing/node_test_rig/src/lib.rs | 1 + testing/state_transition_vectors/Makefile | 2 +- .../src/mock_beacon_node.rs | 3 +- .../src/beacon_head_monitor.rs | 6 +- .../beacon_node_fallback/src/lib.rs | 32 +- .../doppelganger_service/src/lib.rs | 2 + validator_client/http_api/src/lib.rs | 4 +- validator_client/src/lib.rs | 9 +- .../validator_services/Cargo.toml | 5 + .../src/attestation_data_service.rs | 312 +++++++++++++++++ .../src/attestation_service.rs | 68 +++- .../validator_services/src/block_service.rs | 171 +++++---- .../validator_services/src/duties_service.rs | 8 +- .../validator_services/src/lib.rs | 1 + .../validator_services/src/sync.rs | 4 +- .../src/sync_committee_service.rs | 35 +- validator_manager/src/common.rs | 2 + validator_manager/src/create_validators.rs | 7 +- validator_manager/src/exit_validators.rs | 2 + validator_manager/src/list_validators.rs | 2 + wordlist.txt | 2 + 72 files changed, 1282 insertions(+), 964 deletions(-) create mode 100644 .github/workflows/docker-reproducible.yml create mode 100644 .vscode/settings.json create mode 100644 beacon_node/lighthouse_network/tests/main.rs create mode 100644 common/eth2_interop_keypairs/tests/main.rs delete mode 100644 consensus/context_deserialize/context_deserialize/Cargo.toml delete mode 100644 consensus/context_deserialize/context_deserialize/src/impls/core.rs delete mode 100644 consensus/context_deserialize/context_deserialize/src/impls/milhouse.rs delete mode 100644 consensus/context_deserialize/context_deserialize/src/impls/mod.rs delete mode 100644 consensus/context_deserialize/context_deserialize/src/impls/ssz.rs delete mode 100644 consensus/context_deserialize/context_deserialize/src/lib.rs delete mode 100644 consensus/context_deserialize/context_deserialize_derive/Cargo.toml delete mode 100644 consensus/context_deserialize/context_deserialize_derive/src/lib.rs delete mode 100644 consensus/context_deserialize/context_deserialize_derive/tests/context_deserialize_derive.rs create mode 100644 crypto/eth2_key_derivation/tests/main.rs create mode 100644 crypto/eth2_keystore/tests/main.rs create mode 100644 crypto/eth2_wallet/tests/main.rs create mode 100644 slasher/tests/main.rs create mode 100644 validator_client/validator_services/src/attestation_data_service.rs diff --git a/.github/workflows/docker-reproducible.yml b/.github/workflows/docker-reproducible.yml new file mode 100644 index 00000000000..f3479e9468d --- /dev/null +++ b/.github/workflows/docker-reproducible.yml @@ -0,0 +1,176 @@ +name: docker-reproducible + +on: + push: + branches: + - unstable + - stable + tags: + - v* + workflow_dispatch: # allows manual triggering for testing purposes and skips publishing an image + +env: + DOCKER_REPRODUCIBLE_IMAGE_NAME: >- + ${{ github.repository_owner }}/lighthouse-reproducible + DOCKER_PASSWORD: ${{ secrets.DH_KEY }} + DOCKER_USERNAME: ${{ secrets.DH_ORG }} + +jobs: + extract-version: + name: extract version + runs-on: ubuntu-22.04 + steps: + - name: Extract version + run: | + if [[ "${{ github.ref }}" == refs/tags/* ]]; then + # It's a tag (e.g., v1.2.3) + VERSION="${GITHUB_REF#refs/tags/}" + elif [[ "${{ github.ref }}" == refs/heads/stable ]]; then + # stable branch -> latest + VERSION="latest" + elif [[ "${{ github.ref }}" == refs/heads/unstable ]]; then + # unstable branch -> latest-unstable + VERSION="latest-unstable" + else + # For manual triggers from other branches and will not publish any image + VERSION="test-build" + fi + echo "VERSION=$VERSION" >> $GITHUB_OUTPUT + id: extract_version + outputs: + VERSION: ${{ steps.extract_version.outputs.VERSION }} + + verify-and-build: + name: verify reproducibility and build + needs: extract-version + strategy: + matrix: + arch: [amd64, arm64] + include: + - arch: amd64 + rust_target: x86_64-unknown-linux-gnu + rust_image: >- + rust:1.88-bullseye@sha256:8e3c421122bf4cd3b2a866af41a4dd52d87ad9e315fd2cb5100e87a7187a9816 + platform: linux/amd64 + runner: ubuntu-22.04 + - arch: arm64 + rust_target: aarch64-unknown-linux-gnu + rust_image: >- + rust:1.88-bullseye@sha256:8b22455a7ce2adb1355067638284ee99d21cc516fab63a96c4514beaf370aa94 + platform: linux/arm64 + runner: ubuntu-22.04-arm + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + driver: docker + + - name: Verify reproducible builds (${{ matrix.arch }}) + run: | + # Build first image + docker build -f Dockerfile.reproducible \ + --platform ${{ matrix.platform }} \ + --build-arg RUST_TARGET="${{ matrix.rust_target }}" \ + --build-arg RUST_IMAGE="${{ matrix.rust_image }}" \ + -t lighthouse-verify-1-${{ matrix.arch }} . + + # Extract binary from first build + docker create --name extract-1-${{ matrix.arch }} lighthouse-verify-1-${{ matrix.arch }} + docker cp extract-1-${{ matrix.arch }}:/lighthouse ./lighthouse-1-${{ matrix.arch }} + docker rm extract-1-${{ matrix.arch }} + + # Clean state for second build + docker buildx prune -f + docker system prune -f + + # Build second image + docker build -f Dockerfile.reproducible \ + --platform ${{ matrix.platform }} \ + --build-arg RUST_TARGET="${{ matrix.rust_target }}" \ + --build-arg RUST_IMAGE="${{ matrix.rust_image }}" \ + -t lighthouse-verify-2-${{ matrix.arch }} . + + # Extract binary from second build + docker create --name extract-2-${{ matrix.arch }} lighthouse-verify-2-${{ matrix.arch }} + docker cp extract-2-${{ matrix.arch }}:/lighthouse ./lighthouse-2-${{ matrix.arch }} + docker rm extract-2-${{ matrix.arch }} + + # Compare binaries + echo "=== Comparing binaries ===" + echo "Build 1 SHA256: $(sha256sum lighthouse-1-${{ matrix.arch }})" + echo "Build 2 SHA256: $(sha256sum lighthouse-2-${{ matrix.arch }})" + + if cmp lighthouse-1-${{ matrix.arch }} lighthouse-2-${{ matrix.arch }}; then + echo "Reproducible build verified for ${{ matrix.arch }}" + else + echo "Reproducible build FAILED for ${{ matrix.arch }}" + echo "BLOCKING RELEASE: Builds are not reproducible!" + echo "First 10 differences:" + cmp -l lighthouse-1-${{ matrix.arch }} lighthouse-2-${{ matrix.arch }} | head -10 + exit 1 + fi + + # Clean up verification artifacts but keep one image for publishing + rm -f lighthouse-*-${{ matrix.arch }} + docker rmi lighthouse-verify-1-${{ matrix.arch }} || true + + # Re-tag the second image for publishing (we verified it's identical to first) + VERSION=${{ needs.extract-version.outputs.VERSION }} + FINAL_TAG="${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${VERSION}-${{ matrix.arch }}" + docker tag lighthouse-verify-2-${{ matrix.arch }} "$FINAL_TAG" + + - name: Log in to Docker Hub + if: ${{ github.event_name != 'workflow_dispatch' }} + uses: docker/login-action@v3 + with: + username: ${{ env.DOCKER_USERNAME }} + password: ${{ env.DOCKER_PASSWORD }} + + - name: Push verified image (${{ matrix.arch }}) + if: ${{ github.event_name != 'workflow_dispatch' }} + run: | + VERSION=${{ needs.extract-version.outputs.VERSION }} + IMAGE_TAG="${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${VERSION}-${{ matrix.arch }}" + docker push "$IMAGE_TAG" + + - name: Clean up local images + run: | + docker rmi lighthouse-verify-2-${{ matrix.arch }} || true + VERSION=${{ needs.extract-version.outputs.VERSION }} + docker rmi "${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${VERSION}-${{ matrix.arch }}" || true + + - name: Upload verification artifacts (on failure) + if: failure() + uses: actions/upload-artifact@v4 + with: + name: verification-failure-${{ matrix.arch }} + path: | + lighthouse-*-${{ matrix.arch }} + + create-manifest: + name: create multi-arch manifest + runs-on: ubuntu-22.04 + needs: [extract-version, verify-and-build] + if: ${{ github.event_name != 'workflow_dispatch' }} + steps: + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ env.DOCKER_USERNAME }} + password: ${{ env.DOCKER_PASSWORD }} + + - name: Create and push multi-arch manifest + run: | + IMAGE_NAME=${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }} + VERSION=${{ needs.extract-version.outputs.VERSION }} + + # Create manifest for the version tag + docker manifest create \ + ${IMAGE_NAME}:${VERSION} \ + ${IMAGE_NAME}:${VERSION}-amd64 \ + ${IMAGE_NAME}:${VERSION}-arm64 + + docker manifest push ${IMAGE_NAME}:${VERSION} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000000..65447c4390a --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "rust-analyzer.cargo.cfgs": [ + "!debug_assertions" + ] +} diff --git a/Cargo.lock b/Cargo.lock index a1ad2ab5ba7..15985b22d2e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2076,22 +2076,21 @@ checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" [[package]] name = "context_deserialize" -version = "0.1.0" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5f9ea0a0ae2de4943f5ca71590b6dbd0b952475f0a0cafb30a470cec78c8b9" dependencies = [ "context_deserialize_derive", - "milhouse", "serde", - "ssz_types", ] [[package]] name = "context_deserialize_derive" -version = "0.1.0" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c57b2db1e4e3ed804dcc49894a144b68fe6c754b8f545eb1dda7ad3c7dbe7e6" dependencies = [ - "context_deserialize", "quote", - "serde", - "serde_json", "syn 1.0.109", ] @@ -3084,16 +3083,6 @@ dependencies = [ "syn 2.0.110", ] -[[package]] -name = "env_logger" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19187fea3ac7e84da7dacf48de0c45d63c6a76f9490dae389aead16c243fce3" -dependencies = [ - "log", - "regex", -] - [[package]] name = "environment" version = "0.1.2" @@ -3268,9 +3257,9 @@ dependencies = [ [[package]] name = "ethereum_hashing" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c853bd72c9e5787f8aafc3df2907c2ed03cff3150c3acd94e2e53a98ab70a8ab" +checksum = "5aa93f58bb1eb3d1e556e4f408ef1dac130bad01ac37db4e7ade45de40d1c86a" dependencies = [ "cpufeatures", "ring", @@ -3292,12 +3281,13 @@ dependencies = [ [[package]] name = "ethereum_ssz" -version = "0.9.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dcddb2554d19cde19b099fadddde576929d7a4d0c1cd3512d1fd95cf174375c" +checksum = "7e8cd8c4f47dfb947dbfe3cdf2945ae1da808dbedc592668658e827a12659ba1" dependencies = [ "alloy-primitives", "arbitrary", + "context_deserialize", "ethereum_serde_utils", "itertools 0.13.0", "serde", @@ -3308,9 +3298,9 @@ dependencies = [ [[package]] name = "ethereum_ssz_derive" -version = "0.9.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a657b6b3b7e153637dc6bdc6566ad9279d9ee11a15b12cfb24a2e04360637e9f" +checksum = "78d247bc40823c365a62e572441a8f8b12df03f171713f06bc76180fcd56ab71" dependencies = [ "darling 0.20.11", "proc-macro2", @@ -5525,8 +5515,7 @@ dependencies = [ "network_utils", "parking_lot", "prometheus-client", - "quickcheck", - "quickcheck_macros", + "proptest", "rand 0.9.2", "regex", "serde", @@ -5833,8 +5822,7 @@ dependencies = [ "alloy-primitives", "ethereum_hashing", "fixed_bytes", - "quickcheck", - "quickcheck_macros", + "proptest", "safe_arith", ] @@ -5870,12 +5858,13 @@ dependencies = [ [[package]] name = "milhouse" -version = "0.7.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bdb104e38d3a8c5ffb7e9d2c43c522e6bcc34070edbadba565e722f0dee56c7" +checksum = "259dd9da2ae5e0278b95da0b7ecef9c18c309d0a2d9e6db57ed33b9e8910c5e7" dependencies = [ "alloy-primitives", "arbitrary", + "context_deserialize", "educe", "ethereum_hashing", "ethereum_ssz", @@ -7222,28 +7211,6 @@ dependencies = [ "unsigned-varint 0.8.0", ] -[[package]] -name = "quickcheck" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6" -dependencies = [ - "env_logger", - "log", - "rand 0.8.5", -] - -[[package]] -name = "quickcheck_macros" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f71ee38b42f8459a88d3362be6f9b841ad2d5421844f61eb1c59c11bff3ac14a" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.110", -] - [[package]] name = "quinn" version = "0.11.9" @@ -8573,11 +8540,12 @@ dependencies = [ [[package]] name = "ssz_types" -version = "0.12.2" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "704671195db617afa3d919da8f220f2535f20d0fa8dad96a1c27a38a5f8f6e9c" +checksum = "1fc20a89bab2dabeee65e9c9eb96892dc222c23254b401e1319b85efd852fa31" dependencies = [ "arbitrary", + "context_deserialize", "ethereum_serde_utils", "ethereum_ssz", "itertools 0.14.0", @@ -9475,9 +9443,9 @@ dependencies = [ [[package]] name = "tree_hash" -version = "0.10.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee44f4cef85f88b4dea21c0b1f58320bdf35715cf56d840969487cff00613321" +checksum = "2db21caa355767db4fd6129876e5ae278a8699f4a6959b1e3e7aff610b532d52" dependencies = [ "alloy-primitives", "ethereum_hashing", @@ -9488,11 +9456,11 @@ dependencies = [ [[package]] name = "tree_hash_derive" -version = "0.10.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bee2ea1551f90040ab0e34b6fb7f2fa3bad8acc925837ac654f2c78a13e3089" +checksum = "711cc655fcbb48384a87dc2bf641b991a15c5ad9afc3caa0b1ab1df3b436f70f" dependencies = [ - "darling 0.20.11", + "darling 0.21.3", "proc-macro2", "quote", "syn 2.0.110", @@ -9903,8 +9871,11 @@ dependencies = [ "futures", "graffiti_file", "logging", + "mockito", "parking_lot", + "regex", "safe_arith", + "serde_json", "slot_clock", "task_executor", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 713fbf25d8c..6ccf429b6c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,8 +47,6 @@ members = [ "common/validator_dir", "common/warp_utils", "common/workspace_members", - "consensus/context_deserialize/context_deserialize", - "consensus/context_deserialize/context_deserialize_derive", "consensus/fixed_bytes", "consensus/fork_choice", "consensus/int_to_bytes", @@ -122,10 +120,7 @@ clap = { version = "4.5.4", features = ["derive", "cargo", "wrap_help"] } clap_utils = { path = "common/clap_utils" } compare_fields = "0.1" console-subscriber = "0.4" -context_deserialize = { path = "consensus/context_deserialize/context_deserialize", features = [ - "all", -] } -context_deserialize_derive = { path = "consensus/context_deserialize/context_deserialize_derive" } +context_deserialize = "0.2" criterion = "0.5" delay_map = "0.4" deposit_contract = { path = "common/deposit_contract" } @@ -143,10 +138,10 @@ eth2_key_derivation = { path = "crypto/eth2_key_derivation" } eth2_keystore = { path = "crypto/eth2_keystore" } eth2_network_config = { path = "common/eth2_network_config" } eth2_wallet = { path = "crypto/eth2_wallet" } -ethereum_hashing = "0.7.0" +ethereum_hashing = "0.8.0" ethereum_serde_utils = "0.8.0" -ethereum_ssz = "0.9.0" -ethereum_ssz_derive = "0.9.0" +ethereum_ssz = { version = "0.10.0", features = ["context_deserialize"] } +ethereum_ssz_derive = "0.10.0" execution_layer = { path = "beacon_node/execution_layer" } exit-future = "0.2" filesystem = { path = "common/filesystem" } @@ -183,7 +178,7 @@ malloc_utils = { path = "common/malloc_utils" } maplit = "1" merkle_proof = { path = "consensus/merkle_proof" } metrics = { path = "common/metrics" } -milhouse = { version = "0.7", default-features = false } +milhouse = { version = "0.9", default-features = false, features = ["context_deserialize"] } mockall = "0.13" mockall_double = "0.3" mockito = "1.5.0" @@ -201,9 +196,8 @@ parking_lot = "0.12" paste = "1" pretty_reqwest_error = { path = "common/pretty_reqwest_error" } prometheus = { version = "0.13", default-features = false } +proptest = "1" proto_array = { path = "consensus/proto_array" } -quickcheck = "1" -quickcheck_macros = "1" quote = "1" r2d2 = "0.8" rand = "0.9.0" @@ -233,7 +227,7 @@ slashing_protection = { path = "validator_client/slashing_protection" } slot_clock = { path = "common/slot_clock" } smallvec = { version = "1.11.2", features = ["arbitrary"] } snap = "1" -ssz_types = "0.12.2" +ssz_types = { version = "0.14.0", features = ["context_deserialize"] } state_processing = { path = "consensus/state_processing" } store = { path = "beacon_node/store" } strum = { version = "0.24", features = ["derive"] } @@ -258,8 +252,8 @@ tracing-core = "0.1" tracing-log = "0.2" tracing-opentelemetry = "0.31.0" tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } -tree_hash = "0.10.0" -tree_hash_derive = "0.10.0" +tree_hash = "0.12.0" +tree_hash_derive = "0.12.0" types = { path = "consensus/types" } url = "2" uuid = { version = "0.8", features = ["serde", "v4"] } @@ -285,13 +279,6 @@ lto = "fat" codegen-units = 1 incremental = false -[profile.reproducible] -inherits = "release" -debug = false -panic = "abort" -codegen-units = 1 -overflow-checks = true - [profile.release-debug] inherits = "release" debug = true diff --git a/Dockerfile.reproducible b/Dockerfile.reproducible index 24ba5a58a9b..903515373f8 100644 --- a/Dockerfile.reproducible +++ b/Dockerfile.reproducible @@ -3,42 +3,22 @@ ARG RUST_IMAGE="rust:1.88-bullseye@sha256:8e3c421122bf4cd3b2a866af41a4dd52d87ad9 FROM ${RUST_IMAGE} AS builder # Install specific version of the build dependencies -RUN apt-get update && apt-get install -y libclang-dev=1:11.0-51+nmu5 cmake=3.18.4-2+deb11u1 +RUN apt-get update && apt-get install -y libclang-dev=1:11.0-51+nmu5 cmake=3.18.4-2+deb11u1 libjemalloc-dev=5.2.1-3 -# Add target architecture argument with default value ARG RUST_TARGET="x86_64-unknown-linux-gnu" # Copy the project to the container -COPY . /app +COPY ./ /app WORKDIR /app -# Get the latest commit timestamp and set SOURCE_DATE_EPOCH (default it to 0 if not passed) -ARG SOURCE_DATE=0 - -# Set environment variables for reproducibility -ARG RUSTFLAGS="-C link-arg=-Wl,--build-id=none -C metadata='' --remap-path-prefix $(pwd)=." -ENV SOURCE_DATE_EPOCH=$SOURCE_DATE \ - CARGO_INCREMENTAL=0 \ - LC_ALL=C \ - TZ=UTC \ - RUSTFLAGS="${RUSTFLAGS}" - -# Set the default features if not provided -ARG FEATURES="gnosis,slasher-lmdb,slasher-mdbx,slasher-redb,jemalloc" - -# Set the default profile if not provided -ARG PROFILE="reproducible" - # Build the project with the reproducible settings -RUN cargo build --bin lighthouse \ - --features "${FEATURES}" \ - --profile "${PROFILE}" \ - --locked \ - --target "${RUST_TARGET}" +RUN make build-reproducible -RUN mv /app/target/${RUST_TARGET}/${PROFILE}/lighthouse /lighthouse +# Move the binary to a standard location +RUN mv /app/target/${RUST_TARGET}/release/lighthouse /lighthouse # Create a minimal final image with just the binary FROM gcr.io/distroless/cc-debian12:nonroot-6755e21ccd99ddead6edc8106ba03888cbeed41a COPY --from=builder /lighthouse /lighthouse + ENTRYPOINT [ "/lighthouse" ] diff --git a/Makefile b/Makefile index 2edc9f86328..a6891b682f7 100644 --- a/Makefile +++ b/Makefile @@ -81,36 +81,67 @@ build-lcli-aarch64: build-lcli-riscv64: cross build --bin lcli --target riscv64gc-unknown-linux-gnu --features "portable" --profile "$(CROSS_PROFILE)" --locked -# extracts the current source date for reproducible builds -SOURCE_DATE := $(shell git log -1 --pretty=%ct) - -# Default image for x86_64 +# Environment variables for reproducible builds +# Initialize RUSTFLAGS +RUST_BUILD_FLAGS = +# Remove build ID from the binary to ensure reproducibility across builds +RUST_BUILD_FLAGS += -C link-arg=-Wl,--build-id=none +# Remove metadata hash from symbol names to ensure reproducible builds +RUST_BUILD_FLAGS += -C metadata='' + +# Set timestamp from last git commit for reproducible builds +SOURCE_DATE ?= $(shell git log -1 --pretty=%ct) + +# Disable incremental compilation to avoid non-deterministic artifacts +CARGO_INCREMENTAL_VAL = 0 +# Set C locale for consistent string handling and sorting +LOCALE_VAL = C +# Set UTC timezone for consistent time handling across builds +TZ_VAL = UTC + +# Features for reproducible builds +FEATURES_REPRODUCIBLE = $(CROSS_FEATURES),jemalloc-unprefixed + +# Derive the architecture-specific library path from RUST_TARGET +JEMALLOC_LIB_ARCH = $(word 1,$(subst -, ,$(RUST_TARGET))) +JEMALLOC_OVERRIDE = /usr/lib/$(JEMALLOC_LIB_ARCH)-linux-gnu/libjemalloc.a + +# Default target architecture +RUST_TARGET ?= x86_64-unknown-linux-gnu + +# Default images for different architectures RUST_IMAGE_AMD64 ?= rust:1.88-bullseye@sha256:8e3c421122bf4cd3b2a866af41a4dd52d87ad9e315fd2cb5100e87a7187a9816 +RUST_IMAGE_ARM64 ?= rust:1.88-bullseye@sha256:8b22455a7ce2adb1355067638284ee99d21cc516fab63a96c4514beaf370aa94 -# Reproducible build for x86_64 -build-reproducible-x86_64: +.PHONY: build-reproducible +build-reproducible: ## Build the lighthouse binary into `target` directory with reproducible builds + SOURCE_DATE_EPOCH=$(SOURCE_DATE) \ + RUSTFLAGS="${RUST_BUILD_FLAGS} --remap-path-prefix $$(pwd)=." \ + CARGO_INCREMENTAL=${CARGO_INCREMENTAL_VAL} \ + LC_ALL=${LOCALE_VAL} \ + TZ=${TZ_VAL} \ + JEMALLOC_OVERRIDE=${JEMALLOC_OVERRIDE} \ + cargo build --bin lighthouse --features "$(FEATURES_REPRODUCIBLE)" --profile "$(PROFILE)" --locked --target $(RUST_TARGET) + +.PHONY: build-reproducible-x86_64 +build-reproducible-x86_64: ## Build reproducible x86_64 Docker image DOCKER_BUILDKIT=1 docker build \ --build-arg RUST_TARGET="x86_64-unknown-linux-gnu" \ --build-arg RUST_IMAGE=$(RUST_IMAGE_AMD64) \ - --build-arg SOURCE_DATE=$(SOURCE_DATE) \ -f Dockerfile.reproducible \ -t lighthouse:reproducible-amd64 . -# Default image for arm64 -RUST_IMAGE_ARM64 ?= rust:1.88-bullseye@sha256:8b22455a7ce2adb1355067638284ee99d21cc516fab63a96c4514beaf370aa94 - -# Reproducible build for aarch64 -build-reproducible-aarch64: +.PHONY: build-reproducible-aarch64 +build-reproducible-aarch64: ## Build reproducible aarch64 Docker image DOCKER_BUILDKIT=1 docker build \ --platform linux/arm64 \ --build-arg RUST_TARGET="aarch64-unknown-linux-gnu" \ --build-arg RUST_IMAGE=$(RUST_IMAGE_ARM64) \ - --build-arg SOURCE_DATE=$(SOURCE_DATE) \ -f Dockerfile.reproducible \ -t lighthouse:reproducible-arm64 . -# Build both architectures -build-reproducible-all: build-reproducible-x86_64 build-reproducible-aarch64 +.PHONY: build-reproducible-all +build-reproducible-all: build-reproducible-x86_64 build-reproducible-aarch64 ## Build both x86_64 and aarch64 reproducible Docker images # Create a `.tar.gz` containing a binary for a specific target. define tarball_release_binary diff --git a/account_manager/src/validator/exit.rs b/account_manager/src/validator/exit.rs index 5ea77f284e2..62e07d81ed9 100644 --- a/account_manager/src/validator/exit.rs +++ b/account_manager/src/validator/exit.rs @@ -31,6 +31,8 @@ pub const DEFAULT_BEACON_NODE: &str = "http://localhost:5052/"; pub const CONFIRMATION_PHRASE: &str = "Exit my validator"; pub const WEBSITE_URL: &str = "https://lighthouse-book.sigmaprime.io/validator_voluntary_exit.html"; +pub const DEFAULT_BEACON_NODE_INDEX: usize = 0; + pub fn cli_app() -> Command { Command::new("exit") .about("Submits a VoluntaryExit to the beacon chain for a given validator keystore.") @@ -103,6 +105,7 @@ pub fn cli_run(matches: &ArgMatches, env: Environment) -> Result< SensitiveUrl::parse(&server_url) .map_err(|e| format!("Failed to parse beacon http server: {:?}", e))?, Timeouts::set_all(Duration::from_secs(env.eth2_config.spec.seconds_per_slot)), + DEFAULT_BEACON_NODE_INDEX, ); let eth2_network_config = env diff --git a/beacon_node/beacon_chain/src/validator_pubkey_cache.rs b/beacon_node/beacon_chain/src/validator_pubkey_cache.rs index 39d2c2c2d76..a346a649f02 100644 --- a/beacon_node/beacon_chain/src/validator_pubkey_cache.rs +++ b/beacon_node/beacon_chain/src/validator_pubkey_cache.rs @@ -1,12 +1,14 @@ use crate::errors::BeaconChainError; use crate::{BeaconChainTypes, BeaconStore}; use bls::PUBLIC_KEY_UNCOMPRESSED_BYTES_LEN; +use rayon::prelude::*; use smallvec::SmallVec; use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use std::collections::HashMap; use std::marker::PhantomData; use store::{DBColumn, Error as StoreError, StoreItem, StoreOp}; +use tracing::instrument; use types::{BeaconState, FixedBytesExtended, Hash256, PublicKey, PublicKeyBytes}; /// Provides a mapping of `validator_index -> validator_publickey`. @@ -28,6 +30,7 @@ impl ValidatorPubkeyCache { /// Create a new public key cache using the keys in `state.validators`. /// /// The new cache will be updated with the keys from `state` and immediately written to disk. + #[instrument(name = "validator_pubkey_cache_new", skip_all)] pub fn new( state: &BeaconState, store: BeaconStore, @@ -46,6 +49,7 @@ impl ValidatorPubkeyCache { } /// Load the pubkey cache from the given on-disk database. + #[instrument(name = "validator_pubkey_cache_load_from_store", skip_all)] pub fn load_from_store(store: BeaconStore) -> Result { let mut pubkeys = vec![]; let mut indices = HashMap::new(); @@ -77,6 +81,7 @@ impl ValidatorPubkeyCache { /// Does not delete any keys from `self` if they don't appear in `state`. /// /// NOTE: The caller *must* commit the returned I/O batch as part of the block import process. + #[instrument(skip_all)] pub fn import_new_pubkeys( &mut self, state: &BeaconState, @@ -106,29 +111,58 @@ impl ValidatorPubkeyCache { self.indices.reserve(validator_keys.len()); let mut store_ops = Vec::with_capacity(validator_keys.len()); - for pubkey_bytes in validator_keys { - let i = self.pubkeys.len(); - if self.indices.contains_key(&pubkey_bytes) { - return Err(BeaconChainError::DuplicateValidatorPublicKey); + let is_initial_import = self.pubkeys.is_empty(); + + // Helper to insert a decompressed key + let mut insert_key = + |pubkey_bytes: PublicKeyBytes, pubkey: PublicKey| -> Result<(), BeaconChainError> { + let i = self.pubkeys.len(); + + if self.indices.contains_key(&pubkey_bytes) { + return Err(BeaconChainError::DuplicateValidatorPublicKey); + } + + // Stage the new validator key for writing to disk. + // It will be committed atomically when the block that introduced it is written to disk. + // Notably it is NOT written while the write lock on the cache is held. + // See: https://github.com/sigp/lighthouse/issues/2327 + store_ops.push(StoreOp::KeyValueOp( + DatabasePubkey::from_pubkey(&pubkey) + .as_kv_store_op(DatabasePubkey::key_for_index(i)), + )); + + self.pubkeys.push(pubkey); + self.pubkey_bytes.push(pubkey_bytes); + self.indices.insert(pubkey_bytes, i); + Ok(()) + }; + + if is_initial_import { + // On first startup, decompress keys in parallel for better performance + let validator_keys_vec: Vec = validator_keys.collect(); + + let decompressed: Vec<(PublicKeyBytes, PublicKey)> = validator_keys_vec + .into_par_iter() + .map(|pubkey_bytes| { + let pubkey = (&pubkey_bytes) + .try_into() + .map_err(BeaconChainError::InvalidValidatorPubkeyBytes)?; + Ok((pubkey_bytes, pubkey)) + }) + .collect::, BeaconChainError>>()?; + + for (pubkey_bytes, pubkey) in decompressed { + insert_key(pubkey_bytes, pubkey)?; + } + } else { + // Sequential path for incremental updates + for pubkey_bytes in validator_keys { + let pubkey = (&pubkey_bytes) + .try_into() + .map_err(BeaconChainError::InvalidValidatorPubkeyBytes)?; + insert_key(pubkey_bytes, pubkey)?; } - - let pubkey = (&pubkey_bytes) - .try_into() - .map_err(BeaconChainError::InvalidValidatorPubkeyBytes)?; - - // Stage the new validator key for writing to disk. - // It will be committed atomically when the block that introduced it is written to disk. - // Notably it is NOT written while the write lock on the cache is held. - // See: https://github.com/sigp/lighthouse/issues/2327 - store_ops.push(StoreOp::KeyValueOp( - DatabasePubkey::from_pubkey(&pubkey) - .as_kv_store_op(DatabasePubkey::key_for_index(i)), - )); - - self.pubkeys.push(pubkey); - self.pubkey_bytes.push(pubkey_bytes); - self.indices.insert(pubkey_bytes, i); } Ok(store_ops) @@ -324,4 +358,39 @@ mod test { let cache = ValidatorPubkeyCache::load_from_store(store).expect("should open cache"); check_cache_get(&cache, &keypairs[..]); } + + #[test] + fn parallel_import_maintains_order() { + // Test that parallel decompression on first startup maintains correct order and indices + let (state, keypairs) = get_state(100); + let store = get_store(); + + // Create cache from empty state (triggers parallel path) + let cache: ValidatorPubkeyCache = + ValidatorPubkeyCache::new(&state, store).expect("should create cache"); + + check_cache_get(&cache, &keypairs[..]); + } + + #[test] + fn incremental_import_maintains_order() { + // Test that incremental imports maintain correct order (triggers sequential path) + let store = get_store(); + + // Start with 50 validators + let (state1, keypairs1) = get_state(50); + let mut cache = + ValidatorPubkeyCache::new(&state1, store.clone()).expect("should create cache"); + check_cache_get(&cache, &keypairs1[..]); + + // Add 50 more validators + let (state2, keypairs2) = get_state(100); + let ops = cache + .import_new_pubkeys(&state2) + .expect("should import pubkeys"); + store.do_atomically_with_block_and_blobs_cache(ops).unwrap(); + + // Verify all 100 validators are correctly indexed + check_cache_get(&cache, &keypairs2[..]); + } } diff --git a/beacon_node/beacon_chain/tests/store_tests.rs b/beacon_node/beacon_chain/tests/store_tests.rs index cf175a56d74..0733d901fc3 100644 --- a/beacon_node/beacon_chain/tests/store_tests.rs +++ b/beacon_node/beacon_chain/tests/store_tests.rs @@ -2705,7 +2705,7 @@ async fn weak_subjectivity_sync_easy() { let num_initial_slots = E::slots_per_epoch() * 11; let checkpoint_slot = Slot::new(E::slots_per_epoch() * 9); let slots = (1..num_initial_slots).map(Slot::new).collect(); - weak_subjectivity_sync_test(slots, checkpoint_slot, None).await + weak_subjectivity_sync_test(slots, checkpoint_slot, None, true).await } #[tokio::test] @@ -2713,7 +2713,7 @@ async fn weak_subjectivity_sync_single_block_batches() { let num_initial_slots = E::slots_per_epoch() * 11; let checkpoint_slot = Slot::new(E::slots_per_epoch() * 9); let slots = (1..num_initial_slots).map(Slot::new).collect(); - weak_subjectivity_sync_test(slots, checkpoint_slot, Some(1)).await + weak_subjectivity_sync_test(slots, checkpoint_slot, Some(1), true).await } #[tokio::test] @@ -2727,7 +2727,7 @@ async fn weak_subjectivity_sync_unaligned_advanced_checkpoint() { slot <= checkpoint_slot - 3 || slot > checkpoint_slot }) .collect(); - weak_subjectivity_sync_test(slots, checkpoint_slot, None).await + weak_subjectivity_sync_test(slots, checkpoint_slot, None, true).await } #[tokio::test] @@ -2741,7 +2741,7 @@ async fn weak_subjectivity_sync_unaligned_unadvanced_checkpoint() { slot <= checkpoint_slot || slot > checkpoint_slot + 3 }) .collect(); - weak_subjectivity_sync_test(slots, checkpoint_slot, None).await + weak_subjectivity_sync_test(slots, checkpoint_slot, None, true).await } // Regression test for https://github.com/sigp/lighthouse/issues/4817 @@ -2753,7 +2753,7 @@ async fn weak_subjectivity_sync_skips_at_genesis() { let end_slot = E::slots_per_epoch() * 4; let slots = (start_slot..end_slot).map(Slot::new).collect(); let checkpoint_slot = Slot::new(E::slots_per_epoch() * 2); - weak_subjectivity_sync_test(slots, checkpoint_slot, None).await + weak_subjectivity_sync_test(slots, checkpoint_slot, None, true).await } // Checkpoint sync from the genesis state. @@ -2766,13 +2766,24 @@ async fn weak_subjectivity_sync_from_genesis() { let end_slot = E::slots_per_epoch() * 2; let slots = (start_slot..end_slot).map(Slot::new).collect(); let checkpoint_slot = Slot::new(0); - weak_subjectivity_sync_test(slots, checkpoint_slot, None).await + weak_subjectivity_sync_test(slots, checkpoint_slot, None, true).await +} + +// Test checkpoint sync without providing blobs - backfill should fetch them. +#[tokio::test] +async fn weak_subjectivity_sync_without_blobs() { + let start_slot = 4; + let end_slot = E::slots_per_epoch() * 4; + let slots = (start_slot..end_slot).map(Slot::new).collect(); + let checkpoint_slot = Slot::new(E::slots_per_epoch() * 2); + weak_subjectivity_sync_test(slots, checkpoint_slot, None, false).await } async fn weak_subjectivity_sync_test( slots: Vec, checkpoint_slot: Slot, backfill_batch_size: Option, + provide_blobs: bool, ) { // Build an initial chain on one harness, representing a synced node with full history. let num_final_blocks = E::slots_per_epoch() * 2; @@ -2874,7 +2885,11 @@ async fn weak_subjectivity_sync_test( .weak_subjectivity_state( wss_state, wss_block.clone(), - wss_blobs_opt.clone(), + if provide_blobs { + wss_blobs_opt.clone() + } else { + None + }, genesis_state, ) .unwrap() diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index 380e0c114a4..21f7618443c 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -42,7 +42,7 @@ use std::time::Duration; use std::time::{SystemTime, UNIX_EPOCH}; use store::database::interface::BeaconNodeBackend; use timer::spawn_timer; -use tracing::{debug, info, warn}; +use tracing::{debug, info, instrument, warn}; use types::data_column_custody_group::compute_ordered_custody_column_indices; use types::{ BeaconState, BlobSidecarList, ChainSpec, EthSpec, ExecutionBlockHash, Hash256, @@ -151,6 +151,7 @@ where /// Initializes the `BeaconChainBuilder`. The `build_beacon_chain` method will need to be /// called later in order to actually instantiate the `BeaconChain`. + #[instrument(skip_all)] pub async fn beacon_chain_builder( mut self, client_genesis: ClientGenesis, @@ -354,15 +355,10 @@ where let anchor_block = SignedBeaconBlock::from_ssz_bytes(&anchor_block_bytes, &spec) .map_err(|e| format!("Unable to parse weak subj block SSZ: {:?}", e))?; - // `BlobSidecar` is no longer used from Fulu onwards (superseded by `DataColumnSidecar`), - // which will be fetched via rpc instead (unimplemented). - let is_before_fulu = !spec - .fork_name_at_slot::(anchor_block.slot()) - .fulu_enabled(); - let anchor_blobs = if is_before_fulu && anchor_block.message().body().has_blobs() { + // Providing blobs is optional now and not providing them is recommended. + // Backfill can handle downloading the blobs or columns for the checkpoint block. + let anchor_blobs = if let Some(anchor_blobs_bytes) = anchor_blobs_bytes { let max_blobs_len = spec.max_blobs_per_block(anchor_block.epoch()) as usize; - let anchor_blobs_bytes = anchor_blobs_bytes - .ok_or("Blobs for checkpoint must be provided using --checkpoint-blobs")?; Some( BlobSidecarList::from_ssz_bytes(&anchor_blobs_bytes, max_blobs_len) .map_err(|e| format!("Unable to parse weak subj blobs SSZ: {e:?}"))?, @@ -393,6 +389,7 @@ where Timeouts::set_all(Duration::from_secs( config.chain.checkpoint_sync_url_timeout, )), + 0, ); debug!("Downloading finalized state"); @@ -618,6 +615,7 @@ where /// /// If type inference errors are being raised, see the comment on the definition of `Self`. #[allow(clippy::type_complexity)] + #[instrument(name = "build_client", skip_all)] pub fn build( mut self, ) -> Result>, String> { @@ -818,6 +816,7 @@ where TColdStore: ItemStore + 'static, { /// Consumes the internal `BeaconChainBuilder`, attaching the resulting `BeaconChain` to self. + #[instrument(skip_all)] pub fn build_beacon_chain(mut self) -> Result { let context = self .runtime_context diff --git a/beacon_node/execution_layer/src/test_utils/mock_builder.rs b/beacon_node/execution_layer/src/test_utils/mock_builder.rs index 9add1369194..12eadefa6fb 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_builder.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_builder.rs @@ -368,7 +368,7 @@ impl MockBuilder { let builder = MockBuilder::new( el, - BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(Duration::from_secs(1))), + BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(Duration::from_secs(1)), 0), validate_pubkey, apply_operations, broadcast_to_bn, diff --git a/beacon_node/http_api/src/test_utils.rs b/beacon_node/http_api/src/test_utils.rs index 27e2a27d35c..dd2247b2a8a 100644 --- a/beacon_node/http_api/src/test_utils.rs +++ b/beacon_node/http_api/src/test_utils.rs @@ -169,7 +169,7 @@ impl InteractiveTester { default: Duration::from_secs(5), ..Timeouts::set_all(Duration::from_secs(5)) }; - let client = BeaconNodeHttpClient::new(beacon_url.clone(), timeouts); + let client = BeaconNodeHttpClient::new(beacon_url.clone(), timeouts, 0); Self { ctx, diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 8d99e696cf7..9966e7de84a 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -322,6 +322,7 @@ impl ApiTester { let client = BeaconNodeHttpClient::new( beacon_url, Timeouts::set_all(Duration::from_secs(SECONDS_PER_SLOT)), + 0, ); Self { @@ -410,6 +411,7 @@ impl ApiTester { )) .unwrap(), Timeouts::set_all(Duration::from_secs(SECONDS_PER_SLOT)), + 0, ); Self { diff --git a/beacon_node/lighthouse_network/Cargo.toml b/beacon_node/lighthouse_network/Cargo.toml index 035452e4b2f..a6dd276c197 100644 --- a/beacon_node/lighthouse_network/Cargo.toml +++ b/beacon_node/lighthouse_network/Cargo.toml @@ -3,6 +3,7 @@ name = "lighthouse_network" version = "0.2.0" authors = ["Sigma Prime "] edition = { workspace = true } +autotests = false [features] libp2p-websocket = [] @@ -72,6 +73,9 @@ features = [ [dev-dependencies] async-channel = { workspace = true } logging = { workspace = true } -quickcheck = { workspace = true } -quickcheck_macros = { workspace = true } +proptest = { workspace = true } tempfile = { workspace = true } + +[[test]] +name = "lighthouse_network_tests" +path = "tests/main.rs" diff --git a/beacon_node/lighthouse_network/src/peer_manager/mod.rs b/beacon_node/lighthouse_network/src/peer_manager/mod.rs index ad16bb0421c..dfa8b374e9c 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/mod.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/mod.rs @@ -2975,8 +2975,7 @@ mod tests { use crate::peer_manager::tests::build_peer_manager_with_trusted_peers; use crate::rpc::{MetaData, MetaDataV3}; use libp2p::PeerId; - use quickcheck::{Arbitrary, Gen, TestResult}; - use quickcheck_macros::quickcheck; + use proptest::prelude::*; use std::collections::HashSet; use tokio::runtime::Runtime; use types::{DataColumnSubnetId, Unsigned}; @@ -2994,159 +2993,202 @@ mod tests { custody_subnets: HashSet, } - impl Arbitrary for PeerCondition { - fn arbitrary(g: &mut Gen) -> Self { - let attestation_net_bitfield = { - let len = ::SubnetBitfieldLength::to_usize(); - let mut bitfield = Vec::with_capacity(len); - for _ in 0..len { - bitfield.push(bool::arbitrary(g)); - } - bitfield - }; - - let sync_committee_net_bitfield = { - let len = ::SyncCommitteeSubnetCount::to_usize(); - let mut bitfield = Vec::with_capacity(len); - for _ in 0..len { - bitfield.push(bool::arbitrary(g)); - } - bitfield - }; - - let spec = E::default_spec(); - let custody_subnets = { - let total_subnet_count = spec.data_column_sidecar_subnet_count; - let custody_subnet_count = u64::arbitrary(g) % (total_subnet_count + 1); // 0 to 128 - (spec.custody_requirement..total_subnet_count) - .filter(|_| bool::arbitrary(g)) - .map(DataColumnSubnetId::new) - .take(custody_subnet_count as usize) - .collect() - }; - - PeerCondition { - peer_id: PeerId::random(), - outgoing: bool::arbitrary(g), - attestation_net_bitfield, - sync_committee_net_bitfield, - score: f64::arbitrary(g), - trusted: bool::arbitrary(g), - gossipsub_score: f64::arbitrary(g), - custody_subnets, - } - } - } - - #[quickcheck] - fn prune_excess_peers(peer_conditions: Vec) -> TestResult { - let target_peer_count = DEFAULT_TARGET_PEERS; + fn peer_condition_strategy() -> impl Strategy { + let attestation_len = ::SubnetBitfieldLength::to_usize(); + let sync_committee_len = ::SyncCommitteeSubnetCount::to_usize(); let spec = E::default_spec(); - if peer_conditions.len() < target_peer_count { - return TestResult::discard(); - } - let trusted_peers: Vec<_> = peer_conditions - .iter() - .filter_map(|p| if p.trusted { Some(p.peer_id) } else { None }) - .collect(); - // If we have a high percentage of trusted peers, it is very difficult to reason about - // the expected results of the pruning. - if trusted_peers.len() > peer_conditions.len() / 3_usize { - return TestResult::discard(); - } - let rt = Runtime::new().unwrap(); - - rt.block_on(async move { - // Collect all the trusted peers - let mut peer_manager = - build_peer_manager_with_trusted_peers(trusted_peers, target_peer_count).await; + let total_subnet_count = spec.data_column_sidecar_subnet_count; + let custody_requirement = spec.custody_requirement; + + // Create the pool of available subnet IDs + let available_subnets: Vec = (custody_requirement..total_subnet_count).collect(); + let max_custody_subnets = available_subnets.len(); + + // Trusted peer probability constants - 1 in 5 peers should be trusted (20%) + const TRUSTED_PEER_WEIGHT_FALSE: u32 = 4; + const TRUSTED_PEER_WEIGHT_TRUE: u32 = 1; + + ( + proptest::collection::vec(any::(), attestation_len), + proptest::collection::vec(any::(), sync_committee_len), + any::(), + any::(), + any::(), + // Weight trusted peers to avoid test rejection due to too many trusted peers + prop_oneof![ + TRUSTED_PEER_WEIGHT_FALSE => Just(false), + TRUSTED_PEER_WEIGHT_TRUE => Just(true), + ], + 0..=max_custody_subnets, + ) + .prop_flat_map( + move |( + attestation_net_bitfield, + sync_committee_net_bitfield, + score, + outgoing, + gossipsub_score, + trusted, + custody_subnet_count, + )| { + // Use proptest's subsequence to select a random subset of subnets + let custody_subnets_strategy = proptest::sample::subsequence( + available_subnets.clone(), + custody_subnet_count, + ); - // Create peers based on the randomly generated conditions. - for condition in &peer_conditions { - let mut attnets = crate::types::EnrAttestationBitfield::::new(); - let mut syncnets = crate::types::EnrSyncCommitteeBitfield::::new(); + ( + Just(attestation_net_bitfield), + Just(sync_committee_net_bitfield), + Just(score), + Just(outgoing), + Just(gossipsub_score), + Just(trusted), + custody_subnets_strategy, + ) + }, + ) + .prop_map( + |( + attestation_net_bitfield, + sync_committee_net_bitfield, + score, + outgoing, + gossipsub_score, + trusted, + custody_subnets_vec, + )| { + let custody_subnets: HashSet = custody_subnets_vec + .into_iter() + .map(DataColumnSubnetId::new) + .collect(); + + PeerCondition { + peer_id: PeerId::random(), + outgoing, + attestation_net_bitfield, + sync_committee_net_bitfield, + score, + trusted, + gossipsub_score, + custody_subnets, + } + }, + ) + } - if condition.outgoing { - peer_manager.inject_connect_outgoing( - &condition.peer_id, - "/ip4/0.0.0.0".parse().unwrap(), - None, - ); - } else { - peer_manager.inject_connect_ingoing( - &condition.peer_id, - "/ip4/0.0.0.0".parse().unwrap(), - None, - ); - } + // Upper bound for testing peer pruning - we test with at least the target number + // and up to 50% more than the target to verify pruning behavior. + const MAX_TEST_PEERS: usize = 300; - for (i, value) in condition.attestation_net_bitfield.iter().enumerate() { - attnets.set(i, *value).unwrap(); - } + proptest! { + #[test] + fn prune_excess_peers(peer_conditions in proptest::collection::vec(peer_condition_strategy(), DEFAULT_TARGET_PEERS..=MAX_TEST_PEERS)) { + let target_peer_count = DEFAULT_TARGET_PEERS; + let spec = E::default_spec(); - for (i, value) in condition.sync_committee_net_bitfield.iter().enumerate() { - syncnets.set(i, *value).unwrap(); - } + let trusted_peers: Vec<_> = peer_conditions + .iter() + .filter_map(|p| if p.trusted { Some(p.peer_id) } else { None }) + .collect(); + // If we have a high percentage of trusted peers, it is very difficult to reason about + // the expected results of the pruning. + prop_assume!(trusted_peers.len() <= peer_conditions.len() / 3_usize); + + let rt = Runtime::new().unwrap(); + + let result = rt.block_on(async move { + // Collect all the trusted peers + let mut peer_manager = + build_peer_manager_with_trusted_peers(trusted_peers, target_peer_count).await; + + // Create peers based on the randomly generated conditions. + for condition in &peer_conditions { + let mut attnets = crate::types::EnrAttestationBitfield::::new(); + let mut syncnets = crate::types::EnrSyncCommitteeBitfield::::new(); + + if condition.outgoing { + peer_manager.inject_connect_outgoing( + &condition.peer_id, + "/ip4/0.0.0.0".parse().unwrap(), + None, + ); + } else { + peer_manager.inject_connect_ingoing( + &condition.peer_id, + "/ip4/0.0.0.0".parse().unwrap(), + None, + ); + } - let subnets_per_custody_group = - spec.data_column_sidecar_subnet_count / spec.number_of_custody_groups; - let metadata = MetaDataV3 { - seq_number: 0, - attnets, - syncnets, - custody_group_count: condition.custody_subnets.len() as u64 - / subnets_per_custody_group, - }; + for (i, value) in condition.attestation_net_bitfield.iter().enumerate() { + attnets.set(i, *value).unwrap(); + } - let mut peer_db = peer_manager.network_globals.peers.write(); - let peer_info = peer_db.peer_info_mut(&condition.peer_id).unwrap(); - peer_info.set_meta_data(MetaData::V3(metadata)); - peer_info.set_gossipsub_score(condition.gossipsub_score); - peer_info.add_to_score(condition.score); - peer_info.set_custody_subnets(condition.custody_subnets.clone()); + for (i, value) in condition.sync_committee_net_bitfield.iter().enumerate() { + syncnets.set(i, *value).unwrap(); + } - for subnet in peer_info.long_lived_subnets() { - peer_db.add_subscription(&condition.peer_id, subnet); + let subnets_per_custody_group = + spec.data_column_sidecar_subnet_count / spec.number_of_custody_groups; + let metadata = MetaDataV3 { + seq_number: 0, + attnets, + syncnets, + custody_group_count: condition.custody_subnets.len() as u64 + / subnets_per_custody_group, + }; + + let mut peer_db = peer_manager.network_globals.peers.write(); + let peer_info = peer_db.peer_info_mut(&condition.peer_id).unwrap(); + peer_info.set_meta_data(MetaData::V3(metadata)); + peer_info.set_gossipsub_score(condition.gossipsub_score); + peer_info.add_to_score(condition.score); + peer_info.set_custody_subnets(condition.custody_subnets.clone()); + + for subnet in peer_info.long_lived_subnets() { + peer_db.add_subscription(&condition.peer_id, subnet); + } } - } - - // Perform the heartbeat. - peer_manager.heartbeat(); - // The minimum number of connected peers cannot be less than the target peer count - // or submitted peers. + // Perform the heartbeat. + peer_manager.heartbeat(); + + // The minimum number of connected peers cannot be less than the target peer count + // or submitted peers. + + let expected_peer_count = target_peer_count.min(peer_conditions.len()); + // Trusted peers could make this larger however. + let no_of_trusted_peers = peer_conditions + .iter() + .filter(|condition| condition.trusted) + .count(); + let expected_peer_count = expected_peer_count.max(no_of_trusted_peers); + + let target_peer_condition = + peer_manager.network_globals.connected_or_dialing_peers() + == expected_peer_count; + + // It could be that we reach our target outbound limit and are unable to prune any + // extra, which violates the target_peer_condition. + let outbound_peers = peer_manager.network_globals.connected_outbound_only_peers(); + let hit_outbound_limit = outbound_peers == peer_manager.target_outbound_peers(); + + // No trusted peers should be disconnected + let trusted_peer_disconnected = peer_conditions.iter().any(|condition| { + condition.trusted + && !peer_manager + .network_globals + .peers + .read() + .is_connected(&condition.peer_id) + }); - let expected_peer_count = target_peer_count.min(peer_conditions.len()); - // Trusted peers could make this larger however. - let no_of_trusted_peers = peer_conditions - .iter() - .filter(|condition| condition.trusted) - .count(); - let expected_peer_count = expected_peer_count.max(no_of_trusted_peers); - - let target_peer_condition = - peer_manager.network_globals.connected_or_dialing_peers() - == expected_peer_count; - - // It could be that we reach our target outbound limit and are unable to prune any - // extra, which violates the target_peer_condition. - let outbound_peers = peer_manager.network_globals.connected_outbound_only_peers(); - let hit_outbound_limit = outbound_peers == peer_manager.target_outbound_peers(); - - // No trusted peers should be disconnected - let trusted_peer_disconnected = peer_conditions.iter().any(|condition| { - condition.trusted - && !peer_manager - .network_globals - .peers - .read() - .is_connected(&condition.peer_id) + (target_peer_condition || hit_outbound_limit) && !trusted_peer_disconnected }); - TestResult::from_bool( - (target_peer_condition || hit_outbound_limit) && !trusted_peer_disconnected, - ) - }) + prop_assert!(result); + } } } diff --git a/beacon_node/lighthouse_network/tests/main.rs b/beacon_node/lighthouse_network/tests/main.rs new file mode 100644 index 00000000000..2ed0eabaff7 --- /dev/null +++ b/beacon_node/lighthouse_network/tests/main.rs @@ -0,0 +1,2 @@ +mod common; +mod rpc_tests; diff --git a/beacon_node/lighthouse_network/tests/rpc_tests.rs b/beacon_node/lighthouse_network/tests/rpc_tests.rs index 81d08764a5f..60e3e3da972 100644 --- a/beacon_node/lighthouse_network/tests/rpc_tests.rs +++ b/beacon_node/lighthouse_network/tests/rpc_tests.rs @@ -1,9 +1,8 @@ #![cfg(test)] -mod common; - +use crate::common; use crate::common::spec_with_all_forks_enabled; -use common::{Protocol, build_tracing_subscriber}; +use crate::common::{Protocol, build_tracing_subscriber}; use lighthouse_network::rpc::{RequestType, methods::*}; use lighthouse_network::service::api_types::AppRequestId; use lighthouse_network::{NetworkEvent, ReportSource, Response}; diff --git a/beacon_node/operation_pool/src/bls_to_execution_changes.rs b/beacon_node/operation_pool/src/bls_to_execution_changes.rs index cc8809c43e6..485f21b5c8b 100644 --- a/beacon_node/operation_pool/src/bls_to_execution_changes.rs +++ b/beacon_node/operation_pool/src/bls_to_execution_changes.rs @@ -19,7 +19,7 @@ pub enum ReceivedPreCapella { /// /// Using the LIFO queue for block production disincentivises spam on P2P at the Capella fork, /// and is less-relevant after that. -#[derive(Debug, Default)] +#[derive(Debug, Default, PartialEq, Eq)] pub struct BlsToExecutionChanges { /// Map from validator index to BLS to execution change. by_validator_index: HashMap>>, diff --git a/beacon_node/operation_pool/src/lib.rs b/beacon_node/operation_pool/src/lib.rs index 24e2cfbbb5d..e92d381bacc 100644 --- a/beacon_node/operation_pool/src/lib.rs +++ b/beacon_node/operation_pool/src/lib.rs @@ -782,6 +782,7 @@ impl PartialEq for OperationPool { && *self.attester_slashings.read() == *other.attester_slashings.read() && *self.proposer_slashings.read() == *other.proposer_slashings.read() && *self.voluntary_exits.read() == *other.voluntary_exits.read() + && *self.bls_to_execution_changes.read() == *other.bls_to_execution_changes.read() } } diff --git a/book/src/advanced_blobs.md b/book/src/advanced_blobs.md index 6d9ecdf72f8..e06bdb9fb9a 100644 --- a/book/src/advanced_blobs.md +++ b/book/src/advanced_blobs.md @@ -17,7 +17,7 @@ While both `--supernode` and `--semi-supernode` can serve blobs, a supernode wil Combining `--prune-blobs false` and `--supernode` (or `--semi-supernode`) implies that no data columns will be pruned, and the node will be able to serve blobs since using the flag. -If you want historical blob data beyond the data availability period (18 days), you can backfill blobs or data columns with the experimental flag `--complete-blob-backfill`. However, do note that this is an experimental feature and it only works when the flag is present during a fresh checkpoint sync when the database is initialised. The flag will have no effect if the node is already running (with an existing database). During blob backfill, the feature may cause some issues, e.g., the node may block most of its peers. +If you want historical blob data beyond the data availability period (18 days), you can backfill blobs or data columns with the experimental flag `--complete-blob-backfill`. However, do note that this is an experimental feature and it only works when the flag is present during a fresh checkpoint sync when the database is initialised. The flag will not backfill blobs if the node is already running (with an existing database). During blob backfill, the feature may cause some issues, e.g., the node may block most of its peers. **⚠️ The following section on Blobs is archived and not maintained as blobs are stored in the form of data columns after the Fulu fork ⚠️** diff --git a/book/src/advanced_checkpoint_sync.md b/book/src/advanced_checkpoint_sync.md index 9cc18dda8c3..7c30598928b 100644 --- a/book/src/advanced_checkpoint_sync.md +++ b/book/src/advanced_checkpoint_sync.md @@ -160,7 +160,7 @@ curl -H "Accept: application/octet-stream" "http://localhost:5052/eth/v1/beacon/ where `$SLOT` is the slot number. A slot which is an epoch boundary slot (i.e., first slot of an epoch) should always be used for manual checkpoint sync. -If the block contains blobs, all state, block and blobs must be provided and must point to the same slot. The +If the block contains blobs, all state, block and blobs must be provided and must point to the same slot (only applies for slots before Fulu). The state may be from the same slot as the block (unadvanced), or advanced to an epoch boundary, in which case it will be assumed to be finalized at that epoch. diff --git a/book/src/contributing_setup.md b/book/src/contributing_setup.md index b817faad879..958e8f71f6e 100644 --- a/book/src/contributing_setup.md +++ b/book/src/contributing_setup.md @@ -71,6 +71,47 @@ $ cargo nextest run -p safe_arith Summary [ 0.012s] 8 tests run: 8 passed, 0 skipped ``` +### Integration tests + +Due to the size and complexity of the test suite, Lighthouse uses a pattern that differs from how +[integration tests are usually defined](https://doc.rust-lang.org/rust-by-example/testing/integration_testing.html). +This pattern helps manage large test suites more effectively and ensures tests only run in release +mode to avoid stack overflow issues. + +#### The "main pattern" + +For packages with integration tests that require more than one file, Lighthouse uses the following +structure: + +- A `main.rs` file is defined at `package/tests/main.rs` that declares other test files as modules +- In `package/Cargo.toml`, integration tests are explicitly configured: + + ```toml + [package] + autotests = false + + [[test]] + name = "package_tests" + path = "tests/main.rs" + ``` + +#### Rust Analyzer configuration + +This pattern, combined with `#![cfg(not(debug_assertions))]` directives in test files (which +prevent tests from running in debug mode), causes Rust Analyzer to not provide IDE services like +autocomplete and error checking in integration test files by default. + +To enable IDE support for these test files, configure Rust Analyzer to disable debug assertions. +For VSCode users, this is already configured in the repository's `.vscode/settings.json` file: + +```json +{ + "rust-analyzer.cargo.cfgs": [ + "!debug_assertions" + ] +} +``` + ### test_logger The test_logger, located in `/common/logging/` can be used to create a `Logger` that by diff --git a/book/src/ui_installation.md b/book/src/ui_installation.md index 5a785650049..82f5d755bcb 100644 --- a/book/src/ui_installation.md +++ b/book/src/ui_installation.md @@ -138,13 +138,13 @@ Navigate to the backend directory `cd backend`. Install all required Node packag After initializing the backend, return to the root directory. Install all frontend dependencies by executing `yarn`. Build the frontend using `yarn build`. Start the frontend production server with `yarn start`. -This will allow you to access siren at `http://localhost:3000` by default. +This will allow you to access siren at `http://localhost:3300` by default. ## Advanced configuration ### About self-signed SSL certificates -By default, internally, Siren is running on port 80 (plain, behind nginx), port 3000 (plain, direct) and port 443 (with SSL, behind nginx)). Siren will generate and use a self-signed certificate on startup. This will generate a security warning when you try to access the interface. We recommend to only disable SSL if you would access Siren over a local LAN or otherwise highly trusted or encrypted network (i.e. VPN). +By default, internally, Siren is running on port 80 (plain, behind nginx), port 3300 (plain, direct) and port 443 (with SSL, behind nginx)). Siren will generate and use a self-signed certificate on startup. This will generate a security warning when you try to access the interface. We recommend to only disable SSL if you would access Siren over a local LAN or otherwise highly trusted or encrypted network (i.e. VPN). #### Generating persistent SSL certificates and installing them to your system diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index bcd979daca6..b0225574617 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -142,6 +142,7 @@ pub struct BeaconNodeHttpClient { client: reqwest::Client, server: SensitiveUrl, timeouts: Timeouts, + pub index: usize, } impl Eq for BeaconNodeHttpClient {} @@ -153,11 +154,12 @@ impl fmt::Display for BeaconNodeHttpClient { } impl BeaconNodeHttpClient { - pub fn new(server: SensitiveUrl, timeouts: Timeouts) -> Self { + pub fn new(server: SensitiveUrl, timeouts: Timeouts, index: usize) -> Self { Self { client: reqwest::Client::new(), server, timeouts, + index, } } @@ -165,11 +167,13 @@ impl BeaconNodeHttpClient { server: SensitiveUrl, client: reqwest::Client, timeouts: Timeouts, + index: usize, ) -> Self { Self { client, server, timeouts, + index, } } // Returns a reference to the `SensitiveUrl` of the server. diff --git a/common/eth2_interop_keypairs/Cargo.toml b/common/eth2_interop_keypairs/Cargo.toml index c19b32014e1..309ff233e62 100644 --- a/common/eth2_interop_keypairs/Cargo.toml +++ b/common/eth2_interop_keypairs/Cargo.toml @@ -3,6 +3,7 @@ name = "eth2_interop_keypairs" version = "0.2.0" authors = ["Paul Hauner "] edition = { workspace = true } +autotests = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] @@ -15,3 +16,7 @@ serde_yaml = { workspace = true } [dev-dependencies] base64 = "0.13.0" + +[[test]] +name = "eth2_interop_keypairs_tests" +path = "tests/main.rs" diff --git a/common/eth2_interop_keypairs/tests/main.rs b/common/eth2_interop_keypairs/tests/main.rs new file mode 100644 index 00000000000..4ee50127f29 --- /dev/null +++ b/common/eth2_interop_keypairs/tests/main.rs @@ -0,0 +1,2 @@ +mod from_file; +mod generation; diff --git a/common/malloc_utils/Cargo.toml b/common/malloc_utils/Cargo.toml index 39c7137d4cb..1052128852a 100644 --- a/common/malloc_utils/Cargo.toml +++ b/common/malloc_utils/Cargo.toml @@ -21,6 +21,8 @@ jemalloc-profiling = ["tikv-jemallocator/profiling"] # Force the use of system malloc (or glibc) rather than jemalloc. # This is a no-op on Windows where jemalloc is always disabled. sysmalloc = [] +# Enable jemalloc with unprefixed malloc (recommended for reproducible builds) +jemalloc-unprefixed = ["jemalloc", "tikv-jemallocator/unprefixed_malloc_on_supported_platforms"] [dependencies] libc = "0.2.79" diff --git a/consensus/context_deserialize/context_deserialize/Cargo.toml b/consensus/context_deserialize/context_deserialize/Cargo.toml deleted file mode 100644 index 0e4a97b9ae3..00000000000 --- a/consensus/context_deserialize/context_deserialize/Cargo.toml +++ /dev/null @@ -1,17 +0,0 @@ -[package] -name = "context_deserialize" -version = "0.1.0" -edition = "2021" - -[features] -default = ["derive"] -derive = ["dep:context_deserialize_derive"] -milhouse = ["dep:milhouse"] -ssz = ["dep:ssz_types"] -all = ["derive", "milhouse", "ssz"] - -[dependencies] -context_deserialize_derive = { version = "0.1.0", path = "../context_deserialize_derive", optional = true } -milhouse = { workspace = true, optional = true } -serde = { workspace = true } -ssz_types = { workspace = true, optional = true } diff --git a/consensus/context_deserialize/context_deserialize/src/impls/core.rs b/consensus/context_deserialize/context_deserialize/src/impls/core.rs deleted file mode 100644 index 803619365f1..00000000000 --- a/consensus/context_deserialize/context_deserialize/src/impls/core.rs +++ /dev/null @@ -1,103 +0,0 @@ -use crate::ContextDeserialize; -use serde::de::{Deserialize, DeserializeSeed, Deserializer, SeqAccess, Visitor}; -use std::marker::PhantomData; -use std::sync::Arc; - -impl<'de, C, T> ContextDeserialize<'de, T> for Arc -where - C: ContextDeserialize<'de, T>, -{ - fn context_deserialize(deserializer: D, context: T) -> Result - where - D: Deserializer<'de>, - { - Ok(Arc::new(C::context_deserialize(deserializer, context)?)) - } -} - -impl<'de, T, C> ContextDeserialize<'de, C> for Vec -where - T: ContextDeserialize<'de, C>, - C: Clone, -{ - fn context_deserialize(deserializer: D, context: C) -> Result - where - D: Deserializer<'de>, - { - // Our Visitor, which owns one copy of the context T - struct ContextVisitor { - context: T, - _marker: PhantomData, - } - - impl<'de, C, T> Visitor<'de> for ContextVisitor - where - C: ContextDeserialize<'de, T>, - T: Clone, - { - type Value = Vec; - - fn expecting(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { - fmt.write_str("a sequence of context‐deserialized elements") - } - - fn visit_seq(self, mut seq: A) -> Result, A::Error> - where - A: SeqAccess<'de>, - { - let mut out = Vec::with_capacity(seq.size_hint().unwrap_or(0)); - // for each element, we clone the context and hand it to the seed - while let Some(elem) = seq.next_element_seed(ContextSeed { - context: self.context.clone(), - _marker: PhantomData, - })? { - out.push(elem); - } - Ok(out) - } - } - - // A little seed that hands the deserializer + context into C::context_deserialize - struct ContextSeed { - context: C, - _marker: PhantomData, - } - - impl<'de, T, C> DeserializeSeed<'de> for ContextSeed - where - T: ContextDeserialize<'de, C>, - C: Clone, - { - type Value = T; - - fn deserialize(self, deserializer: D) -> Result - where - D: Deserializer<'de>, - { - T::context_deserialize(deserializer, self.context) - } - } - - deserializer.deserialize_seq(ContextVisitor { - context, - _marker: PhantomData, - }) - } -} - -macro_rules! trivial_deserialize { - ($($t:ty),* $(,)?) => { - $( - impl<'de, T> ContextDeserialize<'de, T> for $t { - fn context_deserialize(deserializer: D, _context: T) -> Result - where - D: Deserializer<'de>, - { - <$t>::deserialize(deserializer) - } - } - )* - }; -} - -trivial_deserialize!(bool, u8, u16, u32, u64, u128, i8, i16, i32, i64, i128, f32, f64); diff --git a/consensus/context_deserialize/context_deserialize/src/impls/milhouse.rs b/consensus/context_deserialize/context_deserialize/src/impls/milhouse.rs deleted file mode 100644 index 3b86f067a3e..00000000000 --- a/consensus/context_deserialize/context_deserialize/src/impls/milhouse.rs +++ /dev/null @@ -1,45 +0,0 @@ -use crate::ContextDeserialize; -use milhouse::{List, Value, Vector}; -use serde::de::Deserializer; -use ssz_types::typenum::Unsigned; - -impl<'de, C, T, N> ContextDeserialize<'de, C> for List -where - T: ContextDeserialize<'de, C> + Value, - N: Unsigned, - C: Clone, -{ - fn context_deserialize(deserializer: D, context: C) -> Result - where - D: Deserializer<'de>, - { - // First deserialize as a Vec. - // This is not the most efficient implementation as it allocates a temporary Vec. In future - // we could write a more performant implementation using `List::builder()`. - let vec = Vec::::context_deserialize(deserializer, context)?; - - // Then convert to List, which will check the length. - List::new(vec) - .map_err(|e| serde::de::Error::custom(format!("Failed to create List: {:?}", e))) - } -} - -impl<'de, C, T, N> ContextDeserialize<'de, C> for Vector -where - T: ContextDeserialize<'de, C> + Value, - N: Unsigned, - C: Clone, -{ - fn context_deserialize(deserializer: D, context: C) -> Result - where - D: Deserializer<'de>, - { - // First deserialize as a List - let list = List::::context_deserialize(deserializer, context)?; - - // Then convert to Vector, which will check the length - Vector::try_from(list).map_err(|e| { - serde::de::Error::custom(format!("Failed to convert List to Vector: {:?}", e)) - }) - } -} diff --git a/consensus/context_deserialize/context_deserialize/src/impls/mod.rs b/consensus/context_deserialize/context_deserialize/src/impls/mod.rs deleted file mode 100644 index 0225c5e031f..00000000000 --- a/consensus/context_deserialize/context_deserialize/src/impls/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -mod core; - -#[cfg(feature = "milhouse")] -mod milhouse; - -#[cfg(feature = "ssz")] -mod ssz; diff --git a/consensus/context_deserialize/context_deserialize/src/impls/ssz.rs b/consensus/context_deserialize/context_deserialize/src/impls/ssz.rs deleted file mode 100644 index 26813a96fb7..00000000000 --- a/consensus/context_deserialize/context_deserialize/src/impls/ssz.rs +++ /dev/null @@ -1,51 +0,0 @@ -use crate::ContextDeserialize; -use serde::{ - de::{Deserializer, Error}, - Deserialize, -}; -use ssz_types::{ - length::{Fixed, Variable}, - typenum::Unsigned, - Bitfield, FixedVector, -}; - -impl<'de, C, T, N> ContextDeserialize<'de, C> for FixedVector -where - T: ContextDeserialize<'de, C>, - N: Unsigned, - C: Clone, -{ - fn context_deserialize(deserializer: D, context: C) -> Result - where - D: Deserializer<'de>, - { - let vec = Vec::::context_deserialize(deserializer, context)?; - FixedVector::new(vec).map_err(|e| D::Error::custom(format!("{:?}", e))) - } -} - -impl<'de, C, N> ContextDeserialize<'de, C> for Bitfield> -where - N: Unsigned + Clone, -{ - fn context_deserialize(deserializer: D, _context: C) -> Result - where - D: Deserializer<'de>, - { - Bitfield::>::deserialize(deserializer) - .map_err(|e| D::Error::custom(format!("{:?}", e))) - } -} - -impl<'de, C, N> ContextDeserialize<'de, C> for Bitfield> -where - N: Unsigned + Clone, -{ - fn context_deserialize(deserializer: D, _context: C) -> Result - where - D: Deserializer<'de>, - { - Bitfield::>::deserialize(deserializer) - .map_err(|e| D::Error::custom(format!("{:?}", e))) - } -} diff --git a/consensus/context_deserialize/context_deserialize/src/lib.rs b/consensus/context_deserialize/context_deserialize/src/lib.rs deleted file mode 100644 index e5f2bfdba38..00000000000 --- a/consensus/context_deserialize/context_deserialize/src/lib.rs +++ /dev/null @@ -1,13 +0,0 @@ -mod impls; - -#[cfg(feature = "derive")] -pub use context_deserialize_derive::context_deserialize; - -use serde::de::Deserializer; - -/// General-purpose deserialization trait that accepts extra context `C`. -pub trait ContextDeserialize<'de, C>: Sized { - fn context_deserialize(deserializer: D, context: C) -> Result - where - D: Deserializer<'de>; -} diff --git a/consensus/context_deserialize/context_deserialize_derive/Cargo.toml b/consensus/context_deserialize/context_deserialize_derive/Cargo.toml deleted file mode 100644 index eedae30cdfe..00000000000 --- a/consensus/context_deserialize/context_deserialize_derive/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "context_deserialize_derive" -version = "0.1.0" -edition = "2021" - -[lib] -proc-macro = true - -[dependencies] -quote = { workspace = true } -syn = { workspace = true } - -[dev-dependencies] -context_deserialize = { path = "../context_deserialize" } -serde = { workspace = true } -serde_json = "1.0" diff --git a/consensus/context_deserialize/context_deserialize_derive/src/lib.rs b/consensus/context_deserialize/context_deserialize_derive/src/lib.rs deleted file mode 100644 index 0b73a43b0a4..00000000000 --- a/consensus/context_deserialize/context_deserialize_derive/src/lib.rs +++ /dev/null @@ -1,118 +0,0 @@ -extern crate proc_macro; -extern crate quote; -extern crate syn; - -use proc_macro::TokenStream; -use quote::quote; -use syn::{ - parse_macro_input, AttributeArgs, DeriveInput, GenericParam, LifetimeDef, Meta, NestedMeta, - WhereClause, -}; - -#[proc_macro_attribute] -pub fn context_deserialize(attr: TokenStream, item: TokenStream) -> TokenStream { - let args = parse_macro_input!(attr as AttributeArgs); - let input = parse_macro_input!(item as DeriveInput); - let ident = &input.ident; - - let mut ctx_types = Vec::new(); - let mut explicit_where: Option = None; - - for meta in args { - match meta { - NestedMeta::Meta(Meta::Path(p)) => { - ctx_types.push(p); - } - NestedMeta::Meta(Meta::NameValue(nv)) if nv.path.is_ident("bound") => { - if let syn::Lit::Str(lit_str) = &nv.lit { - let where_string = format!("where {}", lit_str.value()); - match syn::parse_str::(&where_string) { - Ok(where_clause) => { - explicit_where = Some(where_clause); - } - Err(err) => { - return syn::Error::new_spanned( - lit_str, - format!("Invalid where clause '{}': {}", lit_str.value(), err), - ) - .to_compile_error() - .into(); - } - } - } else { - return syn::Error::new_spanned( - &nv, - "Expected a string literal for `bound` value", - ) - .to_compile_error() - .into(); - } - } - _ => { - return syn::Error::new_spanned( - &meta, - "Expected paths or `bound = \"...\"` in #[context_deserialize(...)]", - ) - .to_compile_error() - .into(); - } - } - } - - if ctx_types.is_empty() { - return quote! { - compile_error!("Usage: #[context_deserialize(Type1, Type2, ..., bound = \"...\")]"); - } - .into(); - } - - let original_generics = input.generics.clone(); - - // Clone and clean generics for impl use (remove default params) - let mut impl_generics = input.generics.clone(); - for param in impl_generics.params.iter_mut() { - if let GenericParam::Type(ty) = param { - ty.eq_token = None; - ty.default = None; - } - } - - // Ensure 'de lifetime exists in impl generics - let has_de = impl_generics - .lifetimes() - .any(|LifetimeDef { lifetime, .. }| lifetime.ident == "de"); - - if !has_de { - impl_generics.params.insert(0, syn::parse_quote! { 'de }); - } - - let (_, ty_generics, _) = original_generics.split_for_impl(); - let (impl_gens, _, _) = impl_generics.split_for_impl(); - - // Generate: no `'de` applied to the type name - let mut impls = quote! {}; - for ctx in ctx_types { - impls.extend(quote! { - impl #impl_gens context_deserialize::ContextDeserialize<'de, #ctx> - for #ident #ty_generics - #explicit_where - { - fn context_deserialize( - deserializer: D, - _context: #ctx, - ) -> Result - where - D: serde::de::Deserializer<'de>, - { - ::deserialize(deserializer) - } - } - }); - } - - quote! { - #input - #impls - } - .into() -} diff --git a/consensus/context_deserialize/context_deserialize_derive/tests/context_deserialize_derive.rs b/consensus/context_deserialize/context_deserialize_derive/tests/context_deserialize_derive.rs deleted file mode 100644 index 8fb46da9c65..00000000000 --- a/consensus/context_deserialize/context_deserialize_derive/tests/context_deserialize_derive.rs +++ /dev/null @@ -1,93 +0,0 @@ -use context_deserialize::{context_deserialize, ContextDeserialize}; -use serde::{Deserialize, Serialize}; - -#[test] -fn test_context_deserialize_derive() { - type TestContext = (); - - #[context_deserialize(TestContext)] - #[derive(Debug, PartialEq, Serialize, Deserialize)] - struct Test { - field: String, - } - - let test = Test { - field: "test".to_string(), - }; - let serialized = serde_json::to_string(&test).unwrap(); - let deserialized = - Test::context_deserialize(&mut serde_json::Deserializer::from_str(&serialized), ()) - .unwrap(); - assert_eq!(test, deserialized); -} - -#[test] -fn test_context_deserialize_derive_multiple_types() { - #[allow(dead_code)] - struct TestContext1(u64); - #[allow(dead_code)] - struct TestContext2(String); - - // This will derive: - // - ContextDeserialize for Test - // - ContextDeserialize for Test - // by just leveraging the Deserialize impl - #[context_deserialize(TestContext1, TestContext2)] - #[derive(Debug, PartialEq, Serialize, Deserialize)] - struct Test { - field: String, - } - - let test = Test { - field: "test".to_string(), - }; - let serialized = serde_json::to_string(&test).unwrap(); - let deserialized = Test::context_deserialize( - &mut serde_json::Deserializer::from_str(&serialized), - TestContext1(1), - ) - .unwrap(); - assert_eq!(test, deserialized); - - let deserialized = Test::context_deserialize( - &mut serde_json::Deserializer::from_str(&serialized), - TestContext2("2".to_string()), - ) - .unwrap(); - - assert_eq!(test, deserialized); -} - -#[test] -fn test_context_deserialize_derive_bound() { - use std::fmt::Debug; - - struct TestContext; - - #[derive(Debug, PartialEq, Serialize, Deserialize)] - struct Inner { - value: u64, - } - - #[context_deserialize( - TestContext, - bound = "T: Serialize + for<'a> Deserialize<'a> + Debug + PartialEq" - )] - #[derive(Debug, PartialEq, Serialize, Deserialize)] - struct Wrapper { - inner: T, - } - - let val = Wrapper { - inner: Inner { value: 42 }, - }; - - let serialized = serde_json::to_string(&val).unwrap(); - let deserialized = Wrapper::::context_deserialize( - &mut serde_json::Deserializer::from_str(&serialized), - TestContext, - ) - .unwrap(); - - assert_eq!(val, deserialized); -} diff --git a/consensus/merkle_proof/Cargo.toml b/consensus/merkle_proof/Cargo.toml index d750c054060..5ba8a1b949e 100644 --- a/consensus/merkle_proof/Cargo.toml +++ b/consensus/merkle_proof/Cargo.toml @@ -14,5 +14,4 @@ fixed_bytes = { workspace = true } safe_arith = { workspace = true } [dev-dependencies] -quickcheck = { workspace = true } -quickcheck_macros = { workspace = true } +proptest = { workspace = true } diff --git a/consensus/merkle_proof/src/lib.rs b/consensus/merkle_proof/src/lib.rs index bf075ec15a5..494c73d05ce 100644 --- a/consensus/merkle_proof/src/lib.rs +++ b/consensus/merkle_proof/src/lib.rs @@ -413,50 +413,70 @@ impl From for MerkleTreeError { #[cfg(test)] mod tests { use super::*; - use quickcheck::TestResult; - use quickcheck_macros::quickcheck; - - /// Check that we can: - /// 1. Build a MerkleTree from arbitrary leaves and an arbitrary depth. - /// 2. Generate valid proofs for all of the leaves of this MerkleTree. - #[quickcheck] - fn quickcheck_create_and_verify(int_leaves: Vec, depth: usize) -> TestResult { - if depth > MAX_TREE_DEPTH || int_leaves.len() > 2usize.pow(depth as u32) { - return TestResult::discard(); - } - let leaves: Vec<_> = int_leaves.into_iter().map(H256::from_low_u64_be).collect(); - let merkle_tree = MerkleTree::create(&leaves, depth); - let merkle_root = merkle_tree.hash(); + use proptest::prelude::*; + + // Limit test depth to avoid generating huge trees. Depth 10 = 1024 max leaves. + const TEST_MAX_DEPTH: usize = 10; - let proofs_ok = (0..leaves.len()).all(|i| { - let (leaf, branch) = merkle_tree - .generate_proof(i, depth) - .expect("should generate proof"); - leaf == leaves[i] && verify_merkle_proof(leaf, &branch, depth, i, merkle_root) - }); + fn merkle_leaves_strategy(max_depth: usize) -> impl Strategy, usize)> { + (0..=max_depth).prop_flat_map(|depth| { + let max_leaves = 2usize.pow(depth as u32); + ( + proptest::collection::vec(any::(), 0..=max_leaves), + Just(depth), + ) + }) + } - TestResult::from_bool(proofs_ok) + fn merkle_leaves_strategy_min_depth( + max_depth: usize, + min_depth: usize, + ) -> impl Strategy, usize)> { + (min_depth..=max_depth).prop_flat_map(|depth| { + let max_leaves = 2usize.pow(depth as u32); + ( + proptest::collection::vec(any::(), 0..=max_leaves), + Just(depth), + ) + }) } - #[quickcheck] - fn quickcheck_push_leaf_and_verify(int_leaves: Vec, depth: usize) -> TestResult { - if depth == 0 || depth > MAX_TREE_DEPTH || int_leaves.len() > 2usize.pow(depth as u32) { - return TestResult::discard(); + proptest::proptest! { + /// Check that we can: + /// 1. Build a MerkleTree from arbitrary leaves and an arbitrary depth. + /// 2. Generate valid proofs for all of the leaves of this MerkleTree. + #[test] + fn proptest_create_and_verify((int_leaves, depth) in merkle_leaves_strategy(TEST_MAX_DEPTH)) { + let leaves: Vec<_> = int_leaves.into_iter().map(H256::from_low_u64_be).collect(); + let merkle_tree = MerkleTree::create(&leaves, depth); + let merkle_root = merkle_tree.hash(); + + let proofs_ok = (0..leaves.len()).all(|i| { + let (leaf, branch) = merkle_tree + .generate_proof(i, depth) + .expect("should generate proof"); + leaf == leaves[i] && verify_merkle_proof(leaf, &branch, depth, i, merkle_root) + }); + + proptest::prop_assert!(proofs_ok); } - let leaves_iter = int_leaves.into_iter().map(H256::from_low_u64_be); - let mut merkle_tree = MerkleTree::create(&[], depth); + #[test] + fn proptest_push_leaf_and_verify((int_leaves, depth) in merkle_leaves_strategy_min_depth(TEST_MAX_DEPTH, 1)) { + let leaves_iter = int_leaves.into_iter().map(H256::from_low_u64_be); + let mut merkle_tree = MerkleTree::create(&[], depth); - let proofs_ok = leaves_iter.enumerate().all(|(i, leaf)| { - assert_eq!(merkle_tree.push_leaf(leaf, depth), Ok(())); - let (stored_leaf, branch) = merkle_tree - .generate_proof(i, depth) - .expect("should generate proof"); - stored_leaf == leaf && verify_merkle_proof(leaf, &branch, depth, i, merkle_tree.hash()) - }); + let proofs_ok = leaves_iter.enumerate().all(|(i, leaf)| { + assert_eq!(merkle_tree.push_leaf(leaf, depth), Ok(())); + let (stored_leaf, branch) = merkle_tree + .generate_proof(i, depth) + .expect("should generate proof"); + stored_leaf == leaf && verify_merkle_proof(leaf, &branch, depth, i, merkle_tree.hash()) + }); - TestResult::from_bool(proofs_ok) + proptest::prop_assert!(proofs_ok); + } } #[test] diff --git a/crypto/eth2_key_derivation/Cargo.toml b/crypto/eth2_key_derivation/Cargo.toml index a893a9360dc..b8976b8ccb3 100644 --- a/crypto/eth2_key_derivation/Cargo.toml +++ b/crypto/eth2_key_derivation/Cargo.toml @@ -3,6 +3,7 @@ name = "eth2_key_derivation" version = "0.1.0" authors = ["Paul Hauner "] edition = { workspace = true } +autotests = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] @@ -14,3 +15,7 @@ zeroize = { workspace = true } [dev-dependencies] hex = { workspace = true } + +[[test]] +name = "eth2_key_derivation_tests" +path = "tests/main.rs" diff --git a/crypto/eth2_key_derivation/tests/main.rs b/crypto/eth2_key_derivation/tests/main.rs new file mode 100644 index 00000000000..a239eaa6185 --- /dev/null +++ b/crypto/eth2_key_derivation/tests/main.rs @@ -0,0 +1,2 @@ +mod eip2333_vectors; +mod tests; diff --git a/crypto/eth2_keystore/Cargo.toml b/crypto/eth2_keystore/Cargo.toml index 61d2722efbd..290a10adc9a 100644 --- a/crypto/eth2_keystore/Cargo.toml +++ b/crypto/eth2_keystore/Cargo.toml @@ -3,6 +3,7 @@ name = "eth2_keystore" version = "0.1.0" authors = ["Pawan Dhananjay "] edition = { workspace = true } +autotests = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] @@ -24,3 +25,7 @@ zeroize = { workspace = true } [dev-dependencies] tempfile = { workspace = true } + +[[test]] +name = "eth2_keystore_tests" +path = "tests/main.rs" diff --git a/crypto/eth2_keystore/tests/main.rs b/crypto/eth2_keystore/tests/main.rs new file mode 100644 index 00000000000..79b31d5eda5 --- /dev/null +++ b/crypto/eth2_keystore/tests/main.rs @@ -0,0 +1,4 @@ +mod eip2335_vectors; +mod json; +mod params; +mod tests; diff --git a/crypto/eth2_wallet/Cargo.toml b/crypto/eth2_wallet/Cargo.toml index 5327bdc163b..0d454016a6b 100644 --- a/crypto/eth2_wallet/Cargo.toml +++ b/crypto/eth2_wallet/Cargo.toml @@ -3,6 +3,7 @@ name = "eth2_wallet" version = "0.1.0" authors = ["Paul Hauner "] edition = { workspace = true } +autotests = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] @@ -18,3 +19,7 @@ uuid = { workspace = true } [dev-dependencies] hex = { workspace = true } tempfile = { workspace = true } + +[[test]] +name = "eth2_wallet_tests" +path = "tests/main.rs" diff --git a/crypto/eth2_wallet/tests/main.rs b/crypto/eth2_wallet/tests/main.rs new file mode 100644 index 00000000000..d59ccff6392 --- /dev/null +++ b/crypto/eth2_wallet/tests/main.rs @@ -0,0 +1,3 @@ +mod eip2386_vectors; +mod json; +mod tests; diff --git a/lcli/src/block_root.rs b/lcli/src/block_root.rs index 497ce1a4385..5bb953ee486 100644 --- a/lcli/src/block_root.rs +++ b/lcli/src/block_root.rs @@ -69,7 +69,7 @@ pub fn run( } (None, Some(beacon_url)) => { let block_id: BlockId = parse_required(matches, "block-id")?; - let client = BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(HTTP_TIMEOUT)); + let client = BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(HTTP_TIMEOUT), 0); executor .handle() .ok_or("shutdown in progress")? diff --git a/lcli/src/http_sync.rs b/lcli/src/http_sync.rs index dd941cda74e..fe9ba7d86ad 100644 --- a/lcli/src/http_sync.rs +++ b/lcli/src/http_sync.rs @@ -43,8 +43,8 @@ pub async fn run_async( let cache_dir_path: PathBuf = parse_optional(matches, "block-cache-dir")?.unwrap_or(DEFAULT_CACHE_DIR.into()); - let source = BeaconNodeHttpClient::new(source_url, Timeouts::set_all(HTTP_TIMEOUT)); - let target = BeaconNodeHttpClient::new(target_url, Timeouts::set_all(HTTP_TIMEOUT)); + let source = BeaconNodeHttpClient::new(source_url, Timeouts::set_all(HTTP_TIMEOUT), 0); + let target = BeaconNodeHttpClient::new(target_url, Timeouts::set_all(HTTP_TIMEOUT), 1); if !cache_dir_path.exists() { fs::create_dir_all(&cache_dir_path) diff --git a/lcli/src/skip_slots.rs b/lcli/src/skip_slots.rs index 88332c1a850..caac452d493 100644 --- a/lcli/src/skip_slots.rs +++ b/lcli/src/skip_slots.rs @@ -90,7 +90,7 @@ pub fn run( } (None, Some(beacon_url)) => { let state_id: StateId = parse_required(matches, "state-id")?; - let client = BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(HTTP_TIMEOUT)); + let client = BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(HTTP_TIMEOUT), 0); let state = executor .handle() .ok_or("shutdown in progress")? diff --git a/lcli/src/state_root.rs b/lcli/src/state_root.rs index b4bbae36c8b..38ab08f8864 100644 --- a/lcli/src/state_root.rs +++ b/lcli/src/state_root.rs @@ -38,7 +38,7 @@ pub fn run( } (None, Some(beacon_url)) => { let state_id: StateId = parse_required(matches, "state-id")?; - let client = BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(HTTP_TIMEOUT)); + let client = BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(HTTP_TIMEOUT), 0); executor .handle() .ok_or("shutdown in progress")? diff --git a/lcli/src/transition_blocks.rs b/lcli/src/transition_blocks.rs index 69d3975d09b..52b517d3b01 100644 --- a/lcli/src/transition_blocks.rs +++ b/lcli/src/transition_blocks.rs @@ -143,7 +143,7 @@ pub fn run( } (None, None, Some(beacon_url)) => { let block_id: BlockId = parse_required(matches, "block-id")?; - let client = BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(HTTP_TIMEOUT)); + let client = BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(HTTP_TIMEOUT), 0); let inner_spec = spec.clone(); executor .handle() diff --git a/slasher/Cargo.toml b/slasher/Cargo.toml index cca55bcef88..94d048ef72e 100644 --- a/slasher/Cargo.toml +++ b/slasher/Cargo.toml @@ -3,6 +3,7 @@ name = "slasher" version = "0.1.0" authors = ["Michael Sproul "] edition = { workspace = true } +autotests = false [features] default = ["lmdb"] @@ -43,3 +44,7 @@ types = { workspace = true } maplit = { workspace = true } rayon = { workspace = true } tempfile = { workspace = true } + +[[test]] +name = "slasher_tests" +path = "tests/main.rs" diff --git a/slasher/tests/main.rs b/slasher/tests/main.rs new file mode 100644 index 00000000000..fb78dcb917d --- /dev/null +++ b/slasher/tests/main.rs @@ -0,0 +1,5 @@ +mod attester_slashings; +mod backend; +mod proposer_slashings; +mod random; +mod wrap_around; diff --git a/testing/node_test_rig/src/lib.rs b/testing/node_test_rig/src/lib.rs index e49d11ee1eb..6ad5400ab67 100644 --- a/testing/node_test_rig/src/lib.rs +++ b/testing/node_test_rig/src/lib.rs @@ -89,6 +89,7 @@ impl LocalBeaconNode { beacon_node_url, beacon_node_http_client, Timeouts::set_all(HTTP_TIMEOUT), + 0, )) } } diff --git a/testing/state_transition_vectors/Makefile b/testing/state_transition_vectors/Makefile index 437aa50b00a..c90810ad398 100644 --- a/testing/state_transition_vectors/Makefile +++ b/testing/state_transition_vectors/Makefile @@ -5,4 +5,4 @@ test: cargo test --release --features "$(TEST_FEATURES)" clean: - rm -r vectors/ + rm -rf vectors/ diff --git a/testing/validator_test_rig/src/mock_beacon_node.rs b/testing/validator_test_rig/src/mock_beacon_node.rs index ff1e772d544..bb102773355 100644 --- a/testing/validator_test_rig/src/mock_beacon_node.rs +++ b/testing/validator_test_rig/src/mock_beacon_node.rs @@ -18,12 +18,13 @@ pub struct MockBeaconNode { } impl MockBeaconNode { - pub async fn new() -> Self { + pub async fn new(index: usize) -> Self { // mock server logging let server = Server::new_async().await; let beacon_api_client = BeaconNodeHttpClient::new( SensitiveUrl::from_str(&server.url()).unwrap(), Timeouts::set_all(Duration::from_secs(1)), + index, ); Self { server, diff --git a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs index 136a6af6607..8f93cde7a02 100644 --- a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs +++ b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs @@ -106,7 +106,9 @@ pub async fn poll_head_event_from_beacon_nodes>>, } impl PartialEq for CandidateBeaconNode { fn eq(&self, other: &Self) -> bool { - self.index == other.index && self.beacon_node == other.beacon_node + self.beacon_node == other.beacon_node } } @@ -226,9 +225,8 @@ impl Eq for CandidateBeaconNode {} impl CandidateBeaconNode { /// Instantiate a new node. - pub fn new(beacon_node: BeaconNodeHttpClient, index: usize) -> Self { + pub fn new(beacon_node: BeaconNodeHttpClient) -> Self { Self { - index, beacon_node, health: Arc::new(RwLock::new(Err(CandidateError::Uninitialized))), } @@ -283,7 +281,7 @@ impl CandidateBeaconNode { }; let new_health = BeaconNodeHealth::from_status( - self.index, + self.beacon_node.index, sync_distance, head, optimistic_status, @@ -492,7 +490,7 @@ impl BeaconNodeFallback { } candidate_info.push(CandidateInfo { - index: candidate.index, + index: candidate.beacon_node.index, endpoint: candidate.beacon_node.to_string(), health, }); @@ -524,7 +522,7 @@ impl BeaconNodeFallback { .into_iter() .enumerate() .map(|(index, url)| { - CandidateBeaconNode::new(BeaconNodeHttpClient::new(url, timeouts.clone()), index) + CandidateBeaconNode::new(BeaconNodeHttpClient::new(url, timeouts.clone(), index)) }) .collect(); @@ -640,7 +638,7 @@ impl BeaconNodeFallback { /// Run `func` against each candidate in `self`, returning immediately if a result is found. /// Otherwise, return all the errors encountered along the way. - pub async fn first_success(&self, func: F) -> Result> + pub async fn first_success(&self, func: F) -> Result<(O, usize), Errors> where F: Fn(BeaconNodeHttpClient) -> R, R: Future>, @@ -695,7 +693,7 @@ impl BeaconNodeFallback { &self, preferred_index: Option, func: F, - ) -> Result> + ) -> Result<(O, usize), Errors> where F: Fn(BeaconNodeHttpClient) -> R + Clone, R: Future>, @@ -704,7 +702,9 @@ impl BeaconNodeFallback { // Try the preferred beacon node first if it exists if let Some(preferred_idx) = preferred_index && let candidates = self.candidates.read().await - && let Some(preferred_candidate) = candidates.iter().find(|c| c.index == preferred_idx) + && let Some(preferred_candidate) = candidates + .iter() + .find(|c| c.beacon_node.index == preferred_idx) { let preferred_node = preferred_candidate.beacon_node.clone(); drop(candidates); @@ -725,7 +725,7 @@ impl BeaconNodeFallback { async fn run_on_candidate( candidate: BeaconNodeHttpClient, func: F, - ) -> Result)> + ) -> Result<(O, usize), (String, Error)> where F: Fn(BeaconNodeHttpClient) -> R, R: Future>, @@ -736,7 +736,7 @@ impl BeaconNodeFallback { // There exists a race condition where `func` may be called when the candidate is // actually not ready. We deem this an acceptable inefficiency. match func(candidate.clone()).await { - Ok(val) => Ok(val), + Ok(val) => Ok((val, candidate.index)), Err(e) => { debug!( node = %candidate, @@ -884,8 +884,9 @@ mod tests { let beacon_node = BeaconNodeHttpClient::new( SensitiveUrl::parse(&format!("http://example_{index}.com")).unwrap(), Timeouts::set_all(Duration::from_secs(index as u64)), + index, ); - CandidateBeaconNode::new(beacon_node, index) + CandidateBeaconNode::new(beacon_node) } let candidate_1 = new_candidate(1); @@ -988,11 +989,10 @@ mod tests { index: usize, spec: &ChainSpec, ) -> (MockBeaconNode, CandidateBeaconNode) { - let mut mock_beacon_node = MockBeaconNode::::new().await; + let mut mock_beacon_node = MockBeaconNode::::new(index).await; mock_beacon_node.mock_config_spec(spec); - let beacon_node = - CandidateBeaconNode::new(mock_beacon_node.beacon_api_client.clone(), index); + let beacon_node = CandidateBeaconNode::new(mock_beacon_node.beacon_api_client.clone()); (mock_beacon_node, beacon_node) } diff --git a/validator_client/doppelganger_service/src/lib.rs b/validator_client/doppelganger_service/src/lib.rs index b0ed78e9965..5e493ab4184 100644 --- a/validator_client/doppelganger_service/src/lib.rs +++ b/validator_client/doppelganger_service/src/lib.rs @@ -133,6 +133,7 @@ async fn beacon_node_liveness( } }) .await + .map(|(data, _)| data) .unwrap_or_else(|e| { crit!( error = %e, @@ -168,6 +169,7 @@ async fn beacon_node_liveness( } }) .await + .map(|(data, _)| data) .unwrap_or_else(|e| { crit!( error = %e, diff --git a/validator_client/http_api/src/lib.rs b/validator_client/http_api/src/lib.rs index 4494fca9574..6ab1fbc693e 100644 --- a/validator_client/http_api/src/lib.rs +++ b/validator_client/http_api/src/lib.rs @@ -436,7 +436,7 @@ pub fn serve( let mut beacon_nodes = Vec::new(); for node in &*block_filter.beacon_nodes.candidates.read().await { beacon_nodes.push(CandidateInfo { - index: node.index, + index: node.beacon_node.index, endpoint: node.beacon_node.to_string(), health: *node.health.read().await, }); @@ -447,7 +447,7 @@ pub fn serve( let mut proposer_nodes = Vec::new(); for node in &*proposer_nodes_list.candidates.read().await { proposer_nodes.push(CandidateInfo { - index: node.index, + index: node.beacon_node.index, endpoint: node.beacon_node.to_string(), health: *node.health.read().await, }); diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 71f637f2873..7719080aa52 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -304,6 +304,7 @@ impl ProductionValidatorClient { url.clone(), beacon_node_http_client, timeouts, + i, )) }; @@ -326,8 +327,7 @@ impl ProductionValidatorClient { // the node in `--beacon_nodes`. let candidates = beacon_nodes .into_iter() - .enumerate() - .map(|(index, node)| CandidateBeaconNode::new(node, index)) + .map(CandidateBeaconNode::new) .collect(); let proposer_nodes_num = proposer_nodes.len(); @@ -335,8 +335,7 @@ impl ProductionValidatorClient { // the node in `--proposer_nodes`. let proposer_candidates = proposer_nodes .into_iter() - .enumerate() - .map(|(index, node)| CandidateBeaconNode::new(node, index)) + .map(CandidateBeaconNode::new) .collect(); // Set the count for beacon node fallbacks excluding the primary beacon node. @@ -712,7 +711,7 @@ async fn init_from_beacon_node( .first_success(|node| async move { node.get_beacon_genesis().await }) .await { - Ok(genesis) => break genesis.data, + Ok((genesis, _)) => break genesis.data, Err(errors) => { // Search for a 404 error which indicates that genesis has not yet // occurred. diff --git a/validator_client/validator_services/Cargo.toml b/validator_client/validator_services/Cargo.toml index c9149409148..5fc245cd9b9 100644 --- a/validator_client/validator_services/Cargo.toml +++ b/validator_client/validator_services/Cargo.toml @@ -22,3 +22,8 @@ tree_hash = { workspace = true } types = { workspace = true } validator_metrics = { workspace = true } validator_store = { workspace = true } + +[dev-dependencies] +mockito = { workspace = true } +regex = { workspace = true } +serde_json = { workspace = true } diff --git a/validator_client/validator_services/src/attestation_data_service.rs b/validator_client/validator_services/src/attestation_data_service.rs new file mode 100644 index 00000000000..b2d0f4c0ade --- /dev/null +++ b/validator_client/validator_services/src/attestation_data_service.rs @@ -0,0 +1,312 @@ +use std::sync::Arc; + +use beacon_node_fallback::BeaconNodeFallback; +use slot_clock::SlotClock; +use tracing::{Instrument, info_span}; +use types::{AttestationData, Slot}; + +/// The AttestationDataService is responsible for downloading and caching attestation data at a given slot. +/// It also helps prevent us from re-downloading identical attestation data. +pub struct AttestationDataService { + attestation_data: Option<(Slot, AttestationData, usize)>, + beacon_nodes: Arc>, +} + +impl AttestationDataService { + pub fn new(beacon_nodes: Arc>) -> Self { + Self { + attestation_data: None, + beacon_nodes, + } + } + + /// Get previously downloaded attestation data. + pub fn get_cached_attestation_data( + &self, + requested_slot: &Slot, + ) -> Option<(AttestationData, usize)> { + if let Some((cached_slot, attestation_data, node_index)) = &self.attestation_data + && cached_slot == requested_slot + { + return Some((attestation_data.clone(), *node_index)); + } + None + } + + pub async fn download_data( + &mut self, + request_slot: &Slot, + candidate_beacon_node: Option, + ) -> Result<(AttestationData, usize), String> { + // If we've already downloaded attestation data for `request_slot`, there's no need to re-download the data. + if let Some((attestation_data, node_index)) = self.get_cached_attestation_data(request_slot) + { + return Ok((attestation_data, node_index)); + } + + let (attestation_data, node_index) = self + .beacon_nodes + .first_success_from_index(candidate_beacon_node, |beacon_node| async move { + let _timer = validator_metrics::start_timer_vec( + &validator_metrics::ATTESTATION_SERVICE_TIMES, + &[validator_metrics::ATTESTATIONS_HTTP_GET], + ); + beacon_node + .get_validator_attestation_data(*request_slot, 0) + .await + .map_err(|e| format!("Failed to produce attestation data: {:?}", e)) + .map(|result| result.data) + }) + .instrument(info_span!("fetch_attestation_data")) + .await + .map_err(|e| e.to_string())?; + + self.attestation_data = Some((*request_slot, attestation_data.clone(), node_index)); + + Ok((attestation_data, node_index)) + } +} + +#[cfg(test)] +mod tests { + use std::{sync::Arc, time::Duration}; + + use beacon_node_fallback::{BeaconNodeFallback, CandidateBeaconNode, Config as FallbackConfig}; + use eth2::{SensitiveUrl, Timeouts}; + use slot_clock::{SlotClock, TestingSlotClock}; + use types::{ + AttestationData, Checkpoint, Epoch, EthSpec, FixedBytesExtended, Hash256, MainnetEthSpec, + MinimalEthSpec, Slot, + }; + + use crate::attestation_data_service::AttestationDataService; + + fn create_attestation_data( + slot: Slot, + source_epoch: Epoch, + target_epoch: Epoch, + ) -> AttestationData { + AttestationData { + slot, + index: 0, + beacon_block_root: Hash256::ZERO, + source: Checkpoint { + epoch: source_epoch, + root: Hash256::ZERO, + }, + target: Checkpoint { + epoch: target_epoch, + root: Hash256::from_low_u64_be(target_epoch.as_u64()), + }, + } + } + + fn create_test_beacon_node_fallback() -> BeaconNodeFallback { + let spec = Arc::new(MainnetEthSpec::default_spec()); + let url = SensitiveUrl::parse("http://localhost:5052").unwrap(); + let client = + eth2::BeaconNodeHttpClient::new(url, Timeouts::set_all(Duration::from_secs(1)), 0); + let candidate = CandidateBeaconNode::new(client); + + let mut fallback = + BeaconNodeFallback::new(vec![candidate], FallbackConfig::default(), vec![], spec); + + fallback.set_slot_clock(TestingSlotClock::new( + Slot::new(1), + Duration::from_secs(0), + Duration::from_secs(12), + )); + + fallback + } + + // Helper to create a beacon node with mocked attestation endpoint + async fn create_mocked_beacon_node( + index: usize, + slot: Slot, + attestation_data: AttestationData, + ) -> (mockito::ServerGuard, CandidateBeaconNode) { + use eth2::types::GenericResponse; + use mockito::{Matcher, Server}; + use regex::Regex; + + let mut server = Server::new_async().await; + let data = GenericResponse::from(attestation_data); + + let path_pattern = Regex::new(&format!( + r"^/eth/v1/validator/attestation_data\?slot={}&committee_index=0$", + slot.as_u64() + )) + .unwrap(); + + server + .mock("GET", Matcher::Regex(path_pattern.to_string())) + .with_status(200) + .with_body(serde_json::to_string(&data).unwrap()) + .create(); + + let url = SensitiveUrl::parse(&server.url()).unwrap(); + let client = + eth2::BeaconNodeHttpClient::new(url, Timeouts::set_all(Duration::from_secs(1)), index); + let candidate = CandidateBeaconNode::new(client); + + (server, candidate) + } + + async fn create_offline_beacon_node( + index: usize, + ) -> (mockito::ServerGuard, CandidateBeaconNode) { + use mockito::{Matcher, Server}; + use regex::Regex; + + let mut server = Server::new_async().await; + let path_pattern = Regex::new(r"^/eth/v1/validator/attestation_data").unwrap(); + + server + .mock("GET", Matcher::Regex(path_pattern.to_string())) + .with_status(500) + .create(); + + let url = SensitiveUrl::parse(&server.url()).unwrap(); + let client = + eth2::BeaconNodeHttpClient::new(url, Timeouts::set_all(Duration::from_secs(1)), index); + let candidate = CandidateBeaconNode::new(client); + + (server, candidate) + } + + #[test] + fn test_new_service() { + let beacon_node_fallback = create_test_beacon_node_fallback(); + let service = + AttestationDataService::::new(Arc::new(beacon_node_fallback)); + + assert!(service.attestation_data.is_none()); + assert!(service.get_cached_attestation_data(&Slot::new(1)).is_none()); + } + + #[test] + fn test_get_cached_attestation_data_returns_cached() { + let beacon_node_fallback = create_test_beacon_node_fallback(); + let mut service = + AttestationDataService::::new(Arc::new(beacon_node_fallback)); + + let slot = Slot::new(10); + let beacon_node_index = 0; + let attestation_data = create_attestation_data(slot, Epoch::new(0), Epoch::new(1)); + service.attestation_data = Some((slot, attestation_data.clone(), beacon_node_index)); + + let cached = service.get_cached_attestation_data(&slot); + assert!(cached.is_some()); + assert_eq!(cached.unwrap(), (attestation_data, beacon_node_index)); + } + + #[tokio::test] + async fn test_download_attestation_data() { + let spec = Arc::new(MinimalEthSpec::default_spec()); + let slot = Slot::new(10); + let attestation_data = create_attestation_data(slot, Epoch::new(0), Epoch::new(1)); + + let (_server, beacon_node) = + create_mocked_beacon_node(0, slot, attestation_data.clone()).await; + + let mut fallback = + BeaconNodeFallback::new(vec![beacon_node], FallbackConfig::default(), vec![], spec); + + fallback.set_slot_clock(TestingSlotClock::new( + Slot::new(1), + Duration::from_secs(0), + Duration::from_secs(12), + )); + + let mut service = AttestationDataService::::new(Arc::new(fallback)); + let result = service.download_data(&slot, None).await; + + // Verify download is successful + assert!(result.is_ok()); + assert_eq!(result.unwrap(), (attestation_data.clone(), 0)); + + // Verify data is cached after successful download + assert_eq!( + service.get_cached_attestation_data(&slot), + Some((attestation_data, 0)) + ); + } + + #[tokio::test] + async fn test_download_attestation_data_all_nodes_offline() { + let spec = Arc::new(MainnetEthSpec::default_spec()); + let slot = Slot::new(10); + + // Create two offline nodes + let (_server1, beacon_node_1) = create_offline_beacon_node(0).await; + let (_server2, beacon_node_2) = create_offline_beacon_node(1).await; + + let mut fallback = BeaconNodeFallback::new( + vec![beacon_node_1, beacon_node_2], + FallbackConfig::default(), + vec![], + spec, + ); + + fallback.set_slot_clock(TestingSlotClock::new( + Slot::new(1), + Duration::from_secs(0), + Duration::from_secs(12), + )); + + let mut service = AttestationDataService::::new(Arc::new(fallback)); + let result = service.download_data(&slot, None).await; + + // Verify all nodes offline + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .contains("Failed to produce attestation data") + ); + + // Verify no data was cached since all nodes failed + assert_eq!(service.get_cached_attestation_data(&slot), None); + } + + #[tokio::test] + async fn test_download_attestation_data_node_fallback() { + let spec = Arc::new(MainnetEthSpec::default_spec()); + let slot = Slot::new(10); + let attestation_data = create_attestation_data(slot, Epoch::new(0), Epoch::new(1)); + + // Create one offline node and one working node + let (_server1, beacon_node_1) = create_offline_beacon_node(0).await; + let (_server2, beacon_node_2) = + create_mocked_beacon_node(1, slot, attestation_data.clone()).await; + let (_server2, beacon_node_3) = + create_mocked_beacon_node(2, slot, attestation_data.clone()).await; + + let mut fallback = BeaconNodeFallback::new( + vec![beacon_node_1, beacon_node_2, beacon_node_3], + FallbackConfig::default(), + vec![], + spec, + ); + + fallback.set_slot_clock(TestingSlotClock::new( + Slot::new(1), + Duration::from_secs(0), + Duration::from_secs(12), + )); + + let mut service = AttestationDataService::::new(Arc::new(fallback)); + let result = service.download_data(&slot, None).await; + + // Verify download is successful and we fell back to the next node + assert!(result.is_ok()); + assert_eq!(result.unwrap(), (attestation_data.clone(), 1)); + + // Verify data is cached after successful download + assert_eq!( + service.get_cached_attestation_data(&slot), + Some((attestation_data, 1)) + ); + } +} diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index ed650b5e624..b8f00144ca2 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -1,6 +1,7 @@ use crate::duties_service::{DutiesService, DutyAndProof}; use tokio::sync::Mutex; +use crate::attestation_data_service::AttestationDataService; use beacon_node_fallback::{ApiTopic, BeaconNodeFallback, beacon_head_monitor::HeadEvent}; use futures::future::join_all; use logging::crit; @@ -9,9 +10,10 @@ use std::collections::HashMap; use std::ops::Deref; use std::sync::Arc; use task_executor::TaskExecutor; +use tokio::sync::RwLock; use tokio::sync::mpsc; use tokio::time::{Duration, Instant, sleep, sleep_until}; -use tracing::{debug, error, info, warn}; +use tracing::{Instrument, debug, error, info, info_span, instrument, warn}; use tree_hash::TreeHash; use types::{Attestation, AttestationData, ChainSpec, CommitteeIndex, EthSpec, Slot}; use validator_store::{Error as ValidatorStoreError, ValidatorStore}; @@ -26,6 +28,7 @@ pub struct AttestationServiceBuilder executor: Option, chain_spec: Option>, head_monitor_rx: Option>>>, + attestation_data_service: Option>>>, disable: bool, } @@ -39,6 +42,7 @@ impl AttestationServiceBuil executor: None, chain_spec: None, head_monitor_rx: None, + attestation_data_service: None, disable: false, } } @@ -59,7 +63,11 @@ impl AttestationServiceBuil } pub fn beacon_nodes(mut self, beacon_nodes: Arc>) -> Self { - self.beacon_nodes = Some(beacon_nodes); + self.beacon_nodes = Some(beacon_nodes.clone()); + self.attestation_data_service = Some(Arc::new(RwLock::new(AttestationDataService::new( + beacon_nodes, + )))); + self } @@ -106,6 +114,9 @@ impl AttestationServiceBuil chain_spec: self .chain_spec .ok_or("Cannot build AttestationService without chain_spec")?, + attestation_data_service: self + .attestation_data_service + .ok_or("Cannot build AttestationService without attestation_data_service")?, head_monitor_rx: self.head_monitor_rx, disable: self.disable, latest_attested_slot: Mutex::new(Slot::default()), @@ -123,6 +134,7 @@ pub struct Inner { executor: TaskExecutor, chain_spec: Arc, head_monitor_rx: Option>>>, + attestation_data_service: Arc>>, disable: bool, latest_attested_slot: Mutex, } @@ -305,6 +317,11 @@ impl AttestationService AttestationService AttestationService attestation_data, + None => { + let mut attestation_data_service = self.attestation_data_service.write().await; + attestation_data_service + .download_data(&slot, candidate_beacon_node) .await - .map_err(|e| format!("Failed to produce attestation data: {:?}", e)) - .map(|result| result.data) - }) - .await - .map_err(|e| e.to_string())?; + .map(|(data, _)| data)? + } + }; // Create futures to produce signed `Attestation` objects. let attestation_data_ref = &attestation_data; @@ -508,6 +523,10 @@ impl AttestationService, Vec<_>) = join_all(signing_futures) + .instrument(info_span!( + "sign_attestations", + count = validator_duties.len() + )) .await .into_iter() .flatten() @@ -556,6 +575,10 @@ impl AttestationService(single_attestations, fork_name) .await }) + .instrument(info_span!( + "publish_attestations", + count = attestations.len() + )) .await { Ok(()) => info!( @@ -592,6 +615,7 @@ impl AttestationService AttestationService AttestationService AttestationService { for signed_aggregate_and_proof in signed_aggregate_and_proofs { diff --git a/validator_client/validator_services/src/block_service.rs b/validator_client/validator_services/src/block_service.rs index c111b1f22eb..564c35611e5 100644 --- a/validator_client/validator_services/src/block_service.rs +++ b/validator_client/validator_services/src/block_service.rs @@ -1,5 +1,4 @@ use beacon_node_fallback::{ApiTopic, BeaconNodeFallback, Error as FallbackError, Errors}; -use bls::SignatureBytes; use eth2::{BeaconNodeHttpClient, StatusCode}; use graffiti_file::{GraffitiFile, determine_graffiti}; use logging::crit; @@ -11,7 +10,7 @@ use std::sync::Arc; use std::time::Duration; use task_executor::TaskExecutor; use tokio::sync::mpsc; -use tracing::{debug, error, info, trace, warn}; +use tracing::{Instrument, debug, error, info, info_span, instrument, trace, warn}; use types::{BlockType, ChainSpec, EthSpec, Graffiti, PublicKeyBytes, Slot}; use validator_store::{Error as ValidatorStoreError, SignedBlock, UnsignedBlock, ValidatorStore}; @@ -173,11 +172,14 @@ impl ProposerFallback { match (beacon_nodes_result, &self.proposer_nodes) { // The non-proposer node call succeed, return the result. - (Ok(success), _) => Ok(success), + (Ok((data, _)), _) => Ok(data), // The non-proposer node call failed, but we don't have any proposer nodes. Return an error. (Err(e), None) => Err(e), // The non-proposer node call failed, try the same call on the proposer nodes. - (Err(_), Some(proposer_nodes)) => proposer_nodes.first_success(func).await, + (Err(_), Some(proposer_nodes)) => proposer_nodes + .first_success(func) + .await + .map(|(data, _)| data), } } } @@ -298,7 +300,7 @@ impl BlockService { self.inner.executor.spawn( async move { let result = service - .publish_block(slot, validator_pubkey, builder_boost_factor) + .get_validator_block_and_publish_block(slot, validator_pubkey, builder_boost_factor) .await; match result { @@ -320,6 +322,7 @@ impl BlockService { } #[allow(clippy::too_many_arguments)] + #[instrument(skip_all, fields(%slot, ?validator_pubkey))] async fn sign_and_publish_block( &self, proposer_fallback: ProposerFallback, @@ -333,6 +336,7 @@ impl BlockService { let res = self .validator_store .sign_block(*validator_pubkey, unsigned_block, slot) + .instrument(info_span!("sign_block")) .await; let signed_block = match res { @@ -389,7 +393,12 @@ impl BlockService { Ok(()) } - async fn publish_block( + #[instrument( + name = "block_proposal_duty_cycle", + skip_all, + fields(%slot, ?validator_pubkey) + )] + async fn get_validator_block_and_publish_block( self, slot: Slot, validator_pubkey: PublicKeyBytes, @@ -442,33 +451,80 @@ impl BlockService { info!(slot = slot.as_u64(), "Requesting unsigned block"); - // Request block from first responsive beacon node. + // Request an SSZ block from all beacon nodes in order, returning on the first successful response. + // If all nodes fail, run a second pass falling back to JSON. // - // Try the proposer nodes last, since it's likely that they don't have a + // Proposer nodes will always be tried last during each pass since it's likely that they don't have a // great view of attestations on the network. - let unsigned_block = proposer_fallback + let ssz_block_response = proposer_fallback .request_proposers_last(|beacon_node| async move { let _get_timer = validator_metrics::start_timer_vec( &validator_metrics::BLOCK_SERVICE_TIMES, &[validator_metrics::BEACON_BLOCK_HTTP_GET], ); - Self::get_validator_block( - &beacon_node, - slot, - randao_reveal_ref, - graffiti, - proposer_index, - builder_boost_factor, - ) - .await - .map_err(|e| { - BlockError::Recoverable(format!( - "Error from beacon node when producing block: {:?}", - e - )) - }) + beacon_node + .get_validator_blocks_v3_ssz::( + slot, + randao_reveal_ref, + graffiti.as_ref(), + builder_boost_factor, + ) + .await }) - .await?; + .await; + + let block_response = match ssz_block_response { + Ok((ssz_block_response, _metadata)) => ssz_block_response, + Err(e) => { + warn!( + slot = slot.as_u64(), + error = %e, + "SSZ block production failed, falling back to JSON" + ); + + proposer_fallback + .request_proposers_last(|beacon_node| async move { + let _get_timer = validator_metrics::start_timer_vec( + &validator_metrics::BLOCK_SERVICE_TIMES, + &[validator_metrics::BEACON_BLOCK_HTTP_GET], + ); + let (json_block_response, _metadata) = beacon_node + .get_validator_blocks_v3::( + slot, + randao_reveal_ref, + graffiti.as_ref(), + builder_boost_factor, + ) + .await + .map_err(|e| { + BlockError::Recoverable(format!( + "Error from beacon node when producing block: {:?}", + e + )) + })?; + + Ok(json_block_response.data) + }) + .await + .map_err(BlockError::from)? + } + }; + + let (block_proposer, unsigned_block) = match block_response { + eth2::types::ProduceBlockV3Response::Full(block) => { + (block.block().proposer_index(), UnsignedBlock::Full(block)) + } + eth2::types::ProduceBlockV3Response::Blinded(block) => { + (block.proposer_index(), UnsignedBlock::Blinded(block)) + } + }; + + info!(slot = slot.as_u64(), "Received unsigned block"); + if proposer_index != Some(block_proposer) { + return Err(BlockError::Recoverable( + "Proposer index does not match block proposer. Beacon chain re-orged".to_string(), + )); + } self_ref .sign_and_publish_block( @@ -483,6 +539,7 @@ impl BlockService { Ok(()) } + #[instrument(skip_all)] async fn publish_signed_block_contents( &self, signed_block: &SignedBlock, @@ -517,70 +574,6 @@ impl BlockService { } Ok::<_, BlockError>(()) } - - async fn get_validator_block( - beacon_node: &BeaconNodeHttpClient, - slot: Slot, - randao_reveal_ref: &SignatureBytes, - graffiti: Option, - proposer_index: Option, - builder_boost_factor: Option, - ) -> Result, BlockError> { - let block_response = match beacon_node - .get_validator_blocks_v3_ssz::( - slot, - randao_reveal_ref, - graffiti.as_ref(), - builder_boost_factor, - ) - .await - { - Ok((ssz_block_response, _)) => ssz_block_response, - Err(e) => { - warn!( - slot = slot.as_u64(), - error = %e, - "Beacon node does not support SSZ in block production, falling back to JSON" - ); - - let (json_block_response, _) = beacon_node - .get_validator_blocks_v3::( - slot, - randao_reveal_ref, - graffiti.as_ref(), - builder_boost_factor, - ) - .await - .map_err(|e| { - BlockError::Recoverable(format!( - "Error from beacon node when producing block: {:?}", - e - )) - })?; - - // Extract ProduceBlockV3Response (data field of the struct ForkVersionedResponse) - json_block_response.data - } - }; - - let (block_proposer, unsigned_block) = match block_response { - eth2::types::ProduceBlockV3Response::Full(block) => { - (block.block().proposer_index(), UnsignedBlock::Full(block)) - } - eth2::types::ProduceBlockV3Response::Blinded(block) => { - (block.proposer_index(), UnsignedBlock::Blinded(block)) - } - }; - - info!(slot = slot.as_u64(), "Received unsigned block"); - if proposer_index != Some(block_proposer) { - return Err(BlockError::Recoverable( - "Proposer index does not match block proposer. Beacon chain re-orged".to_string(), - )); - } - - Ok::<_, BlockError>(unsigned_block) - } } /// Wrapper for values we want to log about a block we signed, for easy extraction from the possible diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index 7569d3946ab..085fbf3c421 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -179,6 +179,7 @@ async fn make_selection_proof( .await; let response_data = middleware_response + .map(|(data, _)| data) .map_err(|e| { Error::FailedToProduceSelectionProof(ValidatorStoreError::Middleware(e.to_string())) })? @@ -727,7 +728,7 @@ async fn poll_validator_indices( .to_string() }); match download_result { - Ok(Some(response)) => { + Ok((Some(response), _)) => { info!( ?pubkey, validator_index = response.data.index, @@ -745,7 +746,7 @@ async fn poll_validator_indices( } // This is not necessarily an error, it just means the validator is not yet known to // the beacon chain. - Ok(None) => { + Ok((None, _)) => { if let Some(current_slot) = current_slot_opt { let next_poll_slot = current_slot.saturating_add(S::E::slots_per_epoch()); duties_service @@ -1202,6 +1203,7 @@ async fn post_validator_duties_attester( .await; match download_result { - Ok(response) => { + Ok((response, _)) => { let dependent_root = response.dependent_root; let relevant_duties = response diff --git a/validator_client/validator_services/src/lib.rs b/validator_client/validator_services/src/lib.rs index 3b8bd9ae14b..e0e05711786 100644 --- a/validator_client/validator_services/src/lib.rs +++ b/validator_client/validator_services/src/lib.rs @@ -1,3 +1,4 @@ +pub mod attestation_data_service; pub mod attestation_service; pub mod block_service; pub mod duties_service; diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index 77032ed15b4..d188e44aae8 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -437,7 +437,7 @@ pub async fn poll_sync_committee_duties_for_period res.data, + Ok((res, _)) => res.data, Err(e) => { warn!( sync_committee_period, @@ -557,7 +557,7 @@ pub async fn make_sync_selection_proof( .await; match middleware_response { - Ok(mut response) => { + Ok((mut response, _)) => { let Some(response_data) = response.data.pop() else { error!( validator_index = duty.validator_index, diff --git a/validator_client/validator_services/src/sync_committee_service.rs b/validator_client/validator_services/src/sync_committee_service.rs index 02f9f24c8a1..95c05f49d95 100644 --- a/validator_client/validator_services/src/sync_committee_service.rs +++ b/validator_client/validator_services/src/sync_committee_service.rs @@ -11,7 +11,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use task_executor::TaskExecutor; use tokio::time::{Duration, Instant, sleep, sleep_until}; -use tracing::{debug, error, info, trace, warn}; +use tracing::{Instrument, debug, error, info, info_span, instrument, trace, warn}; use types::{ ChainSpec, EthSpec, Hash256, PublicKeyBytes, Slot, SyncCommitteeSubscription, SyncContributionData, SyncDuty, SyncSelectionProof, SyncSubnetId, @@ -187,7 +187,7 @@ impl SyncCommitteeService block.data.root, + Ok((block, _)) => block.data.root, Err(errs) => { warn!( errors = errs.to_string(), @@ -208,7 +208,8 @@ impl SyncCommitteeService SyncCommitteeService SyncCommitteeService SyncCommitteeService SyncCommitteeService SyncCommitteeService SyncCommitteeService SyncCommitteeService SyncCommitteeService SyncCommitteeService(config: ExitConfig) -> Result<(), String> { let beacon_node = BeaconNodeHttpClient::new( beacon_url.clone(), Timeouts::set_all(Duration::from_secs(12)), + DEFAULT_BEACON_NODE_INDEX, ); if beacon_node diff --git a/validator_manager/src/list_validators.rs b/validator_manager/src/list_validators.rs index 082894a995d..4e57d3e845e 100644 --- a/validator_manager/src/list_validators.rs +++ b/validator_manager/src/list_validators.rs @@ -7,6 +7,7 @@ use std::path::PathBuf; use std::time::Duration; use types::{ChainSpec, EthSpec, PublicKeyBytes}; +use crate::common::DEFAULT_BEACON_NODE_INDEX; use crate::exit_validators::get_current_epoch; use crate::{DumpConfig, common::vc_http_client}; @@ -136,6 +137,7 @@ async fn run(config: ListConfig) -> Result Date: Mon, 1 Dec 2025 18:19:45 -0300 Subject: [PATCH 28/41] Fix --- validator_client/beacon_node_fallback/src/lib.rs | 16 +++++++++++++--- validator_client/doppelganger_service/src/lib.rs | 2 -- validator_client/src/lib.rs | 2 +- .../src/attestation_service.rs | 2 -- .../validator_services/src/block_service.rs | 3 +-- .../validator_services/src/duties_service.rs | 10 ++++------ validator_client/validator_services/src/sync.rs | 4 ++-- .../src/sync_committee_service.rs | 5 ++--- 8 files changed, 23 insertions(+), 21 deletions(-) diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index d5a9f3096b1..64c241557f0 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -636,9 +636,19 @@ impl BeaconNodeFallback { .collect() } + /// A wrapper for `first_success_with_index` when the beacon node `index` is not needed. + pub async fn first_success(&self, func: F) -> Result> + where + F: Fn(BeaconNodeHttpClient) -> R, + R: Future>, + Err: Debug, + { + self.first_success_with_index(func).await.map(|(val, _)| val) + } + /// Run `func` against each candidate in `self`, returning immediately if a result is found. /// Otherwise, return all the errors encountered along the way. - pub async fn first_success(&self, func: F) -> Result<(O, usize), Errors> + pub async fn first_success_with_index(&self, func: F) -> Result<(O, usize), Errors> where F: Fn(BeaconNodeHttpClient) -> R, R: Future>, @@ -712,13 +722,13 @@ impl BeaconNodeFallback { match Self::run_on_candidate(preferred_node, &func).await { Ok(val) => return Ok(val), Err(_) => { - return self.first_success(func).await; + return self.first_success_with_index(func).await; } } } // Fall back to normal first_success behavior - self.first_success(func).await + self.first_success_with_index(func).await } /// Run the future `func` on `candidate` while reporting metrics. diff --git a/validator_client/doppelganger_service/src/lib.rs b/validator_client/doppelganger_service/src/lib.rs index 5e493ab4184..b0ed78e9965 100644 --- a/validator_client/doppelganger_service/src/lib.rs +++ b/validator_client/doppelganger_service/src/lib.rs @@ -133,7 +133,6 @@ async fn beacon_node_liveness( } }) .await - .map(|(data, _)| data) .unwrap_or_else(|e| { crit!( error = %e, @@ -169,7 +168,6 @@ async fn beacon_node_liveness( } }) .await - .map(|(data, _)| data) .unwrap_or_else(|e| { crit!( error = %e, diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 7719080aa52..4ef560c4049 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -711,7 +711,7 @@ async fn init_from_beacon_node( .first_success(|node| async move { node.get_beacon_genesis().await }) .await { - Ok((genesis, _)) => break genesis.data, + Ok(genesis) => break genesis.data, Err(errors) => { // Search for a 404 error which indicates that genesis has not yet // occurred. diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index b8f00144ca2..3182e169574 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -670,7 +670,6 @@ impl AttestationService AttestationService { for signed_aggregate_and_proof in signed_aggregate_and_proofs { diff --git a/validator_client/validator_services/src/block_service.rs b/validator_client/validator_services/src/block_service.rs index 564c35611e5..5f87858daa1 100644 --- a/validator_client/validator_services/src/block_service.rs +++ b/validator_client/validator_services/src/block_service.rs @@ -172,14 +172,13 @@ impl ProposerFallback { match (beacon_nodes_result, &self.proposer_nodes) { // The non-proposer node call succeed, return the result. - (Ok((data, _)), _) => Ok(data), + (Ok(data), _) => Ok(data), // The non-proposer node call failed, but we don't have any proposer nodes. Return an error. (Err(e), None) => Err(e), // The non-proposer node call failed, try the same call on the proposer nodes. (Err(_), Some(proposer_nodes)) => proposer_nodes .first_success(func) .await - .map(|(data, _)| data), } } } diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index 085fbf3c421..fea08a09ef9 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -179,7 +179,6 @@ async fn make_selection_proof( .await; let response_data = middleware_response - .map(|(data, _)| data) .map_err(|e| { Error::FailedToProduceSelectionProof(ValidatorStoreError::Middleware(e.to_string())) })? @@ -716,7 +715,7 @@ async fn poll_validator_indices( ) .await }) - .await; + .await?; let fee_recipient = duties_service .validator_store @@ -728,7 +727,7 @@ async fn poll_validator_indices( .to_string() }); match download_result { - Ok((Some(response), _)) => { + Ok(response) => { info!( ?pubkey, validator_index = response.data.index, @@ -746,7 +745,7 @@ async fn poll_validator_indices( } // This is not necessarily an error, it just means the validator is not yet known to // the beacon chain. - Ok((None, _)) => { + Ok(None) => { if let Some(current_slot) = current_slot_opt { let next_poll_slot = current_slot.saturating_add(S::E::slots_per_epoch()); duties_service @@ -1203,7 +1202,6 @@ async fn post_validator_duties_attester( .await; match download_result { - Ok((response, _)) => { + Ok(response) => { let dependent_root = response.dependent_root; let relevant_duties = response diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index d188e44aae8..77032ed15b4 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -437,7 +437,7 @@ pub async fn poll_sync_committee_duties_for_period res.data, + Ok(res) => res.data, Err(e) => { warn!( sync_committee_period, @@ -557,7 +557,7 @@ pub async fn make_sync_selection_proof( .await; match middleware_response { - Ok((mut response, _)) => { + Ok(mut response) => { let Some(response_data) = response.data.pop() else { error!( validator_index = duty.validator_index, diff --git a/validator_client/validator_services/src/sync_committee_service.rs b/validator_client/validator_services/src/sync_committee_service.rs index 95c05f49d95..5f6b1cb710f 100644 --- a/validator_client/validator_services/src/sync_committee_service.rs +++ b/validator_client/validator_services/src/sync_committee_service.rs @@ -187,7 +187,7 @@ impl SyncCommitteeService block.data.root, + Ok(block) => block.data.root, Err(errs) => { warn!( errors = errs.to_string(), @@ -378,8 +378,7 @@ impl SyncCommitteeService Date: Mon, 1 Dec 2025 18:21:00 -0300 Subject: [PATCH 29/41] fix --- validator_client/validator_services/src/duties_service.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index fea08a09ef9..7569d3946ab 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -715,7 +715,7 @@ async fn poll_validator_indices( ) .await }) - .await?; + .await; let fee_recipient = duties_service .validator_store @@ -727,7 +727,7 @@ async fn poll_validator_indices( .to_string() }); match download_result { - Ok(response) => { + Ok(Some(response)) => { info!( ?pubkey, validator_index = response.data.index, From 6ffcfdb1c25f5c7dea58fa16ffc76fa7d1cffdb4 Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Mon, 1 Dec 2025 18:26:06 -0300 Subject: [PATCH 30/41] Clean up --- account_manager/src/validator/exit.rs | 3 --- beacon_node/client/src/builder.rs | 1 - .../src/test_utils/mock_builder.rs | 2 +- beacon_node/http_api/src/test_utils.rs | 2 +- beacon_node/http_api/tests/tests.rs | 2 -- common/eth2/src/lib.rs | 17 +++++++++++-- lcli/src/block_root.rs | 2 +- lcli/src/http_sync.rs | 4 +-- lcli/src/skip_slots.rs | 2 +- lcli/src/state_root.rs | 2 +- lcli/src/transition_blocks.rs | 2 +- .../src/mock_beacon_node.rs | 2 +- .../src/beacon_head_monitor.rs | 4 +-- .../beacon_node_fallback/src/lib.rs | 25 +++++++++++++------ validator_client/http_api/src/lib.rs | 4 +-- .../src/attestation_data_service.rs | 16 ++++++++---- .../validator_services/src/block_service.rs | 4 +-- validator_manager/src/common.rs | 2 -- validator_manager/src/create_validators.rs | 7 ++---- validator_manager/src/exit_validators.rs | 2 -- validator_manager/src/list_validators.rs | 2 -- 21 files changed, 59 insertions(+), 48 deletions(-) diff --git a/account_manager/src/validator/exit.rs b/account_manager/src/validator/exit.rs index 62e07d81ed9..5ea77f284e2 100644 --- a/account_manager/src/validator/exit.rs +++ b/account_manager/src/validator/exit.rs @@ -31,8 +31,6 @@ pub const DEFAULT_BEACON_NODE: &str = "http://localhost:5052/"; pub const CONFIRMATION_PHRASE: &str = "Exit my validator"; pub const WEBSITE_URL: &str = "https://lighthouse-book.sigmaprime.io/validator_voluntary_exit.html"; -pub const DEFAULT_BEACON_NODE_INDEX: usize = 0; - pub fn cli_app() -> Command { Command::new("exit") .about("Submits a VoluntaryExit to the beacon chain for a given validator keystore.") @@ -105,7 +103,6 @@ pub fn cli_run(matches: &ArgMatches, env: Environment) -> Result< SensitiveUrl::parse(&server_url) .map_err(|e| format!("Failed to parse beacon http server: {:?}", e))?, Timeouts::set_all(Duration::from_secs(env.eth2_config.spec.seconds_per_slot)), - DEFAULT_BEACON_NODE_INDEX, ); let eth2_network_config = env diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index 21f7618443c..c48021e45d4 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -389,7 +389,6 @@ where Timeouts::set_all(Duration::from_secs( config.chain.checkpoint_sync_url_timeout, )), - 0, ); debug!("Downloading finalized state"); diff --git a/beacon_node/execution_layer/src/test_utils/mock_builder.rs b/beacon_node/execution_layer/src/test_utils/mock_builder.rs index 12eadefa6fb..9add1369194 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_builder.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_builder.rs @@ -368,7 +368,7 @@ impl MockBuilder { let builder = MockBuilder::new( el, - BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(Duration::from_secs(1)), 0), + BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(Duration::from_secs(1))), validate_pubkey, apply_operations, broadcast_to_bn, diff --git a/beacon_node/http_api/src/test_utils.rs b/beacon_node/http_api/src/test_utils.rs index dd2247b2a8a..27e2a27d35c 100644 --- a/beacon_node/http_api/src/test_utils.rs +++ b/beacon_node/http_api/src/test_utils.rs @@ -169,7 +169,7 @@ impl InteractiveTester { default: Duration::from_secs(5), ..Timeouts::set_all(Duration::from_secs(5)) }; - let client = BeaconNodeHttpClient::new(beacon_url.clone(), timeouts, 0); + let client = BeaconNodeHttpClient::new(beacon_url.clone(), timeouts); Self { ctx, diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 9966e7de84a..8d99e696cf7 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -322,7 +322,6 @@ impl ApiTester { let client = BeaconNodeHttpClient::new( beacon_url, Timeouts::set_all(Duration::from_secs(SECONDS_PER_SLOT)), - 0, ); Self { @@ -411,7 +410,6 @@ impl ApiTester { )) .unwrap(), Timeouts::set_all(Duration::from_secs(SECONDS_PER_SLOT)), - 0, ); Self { diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index b0225574617..f8f046a9353 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -142,7 +142,7 @@ pub struct BeaconNodeHttpClient { client: reqwest::Client, server: SensitiveUrl, timeouts: Timeouts, - pub index: usize, + index: usize, } impl Eq for BeaconNodeHttpClient {} @@ -154,7 +154,20 @@ impl fmt::Display for BeaconNodeHttpClient { } impl BeaconNodeHttpClient { - pub fn new(server: SensitiveUrl, timeouts: Timeouts, index: usize) -> Self { + pub fn new(server: SensitiveUrl, timeouts: Timeouts) -> Self { + Self { + client: reqwest::Client::new(), + server, + timeouts, + index: 0, + } + } + + pub fn index(&self) -> usize { + self.index + } + + pub fn new_with_index(server: SensitiveUrl, timeouts: Timeouts, index: usize) -> Self { Self { client: reqwest::Client::new(), server, diff --git a/lcli/src/block_root.rs b/lcli/src/block_root.rs index 5bb953ee486..497ce1a4385 100644 --- a/lcli/src/block_root.rs +++ b/lcli/src/block_root.rs @@ -69,7 +69,7 @@ pub fn run( } (None, Some(beacon_url)) => { let block_id: BlockId = parse_required(matches, "block-id")?; - let client = BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(HTTP_TIMEOUT), 0); + let client = BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(HTTP_TIMEOUT)); executor .handle() .ok_or("shutdown in progress")? diff --git a/lcli/src/http_sync.rs b/lcli/src/http_sync.rs index fe9ba7d86ad..dd941cda74e 100644 --- a/lcli/src/http_sync.rs +++ b/lcli/src/http_sync.rs @@ -43,8 +43,8 @@ pub async fn run_async( let cache_dir_path: PathBuf = parse_optional(matches, "block-cache-dir")?.unwrap_or(DEFAULT_CACHE_DIR.into()); - let source = BeaconNodeHttpClient::new(source_url, Timeouts::set_all(HTTP_TIMEOUT), 0); - let target = BeaconNodeHttpClient::new(target_url, Timeouts::set_all(HTTP_TIMEOUT), 1); + let source = BeaconNodeHttpClient::new(source_url, Timeouts::set_all(HTTP_TIMEOUT)); + let target = BeaconNodeHttpClient::new(target_url, Timeouts::set_all(HTTP_TIMEOUT)); if !cache_dir_path.exists() { fs::create_dir_all(&cache_dir_path) diff --git a/lcli/src/skip_slots.rs b/lcli/src/skip_slots.rs index caac452d493..88332c1a850 100644 --- a/lcli/src/skip_slots.rs +++ b/lcli/src/skip_slots.rs @@ -90,7 +90,7 @@ pub fn run( } (None, Some(beacon_url)) => { let state_id: StateId = parse_required(matches, "state-id")?; - let client = BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(HTTP_TIMEOUT), 0); + let client = BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(HTTP_TIMEOUT)); let state = executor .handle() .ok_or("shutdown in progress")? diff --git a/lcli/src/state_root.rs b/lcli/src/state_root.rs index 38ab08f8864..b4bbae36c8b 100644 --- a/lcli/src/state_root.rs +++ b/lcli/src/state_root.rs @@ -38,7 +38,7 @@ pub fn run( } (None, Some(beacon_url)) => { let state_id: StateId = parse_required(matches, "state-id")?; - let client = BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(HTTP_TIMEOUT), 0); + let client = BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(HTTP_TIMEOUT)); executor .handle() .ok_or("shutdown in progress")? diff --git a/lcli/src/transition_blocks.rs b/lcli/src/transition_blocks.rs index 52b517d3b01..69d3975d09b 100644 --- a/lcli/src/transition_blocks.rs +++ b/lcli/src/transition_blocks.rs @@ -143,7 +143,7 @@ pub fn run( } (None, None, Some(beacon_url)) => { let block_id: BlockId = parse_required(matches, "block-id")?; - let client = BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(HTTP_TIMEOUT), 0); + let client = BeaconNodeHttpClient::new(beacon_url, Timeouts::set_all(HTTP_TIMEOUT)); let inner_spec = spec.clone(); executor .handle() diff --git a/testing/validator_test_rig/src/mock_beacon_node.rs b/testing/validator_test_rig/src/mock_beacon_node.rs index bb102773355..7b7fa8e64ca 100644 --- a/testing/validator_test_rig/src/mock_beacon_node.rs +++ b/testing/validator_test_rig/src/mock_beacon_node.rs @@ -21,7 +21,7 @@ impl MockBeaconNode { pub async fn new(index: usize) -> Self { // mock server logging let server = Server::new_async().await; - let beacon_api_client = BeaconNodeHttpClient::new( + let beacon_api_client = BeaconNodeHttpClient::new_with_index( SensitiveUrl::from_str(&server.url()).unwrap(), Timeouts::set_all(Duration::from_secs(1)), index, diff --git a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs index 8f93cde7a02..7a460d290b4 100644 --- a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs +++ b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs @@ -107,7 +107,7 @@ pub async fn poll_head_event_from_beacon_nodes BeaconNodeFallback { } candidate_info.push(CandidateInfo { - index: candidate.beacon_node.index, + index: candidate.beacon_node.index(), endpoint: candidate.beacon_node.to_string(), health, }); @@ -522,7 +522,11 @@ impl BeaconNodeFallback { .into_iter() .enumerate() .map(|(index, url)| { - CandidateBeaconNode::new(BeaconNodeHttpClient::new(url, timeouts.clone(), index)) + CandidateBeaconNode::new(BeaconNodeHttpClient::new_with_index( + url, + timeouts.clone(), + index, + )) }) .collect(); @@ -643,12 +647,17 @@ impl BeaconNodeFallback { R: Future>, Err: Debug, { - self.first_success_with_index(func).await.map(|(val, _)| val) + self.first_success_with_index(func) + .await + .map(|(val, _)| val) } /// Run `func` against each candidate in `self`, returning immediately if a result is found. /// Otherwise, return all the errors encountered along the way. - pub async fn first_success_with_index(&self, func: F) -> Result<(O, usize), Errors> + pub async fn first_success_with_index( + &self, + func: F, + ) -> Result<(O, usize), Errors> where F: Fn(BeaconNodeHttpClient) -> R, R: Future>, @@ -714,7 +723,7 @@ impl BeaconNodeFallback { && let candidates = self.candidates.read().await && let Some(preferred_candidate) = candidates .iter() - .find(|c| c.beacon_node.index == preferred_idx) + .find(|c| c.beacon_node.index() == preferred_idx) { let preferred_node = preferred_candidate.beacon_node.clone(); drop(candidates); @@ -746,7 +755,7 @@ impl BeaconNodeFallback { // There exists a race condition where `func` may be called when the candidate is // actually not ready. We deem this an acceptable inefficiency. match func(candidate.clone()).await { - Ok(val) => Ok((val, candidate.index)), + Ok(val) => Ok((val, candidate.index())), Err(e) => { debug!( node = %candidate, @@ -891,7 +900,7 @@ mod tests { let execution_status = ExecutionEngineHealth::Healthy; fn new_candidate(index: usize) -> CandidateBeaconNode { - let beacon_node = BeaconNodeHttpClient::new( + let beacon_node = BeaconNodeHttpClient::new_with_index( SensitiveUrl::parse(&format!("http://example_{index}.com")).unwrap(), Timeouts::set_all(Duration::from_secs(index as u64)), index, diff --git a/validator_client/http_api/src/lib.rs b/validator_client/http_api/src/lib.rs index 6ab1fbc693e..1ab16330245 100644 --- a/validator_client/http_api/src/lib.rs +++ b/validator_client/http_api/src/lib.rs @@ -436,7 +436,7 @@ pub fn serve( let mut beacon_nodes = Vec::new(); for node in &*block_filter.beacon_nodes.candidates.read().await { beacon_nodes.push(CandidateInfo { - index: node.beacon_node.index, + index: node.beacon_node.index(), endpoint: node.beacon_node.to_string(), health: *node.health.read().await, }); @@ -447,7 +447,7 @@ pub fn serve( let mut proposer_nodes = Vec::new(); for node in &*proposer_nodes_list.candidates.read().await { proposer_nodes.push(CandidateInfo { - index: node.beacon_node.index, + index: node.beacon_node.index(), endpoint: node.beacon_node.to_string(), health: *node.health.read().await, }); diff --git a/validator_client/validator_services/src/attestation_data_service.rs b/validator_client/validator_services/src/attestation_data_service.rs index b2d0f4c0ade..43e58f303ad 100644 --- a/validator_client/validator_services/src/attestation_data_service.rs +++ b/validator_client/validator_services/src/attestation_data_service.rs @@ -105,7 +105,7 @@ mod tests { let spec = Arc::new(MainnetEthSpec::default_spec()); let url = SensitiveUrl::parse("http://localhost:5052").unwrap(); let client = - eth2::BeaconNodeHttpClient::new(url, Timeouts::set_all(Duration::from_secs(1)), 0); + eth2::BeaconNodeHttpClient::new(url, Timeouts::set_all(Duration::from_secs(1))); let candidate = CandidateBeaconNode::new(client); let mut fallback = @@ -146,8 +146,11 @@ mod tests { .create(); let url = SensitiveUrl::parse(&server.url()).unwrap(); - let client = - eth2::BeaconNodeHttpClient::new(url, Timeouts::set_all(Duration::from_secs(1)), index); + let client = eth2::BeaconNodeHttpClient::new_with_index( + url, + Timeouts::set_all(Duration::from_secs(1)), + index, + ); let candidate = CandidateBeaconNode::new(client); (server, candidate) @@ -168,8 +171,11 @@ mod tests { .create(); let url = SensitiveUrl::parse(&server.url()).unwrap(); - let client = - eth2::BeaconNodeHttpClient::new(url, Timeouts::set_all(Duration::from_secs(1)), index); + let client = eth2::BeaconNodeHttpClient::new_with_index( + url, + Timeouts::set_all(Duration::from_secs(1)), + index, + ); let candidate = CandidateBeaconNode::new(client); (server, candidate) diff --git a/validator_client/validator_services/src/block_service.rs b/validator_client/validator_services/src/block_service.rs index 5f87858daa1..cbb9fd4a2e2 100644 --- a/validator_client/validator_services/src/block_service.rs +++ b/validator_client/validator_services/src/block_service.rs @@ -176,9 +176,7 @@ impl ProposerFallback { // The non-proposer node call failed, but we don't have any proposer nodes. Return an error. (Err(e), None) => Err(e), // The non-proposer node call failed, try the same call on the proposer nodes. - (Err(_), Some(proposer_nodes)) => proposer_nodes - .first_success(func) - .await + (Err(_), Some(proposer_nodes)) => proposer_nodes.first_success(func).await, } } } diff --git a/validator_manager/src/common.rs b/validator_manager/src/common.rs index 75f5562e80b..0e93b257734 100644 --- a/validator_manager/src/common.rs +++ b/validator_manager/src/common.rs @@ -28,8 +28,6 @@ pub const COUNT_FLAG: &str = "count"; /// 2. Weird enough to identify Lighthouse. const LIGHTHOUSE_DEPOSIT_CLI_VERSION: &str = "20.18.20"; -pub const DEFAULT_BEACON_NODE_INDEX: usize = 0; - #[derive(Debug)] pub enum UploadError { InvalidPublicKey, diff --git a/validator_manager/src/create_validators.rs b/validator_manager/src/create_validators.rs index a5a1f041fcb..19f78be2ea7 100644 --- a/validator_manager/src/create_validators.rs +++ b/validator_manager/src/create_validators.rs @@ -319,11 +319,8 @@ impl ValidatorsAndDeposits { } let bn_http_client = if let Some(bn_url) = bn_url { - let bn_http_client = BeaconNodeHttpClient::new( - bn_url, - Timeouts::set_all(BEACON_NODE_HTTP_TIMEOUT), - DEFAULT_BEACON_NODE_INDEX, - ); + let bn_http_client = + BeaconNodeHttpClient::new(bn_url, Timeouts::set_all(BEACON_NODE_HTTP_TIMEOUT)); /* * Print the version of the remote beacon node. diff --git a/validator_manager/src/exit_validators.rs b/validator_manager/src/exit_validators.rs index 5f2eff264f4..4a398793ce1 100644 --- a/validator_manager/src/exit_validators.rs +++ b/validator_manager/src/exit_validators.rs @@ -1,4 +1,3 @@ -use crate::common::DEFAULT_BEACON_NODE_INDEX; use crate::{DumpConfig, common::vc_http_client}; use clap::{Arg, ArgAction, ArgMatches, Command}; @@ -194,7 +193,6 @@ async fn run(config: ExitConfig) -> Result<(), String> { let beacon_node = BeaconNodeHttpClient::new( beacon_url.clone(), Timeouts::set_all(Duration::from_secs(12)), - DEFAULT_BEACON_NODE_INDEX, ); if beacon_node diff --git a/validator_manager/src/list_validators.rs b/validator_manager/src/list_validators.rs index 4e57d3e845e..082894a995d 100644 --- a/validator_manager/src/list_validators.rs +++ b/validator_manager/src/list_validators.rs @@ -7,7 +7,6 @@ use std::path::PathBuf; use std::time::Duration; use types::{ChainSpec, EthSpec, PublicKeyBytes}; -use crate::common::DEFAULT_BEACON_NODE_INDEX; use crate::exit_validators::get_current_epoch; use crate::{DumpConfig, common::vc_http_client}; @@ -137,7 +136,6 @@ async fn run(config: ListConfig) -> Result Date: Mon, 1 Dec 2025 16:38:13 -0500 Subject: [PATCH 31/41] testsss --- .../src/beacon_head_monitor.rs | 225 ++++++++++++++++++ .../beacon_node_fallback/src/lib.rs | 129 ++++++++++ 2 files changed, 354 insertions(+) diff --git a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs index 136a6af6607..e838b286e5c 100644 --- a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs +++ b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs @@ -142,3 +142,228 @@ pub async fn poll_head_event_from_beacon_nodes SseHead { + SseHead { + slot: types::Slot::new(slot), + block: Hash256::from_low_u64_be(block_root as u64), + state: Hash256::from_low_u64_be(block_root as u64), + epoch_transition: false, + previous_duty_dependent_root: Hash256::from_low_u64_be(block_root as u64), + current_duty_dependent_root: Hash256::from_low_u64_be(block_root as u64), + execution_optimistic: false, + } + } + + #[tokio::test] + async fn test_beacon_head_cache_insertion_and_retrieval() { + let cache = BeaconHeadCache::new(); + let head_1 = create_sse_head(1, 1); + let head_2 = create_sse_head(2, 2); + + cache.insert(0, head_1.clone()).await; + cache.insert(1, head_2.clone()).await; + + assert_eq!(cache.get(0).await, Some(head_1)); + assert_eq!(cache.get(1).await, Some(head_2)); + assert_eq!(cache.get(2).await, None); + } + + #[tokio::test] + async fn test_beacon_head_cache_update() { + let cache = BeaconHeadCache::new(); + let head_old = create_sse_head(1, 1); + let head_new = create_sse_head(2, 2); + + cache.insert(0, head_old).await; + cache.insert(0, head_new.clone()).await; + + assert_eq!(cache.get(0).await, Some(head_new)); + } + + #[tokio::test] + async fn test_is_latest_with_higher_slot() { + let cache = BeaconHeadCache::new(); + let head_1 = create_sse_head(1, 1); + let head_2 = create_sse_head(2, 2); + let head_3 = create_sse_head(3, 3); + + cache.insert(0, head_1).await; + cache.insert(1, head_2).await; + + assert!(cache.is_latest(&head_3).await); + } + + #[tokio::test] + async fn test_is_latest_with_lower_slot() { + let cache = BeaconHeadCache::new(); + let head_1 = create_sse_head(1, 1); + let head_2 = create_sse_head(2, 2); + let head_older = create_sse_head(1, 99); + + cache.insert(0, head_1).await; + cache.insert(1, head_2).await; + + assert!(!cache.is_latest(&head_older).await); + } + + #[tokio::test] + async fn test_is_latest_with_equal_slot() { + let cache = BeaconHeadCache::new(); + let head_1 = create_sse_head(5, 1); + let head_2 = create_sse_head(5, 2); + let head_equal = create_sse_head(5, 3); + + cache.insert(0, head_1).await; + cache.insert(1, head_2).await; + + assert!(cache.is_latest(&head_equal).await); + } + + #[tokio::test] + async fn test_is_latest_empty_cache() { + let cache = BeaconHeadCache::new(); + let head = create_sse_head(1, 1); + + assert!(cache.is_latest(&head).await); + } + + #[tokio::test] + async fn test_purge_cache_clears_all_entries() { + let cache = BeaconHeadCache::new(); + let head_1 = create_sse_head(1, 1); + let head_2 = create_sse_head(2, 2); + + cache.insert(0, head_1).await; + cache.insert(1, head_2).await; + + assert!(cache.get(0).await.is_some()); + assert!(cache.get(1).await.is_some()); + + cache.purge_cache().await; + + assert!(cache.get(0).await.is_none()); + assert!(cache.get(1).await.is_none()); + } + + #[tokio::test] + async fn test_head_event_creation() { + let event = HeadEvent { + beacon_node_index: 42, + }; + assert_eq!(event.beacon_node_index, 42); + } + + #[tokio::test] + async fn test_cache_caches_multiple_heads_from_different_nodes() { + let cache = BeaconHeadCache::new(); + let head_1 = create_sse_head(10, 1); + let head_2 = create_sse_head(5, 2); + let head_3 = create_sse_head(8, 3); + + cache.insert(0, head_1.clone()).await; + cache.insert(1, head_2.clone()).await; + cache.insert(2, head_3.clone()).await; + + // Verify all are stored + assert_eq!(cache.get(0).await, Some(head_1)); + assert_eq!(cache.get(1).await, Some(head_2)); + assert_eq!(cache.get(2).await, Some(head_3)); + + // The latest should be slot 10 + let head_10 = create_sse_head(10, 99); + assert!(cache.is_latest(&head_10).await); + + // Anything with slot > 10 should be latest + let head_11 = create_sse_head(11, 99); + assert!(cache.is_latest(&head_11).await); + + // Anything with slot < 10 should not be latest + let head_9 = create_sse_head(9, 99); + assert!(!cache.is_latest(&head_9).await); + } + + #[tokio::test] + async fn test_cache_handles_concurrent_operations() { + let cache = Arc::new(BeaconHeadCache::new()); + let mut handles = vec![]; + + // Spawn multiple tasks that insert heads concurrently + for i in 0..10 { + let cache_clone = cache.clone(); + let handle = tokio::spawn(async move { + let head = create_sse_head(i as u64, (i % 256) as u8); + cache_clone.insert(i, head).await; + }); + handles.push(handle); + } + + // Wait for all tasks to complete + for handle in handles { + handle.await.unwrap(); + } + + // Verify all heads are cached + for i in 0..10 { + assert!(cache.get(i).await.is_some()); + } + } + + #[tokio::test] + async fn test_is_latest_after_cache_updates() { + let cache = BeaconHeadCache::new(); + + // Start with head at slot 5 + let head_5 = create_sse_head(5, 1); + cache.insert(0, head_5.clone()).await; + assert!(cache.is_latest(&head_5).await); + + // Add a higher slot + let head_10 = create_sse_head(10, 2); + cache.insert(1, head_10.clone()).await; + + // head_5 should no longer be latest + assert!(!cache.is_latest(&head_5).await); + // head_10 should be latest + assert!(cache.is_latest(&head_10).await); + + // Add an even higher slot + let head_15 = create_sse_head(15, 3); + cache.insert(2, head_15.clone()).await; + + // head_10 should no longer be latest + assert!(!cache.is_latest(&head_10).await); + // head_15 should be latest + assert!(cache.is_latest(&head_15).await); + } + + #[tokio::test] + async fn test_cache_default_is_empty() { + let cache = BeaconHeadCache::default(); + assert!(cache.get(0).await.is_none()); + assert!(cache.get(999).await.is_none()); + } + + #[tokio::test] + async fn test_is_latest_with_multiple_same_slot_heads() { + let cache = BeaconHeadCache::new(); + let head_slot_5_node1 = create_sse_head(5, 1); + let head_slot_5_node2 = create_sse_head(5, 2); + let head_slot_5_node3 = create_sse_head(5, 3); + + cache.insert(0, head_slot_5_node1).await; + cache.insert(1, head_slot_5_node2).await; + + // All heads with slot 5 should be considered latest + assert!(cache.is_latest(&head_slot_5_node3).await); + + // But heads with slot 4 should not be latest + let head_slot_4 = create_sse_head(4, 4); + assert!(!cache.is_latest(&head_slot_4).await); + } +} diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index fbc3798879a..e24750cc7c2 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -1147,4 +1147,133 @@ mod tests { mock1.expect(3).assert(); mock2.expect(3).assert(); } + + #[tokio::test] + async fn first_success_from_index_tries_preferred_node_first() { + let spec = Arc::new(MainnetEthSpec::default_spec()); + let (mut mock_beacon_node_1, beacon_node_1) = new_mock_beacon_node(0, &spec).await; + let (mut mock_beacon_node_2, beacon_node_2) = new_mock_beacon_node(1, &spec).await; + let (mut mock_beacon_node_3, beacon_node_3) = new_mock_beacon_node(2, &spec).await; + + let beacon_node_fallback = create_beacon_node_fallback( + vec![beacon_node_1, beacon_node_2, beacon_node_3], + vec![], + spec.clone(), + ); + + let mock1 = mock_beacon_node_1.mock_offline_node(); + let _mock2 = mock_beacon_node_2.mock_online_node(); + let mock3 = mock_beacon_node_3.mock_online_node(); + + // Request with preferred_index=1 (beacon_node_2) + let result = beacon_node_fallback + .first_success_from_index(Some(1), |client| async move { + client.get_node_version().await + }) + .await; + + // Should succeed since beacon_node_2 is online + assert!(result.is_ok()); + + // mock1 should not be called since preferred node succeeds + mock1.expect(0).assert(); + mock3.expect(0).assert(); + } + + #[tokio::test] + async fn first_success_from_index_falls_back_when_preferred_fails() { + let spec = Arc::new(MainnetEthSpec::default_spec()); + let (mut mock_beacon_node_1, beacon_node_1) = new_mock_beacon_node(0, &spec).await; + let (mut mock_beacon_node_2, beacon_node_2) = new_mock_beacon_node(1, &spec).await; + let (mut mock_beacon_node_3, beacon_node_3) = new_mock_beacon_node(2, &spec).await; + + let beacon_node_fallback = create_beacon_node_fallback( + vec![beacon_node_1, beacon_node_2, beacon_node_3], + vec![], + spec.clone(), + ); + + let _mock1 = mock_beacon_node_1.mock_online_node(); + let mock2 = mock_beacon_node_2.mock_offline_node(); + let _mock3 = mock_beacon_node_3.mock_offline_node(); + + // Request with preferred_index=1 (beacon_node_2), but it's offline + let result = beacon_node_fallback + .first_success_from_index(Some(1), |client| async move { + client.get_node_version().await + }) + .await; + + // Should succeed by falling back to beacon_node_1 + assert!(result.is_ok()); + + // mock2 should be called at least once (the preferred attempt) + mock2.expect(1).assert(); + // since the result was ok we can safely assume that the fallback first_success + // behaviour succeeded instead of checking if either of mock1/mock3 received hits + } + + #[tokio::test] + async fn first_success_from_index_with_none_falls_back_to_first_success() { + let spec = Arc::new(MainnetEthSpec::default_spec()); + let (mut mock_beacon_node_1, beacon_node_1) = new_mock_beacon_node(0, &spec).await; + let (mut mock_beacon_node_2, beacon_node_2) = new_mock_beacon_node(1, &spec).await; + let (mut mock_beacon_node_3, beacon_node_3) = new_mock_beacon_node(2, &spec).await; + + let beacon_node_fallback = create_beacon_node_fallback( + vec![beacon_node_1, beacon_node_2, beacon_node_3], + vec![], + spec.clone(), + ); + + let _mock1 = mock_beacon_node_1.mock_offline_node(); + let _mock2 = mock_beacon_node_2.mock_offline_node(); + let mock3 = mock_beacon_node_3.mock_online_node(); + + // Request with preferred_index=None + let result = beacon_node_fallback + .first_success_from_index(None, |client| async move { + client.get_node_version().await + }) + .await; + + // Should succeed with beacon_node_3 in the first pass + assert!(result.is_ok()); + + // mock3 should be called once in the first pass + mock3.expect(1).assert(); + } + + #[tokio::test] + async fn first_success_from_index_all_offline() { + let spec = Arc::new(MainnetEthSpec::default_spec()); + let (mut mock_beacon_node_1, beacon_node_1) = new_mock_beacon_node(0, &spec).await; + let (mut mock_beacon_node_2, beacon_node_2) = new_mock_beacon_node(1, &spec).await; + let (mut mock_beacon_node_3, beacon_node_3) = new_mock_beacon_node(2, &spec).await; + + let beacon_node_fallback = create_beacon_node_fallback( + vec![beacon_node_1, beacon_node_2, beacon_node_3], + vec![], + spec.clone(), + ); + + let _mock1 = mock_beacon_node_1.mock_offline_node(); + let mock2 = mock_beacon_node_2.mock_offline_node(); + let _mock3 = mock_beacon_node_3.mock_offline_node(); + + // Request with preferred_index=1, but all nodes are offline + let result = beacon_node_fallback + .first_success_from_index(Some(1), |client| async move { + client.get_node_version().await + }) + .await; + + // Should fail since all nodes are offline + assert!(result.is_err()); + + // Preferred node (mock2) should be called 3 times: + // - 1 time for the preferred attempt + // - 2 more times from the fallback to first_success (first and second pass) + mock2.expect(3).assert(); + } } From 84c8291c0c27028d59444cab806804fe654a30d4 Mon Sep 17 00:00:00 2001 From: hopinheimer Date: Mon, 1 Dec 2025 16:38:40 -0500 Subject: [PATCH 32/41] fmt --- .../src/beacon_head_monitor.rs | 2 +- .../beacon_node_fallback/src/lib.rs | 30 +++++++++++-------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs index e838b286e5c..4fd8ca17c1c 100644 --- a/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs +++ b/validator_client/beacon_node_fallback/src/beacon_head_monitor.rs @@ -146,7 +146,7 @@ pub async fn poll_head_event_from_beacon_nodes SseHead { SseHead { diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index e24750cc7c2..33fdc9cf338 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -1167,9 +1167,10 @@ mod tests { // Request with preferred_index=1 (beacon_node_2) let result = beacon_node_fallback - .first_success_from_index(Some(1), |client| async move { - client.get_node_version().await - }) + .first_success_from_index( + Some(1), + |client| async move { client.get_node_version().await }, + ) .await; // Should succeed since beacon_node_2 is online @@ -1199,9 +1200,10 @@ mod tests { // Request with preferred_index=1 (beacon_node_2), but it's offline let result = beacon_node_fallback - .first_success_from_index(Some(1), |client| async move { - client.get_node_version().await - }) + .first_success_from_index( + Some(1), + |client| async move { client.get_node_version().await }, + ) .await; // Should succeed by falling back to beacon_node_1 @@ -1211,7 +1213,7 @@ mod tests { mock2.expect(1).assert(); // since the result was ok we can safely assume that the fallback first_success // behaviour succeeded instead of checking if either of mock1/mock3 received hits - } + } #[tokio::test] async fn first_success_from_index_with_none_falls_back_to_first_success() { @@ -1232,9 +1234,10 @@ mod tests { // Request with preferred_index=None let result = beacon_node_fallback - .first_success_from_index(None, |client| async move { - client.get_node_version().await - }) + .first_success_from_index( + None, + |client| async move { client.get_node_version().await }, + ) .await; // Should succeed with beacon_node_3 in the first pass @@ -1263,9 +1266,10 @@ mod tests { // Request with preferred_index=1, but all nodes are offline let result = beacon_node_fallback - .first_success_from_index(Some(1), |client| async move { - client.get_node_version().await - }) + .first_success_from_index( + Some(1), + |client| async move { client.get_node_version().await }, + ) .await; // Should fail since all nodes are offline From 5ce62b8153644bc496ff533636a8e7c1537a1ae0 Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Mon, 1 Dec 2025 20:53:31 -0300 Subject: [PATCH 33/41] Test --- .../validator_services/src/attestation_service.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 3182e169574..7db1ef33a04 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -437,13 +437,17 @@ impl AttestationService attestation_data, None => { let mut attestation_data_service = self.attestation_data_service.write().await; - attestation_data_service + let attestation_data = attestation_data_service .download_data(&slot, candidate_beacon_node) .await - .map(|(data, _)| data)? + .map(|(data, _)| data)?; + drop(attestation_data_service); + attestation_data } }; + info!(?attestation_data, "GOT ATTESTATION DATA"); + // Create futures to produce signed `Attestation` objects. let attestation_data_ref = &attestation_data; let signing_futures = validator_duties.iter().map(|duty_and_proof| async move { From 69b8128eb1ebd8e34b7bfe087532f1a775923285 Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Wed, 3 Dec 2025 16:36:09 -0300 Subject: [PATCH 34/41] Remove write lock --- .../src/attestation_data_service.rs | 71 ++----------------- .../src/attestation_service.rs | 30 +++----- 2 files changed, 13 insertions(+), 88 deletions(-) diff --git a/validator_client/validator_services/src/attestation_data_service.rs b/validator_client/validator_services/src/attestation_data_service.rs index 43e58f303ad..8e52d04914d 100644 --- a/validator_client/validator_services/src/attestation_data_service.rs +++ b/validator_client/validator_services/src/attestation_data_service.rs @@ -8,42 +8,21 @@ use types::{AttestationData, Slot}; /// The AttestationDataService is responsible for downloading and caching attestation data at a given slot. /// It also helps prevent us from re-downloading identical attestation data. pub struct AttestationDataService { - attestation_data: Option<(Slot, AttestationData, usize)>, beacon_nodes: Arc>, } impl AttestationDataService { pub fn new(beacon_nodes: Arc>) -> Self { Self { - attestation_data: None, beacon_nodes, } } - /// Get previously downloaded attestation data. - pub fn get_cached_attestation_data( - &self, - requested_slot: &Slot, - ) -> Option<(AttestationData, usize)> { - if let Some((cached_slot, attestation_data, node_index)) = &self.attestation_data - && cached_slot == requested_slot - { - return Some((attestation_data.clone(), *node_index)); - } - None - } - pub async fn download_data( - &mut self, + &self, request_slot: &Slot, candidate_beacon_node: Option, ) -> Result<(AttestationData, usize), String> { - // If we've already downloaded attestation data for `request_slot`, there's no need to re-download the data. - if let Some((attestation_data, node_index)) = self.get_cached_attestation_data(request_slot) - { - return Ok((attestation_data, node_index)); - } - let (attestation_data, node_index) = self .beacon_nodes .first_success_from_index(candidate_beacon_node, |beacon_node| async move { @@ -61,8 +40,6 @@ impl AttestationDataService { .await .map_err(|e| e.to_string())?; - self.attestation_data = Some((*request_slot, attestation_data.clone(), node_index)); - Ok((attestation_data, node_index)) } } @@ -181,31 +158,6 @@ mod tests { (server, candidate) } - #[test] - fn test_new_service() { - let beacon_node_fallback = create_test_beacon_node_fallback(); - let service = - AttestationDataService::::new(Arc::new(beacon_node_fallback)); - - assert!(service.attestation_data.is_none()); - assert!(service.get_cached_attestation_data(&Slot::new(1)).is_none()); - } - - #[test] - fn test_get_cached_attestation_data_returns_cached() { - let beacon_node_fallback = create_test_beacon_node_fallback(); - let mut service = - AttestationDataService::::new(Arc::new(beacon_node_fallback)); - - let slot = Slot::new(10); - let beacon_node_index = 0; - let attestation_data = create_attestation_data(slot, Epoch::new(0), Epoch::new(1)); - service.attestation_data = Some((slot, attestation_data.clone(), beacon_node_index)); - - let cached = service.get_cached_attestation_data(&slot); - assert!(cached.is_some()); - assert_eq!(cached.unwrap(), (attestation_data, beacon_node_index)); - } #[tokio::test] async fn test_download_attestation_data() { @@ -225,18 +177,12 @@ mod tests { Duration::from_secs(12), )); - let mut service = AttestationDataService::::new(Arc::new(fallback)); + let service = AttestationDataService::::new(Arc::new(fallback)); let result = service.download_data(&slot, None).await; // Verify download is successful assert!(result.is_ok()); assert_eq!(result.unwrap(), (attestation_data.clone(), 0)); - - // Verify data is cached after successful download - assert_eq!( - service.get_cached_attestation_data(&slot), - Some((attestation_data, 0)) - ); } #[tokio::test] @@ -261,7 +207,7 @@ mod tests { Duration::from_secs(12), )); - let mut service = AttestationDataService::::new(Arc::new(fallback)); + let service = AttestationDataService::::new(Arc::new(fallback)); let result = service.download_data(&slot, None).await; // Verify all nodes offline @@ -271,9 +217,6 @@ mod tests { .unwrap_err() .contains("Failed to produce attestation data") ); - - // Verify no data was cached since all nodes failed - assert_eq!(service.get_cached_attestation_data(&slot), None); } #[tokio::test] @@ -302,17 +245,11 @@ mod tests { Duration::from_secs(12), )); - let mut service = AttestationDataService::::new(Arc::new(fallback)); + let service = AttestationDataService::::new(Arc::new(fallback)); let result = service.download_data(&slot, None).await; // Verify download is successful and we fell back to the next node assert!(result.is_ok()); assert_eq!(result.unwrap(), (attestation_data.clone(), 1)); - - // Verify data is cached after successful download - assert_eq!( - service.get_cached_attestation_data(&slot), - Some((attestation_data, 1)) - ); } } diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 7db1ef33a04..43bd47503bf 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -10,7 +10,6 @@ use std::collections::HashMap; use std::ops::Deref; use std::sync::Arc; use task_executor::TaskExecutor; -use tokio::sync::RwLock; use tokio::sync::mpsc; use tokio::time::{Duration, Instant, sleep, sleep_until}; use tracing::{Instrument, debug, error, info, info_span, instrument, warn}; @@ -28,7 +27,7 @@ pub struct AttestationServiceBuilder executor: Option, chain_spec: Option>, head_monitor_rx: Option>>>, - attestation_data_service: Option>>>, + attestation_data_service: Option>>, disable: bool, } @@ -64,9 +63,9 @@ impl AttestationServiceBuil pub fn beacon_nodes(mut self, beacon_nodes: Arc>) -> Self { self.beacon_nodes = Some(beacon_nodes.clone()); - self.attestation_data_service = Some(Arc::new(RwLock::new(AttestationDataService::new( + self.attestation_data_service = Some(Arc::new(AttestationDataService::new( beacon_nodes, - )))); + ))); self } @@ -134,7 +133,7 @@ pub struct Inner { executor: TaskExecutor, chain_spec: Arc, head_monitor_rx: Option>>>, - attestation_data_service: Arc>>, + attestation_data_service: Arc>, disable: bool, latest_attested_slot: Mutex, } @@ -431,22 +430,11 @@ impl AttestationService attestation_data, - None => { - let mut attestation_data_service = self.attestation_data_service.write().await; - let attestation_data = attestation_data_service - .download_data(&slot, candidate_beacon_node) - .await - .map(|(data, _)| data)?; - drop(attestation_data_service); - attestation_data - } - }; - - info!(?attestation_data, "GOT ATTESTATION DATA"); + let attestation_data = self + .attestation_data_service + .download_data(&slot, candidate_beacon_node) + .await + .map(|(data, _)| data)?; // Create futures to produce signed `Attestation` objects. let attestation_data_ref = &attestation_data; From 17011184fd85b11119561b721560fc079fdfbe82 Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Wed, 3 Dec 2025 16:40:16 -0300 Subject: [PATCH 35/41] Fmt --- .../src/attestation_data_service.rs | 24 +------------------ .../src/attestation_service.rs | 4 +--- 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/validator_client/validator_services/src/attestation_data_service.rs b/validator_client/validator_services/src/attestation_data_service.rs index 8e52d04914d..ab8e5dfad25 100644 --- a/validator_client/validator_services/src/attestation_data_service.rs +++ b/validator_client/validator_services/src/attestation_data_service.rs @@ -13,9 +13,7 @@ pub struct AttestationDataService { impl AttestationDataService { pub fn new(beacon_nodes: Arc>) -> Self { - Self { - beacon_nodes, - } + Self { beacon_nodes } } pub async fn download_data( @@ -78,25 +76,6 @@ mod tests { } } - fn create_test_beacon_node_fallback() -> BeaconNodeFallback { - let spec = Arc::new(MainnetEthSpec::default_spec()); - let url = SensitiveUrl::parse("http://localhost:5052").unwrap(); - let client = - eth2::BeaconNodeHttpClient::new(url, Timeouts::set_all(Duration::from_secs(1))); - let candidate = CandidateBeaconNode::new(client); - - let mut fallback = - BeaconNodeFallback::new(vec![candidate], FallbackConfig::default(), vec![], spec); - - fallback.set_slot_clock(TestingSlotClock::new( - Slot::new(1), - Duration::from_secs(0), - Duration::from_secs(12), - )); - - fallback - } - // Helper to create a beacon node with mocked attestation endpoint async fn create_mocked_beacon_node( index: usize, @@ -158,7 +137,6 @@ mod tests { (server, candidate) } - #[tokio::test] async fn test_download_attestation_data() { let spec = Arc::new(MinimalEthSpec::default_spec()); diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 43bd47503bf..4b09d47eb02 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -63,9 +63,7 @@ impl AttestationServiceBuil pub fn beacon_nodes(mut self, beacon_nodes: Arc>) -> Self { self.beacon_nodes = Some(beacon_nodes.clone()); - self.attestation_data_service = Some(Arc::new(AttestationDataService::new( - beacon_nodes, - ))); + self.attestation_data_service = Some(Arc::new(AttestationDataService::new(beacon_nodes))); self } From 833e07484d30120fcbe42bc4182e00a83a3041c8 Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Tue, 9 Dec 2025 12:09:19 -0300 Subject: [PATCH 36/41] fmt --- .../validator_services/src/attestation_service.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 4a4c79546e1..19223386fbe 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -244,7 +244,11 @@ impl AttestationService,) -> Result<(), String> { + fn spawn_attestation_tasks( + &self, + slot_duration: Duration, + beacon_node_index: Option, + ) -> Result<(), String> { let slot = self.slot_clock.now().ok_or("Failed to read slot clock")?; let duration_to_next_slot = self .slot_clock @@ -431,7 +435,7 @@ impl AttestationService Date: Tue, 9 Dec 2025 14:20:35 -0300 Subject: [PATCH 37/41] Implement consensus mechanism --- consensus/types/src/slot_epoch.rs | 4 + .../beacon_node_fallback/src/lib.rs | 43 ++++++ .../src/attestation_data_service.rs | 123 ++++++++++++++++-- .../src/attestation_service.rs | 98 ++++++++++++-- 4 files changed, 249 insertions(+), 19 deletions(-) diff --git a/consensus/types/src/slot_epoch.rs b/consensus/types/src/slot_epoch.rs index 05af9c5232d..2673d9d01e6 100644 --- a/consensus/types/src/slot_epoch.rs +++ b/consensus/types/src/slot_epoch.rs @@ -57,6 +57,10 @@ impl Slot { pub fn max_value() -> Slot { Slot(u64::MAX) } + + pub fn is_start_slot_in_epoch(&self, slots_per_epoch: u64) -> bool { + self.0.is_multiple_of(slots_per_epoch) + } } impl Epoch { diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index 5f8c8a04b7e..afdc1b8b425 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -652,6 +652,49 @@ impl BeaconNodeFallback { .map(|(val, _)| val) } + pub async fn first_n_responses( + &self, + fetch_func: F, + mut consensus_check: C, + ) -> Result<(O, usize), Errors> + where + F: Fn(BeaconNodeHttpClient) -> R, + R: Future>, + C: FnMut(&(O, usize)) -> bool, + O: Eq + Clone + Debug, + Err: Debug, + { + let mut errors = vec![]; + + // Collect all responses from all candidates + let candidates = self.candidates.read().await; + let mut futures = vec![]; + + for candidate in candidates.iter() { + futures.push(Self::run_on_candidate( + candidate.beacon_node.clone(), + &fetch_func, + )); + } + drop(candidates); + + // Process futures sequentially, checking consensus after each response + for future in futures { + match future.await { + Ok(val) => { + if consensus_check(&val) { + return Ok(val); + } + } + Err(e) => { + errors.push(e); + } + } + } + + Err(Errors(errors)) + } + /// Run `func` against each candidate in `self`, returning immediately if a result is found. /// Otherwise, return all the errors encountered along the way. pub async fn first_success_with_index( diff --git a/validator_client/validator_services/src/attestation_data_service.rs b/validator_client/validator_services/src/attestation_data_service.rs index ab8e5dfad25..ee8078afa4c 100644 --- a/validator_client/validator_services/src/attestation_data_service.rs +++ b/validator_client/validator_services/src/attestation_data_service.rs @@ -1,9 +1,32 @@ -use std::sync::Arc; +use std::{collections::HashMap, sync::Arc}; use beacon_node_fallback::BeaconNodeFallback; use slot_clock::SlotClock; use tracing::{Instrument, info_span}; -use types::{AttestationData, Slot}; +use types::{AttestationData, Checkpoint, Epoch, Slot}; + +#[derive(Debug, Clone)] +pub enum AttestationDataStrategy { + Fallback, + ByIndex(usize), + Consensus((usize, Option<(Checkpoint, usize)>)), + IgnoreEpoch(Epoch), +} + +#[derive(Debug, PartialEq, Eq, Hash, Clone)] +pub struct FFGConsensus { + pub source: Checkpoint, + pub target: Checkpoint, +} + +impl FFGConsensus { + pub fn new(attestation_data: &AttestationData) -> Self { + FFGConsensus { + source: attestation_data.source, + target: attestation_data.target, + } + } +} /// The AttestationDataService is responsible for downloading and caching attestation data at a given slot. /// It also helps prevent us from re-downloading identical attestation data. @@ -16,13 +39,12 @@ impl AttestationDataService { Self { beacon_nodes } } - pub async fn download_data( + async fn data_by_index( &self, request_slot: &Slot, candidate_beacon_node: Option, ) -> Result<(AttestationData, usize), String> { - let (attestation_data, node_index) = self - .beacon_nodes + self.beacon_nodes .first_success_from_index(candidate_beacon_node, |beacon_node| async move { let _timer = validator_metrics::start_timer_vec( &validator_metrics::ATTESTATION_SERVICE_TIMES, @@ -36,9 +58,82 @@ impl AttestationDataService { }) .instrument(info_span!("fetch_attestation_data")) .await - .map_err(|e| e.to_string())?; + .map_err(|e| e.to_string()) + } + + pub async fn data_by_threshold( + &self, + request_slot: &Slot, + threshold: usize, + checkpoint_and_index: Option<(Checkpoint, usize)>, + ) -> Result<(AttestationData, usize), String> { + let mut results = HashMap::new(); + + self.beacon_nodes + .first_n_responses( + |beacon_node| async move { + let _timer = validator_metrics::start_timer_vec( + &validator_metrics::ATTESTATION_SERVICE_TIMES, + &[validator_metrics::ATTESTATIONS_HTTP_GET], + ); + beacon_node + .get_validator_attestation_data(*request_slot, 0) + .await + .map_err(|e| format!("Failed to produce attestation data: {:?}", e)) + .map(|result| result.data) + }, + |(attestation_data, index)| { + if let Some((target_checkpoint, preferred_index)) = checkpoint_and_index { + // If we have a preferred index set, return attestation data from it + // TODO(attestation-consensus) this is a small optimization to immediately return data + // from the preferred index. We shouldn't need to check the target checkpoint, but maybe + // its just safer to do so? + if preferred_index == *index { + return true; + } + // return if fetched data matches the target checkpoint + return attestation_data.target == target_checkpoint; + } + + let ffg_consensus = FFGConsensus::new(attestation_data); + results + .entry(ffg_consensus.clone()) + .or_insert_with(Vec::new) + .push(*index); + if results + .get(&ffg_consensus) + .is_some_and(|servers| servers.len() >= threshold) + { + return true; + } + + false + }, + ) + .instrument(info_span!("fetch_attestation_data")) + .await + .map_err(|e| e.to_string()) + } - Ok((attestation_data, node_index)) + pub async fn download_data( + &self, + request_slot: &Slot, + strategy: &AttestationDataStrategy, + ) -> Result<(AttestationData, usize), String> { + match strategy { + AttestationDataStrategy::Fallback => self.data_by_index(request_slot, None).await, + AttestationDataStrategy::ByIndex(index) => { + self.data_by_index(request_slot, Some(*index)).await + } + AttestationDataStrategy::Consensus((threshold, checkpoint_and_index)) => { + self.data_by_threshold(request_slot, *threshold, checkpoint_and_index.clone()) + .await + } + AttestationDataStrategy::IgnoreEpoch(epoch) => Err(format!( + "Disabled attestation production for epoch {:?}", + epoch + )), + } } } @@ -54,7 +149,7 @@ mod tests { MinimalEthSpec, Slot, }; - use crate::attestation_data_service::AttestationDataService; + use crate::attestation_data_service::{AttestationDataService, AttestationDataStrategy}; fn create_attestation_data( slot: Slot, @@ -156,7 +251,9 @@ mod tests { )); let service = AttestationDataService::::new(Arc::new(fallback)); - let result = service.download_data(&slot, None).await; + let result = service + .download_data(&slot, &AttestationDataStrategy::Fallback) + .await; // Verify download is successful assert!(result.is_ok()); @@ -186,7 +283,9 @@ mod tests { )); let service = AttestationDataService::::new(Arc::new(fallback)); - let result = service.download_data(&slot, None).await; + let result = service + .download_data(&slot, &AttestationDataStrategy::Fallback) + .await; // Verify all nodes offline assert!(result.is_err()); @@ -224,7 +323,9 @@ mod tests { )); let service = AttestationDataService::::new(Arc::new(fallback)); - let result = service.download_data(&slot, None).await; + let result = service + .download_data(&slot, &AttestationDataStrategy::Fallback) + .await; // Verify download is successful and we fell back to the next node assert!(result.is_ok()); diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 19223386fbe..55cd3e01b84 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -1,7 +1,7 @@ use crate::duties_service::{DutiesService, DutyAndProof}; use tokio::sync::Mutex; -use crate::attestation_data_service::AttestationDataService; +use crate::attestation_data_service::{AttestationDataService, AttestationDataStrategy}; use beacon_node_fallback::{ApiTopic, BeaconNodeFallback, beacon_head_monitor::HeadEvent}; use futures::future::join_all; use logging::crit; @@ -14,7 +14,9 @@ use tokio::sync::mpsc; use tokio::time::{Duration, Instant, sleep, sleep_until}; use tracing::{Instrument, Span, debug, error, info, info_span, instrument, warn}; use tree_hash::TreeHash; -use types::{Attestation, AttestationData, ChainSpec, CommitteeIndex, EthSpec, Slot}; +use types::{ + Attestation, AttestationData, ChainSpec, Checkpoint, CommitteeIndex, Epoch, EthSpec, Slot, +}; use validator_store::{Error as ValidatorStoreError, ValidatorStore}; /// Builds an `AttestationService`. @@ -28,6 +30,8 @@ pub struct AttestationServiceBuilder chain_spec: Option>, head_monitor_rx: Option>>>, attestation_data_service: Option>>, + latest_target_checkpoint: Arc>>, + consensus_threshold: Option, disable: bool, } @@ -42,6 +46,8 @@ impl AttestationServiceBuil chain_spec: None, head_monitor_rx: None, attestation_data_service: None, + latest_target_checkpoint: Arc::new(Mutex::new(None)), + consensus_threshold: None, disable: false, } } @@ -83,6 +89,11 @@ impl AttestationServiceBuil self } + pub fn consensus_threshold(mut self, threshold: usize) -> Self { + self.consensus_threshold = Some(threshold); + self + } + pub fn head_monitor_rx( mut self, head_monitor_rx: Option>>>, @@ -115,6 +126,8 @@ impl AttestationServiceBuil .attestation_data_service .ok_or("Cannot build AttestationService without attestation_data_service")?, head_monitor_rx: self.head_monitor_rx, + latest_target_checkpoint: self.latest_target_checkpoint, + consensus_threshold: self.consensus_threshold, disable: self.disable, latest_attested_slot: Mutex::new(Slot::default()), }), @@ -132,6 +145,8 @@ pub struct Inner { chain_spec: Arc, head_monitor_rx: Option>>>, attestation_data_service: Arc>, + latest_target_checkpoint: Arc>>, + consensus_threshold: Option, disable: bool, latest_attested_slot: Mutex, } @@ -170,6 +185,13 @@ impl AttestationService AttestationService AttestationService *last_slot = current_slot, Err(e) => { crit!(error = e, "Failed to spawn attestation tasks") @@ -247,7 +274,7 @@ impl AttestationService, + attestation_data_strategy: AttestationDataStrategy, ) -> Result<(), String> { let slot = self.slot_clock.now().ok_or("Failed to read slot clock")?; let duration_to_next_slot = self @@ -265,11 +292,66 @@ impl AttestationService Date: Tue, 9 Dec 2025 14:26:18 -0300 Subject: [PATCH 38/41] Set default threshold to 3, just for testing --- validator_client/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 4ef560c4049..df9f7dcdeec 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -516,6 +516,7 @@ impl ProductionValidatorClient { .beacon_nodes(beacon_nodes.clone()) .executor(context.executor.clone()) .head_monitor_rx(head_monitor_rx) + .consensus_threshold(3) .chain_spec(context.eth2_config.spec.clone()) .disable(config.disable_attesting); From eb07e8f6c6ed8e31702412a2ec5d42ca8bf38226 Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Tue, 9 Dec 2025 15:21:01 -0300 Subject: [PATCH 39/41] Fix --- .../validator_services/src/attestation_service.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 55cd3e01b84..8935d1d5e12 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -309,7 +309,7 @@ impl AttestationService AttestationService Date: Wed, 10 Dec 2025 12:13:40 -0300 Subject: [PATCH 40/41] Add score aggregation --- .../src/attestation_data_service.rs | 170 +++++++++++++----- .../src/attestation_service.rs | 3 +- 2 files changed, 128 insertions(+), 45 deletions(-) diff --git a/validator_client/validator_services/src/attestation_data_service.rs b/validator_client/validator_services/src/attestation_data_service.rs index ee8078afa4c..f8360383cc0 100644 --- a/validator_client/validator_services/src/attestation_data_service.rs +++ b/validator_client/validator_services/src/attestation_data_service.rs @@ -1,6 +1,7 @@ use std::{collections::HashMap, sync::Arc}; use beacon_node_fallback::BeaconNodeFallback; +use safe_arith::SafeArith; use slot_clock::SlotClock; use tracing::{Instrument, info_span}; use types::{AttestationData, Checkpoint, Epoch, Slot}; @@ -10,22 +11,123 @@ pub enum AttestationDataStrategy { Fallback, ByIndex(usize), Consensus((usize, Option<(Checkpoint, usize)>)), + HighestScore, IgnoreEpoch(Epoch), } -#[derive(Debug, PartialEq, Eq, Hash, Clone)] -pub struct FFGConsensus { - pub source: Checkpoint, - pub target: Checkpoint, +// New trait for aggregation strategies that need parallel queries +trait ResultAggregator { + /// Process a single response and decide whether to continue or stop + fn process_result(&mut self, attestation_data: &AttestationData, index: usize) -> bool; // Returns true if we should stop and return + + /// Get the final result after aggregation + fn get_result(&self) -> Option<(AttestationData, usize)>; +} + +// Consensus aggregator +struct ConsensusAggregator { + results: HashMap>, + threshold: usize, + target_checkpoint_and_index: Option<(Checkpoint, usize)>, + consensus_result: Option<(AttestationData, usize)>, +} + +impl ConsensusAggregator { + fn new(threshold: usize, target_checkpoint_and_index: Option<(Checkpoint, usize)>) -> Self { + Self { + results: HashMap::new(), + threshold, + target_checkpoint_and_index, + consensus_result: None, + } + } +} + +impl ResultAggregator for ConsensusAggregator { + fn process_result(&mut self, attestation_data: &AttestationData, index: usize) -> bool { + if let Some((target_checkpoint, preferred_index)) = self.target_checkpoint_and_index { + // If we have a preferred index set, return attestation data from it + // TODO(attestation-consensus) this is a small optimization to immediately return data + // from the preferred index. We shouldn't need to check the target checkpoint, but maybe + // its just safer to do so? + if preferred_index == index { + self.consensus_result = Some((attestation_data.clone(), index)); + return true; + } + // return if fetched data matches the target checkpoint + if attestation_data.target == target_checkpoint { + self.consensus_result = Some((attestation_data.clone(), index)); + return true; + } + } + self.results + .entry(attestation_data.target) + .or_insert_with(Vec::new) + .push(index); + + if self + .results + .get(&attestation_data.target) + .is_some_and(|servers| servers.len() >= self.threshold) + { + // Consensus has been reached + self.consensus_result = Some((attestation_data.clone(), index)); + return true; + } + + false + } + + fn get_result(&self) -> Option<(AttestationData, usize)> { + self.consensus_result.clone() + } +} + +// Score aggregator +struct ScoreAggregator { + results: HashMap, + // TODO im pretty sure the head slot is just the requested slot + // double check the attestation service before deleting this TODO. + head_slot: Slot, + responses_needed: usize, + responses_received: usize, } -impl FFGConsensus { - pub fn new(attestation_data: &AttestationData) -> Self { - FFGConsensus { - source: attestation_data.source, - target: attestation_data.target, +impl ScoreAggregator { + fn new(head_slot: Slot, responses_needed: usize) -> Self { + Self { + results: HashMap::new(), + head_slot, + responses_needed, + responses_received: 0, } } + + fn calculate_score(&self, attestation_data: &AttestationData) -> u64 { + let checkpoint_value = attestation_data.source.epoch + attestation_data.target.epoch; + let slot_value = 1 + attestation_data.slot.as_u64() - self.head_slot.as_u64(); + // TODO unwrap + checkpoint_value.as_u64() + 1.safe_div(slot_value).unwrap() + } +} + +impl ResultAggregator for ScoreAggregator { + fn process_result(&mut self, attestation_data: &AttestationData, index: usize) -> bool { + let score = self.calculate_score(attestation_data); + self.results + .insert(index, (score, attestation_data.clone())); + self.responses_received += 1; + + // Stop when we've received enough responses + self.responses_received >= self.responses_needed + } + + fn get_result(&self) -> Option<(AttestationData, usize)> { + self.results + .iter() + .max_by_key(|(_, (score, _))| score) + .map(|(idx, (_, data))| (data.clone(), *idx)) + } } /// The AttestationDataService is responsible for downloading and caching attestation data at a given slot. @@ -61,14 +163,11 @@ impl AttestationDataService { .map_err(|e| e.to_string()) } - pub async fn data_by_threshold( + async fn data_with_aggregation( &self, request_slot: &Slot, - threshold: usize, - checkpoint_and_index: Option<(Checkpoint, usize)>, + mut aggregator: impl ResultAggregator, ) -> Result<(AttestationData, usize), String> { - let mut results = HashMap::new(); - self.beacon_nodes .first_n_responses( |beacon_node| async move { @@ -82,37 +181,15 @@ impl AttestationDataService { .map_err(|e| format!("Failed to produce attestation data: {:?}", e)) .map(|result| result.data) }, - |(attestation_data, index)| { - if let Some((target_checkpoint, preferred_index)) = checkpoint_and_index { - // If we have a preferred index set, return attestation data from it - // TODO(attestation-consensus) this is a small optimization to immediately return data - // from the preferred index. We shouldn't need to check the target checkpoint, but maybe - // its just safer to do so? - if preferred_index == *index { - return true; - } - // return if fetched data matches the target checkpoint - return attestation_data.target == target_checkpoint; - } - - let ffg_consensus = FFGConsensus::new(attestation_data); - results - .entry(ffg_consensus.clone()) - .or_insert_with(Vec::new) - .push(*index); - if results - .get(&ffg_consensus) - .is_some_and(|servers| servers.len() >= threshold) - { - return true; - } - - false - }, + |(attestation_data, index)| aggregator.process_result(attestation_data, *index), ) .instrument(info_span!("fetch_attestation_data")) .await - .map_err(|e| e.to_string()) + .map_err(|e| e.to_string())?; + + aggregator + .get_result() + .ok_or_else(|| "No valid attestation data found".to_string()) } pub async fn download_data( @@ -126,13 +203,20 @@ impl AttestationDataService { self.data_by_index(request_slot, Some(*index)).await } AttestationDataStrategy::Consensus((threshold, checkpoint_and_index)) => { - self.data_by_threshold(request_slot, *threshold, checkpoint_and_index.clone()) + let consensus_aggregator = + ConsensusAggregator::new(*threshold, checkpoint_and_index.clone()); + self.data_with_aggregation(request_slot, consensus_aggregator) .await } AttestationDataStrategy::IgnoreEpoch(epoch) => Err(format!( "Disabled attestation production for epoch {:?}", epoch )), + AttestationDataStrategy::HighestScore => { + let aggregator = + ScoreAggregator::new(*request_slot, self.beacon_nodes.num_total().await); + self.data_with_aggregation(request_slot, aggregator).await + } } } } diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index 8935d1d5e12..ab83f7e7a9b 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -344,8 +344,7 @@ impl AttestationService Date: Wed, 10 Dec 2025 16:12:01 -0300 Subject: [PATCH 41/41] fix --- .../src/attestation_service.rs | 76 +++++++++---------- 1 file changed, 34 insertions(+), 42 deletions(-) diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index ab83f7e7a9b..0d2c5772358 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -287,53 +287,45 @@ impl AttestationService = self.duties_service.attesters(slot).into_iter().collect(); let attestation_service = self.clone(); + // If we're using the Consensus strategy we need to handle the following situations: + // - The first slot in the epoch + // - A slot thats within an attestable epoch + // - A slot thats not within an attestable epoch + // - A slot that is not the first in an epoch and no target checkpoint to compare + let attestation_data_strategy = + if let AttestationDataStrategy::Consensus((threshold, _)) = attestation_data_strategy { + if slot.is_start_slot_in_epoch(S::E::slots_per_epoch()) { + // if the current slot is the first slot in the epoch use the default consensus strategy + attestation_data_strategy + } else if let Some((latest_attestable_epoch, target_checkpoint, preferred_index)) = + *attestation_service.latest_target_checkpoint.blocking_lock() + { + if slot.epoch(S::E::slots_per_epoch()) == latest_attestable_epoch { + // If the current slot is within the latest attestable epoch, we can attest + // using the `preferred_index` or nodes that have a matching `target_checkpoint` + AttestationDataStrategy::Consensus(( + threshold, + Some((target_checkpoint, preferred_index)), + )) + } else { + // If the current slot is not within the latest attestable epoch, we cannot attest + AttestationDataStrategy::IgnoreEpoch(slot.epoch(S::E::slots_per_epoch())) + } + } else { + // If the current slot is not the first slot in an epoch and there is no target checkpoint to compare, + // run the default consensus strategy. This can happen if the attestation service was initially + // launched in the middle of an epoch. + attestation_data_strategy + } + } else { + attestation_data_strategy + }; + let attestation_data_handle = self .inner .executor .spawn_handle( async move { - // If we're using the Consensus strategy we need to handle the following situations: - // - The first slot in the epoch - // - A slot thats within an attestable epoch - // - A slot thats not within an attestable epoch - // - A slot that is not the first in an epoch and no target checkpoint to compare - let attestation_data_strategy = - if let AttestationDataStrategy::Consensus((threshold, _)) = - attestation_data_strategy - { - if slot.is_start_slot_in_epoch(S::E::slots_per_epoch()) { - // if the current slot is the first slot in the epoch use the default consensus strategy - attestation_data_strategy - } else if let Some(( - latest_attestable_epoch, - target_checkpoint, - preferred_index, - )) = - *attestation_service.latest_target_checkpoint.lock().await - { - if slot.epoch(S::E::slots_per_epoch()) == latest_attestable_epoch { - // If the current slot is within the latest attestable epoch, we can attest - // using the `preferred_index` or nodes that have a matching `target_checkpoint` - AttestationDataStrategy::Consensus(( - threshold, - Some((target_checkpoint, preferred_index)), - )) - } else { - // If the current slot is not within the latest attestable epoch, we cannot attest - AttestationDataStrategy::IgnoreEpoch( - slot.epoch(S::E::slots_per_epoch()), - ) - } - } else { - // If the current slot is not the first slot in an epoch and there is no target checkpoint to compare, - // run the default consensus strategy. This can happen if the attestation service was initially - // launched in the middle of an epoch. - attestation_data_strategy - } - } else { - attestation_data_strategy - }; - let (attestation_data, index) = attestation_service .attestation_data_service .download_data(&slot, &attestation_data_strategy)