diff --git a/crates/nexum-runtime/src/bootstrap.rs b/crates/nexum-runtime/src/bootstrap.rs index 532e5582..e78120c7 100644 --- a/crates/nexum-runtime/src/bootstrap.rs +++ b/crates/nexum-runtime/src/bootstrap.rs @@ -44,6 +44,7 @@ pub async fn run( wasm, manifest, clocks: None, + supervisor_clock: None, }; let ctx = LaunchContext { tasks: TaskManager::new(), diff --git a/crates/nexum-runtime/src/builder.rs b/crates/nexum-runtime/src/builder.rs index be027497..04a329fe 100644 --- a/crates/nexum-runtime/src/builder.rs +++ b/crates/nexum-runtime/src/builder.rs @@ -18,6 +18,7 @@ use std::future::{Future, IntoFuture}; use std::marker::PhantomData; use std::path::{Path, PathBuf}; +use std::sync::Arc; use std::time::Duration; use nexum_tasks::{DrainOutcome, TaskExit, TaskHandle, TaskManager, TaskSet}; @@ -33,8 +34,8 @@ use crate::host::extension::Extension; use crate::host::logs::LogPipeline; use crate::preset::Runtime; use crate::runtime::event_loop; -pub use crate::supervisor::WasiClockOverride; use crate::supervisor::{self, Supervisor}; +pub use crate::supervisor::{SupervisorClock, WasiClockOverride}; /// Ambient inputs the imperative launcher reads: the task manager every /// runtime task spawns through, and the loaded config. @@ -137,6 +138,10 @@ pub struct AssembledRuntime<'a, T: RuntimeTypes> { pub manifest: Option<&'a Path>, /// Per-store WASI clock override; `None` leaves the ambient host clocks. pub clocks: Option, + /// Clock the supervisor's poison window and restart backoff read + /// `now()` from; `None` uses the real host clock. Distinct from + /// `clocks`, which only virtualizes guest-visible WASI time. + pub supervisor_clock: Option>, } /// An assembled runtime launchable from a [`LaunchContext`]. @@ -154,6 +159,7 @@ impl LaunchRuntime for AssembledRuntime<'_, T> { wasm, manifest, clocks, + supervisor_clock, } = self; let LaunchContext { tasks, @@ -196,6 +202,7 @@ impl LaunchRuntime for AssembledRuntime<'_, T> { &engine_cfg.limits, &extensions, clocks, + supervisor_clock, ) .await? } else if !engine_cfg.modules.is_empty() { @@ -206,6 +213,7 @@ impl LaunchRuntime for AssembledRuntime<'_, T> { &components, &extensions, clocks, + supervisor_clock, ) .await? } else { @@ -347,6 +355,7 @@ impl<'a> RuntimeBuilder<'a> { wasm: None, manifest: None, clocks: None, + supervisor_clock: None, _t: PhantomData, } } @@ -361,6 +370,7 @@ impl<'a> RuntimeBuilder<'a> { wasm: None, manifest: None, clocks: None, + supervisor_clock: None, _r: PhantomData, } } @@ -375,6 +385,7 @@ pub struct PresetBuilder<'a, R: Runtime> { wasm: Option, manifest: Option, clocks: Option, + supervisor_clock: Option>, _r: PhantomData R>, } @@ -405,6 +416,16 @@ impl<'a, R: Runtime> PresetBuilder<'a, R> { self } + /// Override the clock the supervisor's poison window and restart + /// backoff read `now()` from. Distinct from + /// [`with_wasi_clocks`](Self::with_wasi_clocks), which only + /// virtualizes guest-visible WASI time. Omitting it uses the real + /// host clock, which is behaviour-neutral. + pub fn with_supervisor_clock(mut self, clock: Arc) -> Self { + self.supervisor_clock = Some(clock); + self + } + /// Open the preset's backends and launch. Builds the [`Components`] bundle /// from the preset's component builders, installs the preset's add-ons, /// then drives [`LaunchRuntime::launch`] with a fresh [`TaskManager`]. @@ -431,6 +452,7 @@ impl<'a, R: Runtime> PresetBuilder<'a, R> { wasm: self.wasm.as_deref(), manifest: self.manifest.as_deref(), clocks: self.clocks, + supervisor_clock: self.supervisor_clock, }; let ctx = LaunchContext { tasks, @@ -448,6 +470,7 @@ pub struct TypedBuilder<'a, T: RuntimeTypes> { wasm: Option, manifest: Option, clocks: Option, + supervisor_clock: Option>, _t: PhantomData T>, } @@ -474,6 +497,16 @@ impl<'a, T: RuntimeTypes> TypedBuilder<'a, T> { self } + /// Override the clock the supervisor's poison window and restart + /// backoff read `now()` from. Distinct from + /// [`with_wasi_clocks`](Self::with_wasi_clocks), which only + /// virtualizes guest-visible WASI time. Omitting it uses the real + /// host clock, which is behaviour-neutral. + pub fn with_supervisor_clock(mut self, clock: Arc) -> Self { + self.supervisor_clock = Some(clock); + self + } + /// Bind the component builders that open the backends at launch. pub fn with_components( self, @@ -485,6 +518,7 @@ impl<'a, T: RuntimeTypes> TypedBuilder<'a, T> { wasm: self.wasm, manifest: self.manifest, clocks: self.clocks, + supervisor_clock: self.supervisor_clock, components, _t: PhantomData, } @@ -498,6 +532,7 @@ pub struct ComponentsStage<'a, T: RuntimeTypes, C, S, E> { wasm: Option, manifest: Option, clocks: Option, + supervisor_clock: Option>, components: ComponentsBuilder, _t: PhantomData T>, } @@ -511,6 +546,7 @@ impl<'a, T: RuntimeTypes, C, S, E> ComponentsStage<'a, T, C, S, E> { wasm: self.wasm, manifest: self.manifest, clocks: self.clocks, + supervisor_clock: self.supervisor_clock, components: self.components, add_ons, } @@ -525,6 +561,7 @@ pub struct ReadyBuilder<'a, T: RuntimeTypes, C, S, E> { wasm: Option, manifest: Option, clocks: Option, + supervisor_clock: Option>, components: ComponentsBuilder, add_ons: &'a [&'a dyn RuntimeAddOn], } @@ -557,6 +594,7 @@ where wasm: self.wasm.as_deref(), manifest: self.manifest.as_deref(), clocks: self.clocks, + supervisor_clock: self.supervisor_clock, }; let ctx = LaunchContext { tasks, @@ -704,6 +742,7 @@ every_n_blocks = "1" wasm: None, manifest: None, clocks: None, + supervisor_clock: None, }; let ctx = LaunchContext { tasks, diff --git a/crates/nexum-runtime/src/supervisor.rs b/crates/nexum-runtime/src/supervisor.rs index 22400dfb..ac674193 100644 --- a/crates/nexum-runtime/src/supervisor.rs +++ b/crates/nexum-runtime/src/supervisor.rs @@ -73,6 +73,10 @@ pub struct Supervisor { /// including the ones rebuilt on restart. `None` leaves the ambient /// host clocks. clocks: Option, + /// Clock the poison window and restart backoff read `now()` from. + /// Real host clock in production; a harness swaps in a manually + /// driven fake for deterministic timing tests. + clock: Arc, } /// Core-only lattice for the runtime's own tests: the reference core @@ -167,6 +171,28 @@ fn resolve_module_limits(res: &ResourceSection, cfg: &ModuleLimits) -> ResolvedL } } +/// Supervisor-internal clock seam for poison-window accounting and +/// restart-backoff scheduling. Distinct from [`WasiClockOverride`], +/// which only virtualizes guest-visible WASI time: this drives the +/// supervisor's own bookkeeping (`next_attempt`, `failure_timestamps`). +/// Defaults to the real host clock; a harness overrides it with a +/// manually-driven fake so backoff/poison-window tests assert +/// deterministically instead of sleeping real wall-clock time. +pub trait SupervisorClock: Send + Sync { + /// Current instant, per this clock's notion of time. + fn now(&self) -> std::time::Instant; +} + +/// Production default: the real host [`Instant`](std::time::Instant). +#[derive(Default)] +struct SystemClock; + +impl SupervisorClock for SystemClock { + fn now(&self) -> std::time::Instant { + std::time::Instant::now() + } +} + struct LoadedModule { name: String, bindings: EventModule, @@ -245,6 +271,7 @@ impl Supervisor { components: &Components, extensions: &[Extension], clocks: Option, + supervisor_clock: Option>, ) -> Result { let registry = capability_registry(extensions); let mut modules = Vec::with_capacity(engine_cfg.modules.len()); @@ -270,6 +297,7 @@ impl Supervisor { extensions: extensions.to_vec(), poison_policy: engine_cfg.limits.poison(), clocks, + clock: supervisor_clock.unwrap_or_else(|| Arc::new(SystemClock)), }) } @@ -289,6 +317,7 @@ impl Supervisor { limits: &ModuleLimits, extensions: &[Extension], clocks: Option, + supervisor_clock: Option>, ) -> Result { let registry = capability_registry(extensions); let entry = ModuleEntry { @@ -312,6 +341,7 @@ impl Supervisor { extensions: extensions.to_vec(), poison_policy: limits.poison(), clocks, + clock: supervisor_clock.unwrap_or_else(|| Arc::new(SystemClock)), }) } @@ -733,7 +763,7 @@ impl Supervisor { let chain_id = chain.id(); let block_number = block.number; let event = nexum::host::types::Event::Block(block); - let now = std::time::Instant::now(); + let now = self.clock.now(); // Hoist the local-store reference out so the per-module // borrow checker is happy when we write the progress // marker after a successful dispatch. @@ -820,7 +850,7 @@ impl Supervisor { log: alloy_rpc_types_eth::Log, cursor_key: Option<&str>, ) -> bool { - let now = std::time::Instant::now(); + let now = self.clock.now(); let Some(idx) = self.modules.iter().position(|m| m.name == module_name) else { warn!(module = %module_name, "no such module - dropping chain-log"); return false; @@ -907,8 +937,10 @@ impl Supervisor { let chain_id = chain.id(); let poison_policy = self.poison_policy; // Hoisted before the per-module borrow so the trap arm can - // synthesize a panic record without re-borrowing `self`. + // synthesize a panic record and schedule the backoff without + // re-borrowing `self`. let router = self.components.logs.router(); + let now = self.clock.now(); let module = &mut self.modules[idx]; // Dispatch-boundary rate limit: throttle before spending any // fuel or entering the guest, so a flood of cheap-to-dispatch @@ -1010,7 +1042,7 @@ impl Supervisor { let latency_ms = elapsed.as_millis() as u64; module.failure_count = module.failure_count.saturating_add(1); let backoff = crate::runtime::restart_policy::backoff_for(module.failure_count); - let next_attempt = std::time::Instant::now() + backoff; + let next_attempt = now + backoff; error!( module = %module.name, chain_id, @@ -1041,7 +1073,7 @@ impl Supervisor { Level::ERROR, format!("run terminated abnormally: {}", trap.root_cause()), )); - record_failure_and_maybe_poison(module, poison_policy, &trap.to_string()); + record_failure_and_maybe_poison(module, poison_policy, now, &trap.to_string()); DispatchOutcome::Trapped } } @@ -1068,10 +1100,11 @@ impl Supervisor { Err(e) => { // Re-instantiation failed: bump the backoff again so // the next attempt is further out. + let now = self.clock.now(); let m = &mut self.modules[idx]; m.failure_count = m.failure_count.saturating_add(1); let backoff = crate::runtime::restart_policy::backoff_for(m.failure_count); - m.next_attempt = Some(std::time::Instant::now() + backoff); + m.next_attempt = Some(now + backoff); error!( module = %name, failure_count = m.failure_count, @@ -1210,9 +1243,9 @@ enum DispatchOutcome { fn record_failure_and_maybe_poison( module: &mut LoadedModule, policy: crate::runtime::poison_policy::PoisonPolicy, + now: std::time::Instant, last_error: &str, ) { - let now = std::time::Instant::now(); // Prune entries outside the window. while let Some(&front) = module.failure_timestamps.front() { if now.duration_since(front) > policy.window { diff --git a/crates/nexum-runtime/src/supervisor/tests.rs b/crates/nexum-runtime/src/supervisor/tests.rs index 6ce381b5..62df3f17 100644 --- a/crates/nexum-runtime/src/supervisor/tests.rs +++ b/crates/nexum-runtime/src/supervisor/tests.rs @@ -174,7 +174,7 @@ async fn boot_mock_supervisor( let config = EngineConfig::default(); let linker = crate::supervisor::build_linker::(engine, &[]) .expect("build_linker"); - Supervisor::boot(engine, &linker, &config, &components, &[], None) + Supervisor::boot(engine, &linker, &config, &components, &[], None, None) .await .expect("boot mock supervisor") } @@ -202,6 +202,7 @@ async fn e2e_supervisor_boots_example_module() { &limits, &core_extensions(), None, + None, ) .await .expect("boot_single"); @@ -250,6 +251,7 @@ chain_id = 1 &limits, &core_extensions(), None, + None, ) .await .expect("boot_single"); @@ -315,6 +317,7 @@ chain_id = 1 &limits, &core_extensions(), Some(clock.as_override()), + None, ) .await .expect("boot_single with a manual clock override"); @@ -425,6 +428,7 @@ async fn boot_production_module( &limits, &core_extensions(), None, + None, ) .await .expect("boot_single") @@ -460,6 +464,7 @@ async fn twap_monitor_without_cow_extension_fails_to_boot() { &limits, &[], None, + None, ) .await; @@ -800,6 +805,7 @@ chain_id = 1 &components, &core_extensions(), None, + None, ) .await .expect("boot"); @@ -1025,6 +1031,7 @@ async fn boot_fixture(wasm: &Path, manifest_relative: &str) -> DefaultSupervisor &limits, &core_extensions(), None, + None, ) .await .expect("boot_single") @@ -1140,6 +1147,7 @@ chain_id = 1 &components, &core_extensions(), None, + None, ) .await .expect("boot"); @@ -1203,14 +1211,21 @@ async fn resource_limit_memory_bomb_traps_and_marks_module_dead() { // // 1. Dispatch 1: trap -> alive=false, failure_count=1, next_attempt=+1s. // 2. Immediate redispatch: skipped (next_attempt in the future). -// 3. After 1.1s: alive flipped back on, dispatch retried. +// 3. After advancing the clock 1s: alive flipped back on, dispatch retried. // 4. With fail_first_n=1, the second attempt succeeds -> failure_count // resets to 0, next_attempt = None. // -// Asserts the schedule shape end-to-end with real wall-clock. +// Asserts the schedule shape end-to-end against a manually-driven +// clock (issue #284): the backoff eligibility check is `next_attempt <= +// now`, so advancing exactly 1s is enough - no real sleep, no jitter +// fudge factor needed. #[tokio::test] async fn restart_flaky_module_recovers_after_backoff() { + use std::sync::Arc; + + use crate::test_utils::clock::ManualInstantClock; + let Some(wasm) = module_wasm_or_skip("flaky-bomb") else { return; }; @@ -1218,7 +1233,7 @@ async fn restart_flaky_module_recovers_after_backoff() { let dir = tempfile::tempdir().unwrap(); let manifest = dir.path().join("module.toml"); // fail_first_n = 1 so the module traps once and recovers on the - // second dispatch attempt. Keeps the test wall-clock under 2 s. + // second dispatch attempt. std::fs::write( &manifest, r#" @@ -1243,6 +1258,7 @@ fail_first_n = "1" let (_dir, store) = temp_local_store(); let components = test_components(store); let limits = crate::engine_config::ModuleLimits::default(); + let clock = ManualInstantClock::new(); let mut supervisor = Supervisor::boot_single( &engine, &linker, @@ -1252,6 +1268,7 @@ fail_first_n = "1" &limits, &core_extensions(), None, + Some(Arc::new(clock.clone())), ) .await .expect("boot_single"); @@ -1277,9 +1294,8 @@ fail_first_n = "1" ); assert_eq!(supervisor.alive_count(), 0); - // Wait for the 1s backoff window to elapse (+ a small fudge for - // scheduler jitter). - tokio::time::sleep(std::time::Duration::from_millis(1100)).await; + // Advance the clock exactly to the backoff window's edge. + clock.advance(std::time::Duration::from_secs(1)); // Dispatch 3: now eligible. fail_first_n=1 was satisfied on // dispatch 1, so this attempt succeeds. The supervisor flips @@ -1300,13 +1316,13 @@ fail_first_n = "1" // // fuel-bomb traps on every dispatch. With a // tight poison policy (3 failures / 60 s) we can observe the -// supervisor escalate from "retry" to "permanent quarantine" inside -// ~4 s of wall clock: +// supervisor escalate from "retry" to "permanent quarantine" against a +// manually-driven clock (issue #284), no real sleep: // // trap 1: failure_count=1, next_attempt=+1s -// sleep 1.1s +// advance the clock 1s // trap 2: failure_count=2, next_attempt=+2s -// sleep 2.1s +// advance the clock 2s // trap 3: failure_count=3 -> POISONED. Recent failures hit the // window threshold; the supervisor stops attempting // restarts entirely. Subsequent dispatches skip the @@ -1316,6 +1332,10 @@ fail_first_n = "1" #[tokio::test] async fn poison_pill_quarantines_module_after_threshold() { + use std::sync::Arc; + + use crate::test_utils::clock::ManualInstantClock; + let Some(wasm) = module_wasm_or_skip("fuel-bomb") else { return; }; @@ -1325,8 +1345,8 @@ async fn poison_pill_quarantines_module_after_threshold() { let (_dir, store) = temp_local_store(); let components = test_components(store); - // Tight policy: 3 failures in 60 s -> quarantine. Keeps the - // test wall-clock under 4 s. Set through `[limits.poison]`. + // Tight policy: 3 failures in 60 s -> quarantine. Set through + // `[limits.poison]`. let limits = crate::engine_config::ModuleLimits { poison: crate::engine_config::PoisonLimitsSection { max_failures: Some(3), @@ -1334,6 +1354,7 @@ async fn poison_pill_quarantines_module_after_threshold() { }, ..Default::default() }; + let clock = ManualInstantClock::new(); let mut supervisor = Supervisor::boot_single( &engine, &linker, @@ -1343,6 +1364,7 @@ async fn poison_pill_quarantines_module_after_threshold() { &limits, &core_extensions(), None, + Some(Arc::new(clock.clone())), ) .await .expect("boot_single"); @@ -1363,13 +1385,13 @@ async fn poison_pill_quarantines_module_after_threshold() { assert_eq!(dispatched, 0); assert_eq!(supervisor.alive_count(), 0); assert_eq!(supervisor.poisoned_count(), 0, "1 trap < threshold"); - tokio::time::sleep(std::time::Duration::from_millis(1_100)).await; + clock.advance(std::time::Duration::from_secs(1)); // Trap 2. let dispatched = supervisor.dispatch_block(block.clone()).await; assert_eq!(dispatched, 0); assert_eq!(supervisor.poisoned_count(), 0, "2 traps < threshold"); - tokio::time::sleep(std::time::Duration::from_millis(2_100)).await; + clock.advance(std::time::Duration::from_secs(2)); // Trap 3 -> POISONED. let dispatched = supervisor.dispatch_block(block.clone()).await; @@ -1501,6 +1523,7 @@ async fn dying_run_leaves_a_panic_record() { &limits, &core_extensions(), None, + None, ) .await .expect("boot_single"); @@ -1556,6 +1579,7 @@ async fn facade_panic_leaves_stderr_host_interface_and_panic_records() { &limits, &core_extensions(), None, + None, ) .await .expect("boot_single"); @@ -1684,6 +1708,7 @@ chain_id = 100 &components, &core_extensions(), None, + None, ) .await .expect("boot"); @@ -1857,10 +1882,15 @@ chain_id = 100 #[tokio::test] async fn multi_chain_poisoned_module_does_not_affect_other_chains() { + use std::sync::Arc; + + use crate::test_utils::clock::ManualInstantClock; + // fuel-bomb (always-traps) on chain 1, example (healthy) on // chain 100. Trap the bomb a few times with a tight poison // policy so it gets quarantined; verify the example keeps - // dispatching on chain 100 throughout. + // dispatching on chain 100 throughout. Drives a manually-advanced + // clock (issue #284) past the bomb's backoff instead of a real sleep. let Some(bomb_wasm) = module_wasm_or_skip("fuel-bomb") else { return; }; @@ -1923,6 +1953,7 @@ chain_id = 100 ], }; + let clock = ManualInstantClock::new(); let mut supervisor = Supervisor::boot( &engine, &linker, @@ -1930,6 +1961,7 @@ chain_id = 100 &components, &core_extensions(), None, + Some(Arc::new(clock.clone())), ) .await .expect("boot"); @@ -1958,8 +1990,8 @@ chain_id = 100 let dispatched_b = supervisor.dispatch_block(block_healthy_chain.clone()).await; assert_eq!(dispatched_b, 1, "module-b receives chain-100 blocks"); - // Wait out the bomb's backoff so trap #2 can land. - tokio::time::sleep(std::time::Duration::from_millis(1_100)).await; + // Advance past the bomb's backoff so trap #2 can land. + clock.advance(std::time::Duration::from_secs(1)); supervisor.dispatch_block(block_bomb_chain).await; assert_eq!( supervisor.poisoned_count(), diff --git a/crates/nexum-runtime/src/test_utils/clock.rs b/crates/nexum-runtime/src/test_utils/clock.rs index fcda4919..f19dac9b 100644 --- a/crates/nexum-runtime/src/test_utils/clock.rs +++ b/crates/nexum-runtime/src/test_utils/clock.rs @@ -1,11 +1,13 @@ -//! A manually-driven WASI clock for deterministic guest time in tests. +//! Manually-driven clocks for deterministic time in tests: a WASI clock +//! for guest-visible time, and a supervisor clock for poison-window and +//! restart-backoff scheduling. use std::sync::{Arc, Mutex}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use wasmtime_wasi::{HostMonotonicClock, HostWallClock}; -use crate::supervisor::WasiClockOverride; +use crate::supervisor::{SupervisorClock, WasiClockOverride}; /// A shared, manually-advanced clock source. /// @@ -96,6 +98,50 @@ impl HostMonotonicClock for ManualClock { } } +/// A manually-advanced [`SupervisorClock`] for deterministic poison-window +/// and restart-backoff tests. Distinct from [`ManualClock`], which +/// virtualizes guest-visible WASI time: this drives the supervisor's own +/// bookkeeping (`next_attempt`, `failure_timestamps`). Cloning yields +/// another handle onto the same instant. +/// +/// Uses `std::sync::Mutex` explicitly (independent of whichever `Mutex` +/// import is in scope elsewhere in this file): a short, never-`.await`-held +/// critical section same as `ManualClock`'s. +#[derive(Clone)] +pub struct ManualInstantClock { + inner: Arc>, +} + +impl Default for ManualInstantClock { + fn default() -> Self { + Self::new() + } +} + +impl ManualInstantClock { + /// Seeded at the real instant this is called: `std::time::Instant` + /// has no arbitrary constructor, so a real reading is the only way + /// to obtain one; [`advance`](Self::advance) moves it forward from + /// there, never backward. + pub fn new() -> Self { + Self { + inner: Arc::new(std::sync::Mutex::new(std::time::Instant::now())), + } + } + + /// Move the clock forward by `by`. + pub fn advance(&self, by: Duration) { + let mut guard = self.inner.lock().expect("manual instant clock poisoned"); + *guard += by; + } +} + +impl SupervisorClock for ManualInstantClock { + fn now(&self) -> std::time::Instant { + *self.inner.lock().expect("manual instant clock poisoned") + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/nexum-runtime/src/test_utils/harness.rs b/crates/nexum-runtime/src/test_utils/harness.rs index e06e163a..ff051d3b 100644 --- a/crates/nexum-runtime/src/test_utils/harness.rs +++ b/crates/nexum-runtime/src/test_utils/harness.rs @@ -3,14 +3,17 @@ //! //! [`TestRuntime`] wraps the public builder path over [`MockTypes`]: it opens //! a module from a wasm path plus a manifest (a path, or inline TOML written -//! to a temp file), installs a manually-driven [`ManualClock`], and returns a -//! handle bundling the running [`RuntimeHandle`] with the retained mock -//! handles. A test programs chain responses and injects block headers or chain -//! logs through [`chain`](TestRuntime::chain), advances guest time through -//! [`clock`](TestRuntime::clock), reads what a module wrote through -//! [`store`](TestRuntime::store), and reads runs and log pages through -//! [`logs`](TestRuntime::logs), then [`shutdown`](TestRuntime::shutdown) and -//! [`wait`](TestRuntime::wait). +//! to a temp file), installs a manually-driven [`ManualClock`] for +//! guest-visible time and a [`ManualInstantClock`] for the supervisor's own +//! poison-window/restart-backoff timing, and returns a handle bundling the +//! running [`RuntimeHandle`] with the retained mock handles. A test programs +//! chain responses and injects block headers or chain logs through +//! [`chain`](TestRuntime::chain), advances guest time through +//! [`clock`](TestRuntime::clock), advances the supervisor's backoff/poison +//! clock through [`supervisor_clock`](TestRuntime::supervisor_clock), reads +//! what a module wrote through [`store`](TestRuntime::store), and reads runs +//! and log pages through [`logs`](TestRuntime::logs), then +//! [`shutdown`](TestRuntime::shutdown) and [`wait`](TestRuntime::wait). //! //! Events dispatch on the spawned event-loop task, so a test injects an event //! and then awaits an observable effect; @@ -22,11 +25,12 @@ //! crate's backend through the same harness. use std::path::PathBuf; +use std::sync::Arc; use std::time::Duration; use alloy_rpc_types_eth::{Header, Log}; -use super::clock::ManualClock; +use super::clock::{ManualClock, ManualInstantClock}; use super::{MockChainProvider, MockStateStore, MockTypes, Prebuilt}; use crate::builder::{RuntimeBuilder, RuntimeHandle}; use crate::engine_config::{EngineConfig, ModuleLimits}; @@ -60,6 +64,7 @@ where chain: MockChainProvider, store: MockStateStore, clock: ManualClock, + supervisor_clock: ManualInstantClock, } impl TestRuntime<()> { @@ -83,6 +88,7 @@ impl TestRuntime { chain: MockChainProvider::new(), store: MockStateStore::new(), clock: ManualClock::new(), + supervisor_clock: ManualInstantClock::new(), } } } @@ -139,6 +145,13 @@ impl TestRuntimeBuilder { &self.clock } + /// The manual clock installed for the supervisor's own poison-window + /// and restart-backoff timing, distinct from the guest-visible + /// [`clock`](Self::clock). + pub fn supervisor_clock(&self) -> &ManualInstantClock { + &self.supervisor_clock + } + /// Open the module and start the runtime. Composes entirely through the /// public builder path over [`MockTypes`]. pub async fn launch(self) -> anyhow::Result> { @@ -165,6 +178,7 @@ impl TestRuntimeBuilder { .with_extensions(self.extensions) .with_module_source(Some(self.wasm), manifest) .with_wasi_clocks(self.clock.as_override()) + .with_supervisor_clock(Arc::new(self.supervisor_clock.clone())) .with_components(ComponentsBuilder::new( Prebuilt(self.chain.clone()), Prebuilt(self.store.clone()), @@ -179,6 +193,7 @@ impl TestRuntimeBuilder { chain: self.chain, store: self.store, clock: self.clock, + supervisor_clock: self.supervisor_clock, ext: self.ext, _tmp: tmp, }) @@ -193,6 +208,7 @@ pub struct TestRuntime { chain: MockChainProvider, store: MockStateStore, clock: ManualClock, + supervisor_clock: ManualInstantClock, ext: E, // Holds any inline manifest for the lifetime of the harness; dropped // when the `TestRuntime` is dropped (or consumed by `wait`). @@ -217,6 +233,13 @@ impl TestRuntime { &self.clock } + /// The manual clock driving the supervisor's own poison-window and + /// restart-backoff timing, distinct from the guest-visible + /// [`clock`](Self::clock). + pub fn supervisor_clock(&self) -> &ManualInstantClock { + &self.supervisor_clock + } + /// The extension payload bound into the lattice ext slot. pub fn ext(&self) -> &E { &self.ext diff --git a/crates/shepherd-cow-host/tests/cow_boot.rs b/crates/shepherd-cow-host/tests/cow_boot.rs index f3a75d9a..00f0bb95 100644 --- a/crates/shepherd-cow-host/tests/cow_boot.rs +++ b/crates/shepherd-cow-host/tests/cow_boot.rs @@ -137,6 +137,7 @@ async fn boot_production_module( &limits, &cow_extensions(), None, + None, ) .await .expect("boot_single")