Add per-model-call wall-clock ceiling + h2 PING keepalives on the default provider client

Cargo.toml

@@ -48,6 +48,13 @@ reqwest = { version = "0.12", default-features = false, features = [
     "json",
     "rustls-tls",
     "stream",
+    # `http2` lets TLS endpoints negotiate HTTP/2 via ALPN, which is what
+    # enables the transport-liveness PING keepalives the provider clients
+    # configure (`http2_keep_alive_*`): a dead peer fails the in-flight call in
+    # under a minute even when zero application bytes are flowing — e.g. a
+    # provider stream that goes silent during long hidden reasoning. Plaintext
+    # endpoints (local Ollama/LM Studio) have no ALPN and stay on HTTP/1.1.
+    "http2",
 ] }
 
 # Optional embedded SQLite checkpointer backend (`graph::checkpoint::sqlite`).

src/harness/agent_loop/model_call.rs

@@ -6,6 +6,16 @@
 //! Split out of `agent_loop/mod.rs`; see that module's doc comment for
 //! the full loop lifecycle, limits, and backoff design.
 
+/// Timeout-message label for a call bounded by the run's remaining wall-clock
+/// budget: the run is out of time, not (necessarily) this call wedged.
+pub(super) const RUN_BOUND_LABEL: &str = "remaining wall-clock budget";
+
+/// Timeout-message label for a model call bounded by the per-call ceiling
+/// ([`RunLimits::max_model_call_ms`][crate::harness::limits::RunLimits::max_model_call_ms]):
+/// this one call ran past the time any single call is allowed, with run time
+/// still left.
+pub(super) const PER_CALL_BOUND_LABEL: &str = "per-model-call ceiling";
+
 use super::*;
 use crate::harness::cache::{
     CachePolicy, CacheSkipReason, apply_prompt_cache_breakpoints, scoped_cache_key,
@@ -325,14 +335,16 @@ impl<State: Send + Sync, Ctx: Send + Sync> AgentHarness<State, Ctx> {
                 if ctx.cancellation.is_cancelled() {
                     return Err(TinyAgentsError::Cancelled);
                 }
-                // Bound this individual provider call by the run's *remaining*
-                // wall-clock budget so a hung or slow model call is interrupted
+                // Bound this individual provider call by the tighter of the
+                // run's *remaining* wall-clock budget and the per-model-call
+                // ceiling, so a hung or slow model call is interrupted
                 // mid-flight, not merely detected by the between-call deadline
                 // check. reqwest/futures are cancel-safe, so dropping the future
-                // on elapse cancels the underlying request. When neither the run
-                // config nor the harness policy configures a timeout the call is
-                // awaited unbounded.
-                let remaining = self.call_budget(ctx);
+                // on elapse cancels the underlying request. Recomputed here,
+                // inside the attempt loop, so every retry attempt gets a fresh
+                // per-call window. When neither the run config nor the harness
+                // policy configures any timeout the call is awaited unbounded.
+                let (remaining, bound) = self.model_call_budget(ctx);
                 let attempt_result = if streaming {
                     let fut = self.invoke_model_streaming_once(
                         state,
@@ -342,7 +354,8 @@ impl<State: Send + Sync, Ctx: Send + Sync> AgentHarness<State, Ctx> {
                         call_id,
                         &mut deltas_emitted,
                     );
-                    Self::with_call_budget(remaining, run_id.as_str(), "model call", fut).await
+                    Self::with_call_budget(remaining, run_id.as_str(), "model call", bound, fut)
+                        .await
                 } else {
                     // Race the wall-clock-bounded unary call against cooperative
                     // cancellation, mirroring the streaming path: a cancel
@@ -354,8 +367,13 @@ impl<State: Send + Sync, Ctx: Send + Sync> AgentHarness<State, Ctx> {
                     // still short-circuits before the request is ever issued.
                     let cancellation = ctx.cancellation.clone();
                     let fut = model.invoke(state, request.clone());
-                    let budgeted =
-                        Self::with_call_budget(remaining, run_id.as_str(), "model call", fut);
+                    let budgeted = Self::with_call_budget(
+                        remaining,
+                        run_id.as_str(),
+                        "model call",
+                        bound,
+                        fut,
+                    );
                     tokio::select! {
                         biased;
                         _ = cancellation.cancelled() => Err(TinyAgentsError::Cancelled),
@@ -510,6 +528,13 @@ impl<State: Send + Sync, Ctx: Send + Sync> AgentHarness<State, Ctx> {
     /// (`None`). Honoring the policy source lets a sub-agent whose child
     /// [`RunConfig`] carries no per-run timeout still be bounded by its
     /// harness's policy-level wall-clock cap.
+    ///
+    /// This is the budget for **tool calls**, which are bounded only by the
+    /// run's remaining time (a sub-agent delegation is a tool call wrapping an
+    /// entire child run). Model calls go through
+    /// [`model_call_budget`](Self::model_call_budget), which additionally
+    /// clamps to the per-call ceiling
+    /// [`RunLimits::max_model_call_ms`][crate::harness::limits::RunLimits::max_model_call_ms].
     pub(super) fn call_budget(&self, ctx: &RunContext<Ctx>) -> Option<Duration> {
         let config_budget = ctx.remaining_wall_clock();
         let policy_budget = self.policy.limits.max_wall_clock_ms.map(|ms| {
@@ -525,6 +550,35 @@ impl<State: Send + Sync, Ctx: Send + Sync> AgentHarness<State, Ctx> {
         }
     }
 
+    /// Computes the wall-clock budget for the next individual **model** call:
+    /// the tighter of the run's remaining budget ([`call_budget`](Self::call_budget))
+    /// and the per-call ceiling
+    /// [`RunLimits::max_model_call_ms`][crate::harness::limits::RunLimits::max_model_call_ms].
+    ///
+    /// Computed afresh for every call *and every retry attempt*, so each
+    /// attempt gets its own full per-call window rather than inheriting an
+    /// earlier attempt's consumption — while still never overshooting the run
+    /// deadline. Returns the budget together with the label of whichever bound
+    /// is in force, so the timeout error names the ceiling that actually fired
+    /// (a per-call ceiling means "this one call wedged"; the run's remainder
+    /// means "the run is out of time") — field triage needs to tell them apart.
+    pub(super) fn model_call_budget(
+        &self,
+        ctx: &RunContext<Ctx>,
+    ) -> (Option<Duration>, &'static str) {
+        let run_budget = self.call_budget(ctx);
+        let per_call = self
+            .policy
+            .limits
+            .max_model_call_ms
+            .map(Duration::from_millis);
+        match (run_budget, per_call) {
+            (Some(run), Some(cap)) if cap < run => (Some(cap), PER_CALL_BOUND_LABEL),
+            (None, Some(cap)) => (Some(cap), PER_CALL_BOUND_LABEL),
+            (run, _) => (run, RUN_BOUND_LABEL),
+        }
+    }
+
     /// Awaits a single call future (model or tool), optionally bounded by
     /// `budget`.
     ///
@@ -534,14 +588,18 @@ impl<State: Send + Sync, Ctx: Send + Sync> AgentHarness<State, Ctx> {
     /// [`TinyAgentsError::Timeout`] is returned. When `budget` is `None` (no
     /// run timeout configured) the future is awaited without a bound.
     ///
-    /// `budget` is the run's *remaining* wall-clock budget at the time the call
-    /// is issued, so each successive call gets a tighter bound as the deadline
-    /// approaches. `what` names the kind of call in the timeout message (e.g.
-    /// `"model call"`, `"tool call"`).
+    /// `budget` is either the run's *remaining* wall-clock budget at the time
+    /// the call is issued (each successive call gets a tighter bound as the
+    /// deadline approaches) or, for model calls, the per-call ceiling when that
+    /// is tighter. `what` names the kind of call in the timeout message (e.g.
+    /// `"model call"`, `"tool call"`); `bound` names which budget source is in
+    /// force ([`RUN_BOUND_LABEL`] / [`PER_CALL_BOUND_LABEL`]) so the message
+    /// says which ceiling fired.
     pub(super) async fn with_call_budget<T, F>(
         budget: Option<Duration>,
         run_id: &str,
         what: &str,
+        bound: &str,
         fut: F,
     ) -> Result<T>
     where
@@ -551,8 +609,7 @@ impl<State: Send + Sync, Ctx: Send + Sync> AgentHarness<State, Ctx> {
             Some(budget) => match tokio::time::timeout(budget, fut).await {
                 Ok(result) => result,
                 Err(_) => Err(TinyAgentsError::Timeout(format!(
-                    "{what} for run `{run_id}` exceeded its remaining wall-clock budget \
-                     ({} ms)",
+                    "{what} for run `{run_id}` exceeded its {bound} ({} ms)",
                     budget.as_millis()
                 ))),
             },

src/harness/agent_loop/test.rs

@@ -2495,6 +2495,137 @@ async fn slow_tool_call_is_timed_out_by_remaining_budget() {
     assert!(matches!(err, TinyAgentsError::Timeout(_)), "got {err:?}");
 }
 
+#[tokio::test]
+async fn per_model_call_ceiling_times_out_a_slow_call_with_run_time_left() {
+    use std::time::Duration;
+
+    use crate::harness::testkit::SlowModel;
+
+    // The run has plenty of wall clock left (60s), but the per-model-call
+    // ceiling (20ms) is tighter than the model's 200ms sleep, so the ceiling
+    // interrupts the call — and the error must name the ceiling, not the run's
+    // remaining budget, so triage can tell a wedged call from an exhausted run.
+    let mut harness: AgentHarness<()> = AgentHarness::new();
+    harness.register_model(
+        "slow",
+        Arc::new(SlowModel::new(Duration::from_millis(200), "too late")),
+    );
+    harness.with_policy(RunPolicy {
+        limits: RunLimits::default().with_max_model_call_ms(Some(20)),
+        ..RunPolicy::default()
+    });
+
+    let config = RunConfig::new("per-call-cap-run").with_timeout_ms(60_000);
+    let err = harness
+        .invoke(&(), (), config, vec![Message::user("hi")])
+        .await
+        .expect_err("a call slower than the per-call ceiling must time out");
+
+    match &err {
+        TinyAgentsError::Timeout(msg) => {
+            assert!(msg.contains("per-model-call ceiling"), "{msg}");
+        }
+        other => panic!("expected Timeout, got {other:?}"),
+    }
+}
+
+#[tokio::test]
+async fn per_model_call_ceiling_bounds_calls_without_any_run_deadline() {
+    use std::time::Duration;
+
+    use crate::harness::testkit::SlowModel;
+
+    // With no run timeout and no policy wall clock, a model call used to be
+    // awaited unbounded. The per-call ceiling alone must bound it.
+    let mut harness: AgentHarness<()> = AgentHarness::new();
+    harness.register_model(
+        "slow",
+        Arc::new(SlowModel::new(Duration::from_millis(200), "too late")),
+    );
+    harness.with_policy(RunPolicy {
+        limits: RunLimits::default().with_max_model_call_ms(Some(20)),
+        ..RunPolicy::default()
+    });
+
+    let err = harness
+        .invoke_default(&(), vec![Message::user("hi")])
+        .await
+        .expect_err("the ceiling alone must bound an otherwise-unbounded call");
+
+    match &err {
+        TinyAgentsError::Timeout(msg) => {
+            assert!(msg.contains("per-model-call ceiling"), "{msg}");
+        }
+        other => panic!("expected Timeout, got {other:?}"),
+    }
+}
+
+#[tokio::test]
+async fn run_remainder_bounds_a_model_call_when_tighter_than_the_ceiling() {
+    use std::time::Duration;
+
+    use crate::harness::testkit::SlowModel;
+
+    // A generous ceiling (10s) never extends a call past the run's own
+    // deadline (20ms): the tighter source wins, and the error blames the
+    // remaining wall-clock budget.
+    let mut harness: AgentHarness<()> = AgentHarness::new();
+    harness.register_model(
+        "slow",
+        Arc::new(SlowModel::new(Duration::from_millis(200), "too late")),
+    );
+    harness.with_policy(RunPolicy {
+        limits: RunLimits::default().with_max_model_call_ms(Some(10_000)),
+        ..RunPolicy::default()
+    });
+
+    let config = RunConfig::new("remainder-tighter-run").with_timeout_ms(20);
+    let err = harness
+        .invoke(&(), (), config, vec![Message::user("hi")])
+        .await
+        .expect_err("the run deadline must still bound the call");
+
+    match &err {
+        TinyAgentsError::Timeout(msg) => {
+            assert!(msg.contains("remaining wall-clock budget"), "{msg}");
+        }
+        other => panic!("expected Timeout, got {other:?}"),
+    }
+}
+
+#[tokio::test]
+async fn per_model_call_ceiling_does_not_bound_tool_calls() {
+    use std::time::Duration;
+
+    // A 20ms per-model-call ceiling with a 100ms tool: tool calls keep the
+    // remaining-only budget (a sub-agent delegation is a tool call wrapping an
+    // entire child run), so the run completes despite the tool outliving the
+    // model-call ceiling many times over.
+    let mut harness: AgentHarness<()> = AgentHarness::new();
+    harness.register_model(
+        "mock",
+        Arc::new(MockModel::with_responses(vec![
+            tool_call_response("call-1", "slow", json!({})),
+            text_response("done", 0, 0),
+        ])),
+    );
+    harness.register_tool(Arc::new(SlowTool {
+        delay: Duration::from_millis(100),
+    }));
+    harness.with_policy(RunPolicy {
+        limits: RunLimits::default().with_max_model_call_ms(Some(20)),
+        ..RunPolicy::default()
+    });
+
+    let config = RunConfig::new("tool-uncapped-run").with_timeout_ms(60_000);
+    let run = harness
+        .invoke(&(), (), config, vec![Message::user("go")])
+        .await
+        .expect("the tool call must not inherit the per-model-call ceiling");
+
+    assert_eq!(run.text(), Some("done".to_string()));
+}
+
 #[tokio::test]
 async fn inherited_tool_timeout_is_enforced_without_a_run_deadline() {
     use std::time::Duration;