diff --git a/engine/agents/llm_retry.py b/engine/agents/llm_retry.py index 31863e2..acecc7c 100644 --- a/engine/agents/llm_retry.py +++ b/engine/agents/llm_retry.py @@ -7,6 +7,7 @@ from typing import TypeVar import httpx +from agents import ModelBehaviorError from openai import ( APIConnectionError, APIError, @@ -54,7 +55,16 @@ def is_retriable_llm_error(exc: BaseException) -> bool: "The model produced invalid content") - 400s except the terminal codes in ``_TERMINAL_400_CODES`` — a clean rerun from local history fixes stale server-side state (INF-3308) + - the Agents SDK's ``ModelBehaviorError`` ("Model did not produce a + final response!" when a stream truncates without a terminal event, + malformed tool-call JSON, calls to nonexistent tools): every variant + is nondeterministic sampled-output misbehavior, so a rerun from local + history can produce a well-formed turn. Other ``AgentsException`` + subclasses stay non-retriable — ``MaxTurnsExceeded`` must terminate + the run and ``UserError`` / guardrail tripwires are deterministic. """ + if isinstance(exc, ModelBehaviorError): + return True if isinstance(exc, (APIConnectionError, APITimeoutError, RateLimitError)): return True if isinstance(exc, APIStatusError): diff --git a/engine/agents/openai_agent_runner.py b/engine/agents/openai_agent_runner.py index d2826db..d7d8564 100644 --- a/engine/agents/openai_agent_runner.py +++ b/engine/agents/openai_agent_runner.py @@ -78,7 +78,10 @@ async def run( Mid-stream failures (events already processed when the stream dies — dropped connections, incomplete chunked reads, stale ``rs_*`` 400s from - the SDK re-sending Responses-API reasoning items) are recovered by + the SDK re-sending Responses-API reasoning items, streams that end + without a terminal event and surface as the SDK's + ``ModelBehaviorError: Model did not produce a final response!``) are + recovered by rerunning from the LOCAL conversation history (INF-3504 / INF-3308): completed turns stay in ``agent_context``; only a trailing incomplete tool turn is trimmed (see ``AgentContext.trim_incomplete_tool_turn``) diff --git a/tests/unit/agents/test_llm_retry.py b/tests/unit/agents/test_llm_retry.py index a93b661..8c01a9e 100644 --- a/tests/unit/agents/test_llm_retry.py +++ b/tests/unit/agents/test_llm_retry.py @@ -2,6 +2,7 @@ import httpx import pytest +from agents import MaxTurnsExceeded, ModelBehaviorError, UserError from openai import ( APIConnectionError, APIError, @@ -71,6 +72,30 @@ def test_non_400_4xx_is_not_retriable() -> None: assert not is_retriable_llm_error(_status_error(404, "not found")) +@pytest.mark.parametrize( + "message", + [ + # Stream truncated without a terminal event (agents run_loop). + "Model did not produce a final response!", + # Other nondeterministic model-output misbehavior the SDK wraps. + 'Invalid JSON input for tool query_traces: {"q": ', + "Tool nonexistent_tool not found in agent halo", + ], +) +def test_model_behavior_error_is_retriable(message: str) -> None: + # A rerun from local history re-samples the turn, so every + # ``ModelBehaviorError`` variant is worth retrying (bounded by the + # runner circuit breaker). + assert is_retriable_llm_error(ModelBehaviorError(message)) + + +def test_other_agents_exceptions_are_not_retriable() -> None: + # ``MaxTurnsExceeded`` must terminate the run; ``UserError`` is a + # deterministic configuration bug. Only ``ModelBehaviorError`` retries. + assert not is_retriable_llm_error(MaxTurnsExceeded("Max turns (30) exceeded")) + assert not is_retriable_llm_error(UserError("bad SDK usage")) + + def test_unrelated_exception_is_not_retriable() -> None: assert not is_retriable_llm_error(ValueError("nope")) diff --git a/tests/unit/agents/test_openai_agent_runner.py b/tests/unit/agents/test_openai_agent_runner.py index 5bb4eb7..d480699 100644 --- a/tests/unit/agents/test_openai_agent_runner.py +++ b/tests/unit/agents/test_openai_agent_runner.py @@ -2,6 +2,7 @@ import httpx import pytest +from agents import ModelBehaviorError from openai import APIConnectionError, APIError, AsyncOpenAI, BadRequestError from engine.agents.agent_context import AgentContext @@ -748,6 +749,54 @@ async def stream_hits_stale_reasoning_item(*, agent, input, context): assert [item.content for item in ctx.items] == ["worked a bit", "answer"] +@pytest.mark.asyncio +async def test_runner_retries_model_behavior_error_when_stream_has_no_final_response() -> None: + """The SDK raises ``ModelBehaviorError: Model did not produce a final + response!`` when a stream truncates without a terminal event. The runner + reruns from local history instead of failing the run — previously this + escaped as non-retriable and killed the whole run on the root turn.""" + bus = EngineOutputBus() + ctx = _context() + execution = AgentExecution( + agent_id="root", + agent_name="root", + depth=0, + parent_agent_id=None, + parent_tool_call_id=None, + ) + + calls: list[list[dict]] = [] + + async def stream_truncates_without_final_response(*, agent, input, context): + calls.append(input) + if len(calls) == 1: + return _StreamYieldsThenRaises( + [_assistant_event("partial answer")], + ModelBehaviorError("Model did not produce a final response!"), + ) + return _FakeStream([_assistant_event("answer\n")]) + + runner = OpenAiAgentRunner( + run_streamed=stream_truncates_without_final_response, + client=_DUMMY_CLIENT, + retry_backoff_base=0.0, + ) + + await runner.run( + sdk_agent=object(), + agent_context=ctx, + agent_execution=execution, + output_bus=bus, + is_root=True, + ) + + assert len(calls) == 2 + # The retry replays the completed assistant message from local history. + assert calls[1] == [{"role": "assistant", "content": "partial answer"}] + assert [item.content for item in ctx.items] == ["partial answer", "answer"] + assert execution.consecutive_llm_failures == 0 + + @pytest.mark.asyncio async def test_runner_propagates_terminal_400_mid_stream() -> None: """Terminal-code 400s mid-stream stay non-retriable — no clean rerun fixes