From 31110ca9c331b18618d02cb193091f31496ef1f9 Mon Sep 17 00:00:00 2001 From: hallerite Date: Mon, 27 Jul 2026 02:55:18 +0200 Subject: [PATCH] feat(v1): import interaction history --- tests/v1/fixtures/echo_user_sim_v1.py | 30 +++++++--- tests/v1/test_e2e.py | 22 +++++-- verifiers/v1/agent.py | 26 +++++++- verifiers/v1/harness.py | 83 ++++++++++++++++++++++++-- verifiers/v1/harnesses/bash/harness.py | 1 + verifiers/v1/harnesses/null/harness.py | 1 + verifiers/v1/rollout.py | 12 +++- 7 files changed, 152 insertions(+), 23 deletions(-) diff --git a/tests/v1/fixtures/echo_user_sim_v1.py b/tests/v1/fixtures/echo_user_sim_v1.py index f57d0d7dd2..7facbba298 100644 --- a/tests/v1/fixtures/echo_user_sim_v1.py +++ b/tests/v1/fixtures/echo_user_sim_v1.py @@ -1,14 +1,15 @@ -"""Multi-turn echo driven by a scripted user — the single user-sim mechanism. +"""Multi-turn echo driven by a scripted user with imported conversation history. The env's `run()` scripts the user through an interaction. The task is -prompt-less, so the first `turn(phrase)` opens the conversation; each later turn -resumes the harness onto the accreted conversation, and leaving the `async with` -closes the interaction (the trace stops as `user_closed`). +prompt-less, so a seeded assistant greeting precedes the first `turn(phrase)`; +each later turn resumes the harness onto the accreted conversation, and leaving +the `async with` closes the interaction (the trace stops as `user_closed`). """ import verifiers.v1 as vf PHRASES = ["hello world", "goodbye world"] +GREETING = "Hi! How can I help you today?" SYSTEM = "Repeat the user's message back to them exactly, with no extra words." @@ -27,6 +28,15 @@ class EchoUserSimData(vf.TaskData): class EchoUserSimTask(vf.Task[EchoUserSimData, vf.State, vf.TaskConfig]): @vf.reward(weight=1.0) async def echoed(self, trace: vf.Trace) -> float: + imported = [ + node + for node in trace.nodes + if not node.sampled + and isinstance(node.message, vf.AssistantMessage) + and node.message.content == GREETING + ] + if len(imported) != 1: + return 0.0 replies = [m.content for m in trace.assistant_messages] phrases = self.data.phrases if len(replies) < len(phrases): @@ -39,9 +49,15 @@ class EchoUserSimEnv(vf.SingleAgentEnv): """Scripts the user side: opens with the first phrase, follows with the rest.""" async def run(self, task, agents): - # An interaction scripting the user: the task carries no prompt, so the - # first turn opens the conversation. - async with agents.agent.interaction(task) as interaction: + # Imported history is native context, while the first phrase remains the + # caller's live opening turn. + async with agents.agent.interaction( + task, + history=[ + vf.SystemMessage(content=SYSTEM), + vf.AssistantMessage(content=GREETING), + ], + ) as interaction: for phrase in task.data.phrases: if (await interaction.turn(phrase)).terminated: break diff --git a/tests/v1/test_e2e.py b/tests/v1/test_e2e.py index 0c77546ae4..8b2b83e6ae 100644 --- a/tests/v1/test_e2e.py +++ b/tests/v1/test_e2e.py @@ -122,11 +122,15 @@ async def test_single_turn(run_v1, harness, harness_runtime, tmp_path): @pytest.mark.parametrize("harness_runtime", USER_RUNTIMES, indirect=True) async def test_user(run_v1, harness_runtime, tmp_path): """Multi-turn, driven by a scripted user — an interaction loop in the env's - `run()` — across the harness runtime axis. The task is prompt-less, so one - run covers the whole exchange shape: the caller opens (the user speaks first), - each later turn resumes the harness onto the conversation, and leaving the loop - ends the exchange (`user_closed`). The user runs in the eval process itself, so - there is no placement axis.""" + `run()` — across the harness runtime axis. Seeded system context and an + assistant greeting are imported before the caller's first live turn; each later + turn resumes the harness onto the conversation, and leaving the loop ends the + exchange (`user_closed`). The user runs in the eval process itself, so there is + no placement axis.""" + from echo_user_sim_v1 import SYSTEM + + import verifiers.v1 as vf + (trace,) = await run_v1( "echo-user-sim-v1", harness="null", @@ -138,6 +142,14 @@ async def test_user(run_v1, harness_runtime, tmp_path): assert trace.num_turns >= 2 # genuinely multi-turn assert trace.stop_condition == "user_closed" # leaving the interaction ended it assert trace.reward == 1.0 + imported_system = [ + node + for node in trace.nodes + if not node.sampled + and isinstance(node.message, vf.SystemMessage) + and node.message.content == SYSTEM + ] + assert len(imported_system) == 1 @pytest.mark.e2e diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index f99158f3af..4464dc25b0 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -340,7 +340,7 @@ def _interception_for( return self._server return None - def _check_resume_support(self) -> None: + def _check_interaction_support(self, history: Messages | None = None) -> None: # Multi-turn capability is a derived fact, not a flag: an exchange advances # by resuming the harness onto the conversation, so the harness needs either # the default relaunch (a Messages prompt) or its own native continuation. @@ -352,6 +352,16 @@ def _check_resume_support(self) -> None: "default relaunch-on-the-conversation, or a native resume() " "override. Use a harness that has one (e.g. bash or null)." ) + if ( + history + and type(harness).bootstrap is Harness.bootstrap + and not harness.SUPPORTS_HISTORY_IMPORT + ): + raise ValueError( + f"Harness {harness.config.id!r} cannot import conversation history: " + "it needs role-preserving Messages launch support or a native " + "bootstrap() override." + ) async def run( self, @@ -431,6 +441,7 @@ async def interaction( runtime: Runtime | None = None, tools: Mapping[str, SharedToolServer] | None = None, mask_prompt: bool = False, + history: Messages | None = None, on_trace: Callable[[Trace], None] | None = None, ) -> AsyncIterator[Interaction]: """Interact with this agent turn-by-turn: a full rollout of `task` where @@ -452,6 +463,13 @@ async def interaction( they do for `run()`; an env supplies its taskset's shared tools automatically for tasks loaded from that taskset. + `history` imports role-preserving conversation context into the new + interaction before its live opening turn. Imported messages are context, + not sampled turns: a prompted task still opens with bare `turn()`, while a + prompt-less task still opens with `turn(message)`. Stateless harnesses + relaunch from the typed transcript; stateful harnesses must implement a + native `bootstrap()`. + Everything is a real rollout — the trace (live on `interaction.trace`), limits, `@stop`s, and scoring all apply; leaving the context ends the exchange (`user_closed`) and finishes the rollout, hooks and scoring @@ -461,7 +479,8 @@ async def interaction( apply here.""" if self._closed: raise RuntimeError("Agent is closed; create a new agent") - self._check_resume_support() + history = _as_messages(history) if history else None + self._check_interaction_support(history) if mask_prompt and task.data.prompt is None: raise ValueError( "mask_prompt hides a prompt the task doesn't have; a prompt-less " @@ -474,6 +493,7 @@ async def interaction( task.data.model_copy(update={"prompt": None}) if mask_prompt else None ), has_user=True, + history=history, on_trace=on_trace, **params, ) @@ -659,6 +679,7 @@ async def interaction( runtime: Runtime | None = None, tools: Mapping[str, SharedToolServer] | None = None, mask_prompt: bool = False, + history: Messages | None = None, on_trace: Callable[[Trace], None] | None = None, ) -> AsyncIterator[Interaction]: """The agent's `interaction`, with every trace stamped with its standing @@ -680,6 +701,7 @@ def remember(current: Trace) -> None: runtime=runtime, tools=tools if tools is not None else self._shared_for(task), mask_prompt=mask_prompt, + history=history, on_trace=self._watch(remember), ) as interaction: yield interaction diff --git a/verifiers/v1/harness.py b/verifiers/v1/harness.py index 78fdb3a0a3..2e75d1359c 100644 --- a/verifiers/v1/harness.py +++ b/verifiers/v1/harness.py @@ -12,7 +12,7 @@ from verifiers.v1.errors import HarnessError, boundary from verifiers.v1.runtimes import ProgramResult, Runtime from verifiers.v1.task import TaskData -from verifiers.v1.types import Messages +from verifiers.v1.types import Messages, UserMessage if TYPE_CHECKING: # Annotation-only: `Trace` appears in signatures only, so this module stays @@ -32,6 +32,9 @@ class Harness(ABC, Generic[ConfigT]): SUPPORTS_RESUME: ClassVar[bool] = False """Whether the default `resume()` can relaunch this harness from the accumulated Messages transcript.""" + SUPPORTS_HISTORY_IMPORT: ClassVar[bool] = False + """Whether the default `bootstrap()` can launch this harness from typed, + role-preserving Messages history.""" EXECUTES_CODE: ClassVar[bool] = True """Whether the program hands the model local execution in the runtime — true for every real harness; the tool-less chat loops (`null`) override to False. Read @@ -121,13 +124,27 @@ async def run( mcp_urls: dict[str, str], data: TaskData, messages: Messages | None = None, + history: Messages | None = None, ) -> None: - """Run ONE segment of the exchange: the program from launch (or, with - `messages`, the user's next turn(s) via `resume`) until it yields — a segment - ends when the program exits. The rollout loop owns the exchange across - segments (and stamps its end); a harness only ever sees one segment.""" + """Run ONE segment of the exchange: launch on the task prompt, bootstrap a + new session from imported `history`, or resume with the user's next + `messages`. A segment ends when the program exits. The rollout loop owns the + exchange across segments (and stamps its end); a harness only ever sees one + segment.""" async with boundary(HarnessError, f"harness {self.config.id!r}"): - if messages is None: + if history: + result = await self.bootstrap( + ctx, + trace, + runtime, + endpoint, + secret, + mcp_urls, + data, + history, + messages, + ) + elif messages is None: result = await self.launch( ctx, trace, runtime, endpoint, secret, mcp_urls, data ) @@ -144,6 +161,60 @@ async def run( f"harness {self.config.id!r} exited {result.exit_code}: {detail}" ) + async def bootstrap( + self, + ctx: ModelContext, + trace: Trace, + runtime: Runtime, + endpoint: str, + secret: str, + mcp_urls: dict[str, str], + data: TaskData, + history: Messages, + messages: Messages | None, + ) -> ProgramResult: + """Start a new exchange with role-preserving conversation `history`. + + `history` is pre-existing context, not a sampled turn. The task prompt (for + a prompted interaction) or `messages` (when the caller opens it) is the live + user turn that follows. The default relaunches a stateless Messages-capable + harness on that conversation. A stateful harness overrides this hook to + import the history into its own new session before sending the live turn. + """ + if not self.SUPPORTS_HISTORY_IMPORT: + raise HarnessError( + f"harness {self.config.id!r} cannot import conversation history: it " + "neither overrides bootstrap() nor declares role-preserving " + "Messages launch support." + ) + if messages is not None: + live = messages + elif isinstance(data.prompt, str): + live = [UserMessage(content=data.prompt)] + elif data.prompt is not None: + live = data.prompt + else: + raise HarnessError( + "history import needs a live opening turn: set task.prompt or call " + "interaction.turn(message)" + ) + # `resolve_prompt` re-emits `data.system_prompt`; mirror `resume()` so an + # imported transcript cannot duplicate that system context. + imported = [ + message + for message in history + if message.role != "system" or data.system_prompt is None + ] + return await self.launch( + ctx, + trace, + runtime, + endpoint, + secret, + mcp_urls, + data.model_copy(update={"prompt": [*imported, *live]}), + ) + async def score(self, trace: Trace, runtime: Runtime) -> None: """Run this harness's `@metric` methods over the finished trace, recording each into `trace.metrics`. Metrics declare what they need (`task`, `trace`, diff --git a/verifiers/v1/harnesses/bash/harness.py b/verifiers/v1/harnesses/bash/harness.py index b94989e535..31d33d6cf7 100644 --- a/verifiers/v1/harnesses/bash/harness.py +++ b/verifiers/v1/harnesses/bash/harness.py @@ -42,6 +42,7 @@ class BashHarness(Harness[BashHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_RESUME = True + SUPPORTS_HISTORY_IMPORT = True NEEDS_CONTAINER = False async def setup(self, runtime: Runtime) -> None: diff --git a/verifiers/v1/harnesses/null/harness.py b/verifiers/v1/harnesses/null/harness.py index 2da3eb8378..89bfeed825 100644 --- a/verifiers/v1/harnesses/null/harness.py +++ b/verifiers/v1/harnesses/null/harness.py @@ -20,6 +20,7 @@ class NullHarness(Harness[NullHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_RESUME = True + SUPPORTS_HISTORY_IMPORT = True EXECUTES_CODE = False NEEDS_CONTAINER = False diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 8db36ccda9..a177c5d81a 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -117,6 +117,7 @@ def __init__( runtime_config: RuntimeConfig, wire_data: TaskData | None = None, has_user: bool = False, + history: Messages | None = None, setup_timeout: float | None = None, harness_timeout: float | None = None, finalize_timeout: float | None = None, @@ -132,6 +133,7 @@ def __init__( self.ctx = ctx self.runtime_config = runtime_config self._has_user = has_user + self._history = history self._setup_timeout = setup_timeout self._harness_time_remaining = harness_timeout self._finalize_timeout = finalize_timeout @@ -306,7 +308,9 @@ async def step(self, messages: Messages | None = None) -> bool: """Run ONE segment: the harness program to its exit. With `messages`, the segment resumes the exchange with the user's turn(s) (`Harness.resume` — for an exchange the user opens, this is also the first segment, on an - empty conversation); without, it launches on the task's own prompt. + empty conversation); without, it launches on the task's own prompt. On the + first segment only, an interaction's imported history is bootstrapped ahead + of that live turn. Returns whether the exchange can continue — a refused turn (limit, @stop), a timeout, a failure, or a segment that made no progress all end it.""" if not self._opened or self._closed or not self.ok: @@ -331,9 +335,11 @@ async def step(self, messages: Messages | None = None) -> bool: self._endpoint, self._secret, self._urls, - trace.task.data, - messages, + data=trace.task.data, + messages=messages, + history=self._history, ) + self._history = None except TimeoutError as e: # Only the rollout deadline reads as a clean truncation; a TimeoutError # from the harness's own I/O with no expired deadline is a failure —