From ced287502361cd3c2c1632f4fdc029204ba6292e Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sun, 16 Aug 2026 15:06:12 +0530 Subject: [PATCH 01/39] feat(harness): agent source registry and the understand-agent stage --- pyproject.toml | 1 + src/fi/alk/harness/__init__.py | 54 ++++ src/fi/alk/harness/__main__.py | 3 + src/fi/alk/harness/cli.py | 135 ++++++++++ src/fi/alk/harness/config.py | 87 +++++++ src/fi/alk/harness/contract.py | 163 ++++++++++++ src/fi/alk/harness/session.py | 197 +++++++++++++++ .../harness/skills/understand-agent/SKILL.md | 65 +++++ src/fi/alk/harness/sources.py | 131 ++++++++++ src/fi/alk/harness/tools.py | 102 ++++++++ src/fi/alk/harness/understand.py | 86 +++++++ tests/test_harness.py | 236 ++++++++++++++++++ uv.lock | 21 ++ 13 files changed, 1281 insertions(+) create mode 100644 src/fi/alk/harness/__init__.py create mode 100644 src/fi/alk/harness/__main__.py create mode 100644 src/fi/alk/harness/cli.py create mode 100644 src/fi/alk/harness/config.py create mode 100644 src/fi/alk/harness/contract.py create mode 100644 src/fi/alk/harness/session.py create mode 100644 src/fi/alk/harness/skills/understand-agent/SKILL.md create mode 100644 src/fi/alk/harness/sources.py create mode 100644 src/fi/alk/harness/tools.py create mode 100644 src/fi/alk/harness/understand.py create mode 100644 tests/test_harness.py diff --git a/pyproject.toml b/pyproject.toml index dc32475..5283c57 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ classifiers = [ "Topic :: Software Development :: Testing", ] dependencies = [ + "claude-agent-sdk>=0.2.139", "fi-instrumentation-otel>=0.1.16", "gepa>=0.0.17", "httpx>=0.24.0", diff --git a/src/fi/alk/harness/__init__.py b/src/fi/alk/harness/__init__.py new file mode 100644 index 0000000..f022cc8 --- /dev/null +++ b/src/fi/alk/harness/__init__.py @@ -0,0 +1,54 @@ +"""The harness: an agent that builds test environments for other agents. + +It reads an agent, works out what it verifiably is, builds a world its tools can run against, +generates scenarios, runs them, and reads the results back. Each of those is a stage, each stage +is its own session, and stages hand work to each other as artifacts on disk. + +The split that matters: the model does judgement, and code decides outcomes. Reading unfamiliar +source, designing a schema, and choosing what is worth testing are judgement. Executing a tool +call and grading a run are not, and are never delegated to a model. + +Stages are described in files under ``skills/``, so the method is editable without touching +code, and where an agent comes from is a registered source, so a new kind of agent is a class +rather than a new code path. +""" + +from .config import ( + DEFAULT_MODEL, + artifact_dir, + load_skill, + provider_env, + read_only_session, +) +from .contract import AgentContract, ToolSpec, validate_contract +from .session import Stage, Turn +from .sources import ( + AgentSource, + RepoSource, + SpecSource, + register_source, + resolve, + supported, +) +from .understand import open_stage, understand + +__all__ = [ + "AgentContract", + "AgentSource", + "DEFAULT_MODEL", + "RepoSource", + "SpecSource", + "Stage", + "ToolSpec", + "Turn", + "artifact_dir", + "load_skill", + "open_stage", + "provider_env", + "read_only_session", + "register_source", + "resolve", + "supported", + "understand", + "validate_contract", +] diff --git a/src/fi/alk/harness/__main__.py b/src/fi/alk/harness/__main__.py new file mode 100644 index 0000000..eb53e2f --- /dev/null +++ b/src/fi/alk/harness/__main__.py @@ -0,0 +1,3 @@ +from .cli import main + +raise SystemExit(main()) diff --git a/src/fi/alk/harness/cli.py b/src/fi/alk/harness/cli.py new file mode 100644 index 0000000..9151c3c --- /dev/null +++ b/src/fi/alk/harness/cli.py @@ -0,0 +1,135 @@ +"""Run a stage from a terminal. + +This is one renderer over the stage loop, not the product. It prints events as lines and reads +follow-ups from stdin; a browser front end subscribes to the same events and draws them as a +transcript beside the artifact. Keeping the terminal a renderer rather than the interface is what +makes the second one cheap. +""" + +from __future__ import annotations + +import argparse +import asyncio +import sys +from pathlib import Path +from typing import Any + +from .config import DEFAULT_MODEL +from .session import TEXT, Event +from .sources import resolve, supported +from .understand import load, open_stage, opening + + +def _render(event: Event) -> None: + line = event.line() + if event.kind == TEXT: + print(line, end="", flush=True) + else: + print(f"\n{line}", flush=True) + + +async def _prompt(question: str) -> str: + return (await asyncio.to_thread(input, question)).strip() + + +async def _answer_questions( + tool_name: str, payload: dict[str, Any], _context: Any +) -> Any: + """Render the model's clarifying questions and return the operator's answers. + + Anything that is not a question is allowed through: the session is already restricted to + read-only built-ins plus our own tools, so there is nothing here to gate. + """ + from claude_agent_sdk.types import PermissionResultAllow + + if tool_name != "AskUserQuestion": + return PermissionResultAllow(updated_input=payload) + + answers: dict[str, Any] = {} + for question in payload.get("questions", []): + print(f"\n\n {question.get('header', '?')}: {question.get('question', '')}") + options = question.get("options", []) or [] + for index, option in enumerate(options, start=1): + print( + f" {index}. {option.get('label')} - {option.get('description', '')}" + ) + raw = await _prompt(" > ") + chosen = raw + if raw.isdigit() and 1 <= int(raw) <= len(options): + chosen = options[int(raw) - 1].get("label", raw) + answers[question.get("question", "")] = chosen + print() + return PermissionResultAllow( + updated_input={"questions": payload.get("questions", []), "answers": answers} + ) + + +async def _understand(args: argparse.Namespace) -> int: + source = resolve(args.kind, name=args.name, root=args.path) + stage, destination = open_stage( + source, + out=Path(args.out) if args.out else None, + # Unattended, there is nobody to answer, so the model records what it could not + # resolve in open_questions rather than blocking on a prompt nobody will see. + ask=_answer_questions if args.interactive else None, + ) + + print(f"agent: {source.name} ({source.kind})") + print(f"out: {destination}\n") + + async with stage: + await stage.say(opening(source), on_event=_render) + while args.interactive: + try: + said = await _prompt("\nkarthik ") + except (EOFError, KeyboardInterrupt): + break + if not said or said in {"q", "quit", "exit"}: + break + await stage.say(said, on_event=_render) + + contract = load(destination) + if contract is None: + print("\nNo contract was submitted.", file=sys.stderr) + return 1 + print( + f"\ncontract: {len(contract.tools)} tools, " + f"{len(contract.hard_constraints)} rules, " + f"{len(contract.real_use_cases)} use cases, " + f"{len(contract.open_questions)} open questions" + ) + print(f"spent: ${stage.spent_usd:.4f}") + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="fi.alk.harness", description=__doc__) + sub = parser.add_subparsers(dest="stage", required=True) + + understand = sub.add_parser( + "understand", help="read an agent and produce its contract" + ) + understand.add_argument("--name", required=True, help="what to call this agent") + understand.add_argument("--path", required=True, help="where the agent is") + understand.add_argument( + "--kind", default="repo", choices=supported(), help="how the agent is supplied" + ) + understand.add_argument("--out", default=None, help="artifact directory") + understand.add_argument( + "--once", + dest="interactive", + action="store_false", + help="run unattended instead of staying open for corrections", + ) + understand.add_argument("--model", default=DEFAULT_MODEL, help=argparse.SUPPRESS) + understand.set_defaults(run=_understand, interactive=True) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + return asyncio.run(args.run(args)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/fi/alk/harness/config.py b/src/fi/alk/harness/config.py new file mode 100644 index 0000000..156d77b --- /dev/null +++ b/src/fi/alk/harness/config.py @@ -0,0 +1,87 @@ +"""Session configuration for the harness. + +One place decides which model runs, how the session reaches it, and what the agent is allowed to +touch. Every stage builds its options from here so that a change of provider or model is one +edit rather than a search across stages. + +Credentials are never read from source. The Vertex project and credential path come from the +environment, which is also how the rest of the platform resolves them. +""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any, Iterable + +from claude_agent_sdk import ClaudeAgentOptions + +DEFAULT_MODEL = "claude-sonnet-4-6" + +SKILLS_ROOT = Path(__file__).parent / "skills" + +_READ_ONLY_TOOLS = ("Read", "Glob", "Grep") + + +def provider_env(model: str | None = None) -> dict[str, str]: + """The provider block passed to the session. + + Claude Code resolves the GCP project from ``GOOGLE_CLOUD_PROJECT``, the credential file, or + the active gcloud configuration, in that order, so an unset project id is not an error here. + """ + env = { + "CLAUDE_CODE_USE_VERTEX": "1", + "CLOUD_ML_REGION": os.environ.get("CLOUD_ML_REGION", "global"), + "ANTHROPIC_MODEL": model or os.environ.get("ALK_HARNESS_MODEL", DEFAULT_MODEL), + } + for passthrough in ( + "ANTHROPIC_VERTEX_PROJECT_ID", + "GOOGLE_CLOUD_PROJECT", + "GOOGLE_APPLICATION_CREDENTIALS", + ): + value = os.environ.get(passthrough) + if value: + env[passthrough] = value + return env + + +def read_only_session( + *, + system_prompt: str, + cwd: str | Path, + mcp_servers: dict[str, Any] | None = None, + extra_tools: Iterable[str] = (), + max_turns: int = 40, + model: str | None = None, +) -> ClaudeAgentOptions: + """A session that may read the agent under test but never write to it. + + The agent under test is somebody's real repository. The harness reads it and writes its own + artifacts elsewhere, so the built-in write tools are simply not granted; the only way this + session can produce anything is by calling one of ours. + """ + allowed = [*_READ_ONLY_TOOLS, "AskUserQuestion", *extra_tools] + return ClaudeAgentOptions( + system_prompt=system_prompt, + allowed_tools=allowed, + mcp_servers=dict(mcp_servers or {}), + permission_mode="acceptEdits", + cwd=str(cwd), + setting_sources=[], + max_turns=max_turns, + env=provider_env(model), + ) + + +def artifact_dir(agent: str, root: str | Path | None = None) -> Path: + """Where a given agent's generated environment lives.""" + base = Path(root) if root else Path("artifacts/environments") + return base / agent + + +def load_skill(name: str) -> str: + """A stage's instructions, kept as a file so the method is editable without touching code.""" + path = SKILLS_ROOT / name / "SKILL.md" + if not path.exists(): + raise FileNotFoundError(f"no skill at {path}") + return path.read_text(encoding="utf-8") diff --git a/src/fi/alk/harness/contract.py b/src/fi/alk/harness/contract.py new file mode 100644 index 0000000..b9b12a6 --- /dev/null +++ b/src/fi/alk/harness/contract.py @@ -0,0 +1,163 @@ +"""The agent contract: what the agent verifiably is, read from its own source. + +Everything downstream is confined to this. A world may only implement tools listed here, a +scenario may only reference values grounded in here, and a checkpoint may only assert against +what is here. It is the anti-hallucination device for every later stage. + +The harness produces it by reading the agent's code and calling ``submit_contract``. Validation +runs inside that tool, so problems are returned into the conversation and the model tries again +rather than a bad contract reaching disk. +""" + +from __future__ import annotations + +import json +from typing import Any + +from pydantic import BaseModel, Field, model_validator + +_STRING_FIELDS = ( + "agent", + "one_liner", + "modality", + "system_prompt_excerpt", + "grading_notes", +) +_LIST_FIELDS = ( + "hard_constraints", + "real_use_cases", + "signature_cases", + "anti_hallucination", +) +_DICT_FIELDS = ("data_schema", "base_environment") + + +class ToolSpec(BaseModel): + name: str + args: list[str] = Field(default_factory=list) + arg_types: dict[str, str] = Field(default_factory=dict) + arg_values: dict[str, Any] = Field(default_factory=dict) + description: str = "" + + +class AgentContract(BaseModel): + """What the agent verifiably is. Nothing downstream may contradict this.""" + + @model_validator(mode="before") + @classmethod + def _normalize_shapes(cls, payload: Any) -> Any: + """Model JSON varies in benign ways: a list where prose was asked, a bare string where a + list was. Normalize instead of rejecting, because shape variance is not a grounding + error and rejecting it burns turns on something that does not matter.""" + if not isinstance(payload, dict): + return payload + for key in _STRING_FIELDS: + value = payload.get(key) + if isinstance(value, list): + payload[key] = "\n".join(str(item) for item in value) + elif value is not None and not isinstance(value, str): + payload[key] = str(value) + for key in _LIST_FIELDS: + value = payload.get(key) + if isinstance(value, str): + payload[key] = [value] + elif isinstance(value, list): + payload[key] = [ + str(item) if not isinstance(item, str) else item for item in value + ] + for key in _DICT_FIELDS: + value = payload.get(key) + if value is not None and not isinstance(value, dict): + payload[key] = {"value": value} + return payload + + agent: str + one_liner: str = "" + modality: str = "chat" + conversational: bool = True + system_prompt_excerpt: str = "" + hard_constraints: list[str] = Field(default_factory=list) + tools: list[ToolSpec] = Field(default_factory=list) + data_schema: dict[str, Any] = Field(default_factory=dict) + base_environment: dict[str, Any] = Field(default_factory=dict) + real_use_cases: list[str] = Field(default_factory=list) + signature_cases: list[str] = Field(default_factory=list) + grading_notes: str = "" + anti_hallucination: list[str] = Field(default_factory=list) + open_questions: list[str] = Field(default_factory=list) + + def tool_names(self) -> set[str]: + return {tool.name for tool in self.tools} + + def brief(self, *, full_schema: bool = True) -> str: + """The grounding block handed to the model on every downstream call.""" + lines: list[str] = [] + for tool in self.tools: + signature = ", ".join( + f"{arg}: {tool.arg_types[arg]}" if arg in tool.arg_types else arg + for arg in tool.args + ) + values = ( + f" [values: {json.dumps(tool.arg_values)[:300]}]" + if tool.arg_values + else "" + ) + lines.append( + f" - {tool.name}({signature}){values} : {tool.description[:140]}" + ) + parts = [ + f"AGENT: {self.agent} - {self.one_liner}", + f"MODALITY: {self.modality}", + "REAL TOOLS (use ONLY these, with these exact arg names and types):\n" + + ("\n".join(lines) or " (none)"), + ] + if self.hard_constraints: + parts.append( + "HARD CONSTRAINTS the agent MUST follow (nothing may contradict these):\n - " + + "\n - ".join(self.hard_constraints[:14]) + ) + if self.data_schema and full_schema: + parts.append( + "REAL DATA / SCHEMA (ground every value and id in this; never invent):\n" + + json.dumps(self.data_schema)[:2400] + ) + if self.grading_notes: + parts.append(f"GRADING NOTES for this agent:\n{self.grading_notes[:900]}") + if self.anti_hallucination: + parts.append( + "NEVER USE THESE (they do not exist / are wrong): " + + json.dumps(self.anti_hallucination)[:700] + ) + return "\n\n".join(parts) + + +def validate_contract(contract: AgentContract) -> list[str]: + """Structural problems that make a contract unusable downstream. + + Deliberately narrow. This cannot tell whether the model read the agent correctly, only + whether the result is shaped well enough to build a world from. Semantic grounding is the + operator's job, which is why the harness surfaces the contract for review. + """ + problems: list[str] = [] + if not contract.agent.strip(): + problems.append("empty:agent") + if not contract.tools: + problems.append("no-tools") + for index, tool in enumerate(contract.tools): + if not tool.name.strip(): + problems.append(f"tool[{index}]:no-name") + continue + unknown = sorted(set(tool.arg_types) - set(tool.args)) + if unknown: + problems.append( + f"tool[{tool.name}]:types-for-unknown-args:{','.join(unknown)}" + ) + if not contract.real_use_cases: + problems.append("no-use-cases") + # Iterate the tools, not tool_names(): that returns a set, so duplicates collapse before + # they can be counted and the check silently never fires. + names = [tool.name for tool in contract.tools if tool.name.strip()] + duplicates = sorted({name for name in names if names.count(name) > 1}) + if duplicates: + problems.append(f"duplicate-tool-names:{','.join(duplicates)}") + return problems diff --git a/src/fi/alk/harness/session.py b/src/fi/alk/harness/session.py new file mode 100644 index 0000000..8a205a3 --- /dev/null +++ b/src/fi/alk/harness/session.py @@ -0,0 +1,197 @@ +"""A stage as a live conversation, emitting what happened as it happens. + +The operator experiences one continuous session: point at an agent, watch a contract appear, +correct something, move on. Underneath, each stage is its own session so context stays small and +any stage can be re-entered without redoing the ones before it. + +A stage stays open across turns, so a correction is the next thing said rather than a re-run, +and it yields typed events rather than a wall of text. A terminal renders those events as lines; +a browser renders the same events as a transcript on one side and the artifact on the other. +Neither is privileged, which is the point. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, AsyncIterator, Callable + +from claude_agent_sdk import ( + AssistantMessage, + ClaudeAgentOptions, + ClaudeSDKClient, + ResultMessage, + SystemMessage, + TextBlock, + ToolResultBlock, + ToolUseBlock, +) + +TEXT = "text" +TOOL = "tool" +ARTIFACT = "artifact" +DONE = "done" + + +@dataclass +class Event: + """One observable thing the stage did.""" + + kind: str + text: str = "" + tool: str = "" + detail: dict[str, Any] = field(default_factory=dict) + + def line(self) -> str: + """A terminal-friendly rendering.""" + if self.kind == TEXT: + return self.text + if self.kind == TOOL: + target = self.detail.get("target") or "" + return f" [{self.tool}{' ' + target if target else ''}]" + if self.kind == ARTIFACT: + return f" [saved {self.detail.get('path', '')}]" + if self.kind == DONE: + cost = self.detail.get("cost_usd") + spent = f" ${cost:.4f}" if isinstance(cost, float) else "" + return f" [{self.detail.get('outcome', '')} turns={self.detail.get('turns', 0)}{spent}]" + return self.text + + +@dataclass +class Turn: + """What one exchange produced.""" + + text: str = "" + events: list[Event] = field(default_factory=list) + tools_used: list[str] = field(default_factory=list) + artifacts: list[str] = field(default_factory=list) + outcome: str = "" + turns: int = 0 + cost_usd: float | None = None + + +_TARGET_KEYS = ("file_path", "path", "pattern", "agent", "tool", "table") + + +def _target(payload: Any) -> str: + """A short label for what a tool call was aimed at, for display only.""" + if not isinstance(payload, dict): + return "" + for key in _TARGET_KEYS: + value = payload.get(key) + if isinstance(value, str) and value: + return value if len(value) <= 80 else value[:77] + "..." + return "" + + +def _saved_path(block: ToolResultBlock) -> str: + """Our tools report what they wrote; surfacing it lets a UI update the artifact pane.""" + content = block.content + if isinstance(content, list): + content = " ".join( + part.get("text", "") for part in content if isinstance(part, dict) + ) + if not isinstance(content, str): + return "" + for token in content.split(): + if token.endswith((".json", ".py", ".sqlite")): + return token.rstrip(".,") + return "" + + +class Stage: + """One stage of the harness, held open so it can be talked to.""" + + def __init__(self, options: ClaudeAgentOptions, *, name: str = "") -> None: + self._options = options + self._client: ClaudeSDKClient | None = None + self.name = name + self.session_id: str | None = None + self.history: list[Turn] = [] + + async def __aenter__(self) -> "Stage": + self._client = ClaudeSDKClient(options=self._options) + await self._client.connect() + return self + + async def __aexit__(self, *_exc: Any) -> None: + if self._client is not None: + await self._client.disconnect() + self._client = None + + @property + def client(self) -> ClaudeSDKClient: + if self._client is None: + raise RuntimeError("stage is not open; use it as an async context manager") + return self._client + + async def stream(self, message: str) -> AsyncIterator[Event]: + """Send a message and yield events as they arrive.""" + await self.client.query(message) + turn = Turn() + async for received in self.client.receive_response(): + for event in self._events(received, turn): + turn.events.append(event) + yield event + self.history.append(turn) + + def _events(self, received: Any, turn: Turn) -> list[Event]: + if isinstance(received, SystemMessage): + data = received.data if isinstance(received.data, dict) else {} + self.session_id = data.get("session_id") or self.session_id + return [] + if isinstance(received, AssistantMessage): + events: list[Event] = [] + for block in received.content: + if isinstance(block, TextBlock): + turn.text += block.text + events.append(Event(TEXT, text=block.text)) + elif isinstance(block, ToolUseBlock): + turn.tools_used.append(block.name) + events.append( + Event( + TOOL, + tool=block.name, + detail={"target": _target(block.input)}, + ) + ) + return events + if isinstance(received, ResultMessage): + turn.outcome = received.subtype + turn.turns = received.num_turns + turn.cost_usd = received.total_cost_usd + self.session_id = received.session_id or self.session_id + return [ + Event( + DONE, + detail={ + "outcome": received.subtype, + "turns": received.num_turns, + "cost_usd": received.total_cost_usd, + }, + ) + ] + blocks = getattr(received, "content", None) + if isinstance(blocks, list): + events = [] + for block in blocks: + if isinstance(block, ToolResultBlock): + path = _saved_path(block) + if path: + turn.artifacts.append(path) + events.append(Event(ARTIFACT, detail={"path": path})) + return events + return [] + + async def say( + self, message: str, *, on_event: Callable[[Event], None] | None = None + ) -> Turn: + """Send a message and wait for the whole reply.""" + async for event in self.stream(message): + if on_event: + on_event(event) + return self.history[-1] + + @property + def spent_usd(self) -> float: + return sum(turn.cost_usd or 0.0 for turn in self.history) diff --git a/src/fi/alk/harness/skills/understand-agent/SKILL.md b/src/fi/alk/harness/skills/understand-agent/SKILL.md new file mode 100644 index 0000000..f9d313d --- /dev/null +++ b/src/fi/alk/harness/skills/understand-agent/SKILL.md @@ -0,0 +1,65 @@ +--- +name: understand-agent +description: Read an AI agent's source and produce its testing contract. +--- + +# Understand the agent + +You are reading the source of an AI agent so that a test environment can be built for it. Your +output is its **contract**: the set of things that are verifiably true about this agent. Every +later stage is confined to it. A world may only implement tools listed here; a scenario may only +reference values grounded here; a checkpoint may only assert what is here. + +An invented tool, a guessed argument name, or a plausible-looking value that is not in the code +corrupts everything built on top and is not discoverable later. When in doubt, ask or leave it +out. + +## How to read + +Start from the entry point and follow the registrations, not the documentation. README files and +docstrings describe intent; the contract records behaviour. Where they disagree, the code wins +and the disagreement is worth mentioning. + +Find, in roughly this order: + +1. **The tools.** Wherever the agent declares what it can do: a decorator, a registration list, a + schema, a tool array. Record the exact callable name the model would emit, not a friendly + label. +2. **Argument names and types.** Read the signature. `order_id: list[str]` is a different tool + from `order_id: str`, and a world built on the wrong one fails at the first call. Record types + whenever the source states them. +3. **Argument values.** Where an argument is constrained to a set, an enum, a literal union, or a + lookup into fixed data, record the real values. +4. **The rules.** Hard constraints the agent is instructed or coded to obey. Prefer the exact + wording from the system prompt or the validation code. +5. **The data.** Where it lives, its shape, and its real contents. In-memory dicts, fixture + files, a seeded database. Record enough for a working replica to be built. +6. **Real use cases.** What this agent is actually for, as concrete situations, drawn from the + tools and data rather than invented. + +## When you are not sure + +You have `AskUserQuestion`. Use it when the source genuinely does not settle something and the +answer changes what gets built: a required-versus-optional argument, two mutually exclusive +readings of a rule, data that looks like a placeholder. Ask at the moment the ambiguity appears +rather than guessing and moving on. + +Do not use it for anything the code answers. Reading one more file is cheaper than a question. + +Anything you could not resolve, and did not ask about, goes in `open_questions`. + +## Anti-hallucination + +Record in `anti_hallucination` the names and values that a reasonable person would expect this +agent to have but which do **not** exist: a plausible tool name that is not registered, an id +that follows the naming convention but is absent from the data, an argument the API does not +take. Later stages use this list to catch themselves. + +## Finishing + +Call `submit_contract` with the full contract. It is validated when you call it, and if there +are problems they come back to you; fix them and call it again. + +Before you submit, check your own work once: open the source again for every tool you listed and +confirm the name, the arguments, and the types are exactly as written there. A contract that is +structurally valid and factually wrong passes every automatic check and fails everything after. diff --git a/src/fi/alk/harness/sources.py b/src/fi/alk/harness/sources.py new file mode 100644 index 0000000..7c3a6c1 --- /dev/null +++ b/src/fi/alk/harness/sources.py @@ -0,0 +1,131 @@ +"""Where an agent comes from, and how a session reaches it. + +A folder of source code is one kind of agent, not the only kind. The same agent may arrive as a +provider connection with a system prompt and a tool schema, as a platform definition, or as a +spec somebody pasted in. The stage that reads an agent is the same in all of those cases; what +differs is where it looks and what it is allowed to touch. + +So the method stays in the skill and the location lives here. Supporting a new kind of agent is +registering one class, not editing any stage. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Protocol + + +class AgentSource(Protocol): + """Everything a stage needs in order to reach one agent.""" + + kind: str + name: str + + def workdir(self) -> Path: + """The directory the session runs in.""" + + def builtin_tools(self) -> tuple[str, ...]: + """Built-in tools this source needs granted.""" + + def servers(self) -> dict[str, Any]: + """In-process tool servers this source provides, if any.""" + + def briefing(self) -> str: + """What to tell the model about where this agent's truth lives.""" + + +@dataclass +class RepoSource: + """An agent that exists as source code on disk.""" + + name: str + root: Path + kind: str = "repo" + + def workdir(self) -> Path: + return self.root + + def builtin_tools(self) -> tuple[str, ...]: + return ("Read", "Glob", "Grep") + + def servers(self) -> dict[str, Any]: + return {} + + def briefing(self) -> str: + return ( + f"This agent is a repository at {self.root}. Its truth is the source code: the tool " + "registrations, the function signatures, the validation logic, and whatever holds " + "its data. Read it with Read, Glob and Grep. Documentation describes intent; the " + "code describes behaviour, and where they disagree the code wins." + ) + + +@dataclass +class SpecSource: + """An agent supplied directly as a prompt and a tool schema, with no repository. + + This is the shape a hosted provider gives back, so it is also the fallback whenever a + connection can be read once and handed over as text. + """ + + name: str + system_prompt: str + tool_schema: list[dict[str, Any]] = field(default_factory=list) + data: dict[str, Any] = field(default_factory=dict) + scratch: Path = Path(".") + kind: str = "spec" + + def workdir(self) -> Path: + return self.scratch + + def builtin_tools(self) -> tuple[str, ...]: + return () + + def servers(self) -> dict[str, Any]: + return {} + + def briefing(self) -> str: + parts = [ + "This agent is supplied as a definition, not a repository. Everything knowable " + "about it is below; there is no code to open, so do not guess at anything absent.", + f"SYSTEM PROMPT:\n{self.system_prompt}", + ] + if self.tool_schema: + parts.append( + f"TOOL SCHEMA:\n{json.dumps(self.tool_schema, indent=2)[:6000]}" + ) + if self.data: + parts.append(f"DATA:\n{json.dumps(self.data, indent=2)[:6000]}") + return "\n\n".join(parts) + + +_REGISTRY: dict[str, Callable[..., AgentSource]] = { + "repo": lambda **kw: RepoSource(name=kw["name"], root=Path(kw["root"])), + "spec": lambda **kw: SpecSource( + name=kw["name"], + system_prompt=kw.get("system_prompt", ""), + tool_schema=kw.get("tool_schema") or [], + data=kw.get("data") or {}, + scratch=Path(kw.get("scratch", ".")), + ), +} + + +def register_source(kind: str, factory: Callable[..., AgentSource]) -> None: + """Add a kind of agent. A provider connection is a class and one line here.""" + _REGISTRY[kind] = factory + + +def resolve(kind: str, **kwargs: Any) -> AgentSource: + if kind not in _REGISTRY: + raise NotImplementedError( + f"no agent source of kind {kind!r}; registered kinds are " + f"{', '.join(sorted(_REGISTRY))}" + ) + return _REGISTRY[kind](**kwargs) + + +def supported() -> tuple[str, ...]: + return tuple(sorted(_REGISTRY)) diff --git a/src/fi/alk/harness/tools.py b/src/fi/alk/harness/tools.py new file mode 100644 index 0000000..2bfe10c --- /dev/null +++ b/src/fi/alk/harness/tools.py @@ -0,0 +1,102 @@ +"""The tools the harness offers a session, and the gates behind them. + +The model does judgement; these do the parts that must be exact. Validation lives inside the +tool rather than after the session, so a problem is returned into the conversation and fixed on +the next turn instead of surfacing once the session is already over. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from claude_agent_sdk import create_sdk_mcp_server, tool + +from .contract import AgentContract, validate_contract + +CONTRACT_SERVER = "contract" + + +def _ok(text: str) -> dict[str, Any]: + return {"content": [{"type": "text", "text": text}]} + + +def _problems(problems: list[str]) -> dict[str, Any]: + return { + "content": [ + { + "type": "text", + "text": "Not accepted. Fix these and call submit_contract again:\n - " + + "\n - ".join(problems), + } + ], + "is_error": True, + } + + +def accept_contract(payload: dict[str, Any], destination: Path) -> dict[str, Any]: + """The gate itself: validate, and write only if it passes. + + A plain function rather than only a tool body, so the rule that decides whether a contract + is usable can be exercised and reasoned about without standing up a session. + """ + try: + contract = AgentContract.model_validate(payload) + except Exception as invalid: + return _problems([f"schema:{invalid}"[:600]]) + + problems = validate_contract(contract) + if problems: + return _problems(problems) + + destination.mkdir(parents=True, exist_ok=True) + path = destination / "contract.json" + path.write_text( + json.dumps(contract.model_dump(), indent=2, ensure_ascii=False), + encoding="utf-8", + ) + return _ok( + f"Accepted and saved to {path}.\n" + f"{len(contract.tools)} tools: {', '.join(sorted(contract.tool_names()))}\n" + f"{len(contract.hard_constraints)} rules, " + f"{len(contract.real_use_cases)} use cases, " + f"{len(contract.open_questions)} open questions." + ) + + +def contract_tools(destination: Path) -> Any: + """A server exposing ``submit_contract``, writing to ``destination`` on acceptance.""" + + @tool( + "submit_contract", + "Submit the agent's testing contract. Validated on submission; problems are returned " + "to you so you can correct them and submit again.", + { + "agent": str, + "one_liner": str, + "modality": str, + "conversational": bool, + "system_prompt_excerpt": str, + "hard_constraints": list, + "tools": list, + "data_schema": dict, + "base_environment": dict, + "real_use_cases": list, + "signature_cases": list, + "grading_notes": str, + "anti_hallucination": list, + "open_questions": list, + }, + ) + async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: + return accept_contract(args, destination) + + return create_sdk_mcp_server( + name=CONTRACT_SERVER, version="0.1.0", tools=[submit_contract] + ) + + +def qualified(server: str, tool_name: str) -> str: + """The name an in-process MCP tool is granted under.""" + return f"mcp__{server}__{tool_name}" diff --git a/src/fi/alk/harness/understand.py b/src/fi/alk/harness/understand.py new file mode 100644 index 0000000..c0fac78 --- /dev/null +++ b/src/fi/alk/harness/understand.py @@ -0,0 +1,86 @@ +"""Stage one: read an agent and produce its contract. + +The stage is the same whatever the agent is. What changes between a repository, a provider +connection and a pasted definition is where the truth lives, and that comes from the source. + +It stays open after the first answer, because a contract is usually right on the second look and +not the first. Correcting it is the next thing said, not a re-run. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Callable + +from .config import artifact_dir, load_skill, read_only_session +from .contract import AgentContract +from .session import Stage +from .sources import AgentSource +from .tools import CONTRACT_SERVER, contract_tools, qualified + +SKILL = "understand-agent" + + +def open_stage( + source: AgentSource, + *, + out: Path | None = None, + ask: Callable[..., Any] | None = None, + max_turns: int = 40, +) -> tuple[Stage, Path]: + """A live understand-the-agent stage, and where it will write.""" + destination = out or artifact_dir(source.name) + options = read_only_session( + system_prompt=f"{load_skill(SKILL)}\n\n## This agent\n\n{source.briefing()}", + cwd=source.workdir(), + mcp_servers={**source.servers(), CONTRACT_SERVER: contract_tools(destination)}, + extra_tools=[ + *source.builtin_tools(), + qualified(CONTRACT_SERVER, "submit_contract"), + ], + max_turns=max_turns, + ) + if ask is not None: + options.can_use_tool = ask + return Stage(options, name=SKILL), destination + + +def opening(source: AgentSource) -> str: + return ( + f"Read the agent named {source.name!r} and produce its contract.\n\n" + "Work through the tools, their exact argument names and types, the constrained argument " + "values, the rules it enforces, and its data. Ask me if the source genuinely does not " + "settle something that changes what gets built. Call submit_contract when you are done." + ) + + +def load(destination: Path) -> AgentContract | None: + """The contract on disk, if the stage produced one.""" + path = Path(destination) / "contract.json" + if not path.exists(): + return None + return AgentContract.model_validate(json.loads(path.read_text(encoding="utf-8"))) + + +async def understand( + source: AgentSource, + *, + out: Path | None = None, + follow_ups: list[str] | None = None, + on_event: Callable[..., Any] | None = None, + ask: Callable[..., Any] | None = None, + max_turns: int = 40, +) -> AgentContract | None: + """Run the stage start to finish and return the contract. + + ``follow_ups`` are corrections applied in the same session, the scripted equivalent of an + operator typing them. ``ask`` handles clarifying questions; without it the model records what + it could not resolve in ``open_questions`` instead of blocking. + """ + stage, destination = open_stage(source, out=out, ask=ask, max_turns=max_turns) + async with stage: + await stage.say(opening(source), on_event=on_event) + for follow_up in follow_ups or []: + await stage.say(follow_up, on_event=on_event) + return load(destination) diff --git a/tests/test_harness.py b/tests/test_harness.py new file mode 100644 index 0000000..06df6b1 --- /dev/null +++ b/tests/test_harness.py @@ -0,0 +1,236 @@ +"""Offline tests for the harness. No model calls, no network, no credentials. + +Every case here encodes something that must stay true for a generated environment to be +trustworthy: the contract cannot be structurally wrong, an unsupported agent source refuses +rather than half-works, and the submit gate returns its problems instead of writing a bad file. +""" + +from __future__ import annotations + +import json + +import pytest + +from fi.alk.harness import ( + AgentContract, + RepoSource, + SpecSource, + ToolSpec, + artifact_dir, + load_skill, + provider_env, + register_source, + resolve, + supported, + validate_contract, +) +from fi.alk.harness.cli import build_parser +from fi.alk.harness.session import ARTIFACT, DONE, TEXT, TOOL, Event +from fi.alk.harness.tools import accept_contract, qualified +from fi.alk.harness.understand import load, opening + + +def _contract(**overrides) -> AgentContract: + payload = { + "agent": "drive_thru", + "tools": [ToolSpec(name="order", args=["item_id"])], + "real_use_cases": ["order an item"], + } + payload.update(overrides) + return AgentContract(**payload) + + +# --- contract ------------------------------------------------------------------------ + + +def test_valid_contract_has_no_problems(): + assert validate_contract(_contract()) == [] + + +@pytest.mark.parametrize( + "overrides,expected", + [ + ({"agent": " "}, "empty:agent"), + ({"tools": []}, "no-tools"), + ({"real_use_cases": []}, "no-use-cases"), + ], +) +def test_validate_contract_catches_structural_problems(overrides, expected): + assert expected in validate_contract(_contract(**overrides)) + + +def test_duplicate_tool_names_are_rejected_and_named(): + """Names the offender: tool_names() is a set, so a naive length comparison never fires.""" + contract = _contract(tools=[ToolSpec(name="order"), ToolSpec(name="order")]) + assert "duplicate-tool-names:order" in validate_contract(contract) + + +def test_types_declared_for_arguments_that_do_not_exist_are_rejected(): + """A type on an argument the tool does not take means the reader misread the signature, + and a world built from it would be wrong in a way nothing downstream could detect.""" + contract = _contract( + tools=[ToolSpec(name="order", args=["item_id"], arg_types={"size": "str"})] + ) + assert "tool[order]:types-for-unknown-args:size" in validate_contract(contract) + + +def test_brief_carries_argument_types_into_downstream_prompts(): + contract = _contract( + tools=[ + ToolSpec( + name="remove_order_item", + args=["order_id"], + arg_types={"order_id": "list[str]"}, + ) + ] + ) + assert "remove_order_item(order_id: list[str])" in contract.brief() + + +def test_shapes_are_normalised_rather_than_rejected(): + """Benign shape variance is not a grounding error; rejecting it burns turns for nothing.""" + contract = AgentContract.model_validate( + { + "agent": "x", + "one_liner": ["a", "b"], + "hard_constraints": "only one rule", + "data_schema": [1, 2], + } + ) + assert contract.one_liner == "a\nb" + assert contract.hard_constraints == ["only one rule"] + assert contract.data_schema == {"value": [1, 2]} + + +# --- sources ------------------------------------------------------------------------- + + +def test_repo_and_spec_sources_are_registered(): + assert {"repo", "spec"}.issubset(set(supported())) + + +def test_unsupported_source_refuses_and_names_what_exists(): + with pytest.raises(NotImplementedError) as raised: + resolve("browser", name="x") + assert "repo" in str(raised.value) + + +def test_repo_source_gets_read_tools_and_a_briefing_that_points_at_the_code(tmp_path): + source = RepoSource(name="a", root=tmp_path) + assert source.builtin_tools() == ("Read", "Glob", "Grep") + assert str(tmp_path) in source.briefing() + + +def test_spec_source_gets_no_file_tools_because_there_is_nothing_to_read(): + source = SpecSource( + name="a", system_prompt="you are a bot", tool_schema=[{"name": "t"}] + ) + assert source.builtin_tools() == () + briefing = source.briefing() + assert "you are a bot" in briefing and "t" in briefing + + +def test_a_new_kind_of_agent_is_a_registration_not_a_code_change(): + register_source("fake", lambda **kw: RepoSource(name=kw["name"], root=".")) + assert resolve("fake", name="z").name == "z" + + +# --- session events ------------------------------------------------------------------ + + +@pytest.mark.parametrize( + "event,expected", + [ + (Event(TEXT, text="hello"), "hello"), + (Event(TOOL, tool="Read", detail={"target": "agent.py"}), " [Read agent.py]"), + (Event(TOOL, tool="Grep"), " [Grep]"), + ( + Event(ARTIFACT, detail={"path": "a/contract.json"}), + " [saved a/contract.json]", + ), + ], +) +def test_events_render_for_a_terminal(event, expected): + assert event.line() == expected + + +def test_done_event_reports_outcome_turns_and_spend(): + line = Event( + DONE, detail={"outcome": "success", "turns": 9, "cost_usd": 0.36} + ).line() + assert "success" in line and "turns=9" in line and "0.36" in line + + +# --- the submit gate ----------------------------------------------------------------- + + +def test_submit_writes_the_contract_when_it_is_valid(tmp_path): + result = accept_contract( + { + "agent": "drive_thru", + "tools": [{"name": "order", "args": ["item_id"]}], + "real_use_cases": ["order an item"], + }, + tmp_path, + ) + assert not result.get("is_error") + written = json.loads((tmp_path / "contract.json").read_text()) + assert written["agent"] == "drive_thru" + + +def test_submit_returns_problems_and_writes_nothing_when_invalid(tmp_path): + """The gate reports into the conversation so the next turn can fix it, which is the only + reason a bad contract does not reach disk.""" + result = accept_contract( + {"agent": "drive_thru", "tools": [], "real_use_cases": []}, tmp_path + ) + assert result.get("is_error") + text = result["content"][0]["text"] + assert "no-tools" in text and "no-use-cases" in text + assert not (tmp_path / "contract.json").exists() + + +def test_load_returns_none_when_the_stage_produced_nothing(tmp_path): + assert load(tmp_path) is None + + +# --- wiring -------------------------------------------------------------------------- + + +def test_provider_env_pins_the_model_and_never_invents_a_project(monkeypatch): + monkeypatch.delenv("ANTHROPIC_VERTEX_PROJECT_ID", raising=False) + monkeypatch.delenv("GOOGLE_CLOUD_PROJECT", raising=False) + env = provider_env("claude-sonnet-4-6") + assert env["CLAUDE_CODE_USE_VERTEX"] == "1" + assert env["ANTHROPIC_MODEL"] == "claude-sonnet-4-6" + assert "ANTHROPIC_VERTEX_PROJECT_ID" not in env + + +def test_qualified_tool_name_matches_the_mcp_convention(): + assert qualified("contract", "submit_contract") == "mcp__contract__submit_contract" + + +def test_the_skill_exists_and_forbids_guessing(): + text = load_skill("understand-agent") + assert "submit_contract" in text + assert "guess" in text.lower() + + +def test_artifacts_land_under_the_agent_name(): + assert artifact_dir("drive_thru").as_posix().endswith("environments/drive_thru") + + +def test_cli_defaults_to_staying_open_for_corrections(): + args = build_parser().parse_args(["understand", "--name", "a", "--path", "."]) + assert args.interactive is True + assert ( + build_parser() + .parse_args(["understand", "--name", "a", "--path", ".", "--once"]) + .interactive + is False + ) + + +def test_opening_names_the_agent_and_asks_for_the_contract(tmp_path): + text = opening(RepoSource(name="drive_thru", root=tmp_path)) + assert "drive_thru" in text and "submit_contract" in text diff --git a/uv.lock b/uv.lock index c07914c..06a5b2d 100644 --- a/uv.lock +++ b/uv.lock @@ -54,6 +54,7 @@ name = "agent-learning-kit" version = "0.1.0" source = { editable = "." } dependencies = [ + { name = "claude-agent-sdk" }, { name = "fi-instrumentation-otel" }, { name = "gepa" }, { name = "httpx" }, @@ -152,6 +153,7 @@ requires-dist = [ { name = "audioop-lts", marker = "python_full_version >= '3.13' and extra == 'trinity'", specifier = ">=0.2.1" }, { name = "chromadb", marker = "extra == 'all'", specifier = ">=0.4.0" }, { name = "chromadb", marker = "extra == 'feedback'", specifier = ">=0.4.0" }, + { name = "claude-agent-sdk", specifier = ">=0.2.139" }, { name = "fi-instrumentation-otel", specifier = ">=0.1.16" }, { name = "gepa", specifier = ">=0.0.17" }, { name = "httpx", specifier = ">=0.24.0" }, @@ -897,6 +899,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/eb/ce/0f7be6e5d0feafa2cda54b12e6542afeea7dea89d2d411e14da90f8abb96/chromadb-1.5.9-cp39-abi3-win_amd64.whl", hash = "sha256:4fd0b560e56761b7f3cb4d5c6205fd5f20814484b4a3e4e9af9038c2b428fc6c", size = 23542454, upload-time = "2026-05-05T05:54:54.942Z" }, ] +[[package]] +name = "claude-agent-sdk" +version = "0.2.139" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "mcp" }, + { name = "sniffio" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/11/b6/cfcdefed1f866a8ba372ef3884c8020dd54338d15d8b45d5a1ff7432cea1/claude_agent_sdk-0.2.139.tar.gz", hash = "sha256:4395ed541cdd4c13aeb1213b3b414b7e8a94cc060a773137e961882e81c174a7", size = 319519, upload-time = "2026-08-14T22:34:48.038Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/7f/f04c33553cbc69bb96d045dc38a6266726fad72130f22f405dfe9eb54bf1/claude_agent_sdk-0.2.139-py3-none-macosx_11_0_arm64.whl", hash = "sha256:cbc50cc475ec633cabfa36347646097e9b1466d53130e4a04a87308ff830c87b", size = 88043656, upload-time = "2026-08-14T22:34:53.027Z" }, + { url = "https://files.pythonhosted.org/packages/85/d0/a17f5318ca0220479f20fdf83fa54a838a0a13ee203495ff67c72c3f43a7/claude_agent_sdk-0.2.139-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:1c08206b1603444582cd365effaf95d2a8248661f1492281fb2d529b0887c047", size = 93000433, upload-time = "2026-08-14T22:34:58.225Z" }, + { url = "https://files.pythonhosted.org/packages/c5/2e/5bcec31700d76ad2d5b9fc28521a75a66f464063dac11373cf8d61446a4f/claude_agent_sdk-0.2.139-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:e69ae1a0b2af684c64839cc16e10b70800d9d2f57622b8c0d1739dd878cd7346", size = 97396659, upload-time = "2026-08-14T22:35:03.734Z" }, + { url = "https://files.pythonhosted.org/packages/c8/7f/582b3c1936c9f4ebc1bdc55a3923f1b680ef3c01928ffff1ea38eb84f637/claude_agent_sdk-0.2.139-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:34b289b3436fe24013f7b9cfe9f0a4e0806917a9ef8bbe829cda9a7b12d41a77", size = 98391889, upload-time = "2026-08-14T22:35:09.683Z" }, + { url = "https://files.pythonhosted.org/packages/56/54/d94af31d19b4e8d63d1b15002fd333ea77a040b7ab7a388044e511c8f9f6/claude_agent_sdk-0.2.139-py3-none-win_amd64.whl", hash = "sha256:9b76f0ffe216d6ca290d5f4f295ecb030dc496f101986ac99480a89d4abc6426", size = 100746507, upload-time = "2026-08-14T22:35:15.144Z" }, +] + [[package]] name = "click" version = "8.4.1" From 42d3c08636b3cf2767e0c336f32b54fcd357f8f4 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sun, 16 Aug 2026 15:58:06 +0530 Subject: [PATCH 02/39] feat(harness): database-backed generated worlds with a probe gate --- src/fi/alk/harness/build.py | 82 ++++++ src/fi/alk/harness/cli.py | 25 +- .../harness/skills/build-environment/SKILL.md | 85 +++++++ src/fi/alk/harness/world/__init__.py | 28 ++ src/fi/alk/harness/world/probe.py | 240 ++++++++++++++++++ src/fi/alk/harness/world/runtime.py | 211 +++++++++++++++ src/fi/alk/harness/world/snapshot.py | 154 +++++++++++ src/fi/alk/harness/world/tools.py | 211 +++++++++++++++ 8 files changed, 1035 insertions(+), 1 deletion(-) create mode 100644 src/fi/alk/harness/build.py create mode 100644 src/fi/alk/harness/skills/build-environment/SKILL.md create mode 100644 src/fi/alk/harness/world/__init__.py create mode 100644 src/fi/alk/harness/world/probe.py create mode 100644 src/fi/alk/harness/world/runtime.py create mode 100644 src/fi/alk/harness/world/snapshot.py create mode 100644 src/fi/alk/harness/world/tools.py diff --git a/src/fi/alk/harness/build.py b/src/fi/alk/harness/build.py new file mode 100644 index 0000000..bf7a07d --- /dev/null +++ b/src/fi/alk/harness/build.py @@ -0,0 +1,82 @@ +"""Stage two: build the world the agent's tools run against. + +Reads the contract stage one produced and builds a database behind the agent's action space, +then freezes it. The frozen snapshot is the base state every scenario restores from; a scenario +adds only the rows it additionally needs. + +The stage stays open, because a world is usually right on the second look. Correcting a handler +is the next thing said, and the tool is re-run on the spot. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Callable + +from claude_agent_sdk import ClaudeAgentOptions + +from .config import artifact_dir, load_skill, provider_env +from .contract import AgentContract +from .session import Stage +from .tools import qualified +from .world.tools import TOOL_NAMES, WORLD_SERVER, world_tools + +SKILL = "build-environment" + + +def open_stage( + contract: AgentContract, + *, + out: Path | None = None, + ask: Callable[..., Any] | None = None, + max_turns: int = 60, +) -> tuple[Stage, Path]: + """A live build-the-world stage, and where it will write.""" + destination = out or artifact_dir(contract.agent) + server, _world = world_tools(contract, destination) + options = ClaudeAgentOptions( + system_prompt=f"{load_skill(SKILL)}\n\n## This agent\n\n{contract.brief()}", + # No file tools and no shell. Everything this stage can do goes through a tool that + # executes it and reports back, which is what makes the guardrails meaningful. + allowed_tools=[ + "AskUserQuestion", + *(qualified(WORLD_SERVER, name) for name in TOOL_NAMES), + ], + mcp_servers={WORLD_SERVER: server}, + permission_mode="acceptEdits", + cwd=str(destination.parent if destination.parent.exists() else Path.cwd()), + setting_sources=[], + max_turns=max_turns, + env=provider_env(), + ) + if ask is not None: + options.can_use_tool = ask + return Stage(options, name=SKILL), destination + + +def opening(contract: AgentContract) -> str: + return ( + f"Build the world for {contract.agent!r}.\n\n" + "Design the schema, seed it from the contract's real data, and write one handler per " + "tool. Verify the refusals yourself with run_tool: a call naming something that does " + "not exist must be refused, not succeed. Declare at least one sequence where state has " + "to carry across calls, then check_world and save_world." + ) + + +async def build( + contract: AgentContract, + *, + out: Path | None = None, + follow_ups: list[str] | None = None, + on_event: Callable[..., Any] | None = None, + ask: Callable[..., Any] | None = None, + max_turns: int = 60, +) -> Path | None: + """Run the stage start to finish. Returns where the world was written, or None.""" + stage, destination = open_stage(contract, out=out, ask=ask, max_turns=max_turns) + async with stage: + await stage.say(opening(contract), on_event=on_event) + for follow_up in follow_ups or []: + await stage.say(follow_up, on_event=on_event) + return destination if (destination / "world.sqlite").exists() else None diff --git a/src/fi/alk/harness/cli.py b/src/fi/alk/harness/cli.py index 9151c3c..8047eea 100644 --- a/src/fi/alk/harness/cli.py +++ b/src/fi/alk/harness/cli.py @@ -14,7 +14,8 @@ from pathlib import Path from typing import Any -from .config import DEFAULT_MODEL +from .build import build +from .config import DEFAULT_MODEL, artifact_dir from .session import TEXT, Event from .sources import resolve, supported from .understand import load, open_stage, opening @@ -102,6 +103,23 @@ async def _understand(args: argparse.Namespace) -> int: return 0 +async def _build(args: argparse.Namespace) -> int: + destination = Path(args.out) if args.out else artifact_dir(args.name) + contract = load(destination) + if contract is None: + print(f"No contract at {destination}. Run `understand` first.", file=sys.stderr) + return 1 + + print(f"agent: {contract.agent} ({len(contract.tools)} tools)") + print(f"out: {destination}\n") + written = await build(contract, out=destination, on_event=_render) + if written is None: + print("\nNo world was saved.", file=sys.stderr) + return 1 + print(f"\nworld: {written}") + return 0 + + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog="fi.alk.harness", description=__doc__) sub = parser.add_subparsers(dest="stage", required=True) @@ -123,6 +141,11 @@ def build_parser() -> argparse.ArgumentParser: ) understand.add_argument("--model", default=DEFAULT_MODEL, help=argparse.SUPPRESS) understand.set_defaults(run=_understand, interactive=True) + + world = sub.add_parser("build", help="build the world from an agent's contract") + world.add_argument("--name", required=True, help="which agent") + world.add_argument("--out", default=None, help="artifact directory") + world.set_defaults(run=_build) return parser diff --git a/src/fi/alk/harness/skills/build-environment/SKILL.md b/src/fi/alk/harness/skills/build-environment/SKILL.md new file mode 100644 index 0000000..87a3f21 --- /dev/null +++ b/src/fi/alk/harness/skills/build-environment/SKILL.md @@ -0,0 +1,85 @@ +--- +name: build-environment +description: Build a real, database-backed world that an agent's tools run against. +--- + +# Build the environment + +You are building the world an agent will be tested in. Its tools will run against your database +and get back whatever it really says, including a refusal when the agent asks for something that +is not there. + +The contract is the only source of truth. Every table, every row, every id comes from it. If the +contract does not contain something, the world does not have it either. + +## What you are building + +1. **A schema.** The tables the agent's data actually needs, with the keys and constraints that + make wrong states impossible to reach. +2. **Seed data.** The agent's real catalogue: its menu, its records, its inventory, taken from + the contract's data, not invented. +3. **One handler per tool.** Python, `def handle(args, db)`, using `db.query`, `db.one` and + `db.execute`. It returns what the real tool would return. +4. **Sequences.** Series of calls whose end state you assert, so consistency across calls is + checked rather than assumed. + +## The one thing that matters most + +**The world must be able to say no.** + +A canned mock answers every call the same way, so an agent that removes an item that was never +added is told it succeeded, and the test that was supposed to catch that passes. Your handlers +exist to prevent exactly that. + +So for every handler, before you return anything, ask what makes this call impossible and check +for it: + +- the id does not exist +- the item exists but is unavailable +- the argument is outside what the tool accepts +- the operation contradicts the current state, like removing from an empty order + +When one of those holds, `raise ToolError("...")` with a message that says what was wrong. A +refusal is the world working. It is not an error you should be avoiding. + +Never let a handler crash on bad input. `KeyError` and `TypeError` are your bugs; `ToolError` is +the world's answer. They must not be confused, and one of the checks tells them apart. + +## How to work + +Build in this order and check as you go. + +1. `create_schema` with the whole schema. +2. `seed` each table from the contract's data. Seed the real catalogue, not a sample of it: a + scenario about an unavailable item needs the unavailable item to be in there. +3. `define_handler` for each tool, one at a time. Each is executed the moment you define it, so + read what comes back. Pass `smoke_arguments` that should work. +4. `run_tool` to try the refusals yourself. Call a removal with an id that was never created. If + it succeeds, the handler is wrong, and no other check will catch that for you. +5. `declare_sequence` for at least one flow where state has to carry across calls. Add something, + list it, remove it, list again. This is the failure that individual calls cannot reveal. +6. `check_world` to see everything at once, fix what it reports, and repeat. +7. `save_world` when it passes. + +`save_world` refuses a world that has not passed its checks or has no declared sequence. That +refusal is not an obstacle to work around; it is the same guarantee you are building into the +handlers. + +## Seed data + +Use the contract's real values. Real ids, real names, real prices, real availability flags. The +whole point is that a test can reference something and have it be there. + +Where the contract records that something is unavailable, or a typo in an id, or a value that +looks wrong, **keep it as it is**. The world is a replica of what the agent actually has, not a +corrected version of it. A test written against a corrected world will not catch the bug the +real one has. + +Leave the world in its natural starting state: empty carts, no in-flight orders, nothing that +belongs to one particular scenario. Individual scenarios add what they need on top of it. + +## Finishing + +Say what you built: the tables, roughly how many rows, which tools, and which refusals you +verified. Then say plainly anything you were unsure about, especially where the contract was +thin and you had to decide. diff --git a/src/fi/alk/harness/world/__init__.py b/src/fi/alk/harness/world/__init__.py new file mode 100644 index 0000000..eda492e --- /dev/null +++ b/src/fi/alk/harness/world/__init__.py @@ -0,0 +1,28 @@ +"""Generated worlds: a real data store behind an agent's tools. + +The pieces here are the parts that must be exact, so that what gets generated per agent stays +small: the runtime a world executes on, the snapshot every scenario restores from, and the probe +suite that decides whether a world is usable at all. +""" + +from .probe import EDGE, HAPPY, SEQUENCE, ProbeReport, ProbeResult, probe +from .runtime import Call, Db, GeneratedWorld, ToolError, WorldSpec +from .snapshot import apply_overlay, read_manifest, restore, save + +__all__ = [ + "Call", + "Db", + "EDGE", + "GeneratedWorld", + "HAPPY", + "ProbeReport", + "ProbeResult", + "SEQUENCE", + "ToolError", + "WorldSpec", + "apply_overlay", + "probe", + "read_manifest", + "restore", + "save", +] diff --git a/src/fi/alk/harness/world/probe.py b/src/fi/alk/harness/world/probe.py new file mode 100644 index 0000000..1ece4fb --- /dev/null +++ b/src/fi/alk/harness/world/probe.py @@ -0,0 +1,240 @@ +"""Whether a generated world is usable, decided by exercising it. + +Published work on synthesised environments is consistent about two things. Most generated +environments contain bugs, so the gate has to aim at the ones that block rather than at +perfection. And the bugs cluster: edge-case handling first, then state consistency across +several calls. A gate that runs each handler once and calls it done misses both clusters. + +So this exercises every tool three ways, and then exercises the world as a sequence: + +- **happy**: a valid call, built from the values the contract says the argument accepts +- **edge**: an identifier that does not exist, and a required argument left out +- **sequence**: a declared series of calls whose final state is asserted + +The distinction that matters throughout is **refusal versus crash**. A tool that rejects a +nonexistent id is working: that refusal is the entire point of a real world. A tool that raises +``KeyError`` on the same input is broken. They are both failures to a naive check and opposite +outcomes here. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Iterable, Mapping, Sequence + +from ..contract import AgentContract, ToolSpec +from .runtime import GeneratedWorld + +HAPPY = "happy" +EDGE = "edge" +SEQUENCE = "sequence" +COVERAGE = "coverage" + +# A value no generated world should ever have seeded, used to prove a lookup refuses. +ABSENT = "__does_not_exist__" + + +@dataclass +class ProbeResult: + name: str + kind: str + passed: bool + detail: str = "" + + +@dataclass +class ProbeReport: + results: list[ProbeResult] = field(default_factory=list) + + @property + def score(self) -> float: + return ( + sum(1 for result in self.results if result.passed) / len(self.results) + if self.results + else 0.0 + ) + + @property + def failures(self) -> list[ProbeResult]: + return [result for result in self.results if not result.passed] + + def summary(self) -> str: + if not self.results: + return "no probes ran" + lines = [ + f"{len(self.results) - len(self.failures)}/{len(self.results)} probes passed" + ] + for failure in self.failures: + lines.append(f" {failure.kind}:{failure.name}: {failure.detail}") + return "\n".join(lines) + + +def _valid_arguments(tool: ToolSpec) -> dict[str, Any]: + """A plausible call, using the values the contract says each argument accepts.""" + arguments: dict[str, Any] = {} + for arg in tool.args: + options = tool.arg_values.get(arg) + if isinstance(options, (list, tuple)): + usable = [value for value in options if value not in (None, "null", "")] + if usable: + arguments[arg] = usable[0] + continue + declared = tool.arg_types.get(arg, "") + if "list" in declared: + arguments[arg] = [] + elif "int" in declared: + arguments[arg] = 1 + elif "bool" in declared: + arguments[arg] = True + else: + arguments[arg] = ABSENT + return arguments + + +def _identifier_arguments(tool: ToolSpec) -> dict[str, Any] | None: + """The same call with every identifier replaced by one that cannot exist.""" + arguments = _valid_arguments(tool) + swapped = False + for arg in tool.args: + if not tool.arg_values.get(arg): + continue + declared = tool.arg_types.get(arg, "") + arguments[arg] = [ABSENT] if "list" in declared else ABSENT + swapped = True + return arguments if swapped else None + + +def probe( + world: GeneratedWorld, + contract: AgentContract, + *, + sequences: Iterable[Mapping[str, Any]] = (), +) -> ProbeReport: + """Exercise the world and report what it can and cannot do. + + ``sequences`` are declared by whoever built the world, because knowing that adding an item + should make it appear in a listing is judgement about this agent, not something derivable + from a schema. + """ + report = ProbeReport() + + for tool in contract.tools: + if tool.name not in world.handlers: + report.results.append( + ProbeResult(tool.name, COVERAGE, False, "contract tool has no handler") + ) + for name in world.handlers: + if name not in contract.tool_names(): + report.results.append( + ProbeResult( + name, COVERAGE, False, "handler for a tool the agent does not have" + ) + ) + + for tool in contract.tools: + if tool.name not in world.handlers: + continue + + call = world.call(tool.name, _valid_arguments(tool)) + # A refusal here is acceptable: the contract's first listed value may genuinely be + # invalid in the seeded world. A crash never is. + report.results.append( + ProbeResult( + tool.name, + HAPPY, + call.ok or call.refused, + "" if call.ok or call.refused else call.error, + ) + ) + + bogus = _identifier_arguments(tool) + if bogus is not None: + call = world.call(tool.name, bogus) + report.results.append( + ProbeResult( + tool.name, + EDGE, + call.refused, + "" + if call.refused + else ( + "succeeded on an id that does not exist" + if call.ok + else f"crashed instead of refusing: {call.error}" + ), + ) + ) + + if tool.args: + missing = _valid_arguments(tool) + missing.pop(tool.args[0], None) + call = world.call(tool.name, missing) + report.results.append( + ProbeResult( + f"{tool.name}:without-{tool.args[0]}", + EDGE, + call.refused, + "" + if call.refused + else ( + "accepted a call with a required argument missing" + if call.ok + else f"crashed instead of refusing: {call.error}" + ), + ) + ) + + unknown = world.call(ABSENT, {}) + report.results.append( + ProbeResult( + "unknown-tool", + EDGE, + unknown.refused, + "" if unknown.refused else "an unknown tool did not refuse", + ) + ) + + for index, sequence in enumerate(sequences): + report.results.append(_run_sequence(world, sequence, index)) + + return report + + +def _run_sequence( + world: GeneratedWorld, sequence: Mapping[str, Any], index: int +) -> ProbeResult: + """Run a declared series of calls and check the state it leaves behind. + + This is the state-consistency check: the failure mode where each call works on its own and + the world still forgets what the previous one did. + """ + name = str(sequence.get("name") or f"sequence-{index}") + calls: Sequence[Mapping[str, Any]] = sequence.get("calls") or () + for step in calls: + call = world.call(str(step.get("tool", "")), step.get("arguments") or {}) + if step.get("expect") == "refusal": + if not call.refused: + return ProbeResult( + name, SEQUENCE, False, f"{call.name} should have refused" + ) + continue + if not call.ok: + return ProbeResult(name, SEQUENCE, False, f"{call.name}: {call.error}") + + state = world.state() + for path, expected in (sequence.get("expect_state") or {}).items(): + table, _, column = path.partition(".") + rows = state.get(table, []) + if column == "count": + if len(rows) != expected: + return ProbeResult( + name, + SEQUENCE, + False, + f"{table} holds {len(rows)} rows, expected {expected}", + ) + elif not any(str(row.get(column)) == str(expected) for row in rows): + return ProbeResult( + name, SEQUENCE, False, f"no row in {table} has {column}={expected!r}" + ) + return ProbeResult(name, SEQUENCE, True) diff --git a/src/fi/alk/harness/world/runtime.py b/src/fi/alk/harness/world/runtime.py new file mode 100644 index 0000000..198afe6 --- /dev/null +++ b/src/fi/alk/harness/world/runtime.py @@ -0,0 +1,211 @@ +"""The runtime a generated world runs on. + +A generated world is a database plus one handler per tool. The handler decides what a call does; +this decides what a handler is allowed to be, what happens when one fails, and what the world +looks like afterwards. Keeping that here means a generated file stays small enough to read and +correct, and the parts that must be exact are not regenerated every time. + +The contract with the rest of the platform is ``EnvironmentAdapter``: ``reset`` publishes the +tools and the starting state, ``handle_tool_call`` executes one call, and the state afterwards is +what the checks grade. A world is therefore drivable by any loop that already drives an +environment, which is the whole reason we generate against this interface rather than inventing +one. +""" + +from __future__ import annotations + +import json +import sqlite3 +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Mapping, Sequence + +from fi.simulate.environment import ( + EnvironmentAdapter, + EnvironmentSnapshot, + ToolExecutionResult, +) + + +class ToolError(Exception): + """A tool refusing for a real reason the agent should see and recover from. + + Distinct from a crash. A refusal is the world working: the id does not exist, the item is + unavailable, the argument is outside what the tool accepts. A crash is our bug, and the two + must never look the same to a caller deciding whether the agent behaved correctly. + """ + + +@dataclass +class Db: + """The handle a handler gets. Deliberately small: query, execute, one. + + Handlers get a database, not a filesystem and not a network. Anything a handler can reach is + something a generated world could depend on, and a world that depends on the outside is not + reproducible. + """ + + connection: sqlite3.Connection + + def query(self, sql: str, params: Sequence[Any] = ()) -> list[dict[str, Any]]: + cursor = self.connection.execute(sql, tuple(params)) + columns = [column[0] for column in (cursor.description or [])] + return [dict(zip(columns, row)) for row in cursor.fetchall()] + + def one(self, sql: str, params: Sequence[Any] = ()) -> dict[str, Any] | None: + rows = self.query(sql, params) + return rows[0] if rows else None + + def execute(self, sql: str, params: Sequence[Any] = ()) -> int: + cursor = self.connection.execute(sql, tuple(params)) + self.connection.commit() + return cursor.rowcount + + +@dataclass +class Call: + """One tool call and what the world did with it.""" + + name: str + arguments: dict[str, Any] + result: Any = None + ok: bool = True + error: str = "" + refused: bool = False + + +class GeneratedWorld(EnvironmentAdapter): + """A database-backed world whose tools are generated per agent. + + Subclasses declare ``name``, ``tools`` and ``handlers``. Everything about execution, + refusal, and state reporting is here so that a generated subclass carries only the parts + that are specific to one agent. + """ + + name = "generated" + tools: list[dict[str, Any]] = [] + handlers: dict[str, str] = {} + + def __init__(self, database: str | Path = ":memory:") -> None: + self.database = str(database) + self.connection = sqlite3.connect(self.database, check_same_thread=False) + self.connection.execute("PRAGMA foreign_keys = ON") + self.calls: list[Call] = [] + + # -- EnvironmentAdapter ---------------------------------------------------------- + + def reset(self, **_context: Any) -> EnvironmentSnapshot: + self.calls = [] + return EnvironmentSnapshot(tools=list(self.tools), state=self.state()) + + def observe(self, **_context: Any) -> EnvironmentSnapshot: + return EnvironmentSnapshot(tools=list(self.tools), state=self.state()) + + def handle_tool_call( + self, tool_call: Mapping[str, Any], **_context: Any + ) -> ToolExecutionResult | None: + name = str( + tool_call.get("name") or (tool_call.get("function") or {}).get("name") or "" + ) + call_id = tool_call.get("id") or tool_call.get("tool_call_id") + arguments = tool_call.get("arguments") or tool_call.get("args") or {} + if not isinstance(arguments, Mapping): + arguments = {} + + call = self.call(name, arguments) + content = ( + json.dumps(call.result, default=str) + if not isinstance(call.result, str) + else call.result + ) + return ToolExecutionResult( + tool_call_id=call_id, + tool_name=name or "unknown", + content=call.error if not call.ok else content, + result=call.result, + success=call.ok, + error=call.error or None, + state_updates=self.state(), + ) + + # -- execution ------------------------------------------------------------------- + + def call(self, name: str, arguments: Mapping[str, Any] | None = None) -> Call: + """Execute one call and record it. Never raises: a failure is an outcome, not an event. + + An unknown tool is a refusal rather than a silent success. An agent reaching for a tool + that does not exist is a finding, and answering it with an acknowledgement is how a test + passes something it should have caught. + """ + args = dict(arguments or {}) + if name not in self.handlers: + return self._record( + Call( + name=name, + arguments=args, + ok=False, + refused=True, + error=( + f"no such tool {name!r}; this agent has " + f"{', '.join(sorted(self.handlers)) or 'none'}" + ), + ) + ) + + namespace: dict[str, Any] = {"ToolError": ToolError, "json": json} + try: + exec(compile(self.handlers[name], f"", "exec"), namespace) + handle = namespace.get("handle") + if not callable(handle): + raise RuntimeError("handler defines no handle(args, db)") + value = handle(args, Db(self.connection)) + except ToolError as refusal: + return self._record( + Call( + name=name, + arguments=args, + ok=False, + refused=True, + error=str(refusal), + ) + ) + except Exception as crash: + # Our bug, not the agent's. Labelled differently so a run is never scored + # against a world that fell over. + return self._record( + Call( + name=name, + arguments=args, + ok=False, + error=f"{type(crash).__name__}: {crash}", + ) + ) + return self._record(Call(name=name, arguments=args, result=value)) + + def _record(self, call: Call) -> Call: + self.calls.append(call) + return call + + # -- state ----------------------------------------------------------------------- + + def state(self) -> dict[str, Any]: + """Every table and its rows: what the checks compare against after a run.""" + tables = Db(self.connection).query( + "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'" + ) + db = Db(self.connection) + return {row["name"]: db.query(f"SELECT * FROM {row['name']}") for row in tables} + + def close(self) -> None: + self.connection.close() + + +@dataclass +class WorldSpec: + """What a generated world is, before it is written out.""" + + agent: str + schema_sql: str = "" + tools: list[dict[str, Any]] = field(default_factory=list) + handlers: dict[str, str] = field(default_factory=dict) + notes: str = "" diff --git a/src/fi/alk/harness/world/snapshot.py b/src/fi/alk/harness/world/snapshot.py new file mode 100644 index 0000000..6a62f34 --- /dev/null +++ b/src/fi/alk/harness/world/snapshot.py @@ -0,0 +1,154 @@ +"""Freezing a world, and starting every scenario from the same frozen copy. + +The database is built once and snapshotted; that snapshot is the base state. A scenario restores +its own copy and layers on whatever it additionally needs, so scenarios cannot inherit each +other's leftovers and a run is repeatable a week later. + +Which is why the overlay exists: a scenario that needs a customer with three open orders adds +those rows to a restored copy rather than editing the snapshot. The base world stays the shared +starting point instead of drifting toward whichever scenario was written last. +""" + +from __future__ import annotations + +import json +import shutil +import sqlite3 +from pathlib import Path +from typing import Any, Mapping + +from .runtime import GeneratedWorld + +DATABASE = "world.sqlite" +HANDLERS = "handlers" +MANIFEST = "manifest.json" +WORLD_MODULE = "world.py" + +_MODULE = '''"""Generated world for {agent}. Do not edit by hand; regenerate instead. + +{notes} +""" + +from pathlib import Path + +from fi.alk.harness.world.runtime import GeneratedWorld + +_HERE = Path(__file__).parent + +TOOLS = {tools} + + +class World(GeneratedWorld): + name = {agent!r} + tools = TOOLS + handlers = {{ + name: (_HERE / "handlers" / f"{{name}}.py").read_text(encoding="utf-8") + for name in {handler_names} + }} + + +def load(database=None): + """The world, restored from its snapshot unless another database is given.""" + return World(database or (_HERE / "world.sqlite")) +''' + + +def save(world: GeneratedWorld, path: str | Path, *, notes: str = "") -> Path: + """Write the world out: the snapshot, the handlers, the module, and a manifest.""" + root = Path(path) + (root / HANDLERS).mkdir(parents=True, exist_ok=True) + + frozen = sqlite3.connect(root / DATABASE) + with frozen: + world.connection.backup(frozen) + frozen.close() + + for name, source in world.handlers.items(): + (root / HANDLERS / f"{name}.py").write_text(source, encoding="utf-8") + + (root / WORLD_MODULE).write_text( + _MODULE.format( + agent=world.name, + notes=notes or "Generated from the agent's contract.", + tools=json.dumps(world.tools, indent=4), + handler_names=json.dumps(sorted(world.handlers)), + ), + encoding="utf-8", + ) + + state = world.state() + (root / MANIFEST).write_text( + json.dumps( + { + "agent": world.name, + "tools": sorted(world.handlers), + "tables": {name: len(rows) for name, rows in state.items()}, + "notes": notes, + }, + indent=2, + ensure_ascii=False, + ), + encoding="utf-8", + ) + return root + + +def restore(path: str | Path, *, into: str | Path | None = None) -> GeneratedWorld: + """A fresh, independent copy of the frozen world. + + In memory by default, because a scenario should not be able to write back into the snapshot + every later scenario depends on. + """ + root = Path(path) + source = root / DATABASE + if not source.exists(): + raise FileNotFoundError(f"no world snapshot at {root}") + + manifest = read_manifest(root) + handlers = { + name: (root / HANDLERS / f"{name}.py").read_text(encoding="utf-8") + for name in manifest.get("tools", []) + if (root / HANDLERS / f"{name}.py").exists() + } + + if into is None: + world = GeneratedWorld(":memory:") + origin = sqlite3.connect(source) + with world.connection: + origin.backup(world.connection) + origin.close() + else: + target = Path(into) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, target) + world = GeneratedWorld(target) + + world.name = manifest.get("agent", "generated") + world.handlers = handlers + world.tools = manifest.get("tool_specs", []) + return world + + +def read_manifest(path: str | Path) -> dict[str, Any]: + return json.loads((Path(path) / MANIFEST).read_text(encoding="utf-8")) + + +def apply_overlay(world: GeneratedWorld, overlay: Mapping[str, Any] | None) -> int: + """Layer one scenario's own rows onto a restored world. + + ``{"table": [{"column": value}, ...]}``. The only sanctioned way a scenario adds data, so the + base world stays the shared starting point rather than drifting per scenario. + """ + written = 0 + for table, rows in (overlay or {}).items(): + for row in rows or []: + if not isinstance(row, Mapping) or not row: + continue + columns = ", ".join(row) + marks = ", ".join("?" for _ in row) + world.connection.execute( + f"INSERT INTO {table} ({columns}) VALUES ({marks})", list(row.values()) + ) + written += 1 + world.connection.commit() + return written diff --git a/src/fi/alk/harness/world/tools.py b/src/fi/alk/harness/world/tools.py new file mode 100644 index 0000000..2058627 --- /dev/null +++ b/src/fi/alk/harness/world/tools.py @@ -0,0 +1,211 @@ +"""The tools that build a world, and the gate that decides it may be saved. + +A deliberately narrow surface. The builder gets no generic file write, because a guardrail needs +something to sit behind: every action goes through a tool that can execute it, check it, and say +what went wrong. Interface design work on coding agents is consistent that this beats handing +over raw access and hoping. + +Three habits throughout, for the same reason: + +- **execute immediately.** A handler is run the moment it is defined, so a mistake comes back on + the next turn rather than at save time. +- **say what happened, briefly.** Counts and names, never dumps. More context measurably makes + agents worse at this. +- **never answer with nothing.** "0 rows inserted" is a result; an empty string is a puzzle. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from claude_agent_sdk import create_sdk_mcp_server, tool + +from ..contract import AgentContract +from .probe import probe +from .runtime import GeneratedWorld +from .snapshot import save + +WORLD_SERVER = "world" + +# Below this, the world is not good enough to build tests on. Synthesis work that measures this +# converges on roughly this bar, and rejects a quarter to a third of what it generates. +ACCEPTABLE = 0.85 + + +def _ok(text: str) -> dict[str, Any]: + return {"content": [{"type": "text", "text": text}]} + + +def _err(text: str) -> dict[str, Any]: + return {"content": [{"type": "text", "text": text}], "is_error": True} + + +def _brief(value: Any, limit: int = 400) -> str: + rendered = value if isinstance(value, str) else json.dumps(value, default=str) + return rendered if len(rendered) <= limit else rendered[: limit - 3] + "..." + + +def world_tools(contract: AgentContract, destination: Path) -> Any: + """A server exposing the world-building surface for one agent.""" + world = GeneratedWorld(":memory:") + world.name = contract.agent + sequences: list[dict[str, Any]] = [] + + @tool( + "create_schema", + "Run CREATE TABLE statements. Call once with the whole schema; call again to alter it.", + {"sql": str}, + ) + async def create_schema(args: dict[str, Any]) -> dict[str, Any]: + try: + world.connection.executescript(args["sql"]) + world.connection.commit() + except Exception as failed: + return _err(f"schema rejected: {failed}") + tables = sorted(world.state()) + return _ok(f"{len(tables)} tables: {', '.join(tables) or 'none'}") + + @tool( + "seed", + "Insert rows. Rows is a list of objects whose keys are column names.", + {"table": str, "rows": list}, + ) + async def seed(args: dict[str, Any]) -> dict[str, Any]: + table, rows = str(args["table"]), args.get("rows") or [] + written = 0 + for row in rows: + if not isinstance(row, dict) or not row: + continue + columns = ", ".join(row) + marks = ", ".join("?" for _ in row) + try: + world.connection.execute( + f"INSERT INTO {table} ({columns}) VALUES ({marks})", + list(row.values()), + ) + written += 1 + except Exception as failed: + world.connection.rollback() + return _err( + f"{written} rows written, then {table} rejected a row: {failed}" + ) + world.connection.commit() + total = len(world.state().get(table, [])) + return _ok(f"{written} rows inserted into {table}; {total} rows there now") + + @tool( + "define_handler", + "Define one tool's implementation. The source must define handle(args, db) and is run " + "immediately against the seeded world, so errors come straight back.", + {"tool_name": str, "source": str, "smoke_arguments": dict}, + ) + async def define_handler(args: dict[str, Any]) -> dict[str, Any]: + name = str(args["tool_name"]) + if name not in contract.tool_names(): + return _err( + f"{name!r} is not a tool this agent has. It has: " + f"{', '.join(sorted(contract.tool_names()))}" + ) + world.handlers[name] = str(args["source"]) + call = world.call(name, args.get("smoke_arguments") or {}) + if call.refused: + return _ok( + f"{name} defined. Smoke call refused, which is a working refusal: {call.error}" + ) + if not call.ok: + del world.handlers[name] + return _err(f"{name} not kept, it crashed on its smoke call: {call.error}") + return _ok(f"{name} defined and ran. Returned {_brief(call.result)}") + + @tool( + "run_tool", + "Call a defined tool and see what the world does. Use this to check a refusal works.", + {"tool_name": str, "arguments": dict}, + ) + async def run_tool(args: dict[str, Any]) -> dict[str, Any]: + call = world.call(str(args["tool_name"]), args.get("arguments") or {}) + if call.refused: + return _ok(f"refused: {call.error}") + if not call.ok: + return _err(f"crashed: {call.error}") + return _ok(f"ok: {_brief(call.result)}") + + @tool( + "declare_sequence", + "Declare a series of calls whose end state should hold, so consistency across calls is " + "checked. expect_state keys are 'table.column' or 'table.count'.", + {"name": str, "calls": list, "expect_state": dict}, + ) + async def declare_sequence(args: dict[str, Any]) -> dict[str, Any]: + sequences.append( + { + "name": str(args.get("name") or f"sequence-{len(sequences)}"), + "calls": args.get("calls") or [], + "expect_state": args.get("expect_state") or {}, + } + ) + return _ok(f"{len(sequences)} sequences declared") + + @tool( + "check_world", + "Exercise every tool with a valid call, a nonexistent id, and a missing argument, then " + "run the declared sequences. Reports what is wrong without saving anything.", + {}, + ) + async def check_world(_args: dict[str, Any]) -> dict[str, Any]: + report = probe(world, contract, sequences=sequences) + return _ok(f"{report.summary()}\nscore {report.score:.2f}") + + @tool( + "save_world", + "Freeze the world and write it out. Refused unless it passes its own checks.", + {"notes": str}, + ) + async def save_world(args: dict[str, Any]) -> dict[str, Any]: + report = probe(world, contract, sequences=sequences) + if report.score < ACCEPTABLE: + return _err( + f"Not saved, the world does not hold up yet.\n{report.summary()}\n" + f"score {report.score:.2f}, needs {ACCEPTABLE:.2f}" + ) + if not sequences: + return _err( + "Not saved. Declare at least one sequence first: a world whose calls each work " + "alone can still forget what the previous one did." + ) + path = save(world, destination, notes=str(args.get("notes") or "")) + tables = world.state() + return _ok( + f"Saved to {path}.\n" + f"{len(world.handlers)} tools, {len(tables)} tables, " + f"{sum(len(rows) for rows in tables.values())} rows.\n" + f"score {report.score:.2f}" + ) + + server = create_sdk_mcp_server( + name=WORLD_SERVER, + version="0.1.0", + tools=[ + create_schema, + seed, + define_handler, + run_tool, + declare_sequence, + check_world, + save_world, + ], + ) + return server, world + + +TOOL_NAMES = ( + "create_schema", + "seed", + "define_handler", + "run_tool", + "declare_sequence", + "check_world", + "save_world", +) From 320f2719b19faba8b7389c006ed8bbd11a0a3854 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sun, 16 Aug 2026 16:22:27 +0530 Subject: [PATCH 03/39] fix(harness): isolate probes, surface tool results, make sequences correctable --- src/fi/alk/harness/build.py | 4 +- src/fi/alk/harness/contract.py | 21 ++++- src/fi/alk/harness/session.py | 37 +++++++-- src/fi/alk/harness/world/probe.py | 12 +++ src/fi/alk/harness/world/runtime.py | 17 +++++ src/fi/alk/harness/world/tools.py | 61 ++++++++++++++- tests/test_harness.py | 114 ++++++++++++++++++++++++++++ 7 files changed, 252 insertions(+), 14 deletions(-) diff --git a/src/fi/alk/harness/build.py b/src/fi/alk/harness/build.py index bf7a07d..8f80fc0 100644 --- a/src/fi/alk/harness/build.py +++ b/src/fi/alk/harness/build.py @@ -35,7 +35,9 @@ def open_stage( destination = out or artifact_dir(contract.agent) server, _world = world_tools(contract, destination) options = ClaudeAgentOptions( - system_prompt=f"{load_skill(SKILL)}\n\n## This agent\n\n{contract.brief()}", + system_prompt=( + f"{load_skill(SKILL)}\n\n## This agent\n\n{contract.brief(with_data=True)}" + ), # No file tools and no shell. Everything this stage can do goes through a tool that # executes it and reports back, which is what makes the guardrails meaningful. allowed_tools=[ diff --git a/src/fi/alk/harness/contract.py b/src/fi/alk/harness/contract.py index b9b12a6..90f0cb0 100644 --- a/src/fi/alk/harness/contract.py +++ b/src/fi/alk/harness/contract.py @@ -89,8 +89,13 @@ def _normalize_shapes(cls, payload: Any) -> Any: def tool_names(self) -> set[str]: return {tool.name for tool in self.tools} - def brief(self, *, full_schema: bool = True) -> str: - """The grounding block handed to the model on every downstream call.""" + def brief(self, *, full_schema: bool = True, with_data: bool = False) -> str: + """The grounding block handed to the model on every downstream call. + + ``with_data`` includes the agent's real starting records rather than only their shape. + A stage that writes scenarios needs to know a menu exists; a stage that builds the world + has to reproduce it row for row, and a shape without records is not enough to do that. + """ lines: list[str] = [] for tool in self.tools: signature = ", ".join( @@ -118,8 +123,16 @@ def brief(self, *, full_schema: bool = True) -> str: ) if self.data_schema and full_schema: parts.append( - "REAL DATA / SCHEMA (ground every value and id in this; never invent):\n" - + json.dumps(self.data_schema)[:2400] + "DATA SHAPE (the fields each record has):\n" + + json.dumps(self.data_schema)[: 24000 if with_data else 2400] + ) + if self.base_environment and with_data: + parts.append( + "THE AGENT'S REAL STARTING DATA. Reproduce this exactly, including anything\n" + "that looks like a mistake: a misspelled id, an item marked unavailable, an odd\n" + "price. The world is a replica of what the agent has, not a corrected version,\n" + "and a test written against a corrected world will not catch the real bug.\n" + + json.dumps(self.base_environment, ensure_ascii=False) ) if self.grading_notes: parts.append(f"GRADING NOTES for this agent:\n{self.grading_notes[:900]}") diff --git a/src/fi/alk/harness/session.py b/src/fi/alk/harness/session.py index 8a205a3..51db910 100644 --- a/src/fi/alk/harness/session.py +++ b/src/fi/alk/harness/session.py @@ -28,6 +28,7 @@ TEXT = "text" TOOL = "tool" +RESULT = "result" ARTIFACT = "artifact" DONE = "done" @@ -48,6 +49,12 @@ def line(self) -> str: if self.kind == TOOL: target = self.detail.get("target") or "" return f" [{self.tool}{' ' + target if target else ''}]" + if self.kind == RESULT: + marker = "!" if self.detail.get("is_error") else ">" + body = "\n".join( + f" {marker} {row}" for row in self.text.splitlines() if row + ) + return body or f" {marker} (no output)" if self.kind == ARTIFACT: return f" [saved {self.detail.get('path', '')}]" if self.kind == DONE: @@ -84,6 +91,16 @@ def _target(payload: Any) -> str: return "" +def _result_text(block: ToolResultBlock, limit: int = 600) -> str: + content = block.content + if isinstance(content, list): + content = "\n".join( + part.get("text", "") for part in content if isinstance(part, dict) + ) + text = content if isinstance(content, str) else str(content) + return text if len(text) <= limit else text[: limit - 3] + "..." + + def _saved_path(block: ToolResultBlock) -> str: """Our tools report what they wrote; surfacing it lets a UI update the artifact pane.""" content = block.content @@ -175,11 +192,21 @@ def _events(self, received: Any, turn: Turn) -> list[Event]: if isinstance(blocks, list): events = [] for block in blocks: - if isinstance(block, ToolResultBlock): - path = _saved_path(block) - if path: - turn.artifacts.append(path) - events.append(Event(ARTIFACT, detail={"path": path})) + if not isinstance(block, ToolResultBlock): + continue + # What a tool said back is the only view a caller has of whether the work is + # going well. Dropping it leaves a run that can only be diagnosed by guessing. + events.append( + Event( + RESULT, + text=_result_text(block), + detail={"is_error": bool(getattr(block, "is_error", False))}, + ) + ) + path = _saved_path(block) + if path: + turn.artifacts.append(path) + events.append(Event(ARTIFACT, detail={"path": path})) return events return [] diff --git a/src/fi/alk/harness/world/probe.py b/src/fi/alk/harness/world/probe.py index 1ece4fb..c91cc4a 100644 --- a/src/fi/alk/harness/world/probe.py +++ b/src/fi/alk/harness/world/probe.py @@ -118,6 +118,11 @@ def probe( """ report = ProbeReport() + # Every probe runs from the same starting world. Probes mutate, so without reverting + # between them each one inherits the debris of the last and a check expecting three rows + # finds seven. That is a fault in the harness, not in the world being checked. + baseline = world.checkpoint() + for tool in contract.tools: if tool.name not in world.handlers: report.results.append( @@ -135,6 +140,7 @@ def probe( if tool.name not in world.handlers: continue + world.revert(baseline) call = world.call(tool.name, _valid_arguments(tool)) # A refusal here is acceptable: the contract's first listed value may genuinely be # invalid in the seeded world. A crash never is. @@ -149,6 +155,7 @@ def probe( bogus = _identifier_arguments(tool) if bogus is not None: + world.revert(baseline) call = world.call(tool.name, bogus) report.results.append( ProbeResult( @@ -166,6 +173,7 @@ def probe( ) if tool.args: + world.revert(baseline) missing = _valid_arguments(tool) missing.pop(tool.args[0], None) call = world.call(tool.name, missing) @@ -184,6 +192,7 @@ def probe( ) ) + world.revert(baseline) unknown = world.call(ABSENT, {}) report.results.append( ProbeResult( @@ -195,8 +204,11 @@ def probe( ) for index, sequence in enumerate(sequences): + world.revert(baseline) report.results.append(_run_sequence(world, sequence, index)) + # Leave the world as the builder left it, not as the last probe left it. + world.revert(baseline) return report diff --git a/src/fi/alk/harness/world/runtime.py b/src/fi/alk/harness/world/runtime.py index 198afe6..b622ed1 100644 --- a/src/fi/alk/harness/world/runtime.py +++ b/src/fi/alk/harness/world/runtime.py @@ -188,6 +188,23 @@ def _record(self, call: Call) -> Call: # -- state ----------------------------------------------------------------------- + def checkpoint(self) -> sqlite3.Connection: + """A copy of the current data, to come back to. + + Probes mutate: ordering an item inserts a row. Without a way back, each probe runs + against the debris of the ones before it, and a check expecting three rows finds seven. + The same restore-a-fresh-copy discipline scenarios use, applied to the gate itself. + """ + copy = sqlite3.connect(":memory:") + with copy: + self.connection.backup(copy) + return copy + + def revert(self, checkpoint: sqlite3.Connection) -> None: + """Put the data back as it was when the checkpoint was taken.""" + with self.connection: + checkpoint.backup(self.connection) + def state(self) -> dict[str, Any]: """Every table and its rows: what the checks compare against after a run.""" tables = Db(self.connection).query( diff --git a/src/fi/alk/harness/world/tools.py b/src/fi/alk/harness/world/tools.py index 2058627..53c53e3 100644 --- a/src/fi/alk/harness/world/tools.py +++ b/src/fi/alk/harness/world/tools.py @@ -135,18 +135,69 @@ async def run_tool(args: dict[str, Any]) -> dict[str, Any]: @tool( "declare_sequence", "Declare a series of calls whose end state should hold, so consistency across calls is " - "checked. expect_state keys are 'table.column' or 'table.count'.", + "checked. Each call is {tool, arguments}. expect_state keys are 'table.column' or " + "'table.count'. Declaring the same name again replaces it, so a mistake is fixed by " + "redeclaring rather than accumulating.", {"name": str, "calls": list, "expect_state": dict}, ) async def declare_sequence(args: dict[str, Any]) -> dict[str, Any]: + name = str(args.get("name") or f"sequence-{len(sequences)}") + calls = args.get("calls") or [] + + # Checked here rather than at save time. A malformed sequence that only fails three + # tools later reads as a mystery, and there is nothing to learn from it in between. + problems: list[str] = [] + if not calls: + problems.append("no calls: a sequence with no calls checks nothing") + for index, step in enumerate(calls): + if not isinstance(step, dict): + problems.append( + f"call {index} is not an object with a tool and arguments" + ) + continue + called = str(step.get("tool") or "") + if not called: + problems.append(f"call {index} has no tool name") + elif called not in world.handlers: + problems.append( + f"call {index} names {called!r}, which has no handler yet. Defined: " + f"{', '.join(sorted(world.handlers)) or 'none'}" + ) + if problems: + return _err(f"{name} not declared:\n - " + "\n - ".join(problems)) + + replaced = any(existing["name"] == name for existing in sequences) + sequences[:] = [existing for existing in sequences if existing["name"] != name] sequences.append( { - "name": str(args.get("name") or f"sequence-{len(sequences)}"), - "calls": args.get("calls") or [], + "name": name, + "calls": calls, "expect_state": args.get("expect_state") or {}, } ) - return _ok(f"{len(sequences)} sequences declared") + verb = "replaced" if replaced else "declared" + return _ok( + f"{name} {verb}. {len(sequences)} sequences: {', '.join(s['name'] for s in sequences)}" + ) + + @tool( + "drop_sequence", + "Remove a declared sequence by name, or all of them with name '*'.", + {"name": str}, + ) + async def drop_sequence(args: dict[str, Any]) -> dict[str, Any]: + name = str(args.get("name") or "") + if name == "*": + sequences.clear() + return _ok("all sequences dropped") + before = len(sequences) + sequences[:] = [existing for existing in sequences if existing["name"] != name] + if len(sequences) == before: + return _err( + f"no sequence called {name!r}. Declared: " + f"{', '.join(s['name'] for s in sequences) or 'none'}" + ) + return _ok(f"{name} dropped. {len(sequences)} left") @tool( "check_world", @@ -193,6 +244,7 @@ async def save_world(args: dict[str, Any]) -> dict[str, Any]: define_handler, run_tool, declare_sequence, + drop_sequence, check_world, save_world, ], @@ -206,6 +258,7 @@ async def save_world(args: dict[str, Any]) -> dict[str, Any]: "define_handler", "run_tool", "declare_sequence", + "drop_sequence", "check_world", "save_world", ) diff --git a/tests/test_harness.py b/tests/test_harness.py index 06df6b1..1d6f586 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -194,6 +194,120 @@ def test_load_returns_none_when_the_stage_produced_nothing(tmp_path): assert load(tmp_path) is None +# --- the world gate ------------------------------------------------------------------ + + +def _cart_world(): + from fi.alk.harness.world import GeneratedWorld + + class W(GeneratedWorld): + name = "cart" + tools = [{"name": "add"}, {"name": "lst"}] + handlers = { + "add": ( + "def handle(args, db):\n" + " if 'item_id' not in args: raise ToolError('item_id is required')\n" + " m = db.one('SELECT * FROM menu WHERE id=?', [args['item_id']])\n" + " if not m: raise ToolError('no item %r' % args['item_id'])\n" + " db.execute('INSERT INTO cart (item_id) VALUES (?)', [args['item_id']])\n" + " return {'ok': 1}\n" + ), + "lst": "def handle(args, db):\n return db.query('SELECT * FROM cart')\n", + } + + world = W(":memory:") + world.connection.executescript( + "CREATE TABLE menu(id TEXT PRIMARY KEY); CREATE TABLE cart(item_id TEXT);" + ) + world.connection.execute("INSERT INTO menu VALUES ('big_mac')") + world.connection.commit() + contract = AgentContract( + agent="cart", + real_use_cases=["add an item"], + tools=[ + ToolSpec(name="add", args=["item_id"], arg_values={"item_id": ["big_mac"]}), + ToolSpec(name="lst"), + ], + ) + return world, contract + + +_SEQUENCE = [ + { + "name": "add-then-list", + "calls": [ + {"tool": "add", "arguments": {"item_id": "big_mac"}}, + {"tool": "lst", "arguments": {}}, + ], + "expect_state": {"cart.count": 1}, + } +] + + +def test_a_sound_world_passes_every_probe(): + from fi.alk.harness.world import probe + + world, contract = _cart_world() + report = probe(world, contract, sequences=_SEQUENCE) + assert report.score == 1.0, report.summary() + + +def test_probing_leaves_the_world_exactly_as_it_found_it(): + """Probes mutate. Without reverting between them, each inherits the last one's debris and + a sequence expecting one row finds several, which reads as a bug in the world.""" + from fi.alk.harness.world import probe + + world, contract = _cart_world() + probe(world, contract, sequences=_SEQUENCE) + assert world.state()["cart"] == [] + + +def test_probing_is_repeatable(): + from fi.alk.harness.world import probe + + world, contract = _cart_world() + first = probe(world, contract, sequences=_SEQUENCE).score + second = probe(world, contract, sequences=_SEQUENCE).score + assert first == second == 1.0 + + +def test_a_tool_that_succeeds_on_a_nonexistent_id_fails_the_gate(): + """The defect the whole thing exists to catch: a call that should have been refused.""" + from fi.alk.harness.world import probe + + world, contract = _cart_world() + world.handlers["add"] = ( + "def handle(args, db):\n" + " db.execute('INSERT INTO cart (item_id) VALUES (?)', [args.get('item_id')])\n" + " return {'ok': 1}\n" + ) + report = probe(world, contract, sequences=_SEQUENCE) + assert any("does not exist" in failure.detail for failure in report.failures), ( + report.summary() + ) + assert report.score < 0.85 + + +def test_a_crash_is_distinguished_from_a_refusal(): + from fi.alk.harness.world import probe + + world, contract = _cart_world() + world.handlers["add"] = ( + "def handle(args, db):\n return {'id': args['item_id']}\n" + ) + report = probe(world, contract, sequences=_SEQUENCE) + assert any("crashed instead of refusing" in f.detail for f in report.failures) + + +def test_a_world_reverts_to_a_checkpoint(): + world, _ = _cart_world() + mark = world.checkpoint() + world.call("add", {"item_id": "big_mac"}) + assert len(world.state()["cart"]) == 1 + world.revert(mark) + assert world.state()["cart"] == [] + + # --- wiring -------------------------------------------------------------------------- From 008766f45481854a774b693e4ffd88e9de46a463 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sun, 16 Aug 2026 17:02:35 +0530 Subject: [PATCH 04/39] feat(harness): check the world's catalogue and argument reads, one conversation per agent --- src/fi/alk/harness/chat.py | 146 ++++++++++++++++++ src/fi/alk/harness/cli.py | 55 +++++-- src/fi/alk/harness/contract.py | 7 + .../harness/skills/build-environment/SKILL.md | 7 + src/fi/alk/harness/tools.py | 13 +- src/fi/alk/harness/world/probe.py | 140 ++++++++++++++++- src/fi/alk/harness/world/runtime.py | 36 +++-- src/fi/alk/harness/world/tools.py | 12 +- 8 files changed, 384 insertions(+), 32 deletions(-) create mode 100644 src/fi/alk/harness/chat.py diff --git a/src/fi/alk/harness/chat.py b/src/fi/alk/harness/chat.py new file mode 100644 index 0000000..f976d00 --- /dev/null +++ b/src/fi/alk/harness/chat.py @@ -0,0 +1,146 @@ +"""One conversation, from pointing at an agent to a world you can test against. + +You say what you want, it does it, you say the next thing. Stages are not commands you invoke; +they are what the harness moves through while you keep talking. When one produces its artifact +the next opens on the same agent, and anything already built stays correctable by saying so. + +Underneath, each stage is still its own session with its own instructions and its own tools, so +context stays small and a stage can be re-entered later without redoing the ones before it. That +is an implementation detail, not something to make somebody manage. +""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable + +from . import build as build_stage +from . import understand as understand_stage +from .config import artifact_dir +from .contract import AgentContract +from .session import Stage +from .sources import AgentSource, resolve + +UNDERSTAND = "understand" +BUILD = "build" +DONE = "done" + +_NEXT = {UNDERSTAND: BUILD, BUILD: DONE} + + +@dataclass +class Conversation: + """The whole thing, held open.""" + + source: AgentSource + out: Path + ask: Callable[..., Any] | None = None + stage_name: str = UNDERSTAND + stage: Stage | None = None + spent_usd: float = 0.0 + history: list[str] = field(default_factory=list) + + # -- what exists so far ---------------------------------------------------------- + + @property + def contract(self) -> AgentContract | None: + return understand_stage.load(self.out) + + @property + def world_built(self) -> bool: + return (self.out / "world.sqlite").exists() + + def _artifact_for(self, stage_name: str) -> bool: + return { + UNDERSTAND: self.contract is not None, + BUILD: self.world_built, + DONE: True, + }[stage_name] + + # -- moving between stages ------------------------------------------------------- + + async def _close(self) -> None: + if self.stage is not None: + self.spent_usd += self.stage.spent_usd + await self.stage.__aexit__(None, None, None) + self.stage = None + + async def _open(self, stage_name: str) -> str: + """Open a stage and return the message that starts it.""" + await self._close() + self.stage_name = stage_name + if stage_name == UNDERSTAND: + self.stage, _ = understand_stage.open_stage( + self.source, out=self.out, ask=self.ask + ) + opening = understand_stage.opening(self.source) + else: + contract = self.contract + if contract is None: + raise RuntimeError("cannot build a world before there is a contract") + self.stage, _ = build_stage.open_stage(contract, out=self.out, ask=self.ask) + opening = build_stage.opening(contract) + await self.stage.__aenter__() + return opening + + def next_stage(self) -> str | None: + """The stage that follows the current one, once this one has produced its artifact.""" + if not self._artifact_for(self.stage_name): + return None + following = _NEXT.get(self.stage_name) + return None if following in (None, DONE) else following + + # -- talking --------------------------------------------------------------------- + + async def start(self, on_event: Callable[..., Any] | None = None) -> None: + opening = await self._open(self._resume_at()) + await self.stage.say(opening, on_event=on_event) # type: ignore[union-attr] + + def _resume_at(self) -> str: + """Pick up where the artifacts say this agent got to.""" + if self.contract is None: + return UNDERSTAND + return BUILD + + async def say(self, message: str, on_event: Callable[..., Any] | None = None) -> None: + """Send a message to whichever stage is open.""" + self.history.append(message) + if self.stage is None: + await self.start(on_event=on_event) + await self.stage.say(message, on_event=on_event) # type: ignore[union-attr] + + async def advance(self, on_event: Callable[..., Any] | None = None) -> str | None: + """Move to the next stage and start it. Returns the stage entered, or None.""" + following = self.next_stage() + if following is None: + return None + opening = await self._open(following) + await self.stage.say(opening, on_event=on_event) # type: ignore[union-attr] + return following + + async def close(self) -> None: + await self._close() + + +def open_conversation( + *, + name: str, + path: str, + kind: str = "repo", + out: Path | None = None, + ask: Callable[..., Any] | None = None, +) -> Conversation: + source = resolve(kind, name=name, root=path) + return Conversation(source=source, out=out or artifact_dir(name), ask=ask) + + +async def _demo() -> None: # pragma: no cover - convenience for manual runs + conversation = open_conversation(name="demo", path=".") + await conversation.start() + await conversation.close() + + +if __name__ == "__main__": # pragma: no cover + asyncio.run(_demo()) diff --git a/src/fi/alk/harness/cli.py b/src/fi/alk/harness/cli.py index 8047eea..057315c 100644 --- a/src/fi/alk/harness/cli.py +++ b/src/fi/alk/harness/cli.py @@ -14,7 +14,8 @@ from pathlib import Path from typing import Any -from .build import build +from .build import open_stage as build_stage +from .build import opening as build_opening from .config import DEFAULT_MODEL, artifact_dir from .session import TEXT, Event from .sources import resolve, supported @@ -78,16 +79,7 @@ async def _understand(args: argparse.Namespace) -> int: print(f"agent: {source.name} ({source.kind})") print(f"out: {destination}\n") - async with stage: - await stage.say(opening(source), on_event=_render) - while args.interactive: - try: - said = await _prompt("\nkarthik ") - except (EOFError, KeyboardInterrupt): - break - if not said or said in {"q", "quit", "exit"}: - break - await stage.say(said, on_event=_render) + await _converse(stage, opening(source), interactive=args.interactive) contract = load(destination) if contract is None: @@ -103,6 +95,25 @@ async def _understand(args: argparse.Namespace) -> int: return 0 +async def _converse(stage, opening_message: str, *, interactive: bool) -> None: + """Say the opening, then keep the stage open for corrections. + + The same shape for every stage. A world is usually right on the second look, and the point + of holding the session open is that correcting it is the next thing said rather than a + rebuild from nothing. + """ + async with stage: + await stage.say(opening_message, on_event=_render) + while interactive: + try: + said = await _prompt("\nkarthik ") + except (EOFError, KeyboardInterrupt): + break + if not said or said in {"q", "quit", "exit"}: + break + await stage.say(said, on_event=_render) + + async def _build(args: argparse.Namespace) -> int: destination = Path(args.out) if args.out else artifact_dir(args.name) contract = load(destination) @@ -112,11 +123,19 @@ async def _build(args: argparse.Namespace) -> int: print(f"agent: {contract.agent} ({len(contract.tools)} tools)") print(f"out: {destination}\n") - written = await build(contract, out=destination, on_event=_render) - if written is None: + + stage, _ = build_stage( + contract, + out=destination, + ask=_answer_questions if args.interactive else None, + ) + await _converse(stage, build_opening(contract), interactive=args.interactive) + + if not (destination / "world.sqlite").exists(): print("\nNo world was saved.", file=sys.stderr) return 1 - print(f"\nworld: {written}") + print(f"\nworld: {destination}") + print(f"spent: ${stage.spent_usd:.4f}") return 0 @@ -145,7 +164,13 @@ def build_parser() -> argparse.ArgumentParser: world = sub.add_parser("build", help="build the world from an agent's contract") world.add_argument("--name", required=True, help="which agent") world.add_argument("--out", default=None, help="artifact directory") - world.set_defaults(run=_build) + world.add_argument( + "--once", + dest="interactive", + action="store_false", + help="run unattended instead of staying open for corrections", + ) + world.set_defaults(run=_build, interactive=True) return parser diff --git a/src/fi/alk/harness/contract.py b/src/fi/alk/harness/contract.py index 90f0cb0..452a639 100644 --- a/src/fi/alk/harness/contract.py +++ b/src/fi/alk/harness/contract.py @@ -165,6 +165,13 @@ def validate_contract(contract: AgentContract) -> list[str]: problems.append( f"tool[{tool.name}]:types-for-unknown-args:{','.join(unknown)}" ) + # A tool genuinely taking no arguments is ordinary; every tool taking none is not. It means + # the arguments were read and then not recorded, and since the world, the probes and the + # checkpoints are all built from these names, nothing downstream can detect their absence. + if contract.tools and not any(tool.args for tool in contract.tools): + problems.append( + "no-arguments-on-any-tool: list each tool's exact parameter names in args" + ) if not contract.real_use_cases: problems.append("no-use-cases") # Iterate the tools, not tool_names(): that returns a set, so duplicates collapse before diff --git a/src/fi/alk/harness/skills/build-environment/SKILL.md b/src/fi/alk/harness/skills/build-environment/SKILL.md index 87a3f21..87e7b07 100644 --- a/src/fi/alk/harness/skills/build-environment/SKILL.md +++ b/src/fi/alk/harness/skills/build-environment/SKILL.md @@ -42,6 +42,13 @@ for it: When one of those holds, `raise ToolError("...")` with a message that says what was wrong. A refusal is the world working. It is not an error you should be avoiding. +`ToolError` is already available inside a handler. Do not define your own, and do not import +anything: a handler has `args`, `db`, `ToolError` and `json`, and nothing else. + +Use the argument names exactly as the contract gives them. A handler reading `order_ids` when +the tool takes `order_id` finds nothing, quietly does nothing, and reports success, which is +the precise failure this world exists to prevent. + Never let a handler crash on bad input. `KeyError` and `TypeError` are your bugs; `ToolError` is the world's answer. They must not be confused, and one of the checks tells them apart. diff --git a/src/fi/alk/harness/tools.py b/src/fi/alk/harness/tools.py index 2bfe10c..8640070 100644 --- a/src/fi/alk/harness/tools.py +++ b/src/fi/alk/harness/tools.py @@ -71,7 +71,18 @@ def contract_tools(destination: Path) -> Any: @tool( "submit_contract", "Submit the agent's testing contract. Validated on submission; problems are returned " - "to you so you can correct them and submit again.", + "to you so you can correct them and submit again.\n\n" + "Each entry in `tools` is an object:\n" + ' {"name": "remove_order_item",\n' + ' "args": ["order_id"],\n' + ' "arg_types": {"order_id": "list[str]"},\n' + ' "arg_values": {"order_id": []},\n' + ' "description": "..."}\n' + "`args` must list the exact parameter names the model emits when calling the tool, in " + "order. `arg_types` carries the declared type wherever the source states one. " + "`arg_values` carries the real permitted values wherever the argument is constrained to " + "a set, an enum or a lookup. Everything downstream is built from these, so a tool " + "submitted without its arguments cannot be tested.", { "agent": str, "one_liner": str, diff --git a/src/fi/alk/harness/world/probe.py b/src/fi/alk/harness/world/probe.py index c91cc4a..5431c9e 100644 --- a/src/fi/alk/harness/world/probe.py +++ b/src/fi/alk/harness/world/probe.py @@ -19,6 +19,7 @@ from __future__ import annotations +import re from dataclasses import dataclass, field from typing import Any, Iterable, Mapping, Sequence @@ -29,6 +30,7 @@ EDGE = "edge" SEQUENCE = "sequence" COVERAGE = "coverage" +DATA = "data" # A value no generated world should ever have seeded, used to prove a lookup refuses. ABSENT = "__does_not_exist__" @@ -91,15 +93,92 @@ def _valid_arguments(tool: ToolSpec) -> dict[str, Any]: return arguments +def _seeded_values(world: GeneratedWorld) -> set[str]: + """Every value present anywhere in the world, for checking the catalogue is complete.""" + present: set[str] = set() + for rows in world.state().values(): + for row in rows: + for value in row.values(): + if isinstance(value, str) and value: + present.add(value) + return present + + +def _is_a_real_identifier(value: Any) -> bool: + """Whether a permitted value names a record, rather than being an enum like 'M' or 'null'.""" + if not isinstance(value, str) or value in ("", "null", "none", "None"): + return False + return len(value) > 2 and not value.isdigit() + + +def _missing_catalogue(world: GeneratedWorld, contract: AgentContract) -> list[str]: + """Identifiers the contract says a tool accepts that are nowhere in the seeded world. + + The gap this closes is a whole category left unseeded. Every call naming a sauce then fails, + which looks from the outside exactly like a world being correctly strict, and a suite where + nothing can be ordered scores perfectly. Whether the catalogue is complete cannot be settled + by behaviour, so it is checked against the data. + """ + present = _seeded_values(world) + missing: list[str] = [] + for tool in contract.tools: + for arg, values in (tool.arg_values or {}).items(): + if not isinstance(values, (list, tuple)): + continue + if not _looks_like_an_identifier(arg, tool.arg_types.get(arg, "")): + continue + absent = [ + value + for value in values + if _is_a_real_identifier(value) and value not in present + ] + if absent: + shown = ", ".join(absent[:4]) + ( + f" and {len(absent) - 4} more" if len(absent) > 4 else "" + ) + missing.append(f"{tool.name}.{arg}: {shown}") + return missing + + +def _reads_argument(source: str, name: str) -> bool: + """Whether a handler actually takes this argument out of ``args``. + + Looking for the bare name is not enough. A handler that reads ``args['order_ids']`` and then + loops ``for order_id in order_ids`` mentions ``order_id`` all over itself while never reading + the argument the tool is given, so it silently ignores its input and reports success or + refuses everything. Both look fine from the outside, which is why this is checked at the + point of access rather than by behaviour. + """ + pattern = ( + rf"args\s*(?:\[\s*|\.get\s*\(\s*|\.pop\s*\(\s*)" + rf"['\"]{re.escape(name)}['\"]" + ) + return re.search(pattern, source) is not None + + +def _looks_like_an_identifier(name: str, declared: str) -> bool: + """Whether an argument names something that has to exist for the call to make sense.""" + if name.endswith(("_id", "_ids", "id", "_key", "_ref")): + return True + return declared in ("str", "list[str]", "List[str]") + + def _identifier_arguments(tool: ToolSpec) -> dict[str, Any] | None: - """The same call with every identifier replaced by one that cannot exist.""" + """The same call with every identifier replaced by one that cannot exist. + + Deliberately not gated on the contract listing that argument's values. A contract that + failed to record them is exactly the case where nobody has checked what this tool does with + a bad id, so skipping the probe there drops it precisely where it is most needed. + """ arguments = _valid_arguments(tool) swapped = False for arg in tool.args: - if not tool.arg_values.get(arg): - continue declared = tool.arg_types.get(arg, "") - arguments[arg] = [ABSENT] if "list" in declared else ABSENT + if not tool.arg_values.get(arg) and not _looks_like_an_identifier( + arg, declared + ): + continue + arguments[arg] = [ABSENT] if "list" in declared.lower() else ABSENT swapped = True return arguments if swapped else None @@ -136,9 +215,38 @@ def probe( ) ) + for gap in _missing_catalogue(world, contract): + report.results.append( + ProbeResult( + gap.split(":")[0], + DATA, + False, + f"the contract accepts values the world does not have: {gap}", + ) + ) + if not _missing_catalogue(world, contract): + report.results.append( + ProbeResult("catalogue", DATA, True, "every permitted identifier exists") + ) + for tool in contract.tools: if tool.name not in world.handlers: continue + source = world.handlers[tool.name] + unread = [arg for arg in tool.args if not _reads_argument(source, arg)] + report.results.append( + ProbeResult( + tool.name, + COVERAGE, + not unread, + # A handler reading order_ids when the tool takes order_id refuses everything, + # which looks exactly like a handler correctly refusing a bad id. Behaviour + # alone cannot tell those apart, so the argument names are checked directly. + "" + if not unread + else f"never reads {', '.join(unread)}, which the contract says it takes", + ) + ) world.revert(baseline) call = world.call(tool.name, _valid_arguments(tool)) @@ -212,6 +320,30 @@ def probe( return report +def dirty_tables( + world: GeneratedWorld, sequences: Iterable[Mapping[str, Any]] +) -> list[str]: + """Tables a scenario writes to that already hold rows before anything has happened. + + A world is the state every scenario starts from, so an order table with rows in it means + the builder's own testing was frozen into the base state. Every scenario then begins with + somebody else's order already in the cart, and a count check that should read one reads + seven. Which tables are transactional is not guessable from a schema, so it is worked out + by running the declared sequences and seeing what moves. + """ + baseline = world.checkpoint() + before = {name: len(rows) for name, rows in world.state().items()} + touched: set[str] = set() + for index, sequence in enumerate(sequences): + world.revert(baseline) + _run_sequence(world, sequence, index) + for name, rows in world.state().items(): + if len(rows) != before.get(name, 0): + touched.add(name) + world.revert(baseline) + return sorted(name for name in touched if before.get(name, 0) > 0) + + def _run_sequence( world: GeneratedWorld, sequence: Mapping[str, Any], index: int ) -> ProbeResult: diff --git a/src/fi/alk/harness/world/runtime.py b/src/fi/alk/harness/world/runtime.py index b622ed1..cc219d8 100644 --- a/src/fi/alk/harness/world/runtime.py +++ b/src/fi/alk/harness/world/runtime.py @@ -62,6 +62,20 @@ def execute(self, sql: str, params: Sequence[Any] = ()) -> int: return cursor.rowcount +def _is_refusal(raised: BaseException) -> bool: + """Whether an exception is the world saying no, rather than the world falling over. + + Matched by name as well as by identity. A generated handler often declares its own + ``ToolError`` rather than using the one already in scope, which is defensive and sensible + from where it sits, and would otherwise turn every deliberate refusal into a reported crash. + Relying on an invisible convention being followed is not a way to decide something this + load-bearing. + """ + if isinstance(raised, ToolError): + return True + return any(base.__name__ == "ToolError" for base in type(raised).__mro__) + + @dataclass class Call: """One tool call and what the world did with it.""" @@ -159,17 +173,17 @@ def call(self, name: str, arguments: Mapping[str, Any] | None = None) -> Call: if not callable(handle): raise RuntimeError("handler defines no handle(args, db)") value = handle(args, Db(self.connection)) - except ToolError as refusal: - return self._record( - Call( - name=name, - arguments=args, - ok=False, - refused=True, - error=str(refusal), + except Exception as raised: + if _is_refusal(raised): + return self._record( + Call( + name=name, + arguments=args, + ok=False, + refused=True, + error=str(raised), + ) ) - ) - except Exception as crash: # Our bug, not the agent's. Labelled differently so a run is never scored # against a world that fell over. return self._record( @@ -177,7 +191,7 @@ def call(self, name: str, arguments: Mapping[str, Any] | None = None) -> Call: name=name, arguments=args, ok=False, - error=f"{type(crash).__name__}: {crash}", + error=f"{type(raised).__name__}: {raised}", ) ) return self._record(Call(name=name, arguments=args, result=value)) diff --git a/src/fi/alk/harness/world/tools.py b/src/fi/alk/harness/world/tools.py index 53c53e3..24aadde 100644 --- a/src/fi/alk/harness/world/tools.py +++ b/src/fi/alk/harness/world/tools.py @@ -23,7 +23,7 @@ from claude_agent_sdk import create_sdk_mcp_server, tool from ..contract import AgentContract -from .probe import probe +from .probe import dirty_tables, probe from .runtime import GeneratedWorld from .snapshot import save @@ -226,6 +226,16 @@ async def save_world(args: dict[str, Any]) -> dict[str, Any]: "Not saved. Declare at least one sequence first: a world whose calls each work " "alone can still forget what the previous one did." ) + dirty = dirty_tables(world, sequences) + if dirty: + counts = world.state() + listed = ", ".join(f"{name} ({len(counts[name])} rows)" for name in dirty) + return _err( + f"Not saved. These hold rows left over from building: {listed}.\n" + "This is the state every scenario starts from, so those rows would appear in " + "every test as somebody else's order already in the cart. Clear them with " + "create_schema or a delete, keep the catalogue, and save again." + ) path = save(world, destination, notes=str(args.get("notes") or "")) tables = world.state() return _ok( From 986a6a1cc9ad12950ed4b380e57efdbf884c85c9 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Mon, 17 Aug 2026 00:51:17 +0530 Subject: [PATCH 05/39] feat(harness): environment as world+simulator+sub-goals, proved scenarios, live runs as a stage --- .gitignore | 1 + oss/simulation-acceptance/voice_cases.py | 25 +- src/fi/alk/harness/DESIGN.md | 222 ++++ src/fi/alk/harness/HOW-IT-WORKS.md | 286 +++++ src/fi/alk/harness/README.md | 329 ++++++ src/fi/alk/harness/__init__.py | 6 + src/fi/alk/harness/amend.py | 260 +++++ src/fi/alk/harness/build.py | 28 +- src/fi/alk/harness/chat.py | 151 ++- src/fi/alk/harness/checks.py | 79 ++ src/fi/alk/harness/cli.py | 268 ++++- src/fi/alk/harness/config.py | 105 +- src/fi/alk/harness/contract.py | 5 + src/fi/alk/harness/environment.py | 156 +++ src/fi/alk/harness/prove.py | 146 +++ src/fi/alk/harness/reception.py | 140 +++ src/fi/alk/harness/run/__init__.py | 208 ++++ src/fi/alk/harness/run/alk.py | 172 +++ src/fi/alk/harness/run/call.py | 102 ++ src/fi/alk/harness/run/conversation.py | 170 +++ src/fi/alk/harness/run/grade.py | 345 ++++++ src/fi/alk/harness/run/live.py | 153 +++ src/fi/alk/harness/run/stage.py | 99 ++ src/fi/alk/harness/run/targets.py | 215 ++++ src/fi/alk/harness/run/tools.py | 304 ++++++ src/fi/alk/harness/run/voice.py | 205 ++++ src/fi/alk/harness/scenario.py | 122 +++ src/fi/alk/harness/scenario_tools.py | 456 ++++++++ src/fi/alk/harness/scenarios.py | 126 +++ src/fi/alk/harness/session.py | 94 +- .../harness/skills/build-environment/SKILL.md | 168 +-- .../alk/harness/skills/run-scenarios/SKILL.md | 56 + .../harness/skills/understand-agent/SKILL.md | 12 + .../harness/skills/write-scenarios/SKILL.md | 93 ++ src/fi/alk/harness/tools.py | 28 + src/fi/alk/harness/world/__init__.py | 7 +- src/fi/alk/harness/world/expectations.py | 91 ++ src/fi/alk/harness/world/kinds.py | 133 +++ src/fi/alk/harness/world/probe.py | 78 +- src/fi/alk/harness/world/runtime.py | 44 +- src/fi/alk/harness/world/snapshot.py | 14 +- src/fi/alk/harness/world/tools.py | 332 +++++- tests/test_harness.py | 982 ++++++++++++++++++ 43 files changed, 6834 insertions(+), 182 deletions(-) create mode 100644 src/fi/alk/harness/DESIGN.md create mode 100644 src/fi/alk/harness/HOW-IT-WORKS.md create mode 100644 src/fi/alk/harness/README.md create mode 100644 src/fi/alk/harness/amend.py create mode 100644 src/fi/alk/harness/checks.py create mode 100644 src/fi/alk/harness/environment.py create mode 100644 src/fi/alk/harness/prove.py create mode 100644 src/fi/alk/harness/reception.py create mode 100644 src/fi/alk/harness/run/__init__.py create mode 100644 src/fi/alk/harness/run/alk.py create mode 100644 src/fi/alk/harness/run/call.py create mode 100644 src/fi/alk/harness/run/conversation.py create mode 100644 src/fi/alk/harness/run/grade.py create mode 100644 src/fi/alk/harness/run/live.py create mode 100644 src/fi/alk/harness/run/stage.py create mode 100644 src/fi/alk/harness/run/targets.py create mode 100644 src/fi/alk/harness/run/tools.py create mode 100644 src/fi/alk/harness/run/voice.py create mode 100644 src/fi/alk/harness/scenario.py create mode 100644 src/fi/alk/harness/scenario_tools.py create mode 100644 src/fi/alk/harness/scenarios.py create mode 100644 src/fi/alk/harness/skills/run-scenarios/SKILL.md create mode 100644 src/fi/alk/harness/skills/write-scenarios/SKILL.md create mode 100644 src/fi/alk/harness/world/expectations.py create mode 100644 src/fi/alk/harness/world/kinds.py diff --git a/.gitignore b/.gitignore index bee5e2a..1a57868 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ artifacts/ !src/fi/simulate/artifacts/ !src/fi/simulate/artifacts/*.py examples/artifacts/ +harness-ui/ diff --git a/oss/simulation-acceptance/voice_cases.py b/oss/simulation-acceptance/voice_cases.py index 9fa2c4d..a517d1f 100644 --- a/oss/simulation-acceptance/voice_cases.py +++ b/oss/simulation-acceptance/voice_cases.py @@ -197,6 +197,29 @@ def missing_env(case: VoiceCase) -> list[str]: return [name for name in case.required_env if not os.environ.get(name, "").strip()] +def _harness_scenario() -> "simulate.Scenario | None": + """The caller the harness prepared, if this run is driving one of its scenarios. + + ``HARNESS_INSTRUCTION`` is the simulator prompt the environment step wrote with this + scenario's values already filled in, so nothing about how a caller behaves is decided here. + Without it the built-in acceptance persona is used and this file behaves exactly as before. + """ + instruction = os.environ.get("HARNESS_INSTRUCTION", "").strip() + if not instruction: + return None + return simulate.Scenario( + name=os.environ.get("HARNESS_SCENARIO", "harness"), + dataset=[ + simulate.Persona( + persona={"name": "customer"}, + situation=instruction, + outcome=os.environ.get("HARNESS_OUTCOME", "") + or "Do what you came to do, or accept that you cannot.", + ) + ], + ) + + def build_inputs(case_id: str, run_id: str) -> VoiceInputs: case = CASES[case_id] room_override = os.environ.get("ACCEPTANCE_ROOM_NAME_OVERRIDE", "").strip() @@ -206,7 +229,7 @@ def build_inputs(case_id: str, run_id: str) -> VoiceInputs: room_mode="managed", room_name_verbatim=bool(room_override), ) - scenario = simulate.Scenario( + scenario = _harness_scenario() or simulate.Scenario( name=f"acceptance-{case_id}", dataset=[ simulate.Persona( diff --git a/src/fi/alk/harness/DESIGN.md b/src/fi/alk/harness/DESIGN.md new file mode 100644 index 0000000..8a83197 --- /dev/null +++ b/src/fi/alk/harness/DESIGN.md @@ -0,0 +1,222 @@ +# The harness: what it builds and how it proves it + +The reference for the rebuild. Written after the corrections in `_scenario-generation/context/` +7.2, 8.1, 9.1, 10.1 and 12, and it supersedes anything in the code that contradicts it. + +--- + +## The model, in one paragraph + +The **environment step** builds everything that is common to every test of one agent: the world +its tools act on, the prompt that drives a simulated user if it has one, and the catalogue of +sub-goals it can be checked against. Every **scenario** is then only a *delta* on that base — a +few values changed after reset, the values substituted into the simulator's prompt, and which +sub-goals must hold. Nothing about a scenario is a template with slots; the harness writes each +one, and proves it works before keeping it. + +``` +environment step ─────────────────────────────► base: world + simulator prompt + sub-goals + │ +scenario 1 ──► reset → setup delta → run → check ─────────┤ +scenario 2 ──► reset → setup delta → run → check ─────────┤ +scenario N ──► reset → setup delta → run → check ─────────┘ +``` + +--- + +## 1. The environment step + +> *"You have to first understand what the f\*\*\* this agent is, from that you will create +> databases, you'll create a snapshot of the databases first."* — Nikhil, 12 + +It produces four things. All of them are written by the harness. None are hardcoded here. + +### 1.1 The world + +Whatever **this** agent needs, and nothing more. For the drive-thru agent that is a database. +For a browser agent it is a site. For something else it is a filesystem, a queue, a service — the +harness decides from the contract what has to exist. + +It **subclasses ALK's `EnvironmentAdapter`**, so the runners that already exist can drive it: +`reset` publishes the tools and the starting state, `handle_tool_call` executes one call, and the +state afterwards is what gets graded. Nothing the harness writes should re-implement a runner. + +It is **frozen once** as a snapshot. Every scenario restores from that snapshot, so a run is +repeatable and no scenario can inherit another's leftovers. + +### 1.2 The simulator prompt — only where the agent is conversational + +> *"For voice and chat there is a simulator, and the input is an instruction to that simulator +> rather than an input to the agent under test. Where there is no actor, variability comes from +> how the environment is designed."* — 10.1 §4 + +The harness writes one prompt for the simulated user of **this** agent, with variables left open. +Each scenario supplies the values. The prompt is an artifact of the environment step because it +is the same for every scenario; only the substituted instruction differs. + +There is no persona field and no persona library. + +> *"Drop the gimmicky persona characters. Variability comes from real conditions instead: a new +> versus an existing user, whether a payment method is on file, addresses."* — 8.1 + +For a browser or coding agent there is no simulator at all; the instruction goes to the agent +directly. + +### 1.3 The sub-goal catalogue + +> *"Defining the sub-goals is our call. The important property is that they are common across +> scenarios so the results roll up: if a payment step appears in 50 scenarios, the analytics +> should show where payment fails and how often."* — 10.1 §8 + +Defined **once**, here, as a named list. Scenarios reference them; they do not invent their own +wording. That is what makes `order-confirmation fails in 7 of 12 scenarios` a sentence anyone can +say. Each entry carries its own check (see §3). + +### 1.4 The gate + +The environment is not accepted because it looks right. It is exercised: every tool called with a +valid call, a nonexistent id, and a missing argument; sequences where state has to carry across +calls. **A refusal is the environment working; a crash is a defect.** It cannot be saved dirty +(rows left over from building) or unverified. + +--- + +## 2. A scenario is a delta + +``` +name identifier +use_case which branch of the agent's real use cases this belongs to +setup what changes in the world after reset — a few rows, files, state +instruction the task. For a conversational agent this is substituted into the + simulator prompt; for a browser or coding agent it goes to the agent +solution the reference trajectory: what a correct agent would do +sub_goals which catalogue entries must hold, and what each must show +``` + +Gone from the old shape: `persona`, `opening`, `goal`, and free-text `must` / `must_not` as the +primary grading. Scenarios are organised **use case → branch**, not by adversarial flavour. + +> *"A login flow is not one row with happy/edge inside it; it is many rows: login-with-Google, +> login-with-Microsoft, forgot-password, sign-up-with-email."* — Nikhil, 7.2 + +--- + +## 3. Checks: deterministic by default, judge as the fallback + +> *"When you have `==` or a python script, then I'll call that deterministic."* +> *"Most likely we can make things deterministic."* +> *"Deterministic, if possible. And LLM also, obviously."* — Nikhil, 10 + +| | | +|---|---| +| **Deterministic** — an assert, an equality, **a python script** | The default | +| **Non-deterministic** — an LLM judging whether a sub-goal was met | Only where nothing observable settles it | + +The trap, in his words: *"you are judging by LLM [so it is non-deterministic], but if you want an +exact output to be 50, then that is deterministic."* An exact fact checked by a judge is **still +non-deterministic**. What matters is who decides, not how precise the fact is. + +A check is code the harness writes, and it has two observable things to work from: + +1. **the world afterwards** — rows, files, whatever this environment is +2. **the recorded tool calls** — that the call happened, *and with the right arguments* + +That second one answers the question left open in 7.2: a booking made for 10 PM when 11 PM was +asked for is a failure, and it is deterministic to detect. + +The judge is left only with what leaves no trace: whether a refusal was explained, whether a price +was invented, tone. + +> Warning from the previous run: *"Judge checkpoints, about a third of all checkpoints, are +> returned as skipped and not graded."* Leaning on the judge does not merely weaken a result — it +> silently produces holes. + +--- + +## 4. Two gates on every scenario, before it is kept + +Terminal-bench's oracle run, which is the reason its tasks are known to be solvable. + +### Gate 1 — solvable + +``` +reset → apply setup → run the solution → run the checks ⇒ must PASS +``` + +If the checks fail with the reference solution, either the scenario is impossible or the check is +wrong. Both have already happened here: a scenario asserted a value the agent was never permitted +to send, and another demanded confirmation of an item that could not be ordered. This catches +them at write time, with no model involved. + +### Gate 2 — not vacuous + +``` +reset → apply setup → run NOTHING → run the checks ⇒ must FAIL +``` + +A check that passes without the agent doing anything grades nothing while reporting a result. +This is the failure that makes a suite quietly green. + +Neither gate asks a model anything. The environment decides. + +**Three things fall out of the solution for free:** it is the expected trajectory; comparing the +agent's trajectory against it gives efficiency (Nikhil's point about the agent that succeeds on +the 21st call after 20 failures); and a scenario that cannot be run is caught before a call is +ever placed. + +--- + +## 5. Running it + +> *"Use an existing harness. Just for Claude agents. Use any existing harness that is there."* +> — Nikhil, 12 + +The simulation runs through **ALK's own path**, not a loop written here. The world is passed in +as the environment; the agent under test is the real agent, in its real runtime. For the voice +case that means the Vapi assistant we already have, over LiveKit, with the tool webhook answered +by **our world** rather than by canned mocks. + +That last part is the whole point of the environment. The previous run's known issues were: + +- *"Mocked tools always succeed, including removing an item that was never added."* +- *"Mock responses do not vary by argument, so read-after-write flows are wrong."* +- *"World state does not change unless a scenario sets `state_updates`, which is often empty."* + +A world that really holds rows and can really refuse removes all three. + +--- + +## 6. What changes per kind of agent, and what does not + +| | Voice / chat | Browser | Coding | +|---|---|---|---| +| World | database, KB | a site | a filesystem, a repo | +| Simulated user | yes — prompt written by the harness | none | none | +| Instruction goes to | the simulator | the agent | the agent | +| Solution | tool calls | actions | commands | +| Check | code over world + calls | code over the page + actions | code over the tree | + +**What never changes:** the environment is built once and frozen; a scenario is a delta; a +solution proves it is solvable; a check that cannot fail is rejected; deterministic first. + +--- + +## 7. Order of work + +1. **Environment step** — the world, the simulator prompt, the sub-goal catalogue, all written by + the harness rather than by a fixed schema here. +2. **Scenario shape** — setup / instruction / solution / sub-goal references. +3. **The two gates** — solvable, and not vacuous. +4. **Run through ALK** — the world serving the tool calls of the real agent. + +--- + +## 8. Instructions, not code + +> *"This is a flow, this is not a harness. You will give your harness the instructions that you +> are supposed to do all this and then the harness will do all that. It's not a code that your +> harness follows."* — Nikhil, 12 + +Every stage's method lives in a `SKILL.md`, editable without touching code. What stays in code is +only what must be exact: executing a call, restoring a snapshot, running a check, and refusing +something that does not hold up. **The harness decides what to do. Code decides what is true.** diff --git a/src/fi/alk/harness/HOW-IT-WORKS.md b/src/fi/alk/harness/HOW-IT-WORKS.md new file mode 100644 index 0000000..e28ce7b --- /dev/null +++ b/src/fi/alk/harness/HOW-IT-WORKS.md @@ -0,0 +1,286 @@ +# How the harness actually works + +What happens between you typing a sentence and a graded result appearing. Written to be read +alongside the code, so every claim below names the file it lives in. + +The shape is the same at every stage, and worth holding onto: + +> **A stage is a model session with a small set of tools and its instructions in a markdown file. +> The model decides what to do; the tools do anything that must be exact and refuse anything that +> must not happen. Nothing reaches disk except through a tool that checked it first.** + +There is no pipeline. Each stage is a conversation you can interrupt, correct, and resume. + +--- + +## The pieces + +| Piece | Where | What it is | +|---|---|---| +| Stage | `session.py` | A live model session, held open across turns, emitting typed events | +| Instructions | `skills//SKILL.md` | How that stage works, in prose. Editable without touching code | +| Tools | `tools.py`, `world/tools.py`, `scenario_tools.py`, `run/tools.py` | The exact half: they execute, validate, and refuse | +| Artifacts | `artifacts/environments//` | What each stage leaves behind for the next | +| Conversation | `chat.py` | Holds one agent's journey through the stages | + +The artifacts, each the input to the next stage: + +``` +contract.json → world.sqlite + handlers/ + simulator_prompt.md + sub_goals.json + → scenarios.json → runs.json +``` + +--- + +## 1. Reception — which agent is this? + +`reception.py` + +A stage with `Read`, `Glob`, `Grep` and one tool, `point_at_agent`. You say where your agent +lives; it looks, confirms the path exists, picks a short name, and calls that tool. + +It looks from the **workspace root** (the directory holding your repos), not from inside +`agent-learning-kit`, because the agent under test is almost never inside the harness. + +`point_at(name, path, kind)` refuses a path that does not exist, and refuses an unknown kind. +`kind` selects an `AgentSource` from `sources.py` — `repo` (code on disk, gets file tools) or +`spec` (a prompt and tool schema pasted in, gets no file tools). **A new kind of agent is one +registration, not a new code path.** + +--- + +## 2. Understand — what is this agent, verifiably? + +`understand.py`, `skills/understand-agent/SKILL.md`, gate in `tools.py` + +The session gets read-only file tools and one submission tool. It reads the agent's source and +calls `submit_contract`. + +**The contract** (`contract.py`) is the anti-hallucination device for everything downstream: + +| Field | Why it matters | +|---|---| +| `tools[]` — name, `args`, `arg_types`, `arg_values`, description | The agent's action space. `arg_values` are the real permitted values — the menu, the enum, the lookup | +| `hard_constraints[]` | Rules the agent must follow. Told to the agent under test, and graded by the judge | +| `base_environment` | Its real starting data, reproduced row for row | +| `real_use_cases[]`, `signature_cases[]` | What it is actually for | +| `anti_hallucination[]` | Things that do not exist and must never be used | +| `amendments[]` | Anything **not** read from source — see below | + +**How it is written:** `accept_contract` in `tools.py` validates before anything reaches disk. It +refuses a contract with no tools, no use cases, duplicate tool names, types for arguments that do +not exist, or — the one that mattered most in practice — *every* tool having no arguments, which +means the arguments were read and then not recorded. Problems are returned **into the +conversation**, so the model corrects and resubmits rather than a bad contract landing. + +**How it is changed later** (`amend.py`). The contract is not frozen, but every change is +recorded with a reason in `amendments[]`, so months later you can still tell what came from the +agent and what came from us: + +- `amend_contract` — let an argument accept a value it did not before +- `add_rule` / `drop_rule` — a hard constraint the source did not state, or one misread +- `fix_tool` — correct argument names, types, description, or remove a tool that does not exist + +Each demands a `why`. A contract that can be rewritten invisibly is no longer evidence. + +--- + +## 3. Build — the world its tools run against + +`build.py`, `skills/build-environment/SKILL.md`, tools in `world/tools.py` + +**This is the part that makes the whole thing worth doing.** Not mocked tool responses: a real +SQLite database with real handlers, so a call for something that is not there is *refused*, and +the agent has to cope. + +It builds three things, all shared by every scenario: **the world**, **the simulator prompt** for +a conversational agent, and **the sub-goal catalogue**. The stage has sixteen tools and no file +access at all: + +| Tool | Does | +|---|---| +| `create_schema` | Run the CREATE TABLE statements | +| `seed` | Insert rows — the agent's real catalogue | +| `change_data` | One UPDATE or DELETE, for fixing a row put in wrong | +| `define_handler` | One tool's implementation, **executed the moment it is defined** | +| `run_tool` | Call a defined tool and see what the world does | +| `declare_sequence` / `drop_sequence` | A series of calls whose end state must hold | +| `inspect_world` | Look at what is in the world | +| `amend_contract`, `add_rule`, `drop_rule`, `fix_tool` | Correct the contract | +| `check_world` | Run every probe, report without saving | +| `save_world` | Freeze it — refused unless it holds up | + +**A handler** is Python: `def handle(args, db)`, with `db.query` / `db.one` / `db.execute` and +`ToolError` in scope. Nothing else — no filesystem, no network, because a world that depends on +the outside is not reproducible. It is `exec`'d per call in `world/runtime.py`. + +**The distinction the whole design turns on** (`runtime.py`): + +- `ToolError` — the world saying *no*. The id does not exist; the item is unavailable. **This is + the world working.** +- Any other exception — our bug. + +They are recorded differently and never confused. `_is_refusal` matches `ToolError` by name +across the class hierarchy, because generated handlers often declare their own. + +### The gate: what `check_world` and `save_world` actually run + +`world/probe.py`. Every probe restores the world to a frozen baseline first, so probes cannot +inherit each other's rows. + +| Probe | Asks | +|---|---| +| `happy` | A valid call built from the contract's permitted values. A refusal is acceptable; a crash never is | +| `edge` | A nonexistent id → must refuse, not succeed and not crash. A missing required argument → must refuse | +| `coverage` | Every contract tool has a handler; no handler for a tool the agent lacks; **and each handler actually reads the arguments the contract says it takes** | +| `data` | Every identifier the contract permits exists in the world — catches a whole category left unseeded, which otherwise looks exactly like correct strictness | +| `sequence` | Each declared sequence, run from the frozen world, leaves the state it claims | +| unknown tool | Calling a tool that does not exist must refuse | + +`save_world` refuses on three counts: **score below 0.85**; **no declared sequence** (calls that +each work alone can still forget what the last one did); and **a dirty world** — rows left over +from building, which would otherwise appear in every scenario as somebody else's order already +in the cart. + +Then `world/snapshot.py` writes `world.sqlite`, `handlers/*.py`, `world.py` and `manifest.json`. +**The snapshot is the base state every scenario restores from.** + +--- + +## 4. Scenarios — the conversations worth having + +`scenarios.py`, `skills/write-scenarios/SKILL.md`, tools in `scenario_tools.py` + +A scenario is a **delta on the base environment**, not a self-contained script (`scenario.py`): + +```json +{ + "name": "quantity_and_unavailable", + "use_case": "ordering with an item the store does not have", + "tests": "quantity is honoured and an unavailable drink never reaches the order", + "setup": {}, + "instruction": "Order two hamburgers and ask for a sweet tea.", + "variables": {}, + "solution": [ + {"tool": "order_regular_item", "arguments": {"item_id": "hamburger"}}, + {"tool": "order_regular_item", "arguments": {"item_id": "hamburger"}} + ], + "sub_goals": ["quantity_honored", "unavailable_drink_refused", "no_unrequested_items"], + "max_turns": 10 +} +``` + +There is no persona and no opening line. Variability comes from **real conditions**, which live +in `setup`: rows this one scenario needs on top of the frozen world. The base world stays the +shared starting point. + +`sub_goals` are **names from the catalogue the environment step defined**, not restated wording. +That is what makes results roll up: the same sub-goal failing in seven of twelve scenarios is one +sentence. + +`solution` is what a correct agent would do. It is never run against the agent — it exists so the +scenario can be proved. + +The writer can **look** (`inspect_world`) and **rehearse** (`try_calls` — run calls against a +throwaway copy and see what state they leave), so a solution is written from what was observed. + +### The gate: two proofs, no model involved + +`prove.py`, called by `submit_scenario` before a scenario is kept: + +| | Run | Must | +|---|---|---| +| **Solvable** | reset → setup → **the solution** → the checks | **pass** | +| **Not vacuous** | reset → setup → **nothing** → the checks | **fail** | + +If the first fails, either the scenario cannot be passed or the check is wrong. If the second +passes, the checks grade nothing while reporting a result — the failure that makes a suite +quietly green. Vacuity is judged on *all* checks passing, because one check surviving an empty run +("no unavailable item was ordered") is legitimate. + +`save_scenarios` additionally refuses a suite where no sub-goal is shared by two scenarios, +because nothing would roll up. + +--- + +## 5. Run — put someone in front of it + +`run/` + +The simulation is **not a loop written here**. ALK already owns placing a call, driving the +synthetic user, and producing a transcript; the harness supplies the world, the instruction, and +the grading. Against the real hosted agent that is `run/live.py` and `run/call.py`: + +``` +world + setup ──► webhook ──► public url ──► the assistant's OWN tools repointed + │ + ALK's voice case places the call ──┘ + │ + the world afterwards + the calls ──► the sub-goals' checks +``` + +1. **Restore** the frozen world and apply this scenario's `setup`. Its own copy, so nothing leaks + between scenarios. +2. **Stand up the webhook** (`run/voice.py`) and bind that world to it. A hosted voice agent + executes its tools by calling a webhook, so answering that webhook from `handle_tool_call` is + the entire integration. +3. **Expose it** (`cloudflared`, or `HARNESS_WEBHOOK_URL`), because a hosted agent cannot reach + loopback. +4. **Repoint the assistant.** `pointed_at` copies the agent's **own** tools and changes only + `server.url`. Nothing about the agent is redefined — rebuilding its tools would mean testing an + agent we wrote. +5. **Place the call** through ALK's own voice case, with the scenario's filled simulator prompt + driving the caller. +6. **Grade** from the world afterwards plus the recorded calls, through the same checks the gates + used. A sub-goal marked `judged` is reported as judged, never silently counted. + +`run/alk.py` is the same story for the text path: the world goes in as `environment=` to ALK's +`ChatEnvironment`, which owns the turn loop. + +Running is also a **stage of the conversation**, not only a command (`run/stage.py`, +`skills/run-scenarios/SKILL.md`, tools in `run/tools.py`: `preflight`, `list_scenarios`, +`run_scenario`, `read_results`). The stage exists because reading a failure is judgement: it has +to sort every failure into one of four causes — the agent was wrong, the world wrongly refused, +the check is wrong, or the simulated caller never asked for the thing — and only the first is a +finding about the agent. Each run's record lands in `runs.json` with the instruction, the +per-sub-goal verdicts, every tool call, and the transcript. + +This is what the environment was built for. The previous run's known issues — *"mocked tools +always succeed, including removing an item that was never added"*, *"mock responses do not vary by +argument"*, *"world state does not change unless a scenario sets `state_updates`"* — are all the +same defect, and a world that really holds rows and can really refuse answers all three. + +--- + +## What is exact, and what is judgement + +The split is deliberate and worth defending: + +| Judgement (the model) | Exact (code) | +|---|---| +| Reading unfamiliar source | Whether the contract is structurally usable | +| Designing a schema | Whether a handler crashes or refuses | +| Choosing what is worth testing | Whether an expectation resolves against real tables | +| Whether a claim held | Whether the state matches | + +**The model never decides whether something passed.** It decides what to try. + +--- + +## Where to extend it + +- A new **agent kind** → a class in `sources.py` and one registration +- A new **world kind** (browser, filesystem, queue) → a class in `world/kinds.py` implementing + `values_present` / `mutable_state` / `describe`, and one registration. Browser is registered + and stubbed +- A new **place the agent runs** (Vapi, LiveKit, a hosted endpoint) → a class in `run/targets.py` + with `open` / `say` / `close`, whose tool calls reach the same `world.handle_tool_call` +- A change to **how a stage works** → edit its `SKILL.md`. No code + +## What is not built + +- Browser worlds: registered, not implemented +- Snapshots are local files, not S3 +- Judged sub-goals are reported as judged, not actually sent to a judge yet +- Results do not post to the platform +- Nothing reports which of the contract's use cases have no scenario diff --git a/src/fi/alk/harness/README.md b/src/fi/alk/harness/README.md new file mode 100644 index 0000000..6cc899f --- /dev/null +++ b/src/fi/alk/harness/README.md @@ -0,0 +1,329 @@ +# The harness + +Point it at an agent. It reads the agent, builds a real database its tools run against, writes +test scenarios, runs them as conversations, and tells you what held and what did not. + +Nothing here is written for a particular agent. Every stage takes the contract and the world as +input, so a different agent is the same commands with a different name. + +--- + +# Part 1 — Setting up, from nothing + +If you have never run this before, do these five steps in order. They take about ten minutes, +most of which is waiting for the install. + +## Before you start + +You need four things on your machine: + +| What | Check it with | If missing | +|---|---|---| +| Python 3.10 or newer | `python3 --version` | install from python.org, or `brew install python` | +| `uv` (the package manager this repo uses) | `uv --version` | `brew install uv` | +| The `claude` command | `claude --version` | `npm install -g @anthropic-ai/claude-code` | +| A Google Cloud service-account key file (`.json`) for Vertex AI | you were given one, or ask | ask whoever set up your GCP access | + +The `claude` command matters: the harness talks to the model through the Claude Agent SDK, and +that SDK runs the `claude` binary under the hood. If it is not installed, every stage fails +immediately with a connection error. + +## Step 1 — Go to the repo + +Every command in this document is run from the **root of the repo**, not from this folder: + +```bash +cd path/to/agent-learning-kit +``` + +Wherever you cloned it, that directory is the one containing `pyproject.toml`. Check you are in +the right place: + +```bash +ls pyproject.toml # should print: pyproject.toml +``` + +If that errors, you are in the wrong directory. Do not continue until it works. + +## Step 2 — Install the dependencies + +```bash +uv sync --extra livekit --group dev +``` + +This reads `pyproject.toml`, downloads everything, and creates a folder called `.venv` in the +repo root. That folder is the "virtual environment": a private copy of Python with this +project's packages in it, so they do not collide with anything else on your machine. + +The `--extra livekit` matters even though the harness never makes a voice call. The harness +builds on `fi.simulate.environment`, and importing anything from `fi.simulate` runs that +package's `__init__`, which pulls in its LiveKit scenario generator. Plain `uv sync` leaves that +out and every command dies with `No module named 'livekit'`. + +It takes a few minutes the first time. You only do this once. + +## Step 3 — Use the virtual environment + +Two ways. **Pick one and stick with it.** + +**Option A — no activation (what this document uses).** Call the Python inside `.venv` directly: + +```bash +.venv/bin/python -m fi.alk.harness +``` + +Nothing to remember, nothing to undo, works in a fresh terminal every time. Every command below +is written this way. + +**Option B — activate it.** If you prefer typing plain `python`: + +```bash +source .venv/bin/activate # your prompt now shows (agent-learning-kit) +python -m fi.alk.harness # plain "python" now means the one in .venv +deactivate # when you are done +``` + +Activation only lasts for that terminal window. Open a new tab and you must activate again. If a +command ever fails with `No module named fi`, you almost certainly forgot. + +## Step 4 — Credentials + +The harness reaches the model through Vertex AI, which needs your Google Cloud service-account +key. Nothing is hardcoded and no key is ever read from source. + +Create a local env file from the template that ships with the repo: + +```bash +cp oss/simulation-acceptance/.env.example .env.acceptance +``` + +Open `.env.acceptance` in an editor and fill in two lines: + +```bash +GOOGLE_APPLICATION_CREDENTIALS=/absolute/path/to/your-service-account.json +GOOGLE_CLOUD_PROJECT=your-gcp-project-id +``` + +`.env.acceptance` is git-ignored. It holds a path to a private key: **never commit it, never +paste its contents into Slack or a PR.** + +Now load it into your terminal, and pick a model: + +```bash +set -a; . ./.env.acceptance; set +a +export CLOUD_ML_REGION=global +export ALK_HARNESS_MODEL=claude-haiku-4-5 +``` + +- `set -a; . ./file; set +a` means "read this file and export everything in it". The leading + `. ` (dot space) is what runs it in your *current* shell, so the variables stick around. +- `ALK_HARNESS_MODEL` picks the model. `claude-haiku-4-5` is cheapest and fine for trying things + out. Use `claude-sonnet-4-6` when you want better scenarios. + +These last only for the current terminal window. Every new terminal, run these three lines again. + +## Step 5 — Check it works + +```bash +.venv/bin/python -m pytest tests/test_harness.py -q +``` + +These are offline tests: no model calls, no credentials, no network. If they pass, your +install is fine. If they fail, the problem is Step 2, not your credentials. + +Then check the credentials separately, with the cheapest thing that talks to the model: + +```bash +.venv/bin/python -m fi.alk.harness +``` + +Say hello. If it answers, the credentials work; type `q` to leave before it spends anything +real. + +--- + +# Part 2 — Using it + +## The short version + +```bash +cd path/to/agent-learning-kit +set -a; . ./.env.acceptance; set +a +export CLOUD_ML_REGION=global ALK_HARNESS_MODEL=claude-haiku-4-5 + +.venv/bin/python -m fi.alk.harness +``` + +That last line is the whole interface. It opens with "which agent would you like to test, and +where is it?", and everything after that is a conversation. It finds the agent, reads it, builds +the world, writes the scenarios, and runs them, moving on as each stage produces its artifact. + +While you are in it: + +- type what you want and press enter +- press enter on an **empty** line to move to the next stage +- type `q` to leave + +Everything it produces is written to `artifacts/environments//`. + +## The same stages, one at a time + +Useful when you want to redo one thing without walking the whole conversation. Each of these +stays open for corrections until you type `q`; add `--once` to run it unattended and exit. + +```bash +# read an agent's source and write down what it verifiably is +.venv/bin/python -m fi.alk.harness understand --name my_agent --path ../my-agent-repo + +# build the environment: the world, the simulator prompt, the sub-goal catalogue +.venv/bin/python -m fi.alk.harness build --name my_agent + +# write the test scenarios, each proved before it is kept +.venv/bin/python -m fi.alk.harness scenarios --name my_agent --count 10 + +# run them against the world here, and grade +.venv/bin/python -m fi.alk.harness run --name my_agent + +# or run them against the real hosted agent, as a conversation +.venv/bin/python -m fi.alk.harness live --name my_agent +``` + +`--name` is just a label for the folder your artifacts go in. `--path` is where the agent's code +lives — a path to another repo on your disk. + +Useful extras: + +- `run --only [ ...]` runs a single scenario instead of all of them +- `run --quiet` hides the conversation and prints only verdicts +- `scenarios` without `--count` uses however many already exist, because coming back to change + one is not a request for a different number of them + +## What each stage does + +**understand** reads the agent's source and produces `contract.json`: its tools, the exact +argument names and permitted values, its hard rules, its real data. Everything downstream is +confined to this, which is what stops later stages inventing tools or menu items. Anything +changed later goes through an amendment tool and is recorded with its reason, so what came from +the agent and what came from us stay distinguishable. + +**build** produces everything common to every test of this agent: + +- **the world** — a real database behind the agent's tools, with one handler per tool that can + genuinely refuse: a nonexistent id, an unavailable item, an argument outside what the tool + accepts. A refusal is the world working; a crash is a defect, and the two are never confused. +- **the simulator prompt** — for a conversational agent, the person on the other side, written + once with `{{ slot }}` variables each scenario fills. +- **the sub-goal catalogue** — the named things this agent can be checked on, each carrying its + check **as code** wherever the answer is observable, and marked judged only where nothing is. + +It is exercised before it can be saved — every tool probed with a valid call, a bogus id and a +missing argument, plus declared sequences where state must carry across calls — and `save_world` +refuses a world that fails, has no sequences, no sub-goals, only judged sub-goals, no simulator +prompt for a conversational agent, or rows left over from its own testing. + +**scenarios** writes each test as a **delta** on that base: a few rows changed after reset, an +instruction that fills the simulator prompt, a reference solution, and which catalogue sub-goals +must hold. Before a scenario is kept it is **proved**, twice, with no model involved: + +1. reset → setup → run the solution → run the checks — they must **pass** (it is solvable) +2. reset → setup → run **nothing** → run the checks — they must **fail** (they grade something) + +**run** gives each scenario its own restored copy of the world and grades from what is left +behind: the state of the world plus every tool call with its arguments. `run` converses with the +agent locally, rebuilt from its contract. `live` is the same grading against the **real hosted +agent**: the webhook its own tools call is answered by the world, so a call for something that +is not there is refused rather than mocked into success. + +## How it grades + +Deterministic by default, a judge only as the fallback. + +Every sub-goal with a check in code is settled by running that check against two things the run +left behind: the world afterwards, and the recorded tool calls with their arguments — so "booked +10 PM when 11 PM was asked" is caught without any judgement. Sub-goals marked judged are handed +to a model with three kinds of evidence: what was said, what the agent actually did, and the +state afterwards. An unanswered claim counts as failed, never as passed, and judged results are +always reported as judged rather than blended into the code-settled score. + +``` +PASS quantity_and_unavailable 3/3 sub-goals settled by code + [x] quantity_honored + [x] unavailable_drink_refused + [x] regular_item_placed_correctly + [?] no_unrequested_items — judged, not settled by code + +what the agent actually did: + order_regular_item({'item_id': 'hamburger'}) -> ok + order_regular_item({'item_id': 'hamburger'}) -> ok +``` + +A run where the world crashed is `VOID`, not `FAIL` — that says nothing about the agent. A check +that raises is a **broken check**, reported as ours, never scored against the agent. + +## What it refuses to do + +These are the parts worth understanding, because they are what make a result mean something. + +- A world that fails its own probes will not save; nor will one with no sequences, no sub-goals, + only judged sub-goals, or rows left over from building it. +- A scenario is not kept until its own solution passes its own checks, and those checks fail + when nothing is done. Unsolvable scenarios and vacuous checks die here, at write time. +- A scenario naming a sub-goal nobody defined, or a table nobody built, is rejected and told + what does exist. +- A suite where no sub-goal is shared between scenarios will not save, because nothing would + roll up across it. +- Changing the contract is allowed but never silent: every widening, added rule or corrected + tool is recorded with its reason in `amendments[]`. + +If a stage tells you it will not do something, that is the design, not a bug to route around. + +## Rough costs + +On Haiku: reading an agent about $0.15, writing three proved scenarios about $0.12, a graded +local run a few cents per scenario. Building the environment is the expensive stage — about +$1.80 on Sonnet, which is worth using there even when everything else runs on Haiku +(`ALK_HARNESS_MODEL` per stage). + +## When something goes wrong + +| What you see | What it means | +|---|---| +| `No module named fi` | Wrong directory, or you are using system `python` instead of `.venv/bin/python` | +| `command not found: uv` | `brew install uv` | +| `No module named 'livekit'` | You ran plain `uv sync`. Run `uv sync --extra livekit --group dev` | +| Fails instantly on any model call | The `claude` command is not installed, or your env vars are not loaded in this terminal | +| `Could not load the default credentials` | `GOOGLE_APPLICATION_CREDENTIALS` is unset or points at a file that is not there | +| `No contract at ...` | Run `understand` first | +| `No world at ...` | Run `build` first | +| A stage does nothing and exits | It ran out of turns. Look at the last few lines: it usually says what it was stuck on | + +Everything a stage did is printed as it happens, and every run is kept in +`artifacts/environments//runs.json`, including the transcript and every tool call. + +--- + +# Part 3 — For developers + +## Adding to it + +- A new **agent** is nothing: the same stages read its contract. +- A new **kind of world** is a class and a registration in `world/kinds.py`. Browser is registered + and stubbed; sqlite is the one built out. +- A new **place the agent runs** is a class and a registration in `run/targets.py`. `local` runs + the agent here from its contract; the live voice path answers a hosted assistant's webhook from + the same `world.handle_tool_call`, so the world, the scenarios and the grading do not change. +- A change to **how a stage works** is an edit to its `skills//SKILL.md`. The markdown is + the method; code holds only what must be exact. + +## Not done yet + +- Browser worlds are registered but not built. +- Snapshots are local files, not object storage. +- Judged sub-goals on the live path are reported as judged, not yet sent to a judge. +- Nothing reports which of the contract's use cases have no scenario. + +## Tests + +```bash +.venv/bin/python -m pytest tests/test_harness.py -q # offline, no credentials needed +``` diff --git a/src/fi/alk/harness/__init__.py b/src/fi/alk/harness/__init__.py index f022cc8..df8ebea 100644 --- a/src/fi/alk/harness/__init__.py +++ b/src/fi/alk/harness/__init__.py @@ -20,7 +20,9 @@ provider_env, read_only_session, ) +from .chat import Conversation, open_conversation from .contract import AgentContract, ToolSpec, validate_contract +from .scenario import Scenario, validate_scenario from .session import Stage, Turn from .sources import ( AgentSource, @@ -35,14 +37,17 @@ __all__ = [ "AgentContract", "AgentSource", + "Conversation", "DEFAULT_MODEL", "RepoSource", + "Scenario", "SpecSource", "Stage", "ToolSpec", "Turn", "artifact_dir", "load_skill", + "open_conversation", "open_stage", "provider_env", "read_only_session", @@ -51,4 +56,5 @@ "supported", "understand", "validate_contract", + "validate_scenario", ] diff --git a/src/fi/alk/harness/amend.py b/src/fi/alk/harness/amend.py new file mode 100644 index 0000000..63807bb --- /dev/null +++ b/src/fi/alk/harness/amend.py @@ -0,0 +1,260 @@ +"""Changing the contract after the fact, and being honest about having done it. + +The contract is what the agent verifiably is, read from its own source. That makes it the thing +everything downstream is confined to, and it is why the harness cannot invent a tool or a value. + +But it is not permanent. Two situations genuinely require changing it, and they are different: + +- **It was read wrong.** Stage one missed a value the agent really accepts. Correcting that is + restoring the truth, and the correction should come from the source. +- **The agent is being changed.** Somebody adds an item to the world because the real menu is + gaining one. The world and the action space have to move together: an item the world holds but + the agent cannot name is dead data, and a scenario about it can only fail. + +Either way the amendment is recorded on the contract itself rather than blended into what was +read, so that a month later it is still possible to tell what came from the agent and what came +from us. That distinction is the whole value of the contract; quietly widening it would make it +the same kind of guess it exists to prevent. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from .contract import AgentContract, validate_contract + +CONTRACT = "contract.json" + + +def widen( + contract: AgentContract, + destination: Path, + *, + tool_name: str, + argument: str, + values: list[str], + why: str, +) -> tuple[bool, str]: + """Let a tool's argument accept values it did not before. + + Amends the contract the stage is holding, then persists it. Loading a second copy from disk + and writing that back would leave the stage still working from the old one, so the world it + goes on to check would be checked against an action space that no longer matches. + + Returns whether it was amended, and what happened. + """ + spec = next((tool for tool in contract.tools if tool.name == tool_name), None) + if spec is None: + return False, ( + f"{tool_name!r} is not a tool this agent has. It has: " + f"{', '.join(sorted(contract.tool_names()))}" + ) + if argument not in spec.args: + return False, ( + f"{tool_name} takes no argument called {argument!r}. It takes: " + f"{', '.join(spec.args) or 'nothing'}" + ) + if not why.strip(): + return ( + False, + "say why: an unexplained amendment is indistinguishable from a guess", + ) + + existing = spec.arg_values.get(argument) + current = list(existing) if isinstance(existing, (list, tuple)) else [] + fresh = [value for value in values if value and value not in current] + if not fresh: + return False, f"{argument} already accepts {', '.join(values) or 'nothing new'}" + + spec.arg_values[argument] = [*current, *fresh] + contract.amendments.append( + f"{tool_name}.{argument} widened by {', '.join(fresh)}: {why.strip()}" + ) + + problems = validate_contract(contract) + if problems: + spec.arg_values[argument] = current + contract.amendments.pop() + return False, "the amended contract would not be valid: " + "; ".join(problems) + + destination = Path(destination) + destination.mkdir(parents=True, exist_ok=True) + (destination / CONTRACT).write_text( + json.dumps(contract.model_dump(), indent=2, ensure_ascii=False), + encoding="utf-8", + ) + return True, ( + f"{tool_name}.{argument} now accepts {', '.join(fresh)}. " + f"{len(contract.amendments)} amendment(s) recorded on the contract." + ) + + +def add_rule( + contract: AgentContract, destination: Path, *, rule: str, why: str +) -> tuple[bool, str]: + """Give the agent a rule its source did not state. + + A hard constraint is not decoration: the agent under test is told it, and the judge grades + against it. So this is a real change to what is being tested, and like a widened argument it + is recorded rather than blended into what was read from the source. + """ + rule = rule.strip() + if not rule: + return False, "no rule given" + if not why.strip(): + return False, "say why: an unexplained rule is indistinguishable from a guess" + if any(rule.lower() == existing.lower() for existing in contract.hard_constraints): + return False, f"the agent already has that rule: {rule}" + + contract.hard_constraints.append(rule) + contract.amendments.append(f"rule added — {rule}: {why.strip()}") + problems = validate_contract(contract) + if problems: + contract.hard_constraints.pop() + contract.amendments.pop() + return False, "the amended contract would not be valid: " + "; ".join(problems) + + destination = Path(destination) + destination.mkdir(parents=True, exist_ok=True) + (destination / CONTRACT).write_text( + json.dumps(contract.model_dump(), indent=2, ensure_ascii=False), + encoding="utf-8", + ) + return True, ( + f"added. The agent now has {len(contract.hard_constraints)} rules, and this one is " + "graded from here on." + ) + + +def drop_rule( + contract: AgentContract, destination: Path, *, rule: str, why: str +) -> tuple[bool, str]: + """Take away a rule the agent does not really have. + + Stage one can misread a comment as a constraint, and a rule nobody has is worse than a + missing one: the agent under test is told to obey it and the judge fails it for not doing + something it was never supposed to do. + """ + if not why.strip(): + return False, "say why: removing a rule changes what is being graded" + match = next( + ( + existing + for existing in contract.hard_constraints + if existing.lower() == rule.strip().lower() + ), + None, + ) or next( + ( + existing + for existing in contract.hard_constraints + if rule.strip().lower() in existing.lower() + ), + None, + ) + if match is None: + return False, ( + "no rule like that. It has:\n - " + + "\n - ".join(contract.hard_constraints) + ) + contract.hard_constraints.remove(match) + contract.amendments.append(f"rule removed — {match}: {why.strip()}") + _persist(contract, destination) + return True, f"removed. {len(contract.hard_constraints)} rules left" + + +def fix_tool( + contract: AgentContract, + destination: Path, + *, + tool_name: str, + why: str, + args: list[str] | None = None, + arg_types: dict[str, str] | None = None, + description: str = "", + remove: bool = False, +) -> tuple[bool, str]: + """Correct a tool that was read wrong, or take away one the agent does not have. + + The most damaging thing stage one can get wrong. Every argument name flows into the world's + handlers, the probes and the scenarios, so a tool recorded with the wrong argument produces + a world that refuses everything and a suite that blames the agent for it. + """ + if not why.strip(): + return False, "say why: this changes what everything downstream is built from" + spec = next((tool for tool in contract.tools if tool.name == tool_name), None) + if spec is None: + return False, ( + f"{tool_name!r} is not a tool this agent has. It has: " + f"{', '.join(sorted(contract.tool_names()))}" + ) + + if remove: + contract.tools.remove(spec) + contract.amendments.append(f"tool removed — {tool_name}: {why.strip()}") + problems = validate_contract(contract) + if problems: + contract.tools.append(spec) + contract.amendments.pop() + return False, "cannot remove it: " + "; ".join(problems) + _persist(contract, destination) + return True, f"{tool_name} removed. {len(contract.tools)} tools left" + + changed = [] + if args is not None: + # Values recorded against an argument that no longer exists would silently be lost, so + # they are carried across by name and anything orphaned is said out loud. + orphaned = sorted(set(spec.arg_values) - set(args)) + spec.args = list(args) + spec.arg_types = {k: v for k, v in spec.arg_types.items() if k in spec.args} + spec.arg_values = {k: v for k, v in spec.arg_values.items() if k in spec.args} + changed.append(f"arguments are now {', '.join(args)}") + if orphaned: + changed.append(f"dropped values recorded for {', '.join(orphaned)}") + if arg_types: + unknown = sorted(set(arg_types) - set(spec.args)) + if unknown: + return False, f"{tool_name} takes no argument called {', '.join(unknown)}" + spec.arg_types.update(arg_types) + changed.append("types updated") + if description: + spec.description = description + changed.append("description updated") + if not changed: + return False, "nothing to change: give args, arg_types, description, or remove" + + contract.amendments.append(f"tool corrected — {tool_name}: {why.strip()}") + problems = validate_contract(contract) + if problems: + return False, "the amended contract would not be valid: " + "; ".join(problems) + _persist(contract, destination) + return True, f"{tool_name}: {', '.join(changed)}" + + +def _persist(contract: AgentContract, destination: Path) -> None: + destination = Path(destination) + destination.mkdir(parents=True, exist_ok=True) + (destination / CONTRACT).write_text( + json.dumps(contract.model_dump(), indent=2, ensure_ascii=False), + encoding="utf-8", + ) + + +def not_offered(contract: AgentContract, candidates: dict[str, set[str]]) -> list[str]: + """For each argument, the candidate values the contract does not let the agent send. + + ``candidates`` maps an argument name to identifiers found in the world that plausibly belong + to it. Kept as an argument rather than inferred here, because which column feeds which + argument is knowledge about one agent, not something a schema states. + """ + missing: list[str] = [] + for tool in contract.tools: + for argument, values in (tool.arg_values or {}).items(): + if argument not in candidates or not isinstance(values, (list, tuple)): + continue + permitted = {str(value) for value in values} + absent = sorted(candidates[argument] - permitted) + if absent: + missing.append(f"{tool.name}.{argument}: {', '.join(absent)}") + return missing diff --git a/src/fi/alk/harness/build.py b/src/fi/alk/harness/build.py index 8f80fc0..c7c6a26 100644 --- a/src/fi/alk/harness/build.py +++ b/src/fi/alk/harness/build.py @@ -15,7 +15,14 @@ from claude_agent_sdk import ClaudeAgentOptions -from .config import artifact_dir, load_skill, provider_env +from .config import ( + artifact_dir, + gate_hooks, + chosen_model, + load_skill, + permission_gate, + provider_env, +) from .contract import AgentContract from .session import Stage from .tools import qualified @@ -34,25 +41,30 @@ def open_stage( """A live build-the-world stage, and where it will write.""" destination = out or artifact_dir(contract.agent) server, _world = world_tools(contract, destination) + allowed = [ + "AskUserQuestion", + *(qualified(WORLD_SERVER, name) for name in TOOL_NAMES), + ] options = ClaudeAgentOptions( system_prompt=( f"{load_skill(SKILL)}\n\n## This agent\n\n{contract.brief(with_data=True)}" ), # No file tools and no shell. Everything this stage can do goes through a tool that # executes it and reports back, which is what makes the guardrails meaningful. - allowed_tools=[ - "AskUserQuestion", - *(qualified(WORLD_SERVER, name) for name in TOOL_NAMES), - ], + allowed_tools=allowed, mcp_servers={WORLD_SERVER: server}, - permission_mode="acceptEdits", + # Not acceptEdits: that auto-approves Edit and Write before the permission callback is + # consulted, so a stage can rewrite an artifact by hand and skip the tool whose + # whole job is to validate that change. + permission_mode="default", cwd=str(destination.parent if destination.parent.exists() else Path.cwd()), setting_sources=[], max_turns=max_turns, + model=chosen_model(), env=provider_env(), ) - if ask is not None: - options.can_use_tool = ask + options.hooks = gate_hooks(allowed) + options.can_use_tool = permission_gate(ask, allowed) return Stage(options, name=SKILL), destination diff --git a/src/fi/alk/harness/chat.py b/src/fi/alk/harness/chat.py index f976d00..6e5f2ba 100644 --- a/src/fi/alk/harness/chat.py +++ b/src/fi/alk/harness/chat.py @@ -17,45 +17,83 @@ from typing import Any, Callable from . import build as build_stage +from . import reception as reception_stage +from . import scenarios as scenario_stage from . import understand as understand_stage +from .run import stage as run_stage from .config import artifact_dir from .contract import AgentContract from .session import Stage from .sources import AgentSource, resolve +RECEPTION = "reception" UNDERSTAND = "understand" BUILD = "build" +SCENARIOS = "scenarios" +RUN = "run" DONE = "done" -_NEXT = {UNDERSTAND: BUILD, BUILD: DONE} +_NEXT = { + RECEPTION: UNDERSTAND, + UNDERSTAND: BUILD, + BUILD: SCENARIOS, + SCENARIOS: RUN, + RUN: DONE, +} @dataclass class Conversation: """The whole thing, held open.""" - source: AgentSource - out: Path + # Both unknown until somebody says which agent this is about, which is itself a stage. + source: AgentSource | None = None + out: Path | None = None ask: Callable[..., Any] | None = None - stage_name: str = UNDERSTAND + wanted: int = 10 + # Where to look for an agent. Almost never inside this repo: the harness lives in one place + # and the agent being tested lives in another, so looking only at our own root means the + # first thing anybody types cannot be found. + workspace: Path | None = None + stage_name: str = "" stage: Stage | None = None spent_usd: float = 0.0 history: list[str] = field(default_factory=list) + _found: dict[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + # Read off the artifacts rather than defaulting to the first stage. An agent whose world + # is already built is at the scenarios, and saying otherwise before anything has been + # opened makes every question about where this conversation is answer wrongly. + self.stage_name = self.stage_name or self._resume_at() # -- what exists so far ---------------------------------------------------------- @property def contract(self) -> AgentContract | None: - return understand_stage.load(self.out) + return understand_stage.load(self.out) if self.out else None @property def world_built(self) -> bool: - return (self.out / "world.sqlite").exists() + return bool(self.out) and (self.out / "world.sqlite").exists() + + @property + def scenarios_written(self) -> bool: + return bool(self.out) and bool(scenario_stage.load(self.out)) + + @property + def anything_run(self) -> bool: + return bool(self.out) and bool(run_stage.load(self.out)) def _artifact_for(self, stage_name: str) -> bool: return { + # A contract already on disk settles which agent this is just as well as being told, + # so coming back to an agent does not mean pointing at its repository again. + RECEPTION: self.source is not None or self.contract is not None, UNDERSTAND: self.contract is not None, BUILD: self.world_built, + SCENARIOS: self.scenarios_written, + RUN: self.anything_run, DONE: True, }[stage_name] @@ -71,17 +109,50 @@ async def _open(self, stage_name: str) -> str: """Open a stage and return the message that starts it.""" await self._close() self.stage_name = stage_name + if stage_name == RECEPTION: + self.stage, self._found = reception_stage.open_stage( + cwd=self.workspace, ask=self.ask + ) + await self.stage.__aenter__() + return reception_stage.opening() + + # Deliberately not "is there a source": a contract on disk settles which agent this is, + # and every stage after the first works from the contract rather than from the source. + # Only re-reading the agent needs to know where it lives. + if self.source is None and self.contract is None: + raise RuntimeError("nobody has said which agent this is about yet") if stage_name == UNDERSTAND: self.stage, _ = understand_stage.open_stage( self.source, out=self.out, ask=self.ask ) opening = understand_stage.opening(self.source) - else: - contract = self.contract - if contract is None: - raise RuntimeError("cannot build a world before there is a contract") + await self.stage.__aenter__() + return opening + + contract = self.contract + if contract is None: + raise RuntimeError("cannot go further before there is a contract") + if self.source is None and stage_name == UNDERSTAND: + raise RuntimeError( + "cannot re-read the agent without knowing where it lives" + ) + if stage_name == BUILD: self.stage, _ = build_stage.open_stage(contract, out=self.out, ask=self.ask) opening = build_stage.opening(contract) + elif stage_name == RUN: + if not self.scenarios_written: + raise RuntimeError("cannot run anything before there are scenarios") + self.stage, _ = run_stage.open_stage(contract, out=self.out, ask=self.ask) + opening = run_stage.opening(contract, self.out) + else: + if not self.world_built: + raise RuntimeError("cannot write scenarios before there is a world") + written = len(scenario_stage.load(self.out)) + wanted = written or self.wanted + self.stage, _ = scenario_stage.open_stage( + contract, out=self.out, wanted=wanted, ask=self.ask + ) + opening = scenario_stage.opening(contract, wanted, written) await self.stage.__aenter__() return opening @@ -95,21 +166,54 @@ def next_stage(self) -> str | None: # -- talking --------------------------------------------------------------------- async def start(self, on_event: Callable[..., Any] | None = None) -> None: + """Open the stage this agent is up to, and set it going.""" opening = await self._open(self._resume_at()) await self.stage.say(opening, on_event=on_event) # type: ignore[union-attr] + async def open_quietly(self) -> None: + """Open the stage without telling it to start. + + A stage's opening message is an instruction to do the stage's work. Sending it because + somebody said hello means a greeting kicks off a build, so it is only sent when the work + is actually what was asked for. + """ + await self._open(self._resume_at()) + def _resume_at(self) -> str: """Pick up where the artifacts say this agent got to.""" + if self.source is None and self.contract is None: + return RECEPTION if self.contract is None: return UNDERSTAND - return BUILD + if not self.world_built: + return BUILD + if not self.scenarios_written: + return SCENARIOS + return RUN - async def say(self, message: str, on_event: Callable[..., Any] | None = None) -> None: + async def say( + self, message: str, on_event: Callable[..., Any] | None = None + ) -> None: """Send a message to whichever stage is open.""" self.history.append(message) if self.stage is None: - await self.start(on_event=on_event) + await self.open_quietly() await self.stage.say(message, on_event=on_event) # type: ignore[union-attr] + await self._settle(on_event=on_event) + + async def _settle(self, on_event: Callable[..., Any] | None = None) -> None: + """Take up whatever the open stage just established, and keep going. + + Reception is the only stage whose result is not a file, so it is the only one the + conversation has to read back. Once it knows the agent there is nothing to decide, so it + goes straight on rather than making somebody confirm what they already said. + """ + settled = self._found.pop("source", None) + if settled is None: + return + self.source = settled + self.out = self.out or artifact_dir(settled.name) + await self.advance(on_event=on_event) async def advance(self, on_event: Callable[..., Any] | None = None) -> str | None: """Move to the next stage and start it. Returns the stage entered, or None.""" @@ -126,14 +230,27 @@ async def close(self) -> None: def open_conversation( *, - name: str, - path: str, + name: str = "", + path: str = "", kind: str = "repo", out: Path | None = None, ask: Callable[..., Any] | None = None, + wanted: int = 10, + workspace: Path | None = None, ) -> Conversation: - source = resolve(kind, name=name, root=path) - return Conversation(source=source, out=out or artifact_dir(name), ask=ask) + """Open the harness. With nothing, it starts by asking which agent you mean. + + Naming the agent up front is a shortcut for coming back to one already in progress, not the + way in. Everything it needs can be said. + """ + source = resolve(kind, name=name, root=path) if name and path else None + return Conversation( + source=source, + out=out or (artifact_dir(name) if name else None), + ask=ask, + wanted=wanted, + workspace=workspace, + ) async def _demo() -> None: # pragma: no cover - convenience for manual runs diff --git a/src/fi/alk/harness/checks.py b/src/fi/alk/harness/checks.py new file mode 100644 index 0000000..c126515 --- /dev/null +++ b/src/fi/alk/harness/checks.py @@ -0,0 +1,79 @@ +"""Running a check the harness wrote, and deciding what its answer means. + +A check is Python because an environment can be a database, a filesystem or a page, and any +little assertion language invented here would fit only the first. It is given the two things a +run leaves behind and returns a sentence when something is wrong: + + def check(world, calls): + rows = world.state()["orders"] + if len(rows) != 1: + return f"{len(rows)} orders, expected 1" + if not any(c.name == "order_combo_meal" for c in calls): + return "the combo was never ordered" + return None + +``world`` is the environment afterwards. ``calls`` is every tool call that was made, each with +its arguments and whether it succeeded — so a check can insist not only that a call happened but +that it happened with the right arguments, which is the difference between booking 11 PM and +booking 10 PM. + +A check that raises is a broken check, not a failed one, and is reported that way. Confusing the +two would let a typo read as a finding about the agent. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Sequence + +from .world.runtime import Call, GeneratedWorld + + +@dataclass +class Outcome: + """What one check said.""" + + name: str + held: bool + said: str = "" + broken: bool = False + + def line(self) -> str: + mark = "!" if self.broken else ("x" if self.held else " ") + return f" [{mark}] {self.name}" + (f" — {self.said}" if self.said else "") + + +def run_check( + source: str, world: GeneratedWorld, calls: Sequence[Call], *, name: str = "check" +) -> Outcome: + """Execute one check against what the run left behind.""" + namespace: dict[str, Any] = {} + try: + exec(compile(source, f"", "exec"), namespace) + except Exception as failed: + return Outcome(name, False, f"the check would not compile: {failed}", broken=True) + + checker = namespace.get("check") + if not callable(checker): + return Outcome(name, False, "the check defines no check(world, calls)", broken=True) + + try: + said = checker(world, list(calls)) + except Exception as failed: + # The check is at fault, not the agent. A KeyError in an assertion is our bug, and + # scoring it against the agent is how a harness invents findings. + return Outcome( + name, False, f"the check raised {type(failed).__name__}: {failed}", broken=True + ) + + if said is None or said is True: + return Outcome(name, True) + return Outcome(name, False, str(said) if said is not True else "") + + +def all_held(outcomes: Sequence[Outcome]) -> bool: + return all(one.held for one in outcomes) and not any(one.broken for one in outcomes) + + +def broken(outcomes: Sequence[Outcome]) -> list[Outcome]: + return [one for one in outcomes if one.broken] diff --git a/src/fi/alk/harness/cli.py b/src/fi/alk/harness/cli.py index 057315c..4c7922f 100644 --- a/src/fi/alk/harness/cli.py +++ b/src/fi/alk/harness/cli.py @@ -16,8 +16,19 @@ from .build import open_stage as build_stage from .build import opening as build_opening -from .config import DEFAULT_MODEL, artifact_dir +from .chat import open_conversation +from .config import ( + DEFAULT_MODEL, + artifact_dir, + chosen_model, + credentials_hint, + permission_gate, +) +from .scenarios import load as load_written +from .scenarios import open_stage as scenario_stage +from .scenarios import opening as scenario_opening from .session import TEXT, Event +from .run.targets import supported as target_kinds from .sources import resolve, supported from .understand import load, open_stage, opening @@ -34,19 +45,10 @@ async def _prompt(question: str) -> str: return (await asyncio.to_thread(input, question)).strip() -async def _answer_questions( - tool_name: str, payload: dict[str, Any], _context: Any -) -> Any: - """Render the model's clarifying questions and return the operator's answers. - - Anything that is not a question is allowed through: the session is already restricted to - read-only built-ins plus our own tools, so there is nothing here to gate. - """ +async def _ask_operator(_tool_name: str, payload: dict[str, Any], _context: Any) -> Any: + """Render the model's clarifying questions and return the operator's answers.""" from claude_agent_sdk.types import PermissionResultAllow - if tool_name != "AskUserQuestion": - return PermissionResultAllow(updated_input=payload) - answers: dict[str, Any] = {} for question in payload.get("questions", []): print(f"\n\n {question.get('header', '?')}: {question.get('question', '')}") @@ -73,10 +75,11 @@ async def _understand(args: argparse.Namespace) -> int: out=Path(args.out) if args.out else None, # Unattended, there is nobody to answer, so the model records what it could not # resolve in open_questions rather than blocking on a prompt nobody will see. - ask=_answer_questions if args.interactive else None, + ask=permission_gate(_ask_operator) if args.interactive else None, ) print(f"agent: {source.name} ({source.kind})") + print(f"model: {chosen_model()}") print(f"out: {destination}\n") await _converse(stage, opening(source), interactive=args.interactive) @@ -122,12 +125,13 @@ async def _build(args: argparse.Namespace) -> int: return 1 print(f"agent: {contract.agent} ({len(contract.tools)} tools)") + print(f"model: {chosen_model()}") print(f"out: {destination}\n") stage, _ = build_stage( contract, out=destination, - ask=_answer_questions if args.interactive else None, + ask=permission_gate(_ask_operator) if args.interactive else None, ) await _converse(stage, build_opening(contract), interactive=args.interactive) @@ -139,9 +143,172 @@ async def _build(args: argparse.Namespace) -> int: return 0 +async def _scenarios(args: argparse.Namespace) -> int: + destination = Path(args.out) if args.out else artifact_dir(args.name) + contract = load(destination) + if contract is None: + print(f"No contract at {destination}. Run `understand` first.", file=sys.stderr) + return 1 + if not (destination / "world.sqlite").exists(): + print(f"No world at {destination}. Run `build` first.", file=sys.stderr) + return 1 + + # With a suite already written, the target is what is there. Somebody who comes back to + # change one scenario is not asking for a different number of them. + existing = len(load_written(destination)) + wanted = args.count or existing or 10 + + print( + f"agent: {contract.agent} " + + (f"({existing} scenarios, loaded)" if existing else f"(writing {wanted})") + ) + print(f"model: {chosen_model()}") + print(f"out: {destination}\n") + + stage, _ = scenario_stage( + contract, + out=destination, + wanted=wanted, + ask=permission_gate(_ask_operator) if args.interactive else None, + ) + await _converse( + stage, + scenario_opening(contract, wanted, existing), + interactive=args.interactive, + ) + + written = load_written(destination) + if not written: + print("\nNo scenarios were saved.", file=sys.stderr) + return 1 + print(f"\nscenarios: {len(written)} in {destination / 'scenarios.json'}") + print(f"spent: ${stage.spent_usd:.4f}") + return 0 + + +async def _live(args: argparse.Namespace) -> int: + """The run stage as a conversation: it decides what to run and reads what came back.""" + from .run.stage import load as load_results + from .run.stage import open_stage as run_stage + from .run.stage import opening as run_opening + + destination = Path(args.out) if args.out else artifact_dir(args.name) + contract = load(destination) + written = load_written(destination) + if contract is None or not written: + print( + f"Need a contract and scenarios at {destination}. Run `understand`, `build` and " + "`scenarios` first.", + file=sys.stderr, + ) + return 1 + + print(f"agent: {contract.agent} ({len(written)} scenarios)") + print(f"model: {chosen_model()}") + print(f"out: {destination}\n") + + stage, _ = run_stage( + contract, + out=destination, + ask=permission_gate(_ask_operator) if args.interactive else None, + ) + await _converse( + stage, run_opening(contract, destination), interactive=args.interactive + ) + + results = load_results(destination) + passed = sum(1 for record in results if record["passed"]) + print(f"\nruns: {passed} of {len(results)} passed, in {destination / 'runs.json'}") + print(f"spent: ${stage.spent_usd:.4f}") + return 0 + + +async def _run(args: argparse.Namespace) -> int: + from .run import run_suite + from .run.grade import summarise + + destination = Path(args.out) if args.out else artifact_dir(args.name) + contract = load(destination) + written = load_written(destination) + if contract is None or not written: + print( + f"Need a contract and scenarios at {destination}. Run `understand`, `build` " + "and `scenarios` first.", + file=sys.stderr, + ) + return 1 + + chosen = [s for s in written if s.name in args.only] if args.only else written + if not chosen: + print(f"No scenario matching {args.only}.", file=sys.stderr) + return 1 + + print(f"agent: {contract.agent} ({len(chosen)} scenarios, target {args.target})") + print(f"model: {chosen_model()}") + print(f"out: {destination}\n") + + def overheard(exchange: Any) -> None: + if args.quiet: + return + print(f" {exchange.speaker:8} {exchange.text}", flush=True) + + def show(result: Any) -> None: + # Just the verdict as it lands. The detail is in the summary at the end, and printing + # it in both places means every failure is read twice. + print(result.line(), flush=True) + + results = await run_suite( + chosen, + contract, + destination, + target=args.target, + model=args.model, + on_result=show, + on_exchange=overheard, + ) + print("\n" + summarise(results)) + print(f"\nspent: ${sum(result.spent_usd for result in results):.4f}") + return 0 if all(result.passed for result in results) else 2 + + +async def _chat(args: argparse.Namespace) -> int: + """One conversation for the whole thing: point at an agent and keep talking.""" + conversation = open_conversation( + name=args.name or "", + path=args.path or "", + kind=args.kind, + out=Path(args.out) if args.out else None, + ask=permission_gate(_ask_operator), + ) + print(f"model: {chosen_model()}") + print(credentials_hint()) + print("\nSay what you want. Enter on its own moves to the next stage; 'q' ends.\n") + + await conversation.start(on_event=_render) + while True: + try: + said = await _prompt(f"\nkarthik ({conversation.stage_name}) ") + except (EOFError, KeyboardInterrupt): + break + if said in {"q", "quit", "exit"}: + break + if not said: + entered = await conversation.advance(on_event=_render) + if entered is None: + print( + "\n [nothing to move on to yet; this stage has not produced its artifact]" + ) + continue + await conversation.say(said, on_event=_render) + await conversation.close() + print(f"\nspent: ${conversation.spent_usd:.4f}") + return 0 + + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog="fi.alk.harness", description=__doc__) - sub = parser.add_subparsers(dest="stage", required=True) + # Talking to it is the way in, so that is what happens when you just start it. + sub = parser.add_subparsers(dest="stage", required=False) understand = sub.add_parser( "understand", help="read an agent and produce its contract" @@ -171,11 +338,80 @@ def build_parser() -> argparse.ArgumentParser: help="run unattended instead of staying open for corrections", ) world.set_defaults(run=_build, interactive=True) + + scenarios = sub.add_parser( + "scenarios", help="write the scenarios to test the agent with" + ) + scenarios.add_argument("--name", required=True, help="which agent") + scenarios.add_argument("--out", default=None, help="artifact directory") + scenarios.add_argument( + "--count", + type=int, + default=None, + help="how many scenarios to write (defaults to however many already exist)", + ) + scenarios.add_argument( + "--once", + dest="interactive", + action="store_false", + help="run unattended instead of staying open for corrections", + ) + scenarios.set_defaults(run=_scenarios, interactive=True) + + live = sub.add_parser( + "live", help="run the scenarios against the real agent, as a conversation" + ) + live.add_argument("--name", required=True, help="which agent") + live.add_argument("--out", default=None, help="artifact directory") + live.add_argument( + "--once", + dest="interactive", + action="store_false", + help="run unattended instead of staying open", + ) + live.set_defaults(run=_live, interactive=True) + + runs = sub.add_parser("run", help="run the scenarios and grade what happened") + runs.add_argument("--name", required=True, help="which agent") + runs.add_argument("--out", default=None, help="artifact directory") + runs.add_argument( + "--target", + default="local", + choices=target_kinds(), + help="where the agent under test runs", + ) + runs.add_argument( + "--only", nargs="*", default=None, help="run only these scenarios, by name" + ) + runs.add_argument("--model", default=None, help="model for the run") + runs.add_argument( + "--quiet", + action="store_true", + help="only the verdicts, without the conversations as they happen", + ) + runs.set_defaults(run=_run) + + chat = sub.add_parser( + "chat", + help="one conversation: understand, build the world, write the scenarios", + ) + # Nothing is required. Which agent, where it lives and how many scenarios are all things + # you say; naming one here is a shortcut back into work already in progress. + chat.add_argument("--name", default=None, help=argparse.SUPPRESS) + chat.add_argument("--path", default=None, help=argparse.SUPPRESS) + chat.add_argument( + "--kind", default="repo", choices=supported(), help=argparse.SUPPRESS + ) + chat.add_argument("--out", default=None, help=argparse.SUPPRESS) + chat.set_defaults(run=_chat) return parser def main(argv: list[str] | None = None) -> int: - args = build_parser().parse_args(argv) + parser = build_parser() + args = parser.parse_args(argv) + if getattr(args, "run", None) is None: + args = parser.parse_args([*(argv or []), "chat"]) return asyncio.run(args.run(args)) diff --git a/src/fi/alk/harness/config.py b/src/fi/alk/harness/config.py index 156d77b..8eab46c 100644 --- a/src/fi/alk/harness/config.py +++ b/src/fi/alk/harness/config.py @@ -23,6 +23,33 @@ _READ_ONLY_TOOLS = ("Read", "Glob", "Grep") +def credentials_hint() -> str: + """A line saying which credentials a run will use, or a warning that it is guessing. + + Claude Code falls back to the active gcloud login when no service-account file is named, + which is a legitimate setup and an easy accident. The accident produces a provider auth + error several layers down, so it is worth saying out loud which one is in play. + """ + named = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + if named: + return f"credentials: {Path(named).name}" + return ( + "credentials: none named, falling back to your gcloud login. If calls fail to " + "authenticate, load the env file first:\n" + " set -a; . ./.env.acceptance; set +a" + ) + + +def chosen_model(model: str | None = None) -> str: + """The model a session will actually run on. + + Passed to the session explicitly as well as through the environment. The environment alone + does not win: the CLI has its own default and will quietly use it, so a run meant for Haiku + goes out on whatever the CLI felt like and the bill says so afterwards. + """ + return model or os.environ.get("ALK_HARNESS_MODEL", DEFAULT_MODEL) + + def provider_env(model: str | None = None) -> dict[str, str]: """The provider block passed to the session. @@ -32,7 +59,7 @@ def provider_env(model: str | None = None) -> dict[str, str]: env = { "CLAUDE_CODE_USE_VERTEX": "1", "CLOUD_ML_REGION": os.environ.get("CLOUD_ML_REGION", "global"), - "ANTHROPIC_MODEL": model or os.environ.get("ALK_HARNESS_MODEL", DEFAULT_MODEL), + "ANTHROPIC_MODEL": chosen_model(model), } for passthrough in ( "ANTHROPIC_VERTEX_PROJECT_ID", @@ -61,16 +88,88 @@ def read_only_session( session can produce anything is by calling one of ours. """ allowed = [*_READ_ONLY_TOOLS, "AskUserQuestion", *extra_tools] - return ClaudeAgentOptions( + options = ClaudeAgentOptions( system_prompt=system_prompt, allowed_tools=allowed, mcp_servers=dict(mcp_servers or {}), - permission_mode="acceptEdits", + # Not acceptEdits: that auto-approves Edit and Write before the permission callback + # is consulted, which silently defeats the gate below. + permission_mode="default", cwd=str(cwd), setting_sources=[], max_turns=max_turns, + model=chosen_model(model), env=provider_env(model), ) + options.hooks = gate_hooks(allowed) + options.can_use_tool = permission_gate(granted=allowed) + return options + + +def gate_hooks(granted: Iterable[str]) -> dict[str, Any]: + """Deny anything a stage was not given, at the point the SDK actually asks. + + ``can_use_tool`` alone does not do this. An ``allowed_tools`` entry approves those tools + before the callback is consulted, and the SDK then warns that the callback is shadowed — so + the gate never runs for the tools we granted, and in practice does not stop the ones we did + not either. A host ``ToolSearch`` reached every stage, returned nothing, and cost a turn each + time. + + A PreToolUse hook is consulted for every call, which is what the deny-by-default rule needed + in order to be true rather than intended. + """ + from claude_agent_sdk.types import HookMatcher + + permitted = {*granted, "AskUserQuestion"} + + async def refuse(payload: dict[str, Any], _tool_use_id: Any, _context: Any) -> dict[str, Any]: + name = str(payload.get("tool_name") or "") + if not name or name in permitted: + return {} + return { + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "deny", + "permissionDecisionReason": ( + f"{name} is not part of this stage. You have " + f"{', '.join(sorted(permitted)) or 'no other tools'}, and everything you " + "produce goes through those, because those are what check it." + ), + } + } + + return {"PreToolUse": [HookMatcher(hooks=[refuse])]} + + +def permission_gate(ask: Any | None = None, granted: Iterable[str] = ()) -> Any: + """Decide what a stage may do: nothing it was not given. + + Deny by default, not deny-a-list. A session is offered whatever tools its host happens to + expose, and anything not named here is by definition not part of how this stage works. An + allow-by-default gate let a host search tool through, which returned nothing useful and cost + a stage its entire turn budget looping on it; the same hole would let a file write through. + + Tools granted through ``allowed_tools`` are approved before this is consulted, so this only + ever sees the ones that were not. + """ + permitted = set(granted) + + async def gate(tool_name: str, payload: dict[str, Any], context: Any) -> Any: + from claude_agent_sdk.types import PermissionResultAllow, PermissionResultDeny + + if tool_name == "AskUserQuestion" and ask is not None: + return await ask(tool_name, payload, context) + if tool_name in permitted: + return PermissionResultAllow(updated_input=payload) + return PermissionResultDeny( + message=( + f"{tool_name} is not part of this stage. You have " + f"{', '.join(sorted(permitted)) or 'no other tools'}, and everything you " + "produce goes through those, because those are what check it." + ) + ) + + return gate def artifact_dir(agent: str, root: str | Path | None = None) -> Path: diff --git a/src/fi/alk/harness/contract.py b/src/fi/alk/harness/contract.py index 452a639..36b1423 100644 --- a/src/fi/alk/harness/contract.py +++ b/src/fi/alk/harness/contract.py @@ -28,6 +28,7 @@ "real_use_cases", "signature_cases", "anti_hallucination", + "amendments", ) _DICT_FIELDS = ("data_schema", "base_environment") @@ -85,6 +86,10 @@ def _normalize_shapes(cls, payload: Any) -> Any: grading_notes: str = "" anti_hallucination: list[str] = Field(default_factory=list) open_questions: list[str] = Field(default_factory=list) + # Anything in here was not read from the agent's source. The contract is meant to be what + # the agent verifiably is, so when the harness widens it the difference is recorded rather + # than blended in, and whoever reads it later can tell the two apart. + amendments: list[str] = Field(default_factory=list) def tool_names(self) -> set[str]: return {tool.name for tool in self.tools} diff --git a/src/fi/alk/harness/environment.py b/src/fi/alk/harness/environment.py new file mode 100644 index 0000000..c6e653b --- /dev/null +++ b/src/fi/alk/harness/environment.py @@ -0,0 +1,156 @@ +"""What the environment step produces: everything common to every test of one agent. + +Three artifacts, and the harness writes all three. Nothing here decides their content. + +- **the world** — whatever this agent acts on, subclassing ALK's ``EnvironmentAdapter`` so the + runners that already exist can drive it +- **the simulator prompt** — for a conversational agent only, with variables left open for each + scenario to fill +- **the sub-goal catalogue** — the named things this agent can be checked on, each carrying its + own check as code + +Kept together because they share a property: they are written once and every scenario is only a +delta on them. A scenario changes a few values, substitutes the simulator's variables, and says +which sub-goals must hold. That is what makes results roll up across a suite — a sub-goal is the +same sub-goal in all twelve scenarios, so "order confirmation fails in 7 of 12" is sayable. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from pydantic import BaseModel, Field + +CATALOGUE = "sub_goals.json" +SIMULATOR = "simulator_prompt.md" + + +class SubGoal(BaseModel): + """One named thing the agent can be checked on, shared across every scenario that needs it. + + ``check`` is Python, written by the harness. It is given what the run left behind and returns + nothing if the sub-goal held, or a sentence saying what was wrong. Code rather than a mini + language because an environment can be a database, a filesystem or a page, and a language + invented here would fit only the first. + + ``judged`` marks the ones nothing observable can settle — whether a refusal was explained, + whether a price was invented. Those go to a model, and are the exception. + """ + + name: str + what: str = "" + check: str = "" + judged: str = "" + + def deterministic(self) -> bool: + return bool(self.check.strip()) + + +class Catalogue(BaseModel): + """Every sub-goal this agent has, defined once.""" + + sub_goals: list[SubGoal] = Field(default_factory=list) + + def named(self, name: str) -> SubGoal | None: + return next((one for one in self.sub_goals if one.name == name), None) + + def names(self) -> set[str]: + return {one.name for one in self.sub_goals} + + +def validate_sub_goal(sub_goal: SubGoal) -> list[str]: + """Problems that make a sub-goal unusable. + + A sub-goal that settles nothing is the expensive kind of wrong: every scenario referencing it + reports a result nobody should believe. + """ + problems: list[str] = [] + if not sub_goal.name.strip(): + problems.append("no name") + if not sub_goal.what.strip(): + problems.append(f"{sub_goal.name}: no description of what it means") + if not sub_goal.check.strip() and not sub_goal.judged.strip(): + problems.append( + f"{sub_goal.name}: settles nothing. Give a check in code, or say what a judge has " + "to decide and why nothing observable can settle it" + ) + if sub_goal.check.strip() and "def check(" not in sub_goal.check: + problems.append( + f"{sub_goal.name}: a check must define check(world, calls) and return a problem as " + "a string, or None when the sub-goal held" + ) + return problems + + +def save_catalogue(catalogue: Catalogue, destination: Path) -> Path: + destination = Path(destination) + destination.mkdir(parents=True, exist_ok=True) + path = destination / CATALOGUE + path.write_text( + json.dumps(catalogue.model_dump(), indent=2, ensure_ascii=False), encoding="utf-8" + ) + return path + + +def load_catalogue(destination: Path) -> Catalogue: + path = Path(destination) / CATALOGUE + if not path.exists(): + return Catalogue() + return Catalogue.model_validate(json.loads(path.read_text(encoding="utf-8"))) + + +def save_simulator_prompt(prompt: str, destination: Path) -> Path: + destination = Path(destination) + destination.mkdir(parents=True, exist_ok=True) + path = destination / SIMULATOR + path.write_text(prompt, encoding="utf-8") + return path + + +def load_simulator_prompt(destination: Path) -> str: + path = Path(destination) / SIMULATOR + return path.read_text(encoding="utf-8") if path.exists() else "" + + +def variables_in(prompt: str) -> set[str]: + """The slots a scenario has to fill. + + Written ``{{ name }}``, so the prompt stays readable as prose and a missing value is caught + before a call is placed rather than appearing verbatim in what the simulated caller says. + """ + import re + + return set(re.findall(r"\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}", prompt)) + + +def fill(prompt: str, values: dict[str, Any]) -> tuple[str, list[str]]: + """The simulator prompt for one scenario, and anything it left unfilled.""" + import re + + missing = sorted(variables_in(prompt) - set(values)) + + def swap(match: re.Match[str]) -> str: + return str(values.get(match.group(1), match.group(0))) + + filled = re.sub(r"\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}", swap, prompt) + return filled, missing + + +def validate_simulator_prompt(prompt: str) -> list[str]: + """Problems that make a simulator prompt unusable. + + Deliberately thin. What a good simulator prompt says is judgement, and belongs in the skill; + what can be checked here is that it exists and that a scenario has somewhere to put its + instruction, since a prompt with no variables is the same prompt for every scenario. + """ + problems: list[str] = [] + if len(prompt.strip()) < 80: + problems.append("too short to be a simulator prompt") + if not variables_in(prompt): + problems.append( + "no variables: without a slot for the scenario's instruction, every scenario would " + "run the same conversation. Write them as {{ instruction }}" + ) + return problems diff --git a/src/fi/alk/harness/prove.py b/src/fi/alk/harness/prove.py new file mode 100644 index 0000000..cd68976 --- /dev/null +++ b/src/fi/alk/harness/prove.py @@ -0,0 +1,146 @@ +"""Proving a scenario is worth keeping, before anything is ever run against the agent. + +Two gates, both pure code. No model is asked whether a scenario is good; the environment decides. + +**Solvable.** Reset the world, apply the scenario's own setup, run its reference solution, run +its checks. They must pass. If they do not, either the scenario cannot be passed at all or its +checks are wrong, and both have happened here: one scenario asserted a value the agent was never +permitted to send; another demanded confirmation of an item that could not be ordered. Neither +was noticed until a live run failed and read as a finding about the agent. + +**Not vacuous.** Reset, apply the setup, run *nothing*, run the checks. They must fail. A check +that passes with no actions taken grades nothing while reporting a result, which is how a suite +goes quietly green. + +Terminal-bench keeps its tasks honest this way, and it is the cheapest useful thing in the whole +harness: no tokens, no network, a few milliseconds. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path + +from .checks import Outcome, run_check +from .environment import Catalogue +from .scenario import Scenario +from .world.runtime import Call, GeneratedWorld +from .world.snapshot import apply_overlay, restore + + +@dataclass +class Proof: + """Whether a scenario holds up, and what happened when it was tried.""" + + solvable: bool = False + vacuous: bool = True + with_solution: list[Outcome] = field(default_factory=list) + with_nothing: list[Outcome] = field(default_factory=list) + refused: list[str] = field(default_factory=list) + broken: list[str] = field(default_factory=list) + + @property + def holds(self) -> bool: + return self.solvable and not self.vacuous and not self.broken + + def why(self) -> str: + """What to fix, in the order worth fixing it.""" + if self.broken: + return "these checks are broken, not failing:\n - " + "\n - ".join( + self.broken + ) + if not self.solvable: + failed = [one for one in self.with_solution if not one.held] + said = "\n - ".join(f"{one.name}: {one.said}" for one in failed) + refusals = ( + "\n\nThe solution's own calls were refused by the world:\n - " + + "\n - ".join(self.refused) + if self.refused + else "" + ) + return ( + "the reference solution does not pass this scenario's own checks, so either the " + "scenario cannot be passed or the checks are wrong:\n - " + + said + + refusals + ) + if self.vacuous: + passed = [one.name for one in self.with_nothing if one.held] + return ( + "these checks pass without the agent doing anything, so they grade nothing:\n - " + + "\n - ".join(passed) + + "\n\nIf the point of this scenario is that nothing should happen, checking " + "the world alone cannot show it — an untouched world looks identical to one " + "where the agent did nothing at all. Check the calls instead: that the agent " + "tried, and that the attempt was refused rather than succeeding.\n" + " def check(world, calls):\n" + " tried = [c for c in calls if c.name == 'add']\n" + " if not tried: return 'never attempted it'\n" + " if any(c.ok for c in tried): return 'it succeeded'\n" + " return None" + ) + return "holds" + + +def _checks_for(scenario: Scenario, catalogue: Catalogue) -> list[tuple[str, str]]: + """The deterministic checks this scenario is graded by, in catalogue order.""" + chosen: list[tuple[str, str]] = [] + for name in scenario.sub_goals: + sub_goal = catalogue.named(name) + if sub_goal is not None and sub_goal.deterministic(): + chosen.append((name, sub_goal.check)) + return chosen + + +def _run( + scenario: Scenario, world_root: Path, *, with_solution: bool +) -> tuple[GeneratedWorld, list[Call], list[str]]: + """A fresh world with the scenario's setup, optionally with the solution played through it.""" + world = restore(world_root) + apply_overlay(world, scenario.setup) + world.reset() + refused: list[str] = [] + if with_solution: + for step in scenario.solution: + call = world.call(step.tool, step.arguments) + if not call.ok: + refused.append(f"{call.name}({step.arguments}): {call.error}") + return world, list(world.calls), refused + + +def prove(scenario: Scenario, catalogue: Catalogue, world_root: Path) -> Proof: + """Run both gates and say whether this scenario is worth keeping.""" + proof = Proof() + checks = _checks_for(scenario, catalogue) + if not checks: + proof.broken = [ + "none of this scenario's sub-goals has a check in code, so nothing here can be " + "settled without asking a model" + ] + return proof + + world, calls, refused = _run(scenario, world_root, with_solution=True) + try: + proof.with_solution = [ + run_check(source, world, calls, name=name) for name, source in checks + ] + finally: + world.close() + proof.refused = refused + proof.broken = [one.name for one in proof.with_solution if one.broken] + proof.solvable = all(one.held for one in proof.with_solution) and not proof.broken + + untouched, nothing, _ = _run(scenario, world_root, with_solution=False) + try: + proof.with_nothing = [ + run_check(source, untouched, nothing, name=name) for name, source in checks + ] + finally: + untouched.close() + # Vacuous only if *every* check still passes with nothing done. One check that survives an + # empty run is often legitimate — "no order was placed" is a real thing to assert about a + # refusal scenario — but a whole set of them means nothing is being graded. + proof.vacuous = bool(proof.with_nothing) and all( + one.held for one in proof.with_nothing + ) + return proof diff --git a/src/fi/alk/harness/reception.py b/src/fi/alk/harness/reception.py new file mode 100644 index 0000000..27b4c79 --- /dev/null +++ b/src/fi/alk/harness/reception.py @@ -0,0 +1,140 @@ +"""Stage zero: working out which agent you mean. + +Everything the harness does is about one agent, so something has to establish which one. That +used to be two flags on a command line, which is the wrong place for it: the whole point is that +you say what you want and it happens, and "here is my agent, set up a test environment for it" +is a sentence, not an invocation. + +So this is a stage like any other. It can look around the filesystem to find what you are +pointing at, it asks if what you said is ambiguous, and it finishes by naming the agent and where +it lives. Everything after it, including where artifacts are written, follows from that. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Callable + +from claude_agent_sdk import ClaudeAgentOptions, create_sdk_mcp_server, tool + +from .config import chosen_model, gate_hooks, permission_gate, provider_env +from .session import Stage +from .sources import AgentSource, resolve, supported +from .tools import qualified, schema + +RECEPTION_SERVER = "agent" +TOOL_NAMES = ("point_at_agent",) + +_INSTRUCTIONS = """ +You are the front desk of a harness that builds test environments for agents. + +Somebody has arrived with an agent they want tested. Your only job is to work out which agent, +and where it lives, and then call point_at_agent. Nothing else happens until you do. + +Usually they will just tell you: a path, a repository, a folder. Take it. Use Read, Glob and Grep +to check the path exists and to see what is actually there, and to pick a sensible short name if +they did not give one. A name is a label for their artifacts, so lower case and no spaces. + +The agent is usually somewhere else on disk, not inside the harness. A path they give you is +relative to where you are looking from, which is a workspace holding many repositories, so try +it as given before deciding it does not exist. + +If the path really is not there, say so and say what you did find near it. If they gestured +vaguely at a directory holding several agents, look, and ask which one with AskUserQuestion. + +Do not read the agent properly and do not start working anything out about it. That is the next +stage's job and it has its own instructions. Point at the agent, say in one line what you are +about to do, and stop. +""" + + +def point_at( + name: str, path: str, kind: str, found: dict[str, AgentSource] +) -> dict[str, Any]: + """Establish which agent this is, or say why it cannot be. + + A plain function rather than only a tool body, so what counts as a reachable agent can be + exercised without standing up a session. + """ + name, path, kind = name.strip(), path.strip(), (kind.strip() or "repo") + if not name: + return _err("no name: the artifacts have to be filed under something") + if kind not in supported(): + return _err(f"no such kind {kind!r}; there is {', '.join(supported())}") + if kind == "repo" and not Path(path).expanduser().exists(): + return _err( + f"there is nothing at {path!r}. Look again with Glob, and if you cannot find it, " + "ask where the agent actually lives." + ) + try: + found["source"] = resolve(kind, name=name, root=Path(path).expanduser()) + except Exception as failed: + return _err(f"could not reach that agent: {failed}") + return { + "content": [{"type": "text", "text": f"Pointed at {name} ({kind}) at {path}."}] + } + + +def open_stage( + *, + cwd: str | Path | None = None, + ask: Callable[..., Any] | None = None, + max_turns: int = 20, +) -> tuple[Stage, dict[str, AgentSource]]: + """A stage that establishes which agent this conversation is about.""" + found: dict[str, AgentSource] = {} + + @tool( + "point_at_agent", + "Name the agent this conversation is about and say where it is. `kind` is how it is " + f"supplied, one of: {', '.join(supported())}. For a repository, `path` is its directory. " + "Call this once you know what you are pointing at.", + schema({"name": str, "path": str, "kind": str}, ["name", "path"]), + ) + async def point_at_agent(args: dict[str, Any]) -> dict[str, Any]: + return point_at( + str(args.get("name") or ""), + str(args.get("path") or ""), + str(args.get("kind") or "repo"), + found, + ) + + server = create_sdk_mcp_server( + name=RECEPTION_SERVER, version="0.1.0", tools=[point_at_agent] + ) + allowed = [ + "Read", + "Glob", + "Grep", + "AskUserQuestion", + *(qualified(RECEPTION_SERVER, name) for name in TOOL_NAMES), + ] + options = ClaudeAgentOptions( + system_prompt=_INSTRUCTIONS.strip(), + allowed_tools=allowed, + mcp_servers={RECEPTION_SERVER: server}, + # Not acceptEdits: that auto-approves Edit and Write before the permission callback is + # consulted, so a stage can rewrite an artifact by hand and skip the tool whose + # whole job is to validate that change. + permission_mode="default", + cwd=str(cwd or Path.cwd()), + setting_sources=[], + max_turns=max_turns, + model=chosen_model(), + env=provider_env(), + ) + options.hooks = gate_hooks(allowed) + options.can_use_tool = permission_gate(ask, allowed) + return Stage(options, name="reception"), found + + +def opening() -> str: + return ( + "Somebody has just opened the harness and has not said anything yet. Greet them in one " + "short line and ask which agent they want tested and where it lives. Do not list your " + "capabilities." + ) + + +def _err(text: str) -> dict[str, Any]: + return {"content": [{"type": "text", "text": text}], "is_error": True} diff --git a/src/fi/alk/harness/run/__init__.py b/src/fi/alk/harness/run/__init__.py new file mode 100644 index 0000000..e36cbf0 --- /dev/null +++ b/src/fi/alk/harness/run/__init__.py @@ -0,0 +1,208 @@ +"""Stage four: run the scenarios against the world and say what happened. + +Every scenario gets its own world. It is restored from the frozen snapshot, the scenario's own +rows are laid on top, and it is thrown away afterwards. Nothing a scenario does can reach the +next one, which is what makes a result mean something on its own and makes the whole suite +repeatable a week later. + +The shape is the same regardless of what is being tested: restore, converse, grade against the +state that is left behind. Where the agent actually runs is a target, so the same scenarios grade +a hosted agent without any of this changing. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Callable, Sequence + +from ..contract import AgentContract +from ..environment import load_catalogue, load_simulator_prompt +from ..scenario import Scenario +from ..world.snapshot import apply_overlay, restore +from .conversation import FINISHED, Exchange, Transcript, converse +from .grade import ( + Checkpoint, + Result, + as_json, + checkpoints, + grade_sub_goals, + judge, + summarise, +) +from .targets import LocalAgent, Target, register_target, resolve, supported + + +def from_alk(report: Any, world, spent: float) -> Transcript: + """What ALK's report says happened, in the shape the grading already reads.""" + exchanges: list[Exchange] = [] + for case in getattr(report, "results", None) or []: + for message in getattr(case, "transcript", None) or []: + role = (message.get("role") or "") if isinstance(message, dict) else "" + text = (message.get("content") or "") if isinstance(message, dict) else "" + if text: + exchanges.append( + Exchange("agent" if role == "assistant" else "customer", str(text)) + ) + return Transcript( + exchanges=exchanges, + calls=list(world.calls), + ended=FINISHED, + spent_usd=spent, + ) + + +RUNS = "runs.json" +REPORT = "report.txt" + +__all__ = [ + "Checkpoint", + "Exchange", + "LocalAgent", + "Result", + "Target", + "Transcript", + "converse", + "register_target", + "run_scenario", + "run_suite", + "supported", + "summarise", +] + + +async def run_scenario( + scenario: Scenario, + contract: AgentContract, + world_root: Path, + *, + target: str = "local", + model: str | None = None, + through_alk: bool = False, + on_exchange: Callable[[Exchange], Any] | None = None, +) -> Result: + """Run one scenario in its own copy of the world and grade what it left behind.""" + catalogue = load_catalogue(world_root) + world = restore(world_root) + try: + apply_overlay(world, scenario.setup) + # reset() is how an environment is started in ALK: it clears the call log and publishes + # the tools and the starting state. Going through it keeps a generated world drivable by + # anything that already drives an environment. + world.reset() + if through_alk: + # ALK owns the simulation and drives the world through EnvironmentAdapter; the + # harness only grades what it is left with. Nothing here is modality-specific, + # which is the point: the browser and voice runners take the same adapter. + from .alk import simulate + + report, spent = await simulate( + scenario, + contract, + world, + model=model, + simulator_prompt=load_simulator_prompt(world_root), + ) + transcript = from_alk(report, world, spent) + for exchange in transcript.exchanges: + if on_exchange: + on_exchange(exchange) + else: + agent = resolve(target)(contract, world, model=model) + transcript = await converse( + agent, + scenario, + contract, + world_root=world_root, + model=model, + on_exchange=on_exchange, + ) + # Settled by code first. The judge is only handed the sub-goals whose catalogue entry + # says nothing observable decides them. + settled = grade_sub_goals(world, scenario, catalogue, transcript.calls) + ending = ", ".join( + f"{name}: {len(rows)} rows" + for name, rows in sorted(world.observe().state.items()) + ) + judgements, judged_cost = await judge( + scenario, transcript, contract, catalogue, model=model, ending=ending + ) + return Result( + scenario=scenario.name, + tests=scenario.tests, + state_failures=[ + f"{one.name}: {one.said}" for one in settled if not one.held + ], + conduct=judgements, + checkpoints=checkpoints(settled, judgements), + crashes=[f"{call.name}: {call.error}" for call in transcript.crashed()], + ended=transcript.ended, + turns=len(transcript.exchanges), + calls=len(transcript.calls), + spent_usd=transcript.spent_usd + judged_cost, + transcript=transcript.spoken(), + actions=transcript.actions(), + ) + finally: + world.close() + + +async def run_suite( + scenarios: Sequence[Scenario], + contract: AgentContract, + world_root: Path, + *, + target: str = "local", + model: str | None = None, + through_alk: bool = False, + out: Path | None = None, + on_result: Callable[[Result], Any] | None = None, + on_exchange: Callable[[Exchange], Any] | None = None, +) -> list[Result]: + """Run every scenario and write the results out. One failing scenario never stops the rest.""" + destination = Path(out or world_root) + results: list[Result] = [] + for scenario in scenarios: + try: + result = await run_scenario( + scenario, + contract, + world_root, + target=target, + model=model, + through_alk=through_alk, + on_exchange=on_exchange, + ) + except Exception as failed: + # A scenario that could not be run is recorded as unrunnable rather than as a + # failure of the agent, and the rest of the suite still runs. + result = Result( + scenario=scenario.name, + tests=scenario.tests, + crashes=[f"could not run: {type(failed).__name__}: {failed}"], + ended="not-run", + ) + results.append(result) + if on_result: + on_result(result) + + destination.mkdir(parents=True, exist_ok=True) + # Records for scenarios this suite did not run are kept, not clobbered. A live call and a + # local run write to the same file, and re-running two scenarios must not erase the third. + ran = {result.scenario for result in results} + kept = [ + record + for record in load_results(destination) + if isinstance(record, dict) and record.get("scenario") not in ran + ] + merged = kept + json.loads(as_json(results)) + (destination / RUNS).write_text( + json.dumps(merged, indent=2, ensure_ascii=False), encoding="utf-8" + ) + (destination / REPORT).write_text(summarise(results), encoding="utf-8") + return results + + +def load_results(destination: Path) -> list[dict[str, Any]]: + path = Path(destination) / RUNS + return json.loads(path.read_text(encoding="utf-8")) if path.exists() else [] diff --git a/src/fi/alk/harness/run/alk.py b/src/fi/alk/harness/run/alk.py new file mode 100644 index 0000000..22847ed --- /dev/null +++ b/src/fi/alk/harness/run/alk.py @@ -0,0 +1,172 @@ +"""Running a generated world through ALK's own simulation, rather than beside it. + +The whole reason a generated world subclasses ``EnvironmentAdapter`` is so the runners that +already exist can drive it. ``ChatEnvironment`` takes ``environment=`` and owns the +synthetic user, the turn loop, the transcript and the report; the browser and voice paths take +the same adapter. A second loop written here would work for exactly one modality and would have +to be rewritten for the next one, which is the thing this design exists to avoid. + +So the split is: + +- **ALK** drives the simulation: who the customer is, when they speak, when it ends. +- **The world** answers every tool call, through ``handle_tool_call``. +- **The harness** grades afterwards, from the state the world is left in plus the transcript. + +What is written here is only the two adapters between the shapes: a scenario becomes an ALK +``Persona``, and the agent under test becomes an ``AgentWrapper``. +""" + +from __future__ import annotations + +from typing import Any + +from fi.simulate import Persona, Scenario as AlkScenario +from fi.simulate.agent.wrapper import AgentInput, AgentResponse, AgentWrapper +from fi.simulate.environments.chat import ChatEnvironment + +from ..contract import AgentContract +from ..scenario import Scenario +from ..world.runtime import GeneratedWorld +from .targets import LocalAgent + + +def as_persona(scenario: Scenario, simulator_prompt: str = "") -> Persona: + """One of our scenarios, in the shape ALK's simulation consumes. + + ``situation`` is the simulator prompt the harness wrote for this agent with the scenario's + values filled in. ALK wraps it in its own voice-execution rules, so what goes here is only + what changes per scenario, not a second set of instructions about how to behave on a call. + + There is no persona payload beyond a label. Who the caller is does not vary between + scenarios; what varies is what they want and what they know. + """ + from ..environment import fill + + filled = fill(simulator_prompt, scenario.slots())[0] if simulator_prompt else "" + return Persona( + persona={"name": "customer"}, + situation=filled or scenario.instruction, + outcome=scenario.tests + or "complete what you came for, or accept that you cannot", + ) + + +def as_alk_scenario( + scenarios: list[Scenario], name: str = "harness", simulator_prompt: str = "" +) -> AlkScenario: + return AlkScenario( + name=name, + description="generated by the harness", + dataset=[as_persona(one, simulator_prompt) for one in scenarios], + ) + + +def _spoken(input: AgentInput) -> str: + """What the customer just said, as text. + + ALK passes a message as a mapping, not a string, and hands the whole history alongside it. + Passing the mapping straight to a session that expects text fails inside the SDK with a + redacted TypeError, which says nothing about where it came from. + """ + latest = input.new_message + if isinstance(latest, dict): + content = latest.get("content") + if isinstance(content, list): + content = " ".join( + part.get("text", "") for part in content if isinstance(part, dict) + ) + if content: + return str(content) + if isinstance(latest, str) and latest: + return latest + for message in reversed(input.messages or []): + if isinstance(message, dict) and message.get("role") != "assistant": + content = message.get("content") + if content: + return str(content) + return "(the customer said nothing)" + + +class ContractAgent(AgentWrapper): + """The agent under test, in the shape ALK drives agents by. + + It holds the same session ``LocalAgent`` uses, so the agent being graded is identical either + way; what changes is who runs the conversation around it. The tool calls it made are reported + back to ALK so they appear in the transcript, having already gone through the world. + """ + + def __init__( + self, + contract: AgentContract, + world: GeneratedWorld, + *, + model: str | None = None, + ) -> None: + self.agent = LocalAgent(contract, world, model=model) + self.world = world + self._open = False + + async def call(self, input: AgentInput) -> AgentResponse: + if not self._open: + await self.agent.open() + self._open = True + + before = len(self.world.calls) + said = await self.agent.say(_spoken(input)) + made = self.world.calls[before:] + + return AgentResponse( + content=said, + tool_calls=[ + {"name": call.name, "arguments": call.arguments} for call in made + ], + tool_responses=[ + { + "name": call.name, + "content": call.error if not call.ok else str(call.result), + "success": call.ok, + } + for call in made + ], + ) + + async def aclose(self) -> None: + if self._open: + await self.agent.close() + self._open = False + + @property + def spent_usd(self) -> float: + return self.agent.spent_usd + + +async def simulate( + scenario: Scenario, + contract: AgentContract, + world: GeneratedWorld, + *, + model: str | None = None, + simulator_prompt: str = "", +) -> tuple[Any, float]: + """Run one scenario through ALK's chat simulation against this world. + + ``auto_execute_tools`` is off because the agent's tools are bound to the world already and + have run by the time it answers. Turning it on would execute every call a second time, which + for a world that really writes rows means every order placed twice. + """ + agent = ContractAgent(contract, world, model=model) + try: + report = await ChatEnvironment().run( + scenario=as_alk_scenario( + [scenario], name=scenario.name, simulator_prompt=simulator_prompt + ), + agent_callback=agent, + environment=world, + auto_execute_tools=False, + max_turns=max(2, scenario.max_turns), + min_turns=2, + modality=contract.modality or "text", + ) + finally: + await agent.aclose() + return report, agent.spent_usd diff --git a/src/fi/alk/harness/run/call.py b/src/fi/alk/harness/run/call.py new file mode 100644 index 0000000..70e69e9 --- /dev/null +++ b/src/fi/alk/harness/run/call.py @@ -0,0 +1,102 @@ +"""One scenario, against the real hosted agent, end to end. + +Everything the harness built is wired together here and then ALK's own voice case places the +call. The harness does not reimplement any of that: it supplies the world the agent's tools act +on, the caller's instruction, and the grading afterwards. + + world + setup ──► webhook ──► public url ──► assistant's own tools repointed + │ + ALK's voice case places the call ──┘ + │ + the world afterwards + the calls ──► sub-goal checks + +Run it: + + set -a; . ./.env.acceptance; set +a + .venv/bin/python -m fi.alk.harness.run.call --name drive_thru --scenario orders_a_big_mac +""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +from pathlib import Path + +from ..config import artifact_dir +from ..scenario_tools import load_scenarios +from .live import grade, wire + +CASE = os.environ.get("HARNESS_VOICE_CASE", "2.1.2") + + +def place_the_call(case: str, dry_run: bool = False) -> int: + """Hand over to ALK's voice case, which owns everything about placing a call.""" + runner = Path("oss/simulation-acceptance/run_voice_case.py") + if not runner.exists(): + raise RuntimeError(f"no voice runner at {runner}; run from the repo root") + command = [sys.executable, str(runner), case] + (["--dry-run"] if dry_run else []) + return subprocess.call(command) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="fi.alk.harness.run.call", description=__doc__) + parser.add_argument("--name", required=True, help="which agent") + parser.add_argument("--scenario", required=True, help="which scenario, by name") + parser.add_argument("--case", default=CASE, help="ALK voice case id") + parser.add_argument( + "--dry-run", action="store_true", help="wire everything up but do not place the call" + ) + args = parser.parse_args(argv) + + root = artifact_dir(args.name) + written = load_scenarios(root) + scenario = next((one for one in written if one.name == args.scenario), None) + if scenario is None: + print( + f"no scenario called {args.scenario!r}. There is: " + + ", ".join(one.name for one in written), + file=sys.stderr, + ) + return 1 + + world, instruction, webhook, tunnel, url, moved = wire(scenario, root) + try: + print(f"agent: {args.name}") + print(f"scenario: {scenario.name}") + print(f"webhook: {url}/tool") + print(f"repointed: {', '.join(moved)}") + print(f"sub-goals: {', '.join(scenario.sub_goals)}\n") + + # The caller's instruction reaches the voice case through the environment, so nothing + # about how a simulated caller behaves is decided twice. + os.environ["HARNESS_INSTRUCTION"] = instruction + os.environ["HARNESS_SCENARIO"] = scenario.name + os.environ["HARNESS_OUTCOME"] = scenario.tests + + code = place_the_call(args.case, dry_run=args.dry_run) + if args.dry_run: + print("\ndry run: nothing was called, and the world is untouched.") + return code + + result = grade(scenario, world, root) + print() + print(result.line()) + for one in result.settled: + print(one.line()) + for name in result.judged: + print(f" [?] {name} — judged, not graded here") + print("\nwhat the agent actually did:") + for call in result.calls or ["(no tool calls reached the world)"]: + print(f" {call}") + return 0 if result.settled and result.met == len(result.settled) else 2 + finally: + webhook.stop() + if tunnel is not None: + tunnel.terminate() + world.close() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/fi/alk/harness/run/conversation.py b/src/fi/alk/harness/run/conversation.py new file mode 100644 index 0000000..e3b54f3 --- /dev/null +++ b/src/fi/alk/harness/run/conversation.py @@ -0,0 +1,170 @@ +"""Two parties talking: a simulated customer with a goal, and the agent under test. + +The customer is a separate session that can only talk. It has no tools and no view of the world, +which is the point: it knows what it wants and how it behaves, and everything it learns about +what is possible it learns from what the agent tells it. An agent that lies to it gets away with +it here exactly as it would with a person, and that is what makes the transcript worth grading. + +The conversation ends when the customer is done, when it gives up, or when it runs out of turns. +All three are recorded, because how a conversation ended is often the finding. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Callable + +from claude_agent_sdk import ClaudeAgentOptions + +from ..config import chosen_model, provider_env +from ..contract import AgentContract +from ..scenario import Scenario +from ..session import Stage +from ..world.runtime import Call +from .targets import Target + +DONE = "[DONE]" +STUCK = "[STUCK]" + +FINISHED = "finished" +GAVE_UP = "gave-up" +RAN_OUT = "ran-out-of-turns" + + +@dataclass +class Exchange: + speaker: str + text: str + + +@dataclass +class Transcript: + """What happened, in the two forms grading needs: what was said and what was done.""" + + exchanges: list[Exchange] = field(default_factory=list) + calls: list[Call] = field(default_factory=list) + ended: str = "" + spent_usd: float = 0.0 + + def spoken(self) -> str: + return "\n".join(f"{turn.speaker}: {turn.text}" for turn in self.exchanges) + + def actions(self) -> str: + if not self.calls: + return "(the agent called no tools at all)" + lines = [] + for call in self.calls: + outcome = ( + "refused" if call.refused else ("crashed" if not call.ok else "ok") + ) + lines.append( + f"{call.name}({call.arguments}) -> {outcome}: {call.error or call.result}" + ) + return "\n".join(lines) + + def crashed(self) -> list[Call]: + """Calls that failed for our reasons rather than the world's. + + Reported separately and never counted against the agent. A run over a world that fell + over says nothing about the agent, and scoring it as a failure is how a harness invents + findings. + """ + return [call for call in self.calls if not call.ok and not call.refused] + + +def customer_prompt( + scenario: Scenario, contract: AgentContract, written: str = "" +) -> str: + """The simulated person, from the prompt the harness wrote for this agent. + + The prompt belongs to the environment, not to this loop: it is written once for the agent and + each scenario fills its slots. Only the ending convention is added here, because it is how + this particular loop knows a conversation is over. + """ + from ..environment import fill + + if written: + filled, _missing = fill(written, scenario.slots()) + else: + # No simulator prompt was written, which the environment gate refuses for a + # conversational agent. Kept minimal rather than inventing a character. + filled = ( + f"You are contacting {contract.agent}, which is: {contract.one_liner}\n\n" + f"WHAT YOU ARE HERE TO DO:\n{scenario.instruction}" + ) + return ( + filled + + f"\n\nWhen you have got what you came for, or accepted that you cannot, reply with " + f"{DONE} and nothing else. If the agent is going in circles and you would give up, " + f"reply {STUCK} and nothing else." + ) + + +async def converse( + target: Target, + scenario: Scenario, + contract: AgentContract, + *, + world_root: Any = None, + model: str | None = None, + on_exchange: Callable[[Exchange], Any] | None = None, +) -> Transcript: + """Run one scenario as a conversation and return what happened.""" + transcript = Transcript() + from ..environment import load_simulator_prompt + + customer = Stage( + ClaudeAgentOptions( + system_prompt=customer_prompt( + scenario, + contract, + load_simulator_prompt(world_root) if world_root else "", + ), + allowed_tools=[], + setting_sources=[], + max_turns=1, + model=chosen_model(model), + env=provider_env(model), + ), + name="customer", + ) + + def record(speaker: str, text: str) -> None: + exchange = Exchange(speaker, text) + transcript.exchanges.append(exchange) + if on_exchange: + on_exchange(exchange) + + await target.open() + await customer.__aenter__() + try: + # The customer opens, in its own words. The scenario's instruction is written *about* + # the caller ("orders two burgers and asks for..."), so speaking it verbatim would hand + # the agent a stage direction instead of a person. + opening = await customer.say( + "The conversation is starting. Say your opening line, and nothing else." + ) + said = opening.text.strip() or scenario.instruction + record("customer", said) + for _turn in range(max(1, scenario.max_turns)): + reply = await target.say(said) + record("agent", reply or "(said nothing)") + + turn = await customer.say(reply or "(no response)") + said = turn.text.strip() + if DONE in said: + transcript.ended = FINISHED + break + if STUCK in said: + transcript.ended = GAVE_UP + break + record("customer", said) + else: + transcript.ended = RAN_OUT + finally: + await customer.__aexit__(None, None, None) + await target.close() + + transcript.calls = list(target.world.calls) if hasattr(target, "world") else [] + transcript.spent_usd = target.spent_usd + customer.spent_usd + return transcript diff --git a/src/fi/alk/harness/run/grade.py b/src/fi/alk/harness/run/grade.py new file mode 100644 index 0000000..7db5a50 --- /dev/null +++ b/src/fi/alk/harness/run/grade.py @@ -0,0 +1,345 @@ +"""Deciding whether a run passed, in two parts that are never mixed. + +**State** is settled by looking at the database. The order exists or it does not, and no amount of +fluent conversation changes the answer. This is the half worth trusting, and it is checked with +the same code the build stage uses to check its own sequences, so a suite cannot pass its gate +and then be graded by a different rule. + +**Conduct** is what the agent said and what it refused, which needs judgement, so it is judged. +Kept separate and reported separately, so nobody reads a pass as meaning the data is right when +what was actually established is that an opinion was favourable. + +The judge is given the tool calls as well as the transcript, because the failure most worth +catching is an agent that says it did something it never did. Reading only the words makes that +failure invisible; reading both makes it obvious. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from typing import Any + +from claude_agent_sdk import ClaudeAgentOptions, create_sdk_mcp_server, tool + +from ..config import chosen_model, gate_hooks, permission_gate, provider_env +from ..contract import AgentContract +from ..scenario import Scenario +from ..session import Stage +from ..tools import qualified +from ..checks import Outcome, run_check +from ..environment import Catalogue +from ..world.runtime import GeneratedWorld +from .conversation import Transcript + +JUDGE_SERVER = "verdict" + + +@dataclass +class Checkpoint: + """One thing that had to be true, and whether it was. + + Every expectation is named and reported whether it held or not. Reporting only the failures + answers "did it pass" but never "how much of this did it get right", and a scenario that + settles eight things and misses one is a different result from one that misses everything. + """ + + name: str + kind: str + passed: bool + detail: str = "" + + def line(self) -> str: + return f" [{'x' if self.passed else ' '}] {self.kind}: {self.name}" + ( + f"\n {self.detail}" if self.detail and not self.passed else "" + ) + + +@dataclass +class Judgement: + claim: str + kind: str + holds: bool + why: str = "" + + +@dataclass +class Result: + scenario: str + tests: str = "" + state_failures: list[str] = field(default_factory=list) + conduct: list[Judgement] = field(default_factory=list) + crashes: list[str] = field(default_factory=list) + checkpoints: list[Checkpoint] = field(default_factory=list) + ended: str = "" + turns: int = 0 + calls: int = 0 + spent_usd: float = 0.0 + transcript: str = "" + # Kept alongside the transcript because a run is diagnosed by comparing them: what the + # agent said it did against what it actually did. + actions: str = "" + + @property + def conduct_failures(self) -> list[Judgement]: + return [item for item in self.conduct if not item.holds] + + @property + def passed(self) -> bool: + return ( + not self.state_failures and not self.conduct_failures and not self.crashes + ) + + @property + def met(self) -> int: + return sum(1 for check in self.checkpoints if check.passed) + + def line(self) -> str: + mark = "PASS" if self.passed else "FAIL" + if self.crashes: + mark = "VOID" + scored = ( + f"{self.met}/{len(self.checkpoints)} checkpoints" + if self.checkpoints + else "nothing checked" + ) + return ( + f"{mark} {self.scenario} {scored} " + f"({self.turns} turns, {self.calls} calls, {self.ended})" + ) + + +def _claims(scenario: Scenario, catalogue: Catalogue) -> list[tuple[str, str]]: + """The sub-goals of this scenario that nothing observable can settle.""" + judged: list[tuple[str, str]] = [] + for name in scenario.sub_goals: + sub_goal = catalogue.named(name) + if sub_goal is not None and not sub_goal.deterministic(): + judged.append((sub_goal.judged or sub_goal.what, name)) + return judged + + +def _judge_prompt(contract: AgentContract) -> str: + return ( + "You are grading one run of an agent under test. You are given three kinds of evidence: " + "what was said, the actions the agent actually took, and the state of its world " + "afterwards.\n\n" + "Each claim is one sub-goal of the run, named in brackets, that nothing observable could " + "settle. Judge each strictly and independently, and only from the evidence in front of " + "you. A claim holds only if the evidence actually shows it; something merely not " + "contradicted does not hold. Where a claim is that something must not have happened, it " + "holds when the thing did not happen.\n\n" + "Three rules that decide most of these:\n" + " - The actions are the truth about what happened. An agent that claims it did " + "something no action performed has not done it, however convincing it sounds.\n" + " - A refused action did not happen. Trying something and being told no is how an " + "agent finds out what is possible, so judge what it ended up doing, not what it " + "attempted on the way there.\n" + " - Declining something holds only if the agent both declined it and gave a true " + "reason. Refusing while inventing a reason is not a pass.\n\n" + f"THE AGENT UNDER TEST: {contract.agent} - {contract.one_liner}\n" + + ( + "ITS RULES:\n - " + "\n - ".join(contract.hard_constraints[:14]) + if contract.hard_constraints + else "" + ) + + "\n\nCall submit_verdict once, with one entry per claim, in the order given." + ) + + +def _verdict_tool(collected: list[dict[str, Any]]) -> Any: + @tool( + "submit_verdict", + "Your judgement. `items` is a list of {claim, holds, why}, one per claim, in the order " + "you were given them. `why` is one sentence citing what in the transcript or the calls " + "decided it.", + {"items": list}, + ) + async def submit_verdict(args: dict[str, Any]) -> dict[str, Any]: + collected[:] = [ + item for item in (args.get("items") or []) if isinstance(item, dict) + ] + return { + "content": [ + {"type": "text", "text": f"recorded {len(collected)} judgements"} + ] + } + + return create_sdk_mcp_server( + name=JUDGE_SERVER, version="0.1.0", tools=[submit_verdict] + ) + + +async def judge( + scenario: Scenario, + transcript: Transcript, + contract: AgentContract, + catalogue: Catalogue, + *, + model: str | None = None, + ending: str = "", +) -> tuple[list[Judgement], float]: + """Judge only the sub-goals nothing observable settles.""" + claims = _claims(scenario, catalogue) + if not claims: + return [], 0.0 + + collected: list[dict[str, Any]] = [] + allowed = [qualified(JUDGE_SERVER, "submit_verdict")] + options = ClaudeAgentOptions( + system_prompt=_judge_prompt(contract), + allowed_tools=allowed, + mcp_servers={JUDGE_SERVER: _verdict_tool(collected)}, + # Not acceptEdits: that auto-approves Edit and Write before the permission callback is + # consulted, so a session can rewrite an artifact by hand and skip the tool whose whole + # job is to validate that change. + permission_mode="default", + setting_sources=[], + max_turns=6, + model=chosen_model(model), + env=provider_env(model), + ) + options.hooks = gate_hooks(allowed) + options.can_use_tool = permission_gate(granted=allowed) + stage = Stage(options, name="judge") + listed = "\n".join( + f"{index + 1}. [{kind}] {claim}" for index, (claim, kind) in enumerate(claims) + ) + async with stage: + await stage.say( + f"WHAT WAS SAID:\n{transcript.spoken() or '(nothing was said)'}\n\n" + f"WHAT THE AGENT ACTUALLY DID:\n{transcript.actions()}\n\n" + f"THE WORLD AFTERWARDS:\n{ending or '(nothing recorded)'}\n\n" + f"CLAIMS TO JUDGE:\n{listed}" + ) + + return to_judgements(claims, collected), stage.spent_usd + + +def to_judgements( + claims: list[tuple[str, str]], collected: list[dict[str, Any]] +) -> list[Judgement]: + """Line the judge's answers up with the claims, and fail anything it did not answer. + + An unjudged claim is a failure, not a pass. A judge that returned nothing, or fewer answers + than there were claims, is exactly the case where a suite would otherwise report a clean + sweep it never earned. + """ + judgements: list[Judgement] = [] + for index, (claim, kind) in enumerate(claims): + found = collected[index] if index < len(collected) else None + judgements.append( + Judgement( + claim=claim, + kind=kind, + holds=bool(found.get("holds")) if found else False, + why=str( + (found or {}).get("why") or "" + if found + else "the judge did not answer this claim" + ), + ) + ) + return judgements + + +def grade_sub_goals( + world: GeneratedWorld, scenario: Scenario, catalogue: Catalogue, calls: list[Any] +) -> list[Outcome]: + """Every sub-goal settled by code, run against what this run left behind.""" + outcomes: list[Outcome] = [] + for name in scenario.sub_goals: + sub_goal = catalogue.named(name) + if sub_goal is None or not sub_goal.deterministic(): + continue + outcomes.append(run_check(sub_goal.check, world, calls, name=name)) + return outcomes + + +def checkpoints(settled: list[Outcome], judged: list[Judgement]) -> list[Checkpoint]: + """Every sub-goal of this scenario, one at a time, and whether each held. + + Named by the shared catalogue entry rather than restated, so the same sub-goal failing across + a suite can be counted. + """ + checks = [ + Checkpoint( + name=one.name, + kind="broken" if one.broken else "code", + passed=one.held, + detail=one.said, + ) + for one in settled + ] + checks.extend( + Checkpoint(name=item.kind, kind="judged", passed=item.holds, detail=item.why) + for item in judged + ) + return checks + + +def summarise(results: list[Result]) -> str: + passed = [result for result in results if result.passed] + void = [result for result in results if result.crashes] + lines = [ + f"{len(passed)}/{len(results)} scenarios passed" + + (f", {len(void)} void (the world crashed)" if void else ""), + "", + ] + for result in results: + lines.append(result.line()) + lines.extend(check.line() for check in result.checkpoints) + failing = [result for result in results if not result.passed] + if failing: + lines.append("") + for result in failing: + lines.append(f"{result.scenario}:") + for failure in result.state_failures: + lines.append(f" state: {failure}") + for item in result.conduct_failures: + lines.append(f" {item.kind}: {item.claim}\n {item.why}") + for crash in result.crashes: + lines.append(f" the world crashed: {crash}") + return "\n".join(lines) + + +def as_json(results: list[Result]) -> str: + return json.dumps( + [ + { + "scenario": result.scenario, + "tests": result.tests, + "passed": result.passed, + "ended": result.ended, + "turns": result.turns, + "calls": result.calls, + "spent_usd": round(result.spent_usd, 4), + "checkpoints_met": f"{result.met}/{len(result.checkpoints)}", + "checkpoints": [ + { + "name": check.name, + "kind": check.kind, + "passed": check.passed, + "detail": check.detail, + } + for check in result.checkpoints + ], + "state_failures": result.state_failures, + "crashes": result.crashes, + "conduct": [ + { + "claim": item.claim, + "kind": item.kind, + "holds": item.holds, + "why": item.why, + } + for item in result.conduct + ], + "transcript": result.transcript, + "actions": result.actions, + } + for result in results + ], + indent=2, + ensure_ascii=False, + ) diff --git a/src/fi/alk/harness/run/live.py b/src/fi/alk/harness/run/live.py new file mode 100644 index 0000000..41a0cf4 --- /dev/null +++ b/src/fi/alk/harness/run/live.py @@ -0,0 +1,153 @@ +"""One scenario, against the real hosted agent, in the environment the harness built. + +The harness wires the whole thing rather than leaving it to be assembled by hand: + +1. restore the world and apply the scenario's setup +2. stand the webhook up and bind that world to it +3. expose it publicly, because a hosted agent has to reach it +4. point the assistant's **own** tools at that address — nothing about the agent is redefined +5. run ALK's voice case with the scenario's instruction driving the simulated caller +6. grade from the world afterwards and the calls the webhook recorded + +Steps 1, 2, 4 and 6 are the whole difference from what existed before: the agent's tool calls now +land in a database that can refuse, instead of in canned responses that always succeed. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import time +from dataclasses import dataclass, field +from pathlib import Path + +from ..checks import Outcome, run_check +from ..environment import fill, load_catalogue, load_simulator_prompt +from ..scenario import Scenario +from ..world.runtime import GeneratedWorld +from ..world.snapshot import apply_overlay, restore +from .voice import WorldWebhook, repoint_assistant + + +@dataclass +class LiveRun: + """What a live call left behind.""" + + scenario: str + settled: list[Outcome] = field(default_factory=list) + judged: list[str] = field(default_factory=list) + calls: list[str] = field(default_factory=list) + ended: str = "" + problems: list[str] = field(default_factory=list) + + @property + def met(self) -> int: + return sum(1 for one in self.settled if one.held) + + def line(self) -> str: + mark = "PASS" if self.settled and self.met == len(self.settled) else "FAIL" + if self.problems: + mark = "VOID" + return f"{mark} {self.scenario} {self.met}/{len(self.settled)} sub-goals settled by code" + + +def public_url(port: int, *, wait: float = 20.0) -> tuple[str, subprocess.Popen | None]: + """A publicly reachable address for the webhook, and the process holding it open. + + A hosted agent runs on somebody else's infrastructure, so a loopback address is unreachable + to it. ``cloudflared`` is what the previous runs used; anything giving a public URL works, and + ``HARNESS_WEBHOOK_URL`` skips this entirely when a tunnel is already running. + """ + named = os.environ.get("HARNESS_WEBHOOK_URL", "").strip() + if named: + return named, None + if not shutil.which("cloudflared"): + raise RuntimeError( + "no way to expose the webhook publicly. Either install cloudflared " + "(brew install cloudflared) or set HARNESS_WEBHOOK_URL to a tunnel you already have." + ) + process = subprocess.Popen( + ["cloudflared", "tunnel", "--url", f"http://127.0.0.1:{port}"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + deadline = time.time() + wait + while time.time() < deadline: + line = process.stdout.readline() if process.stdout else "" + if "trycloudflare.com" in line: + for word in line.split(): + if word.startswith("https://") and "trycloudflare.com" in word: + return word.strip(), process + process.terminate() + raise RuntimeError("cloudflared did not report a public URL in time") + + +def prepare(scenario: Scenario, world_root: Path) -> tuple[GeneratedWorld, str]: + """The world this scenario runs in, and what the simulated caller is told. + + The instruction is the scenario's values filled into the simulator prompt the environment + step wrote. Nothing about how a caller behaves is decided here; that belongs to the prompt. + """ + world = restore(world_root) + apply_overlay(world, scenario.setup) + world.reset() + + written = load_simulator_prompt(world_root) + if not written: + return world, scenario.instruction + filled, missing = fill(written, scenario.slots()) + if missing: + raise RuntimeError( + f"the simulator prompt asks for {', '.join(missing)}, which {scenario.name} does " + "not supply. An unfilled slot reaches the caller verbatim." + ) + return world, filled + + +def grade(scenario: Scenario, world: GeneratedWorld, world_root: Path) -> LiveRun: + """The same sub-goal checks every other run uses, against what the call left behind.""" + catalogue = load_catalogue(world_root) + run = LiveRun(scenario=scenario.name) + for name in scenario.sub_goals: + sub_goal = catalogue.named(name) + if sub_goal is None: + run.problems.append(f"{name} is not in the catalogue") + elif sub_goal.deterministic(): + run.settled.append(run_check(sub_goal.check, world, world.calls, name=name)) + else: + run.judged.append(name) + run.calls = [ + f"{call.name}({call.arguments}) -> " + + ("refused: " + call.error if call.refused else "ok" if call.ok else "crashed") + for call in world.calls + ] + return run + + +def wire(scenario: Scenario, world_root: Path, *, assistant_id: str = "", api_key: str = ""): + """Everything up to placing the call: world, webhook, tunnel, assistant. + + Returns the bound world, the caller's instruction, the webhook and the tunnel, so whoever + places the call decides how — ALK's voice case, a phone leg, or a web call. + """ + assistant_id = assistant_id or os.environ.get("VAPI_ASSISTANT_ID", "") + api_key = api_key or os.environ.get("VAPI_API_KEY", "") + if not assistant_id or not api_key: + raise RuntimeError( + "VAPI_ASSISTANT_ID and VAPI_API_KEY have to be set. The assistant already exists " + "with the agent's own tools; the harness only changes where those calls are sent." + ) + + world, instruction = prepare(scenario, world_root) + webhook = WorldWebhook().start() + webhook.bind(world) + try: + url, tunnel = public_url(webhook.port) + moved = repoint_assistant(assistant_id, api_key, url) + except Exception: + webhook.stop() + world.close() + raise + return world, instruction, webhook, tunnel, url, moved diff --git a/src/fi/alk/harness/run/stage.py b/src/fi/alk/harness/run/stage.py new file mode 100644 index 0000000..b51f401 --- /dev/null +++ b/src/fi/alk/harness/run/stage.py @@ -0,0 +1,99 @@ +"""Stage four: run the scenarios against the real agent, and say what came back. + +The last stage that was a command rather than a conversation. Nothing about it needed to be: +wiring the world to the assistant and running the checks is already code, and the part worth +having judgement on is which scenario to run and what a failure actually means. + +That second part is why this is a stage at all. A failing check has four possible causes and only +one of them is a finding about the agent — the others are a wrong check, a wrong contract, or a +simulated caller that never asked for the thing. Deciding which is reading, not arithmetic. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Callable + +from claude_agent_sdk import ClaudeAgentOptions + +from ..config import ( + artifact_dir, + gate_hooks, + chosen_model, + load_skill, + permission_gate, + provider_env, +) +from ..contract import AgentContract +from ..scenario_tools import load_scenarios +from ..session import Stage +from ..tools import qualified +from .tools import RUN_SERVER, TOOL_NAMES, load_results, missing_prerequisites, run_tools + +SKILL = "run-scenarios" + + +def open_stage( + contract: AgentContract, + *, + out: Path | None = None, + ask: Callable[..., Any] | None = None, + max_turns: int = 40, +) -> tuple[Stage, Path]: + """A live run-the-scenarios stage, and where it will write its results.""" + destination = out or artifact_dir(contract.agent) + server = run_tools(destination, destination) + allowed = [ + "AskUserQuestion", + *(qualified(RUN_SERVER, name) for name in TOOL_NAMES), + ] + options = ClaudeAgentOptions( + system_prompt=( + f"{load_skill(SKILL)}\n\n## This agent\n\n{contract.brief()}" + ), + allowed_tools=allowed, + mcp_servers={RUN_SERVER: server}, + permission_mode="default", + cwd=str(destination.parent if destination.parent.exists() else Path.cwd()), + setting_sources=[], + max_turns=max_turns, + model=chosen_model(), + env=provider_env(), + ) + options.hooks = gate_hooks(allowed) + options.can_use_tool = permission_gate(ask, allowed) + return Stage(options, name=SKILL), destination + + +def opening(contract: AgentContract, destination: Path) -> str: + """What to tell the stage when it opens. + + Deliberately does not tell it to run everything. Each call costs money and takes minutes, and + a stage that opens by spending the whole suite gives nobody a chance to say which one they + cared about. + """ + written = load_scenarios(destination) + already = load_results(destination) + blocked = missing_prerequisites() + if blocked: + return ( + f"There are {len(written)} scenarios for {contract.agent!r}, but a live call cannot " + "be placed yet:\n - " + "\n - ".join(blocked) + "\n\nSay this plainly and stop." + ) + if already: + passed = sum(1 for record in already if record["passed"]) + return ( + f"{len(already)} of {len(written)} scenarios for {contract.agent!r} have been run, " + f"{passed} passing. Say where things stand with read_results, then ask which to run." + ) + return ( + f"{len(written)} scenarios are ready for {contract.agent!r} and none has been run.\n\n" + "Run preflight, then list_scenarios, then say which ones you would run first and why. " + "Do not start running them until you are asked to — each call takes minutes and costs " + "real money." + ) + + +def load(destination: Path) -> list[dict[str, Any]]: + """What has been run for this agent, if anything has.""" + return load_results(Path(destination)) diff --git a/src/fi/alk/harness/run/targets.py b/src/fi/alk/harness/run/targets.py new file mode 100644 index 0000000..1204f97 --- /dev/null +++ b/src/fi/alk/harness/run/targets.py @@ -0,0 +1,215 @@ +"""What is being tested, and how the harness talks to it. + +The rest of the run does not care what the agent under test is. It says something and gets a +reply back, and whatever tool calls happened in between landed in the world. That is the entire +interface, and keeping it that narrow is what lets the same scenarios, the same world and the +same grading run against an agent hosted anywhere. + +Two things are supplied per target: how to say something to it, and how its tool calls reach the +world. ``LocalAgent`` runs the agent in this process from its contract, which needs nothing +except the contract and is what makes a suite runnable the moment the world is built. A hosted +target is the same class with the transport swapped: the agent runs wherever it runs, its tool +calls arrive over a webhook, and the webhook answers from ``world.handle_tool_call``. The world +does not change, the scenarios do not change, and the grading does not change. +""" + +from __future__ import annotations + +from typing import Any, Callable, Protocol, runtime_checkable + +from claude_agent_sdk import ClaudeAgentOptions, create_sdk_mcp_server, tool + +from ..config import chosen_model, gate_hooks, permission_gate, provider_env +from ..contract import AgentContract +from ..session import Stage +from ..tools import qualified +from ..world.runtime import GeneratedWorld + +AGENT_SERVER = "agent" + +_TYPES: dict[str, type] = { + "str": str, + "string": str, + "int": int, + "integer": int, + "float": float, + "number": float, + "bool": bool, + "boolean": bool, + "list": list, + "dict": dict, +} + + +def _python_type(declared: str) -> type: + """The type a tool's argument is declared with, as something a schema can carry.""" + lowered = (declared or "").strip().lower() + if lowered.startswith(("list", "sequence", "array")): + return list + if lowered.startswith(("dict", "mapping", "object")): + return dict + return _TYPES.get(lowered, str) + + +def describe(spec: Any, contract: AgentContract) -> str: + """What the agent is told a tool takes, including the values it accepts. + + The values matter more than they look. An agent whose real schema enumerates its menu knows + that a Big Mac combo is ``big_mac_combo``; the same agent without them guesses, gets refused, + and reads as broken when what is broken is the harness that withheld them. Anything the + contract recorded as permitted, the agent under test is told. + """ + parts = [spec.description or f"{spec.name} for {contract.agent}"] + for arg in spec.args: + values = spec.arg_values.get(arg) + if isinstance(values, (list, tuple)) and values: + rendered = ", ".join(str(value) for value in values) + parts.append(f" {arg} accepts: {rendered}") + elif arg in spec.arg_types: + parts.append(f" {arg}: {spec.arg_types[arg]}") + return "\n".join(parts) + + +def agent_tools(contract: AgentContract, world: GeneratedWorld) -> Any: + """The agent's own tools, wired to the world so a call really happens. + + Every call goes through ``world.call``, so a refusal comes back as a refusal the agent can + read and recover from, rather than as a success it will happily build on. + """ + + def bind(spec: Any) -> Any: + schema = { + arg: _python_type(spec.arg_types.get(arg, "str")) for arg in spec.args + } + + @tool(spec.name, describe(spec, contract), schema) + async def call_tool( + args: dict[str, Any], _name: str = spec.name + ) -> dict[str, Any]: + # Through handle_tool_call, not straight to world.call. That method is the interface + # ALK's own runners drive an environment by, so going around it would leave the + # claim that a generated world plugs into them untested — and free to drift. + done = world.handle_tool_call({"name": _name, "arguments": args}) + if done is None: + return { + "content": [{"type": "text", "text": f"no such tool {_name}"}], + "is_error": True, + } + return { + "content": [{"type": "text", "text": done.content or ""}], + **({} if done.success else {"is_error": True}), + } + + return call_tool + + return create_sdk_mcp_server( + name=AGENT_SERVER, + version="0.1.0", + tools=[bind(spec) for spec in contract.tools], + ) + + +def agent_prompt(contract: AgentContract) -> str: + """The agent under test, as its contract describes it. + + Only what the contract records, because anything added here is a difference between the agent + being graded and the agent that exists. + """ + parts = [ + f"You are {contract.agent}: {contract.one_liner}".strip(), + contract.system_prompt_excerpt.strip(), + ] + if contract.hard_constraints: + parts.append( + "Rules you must follow:\n - " + "\n - ".join(contract.hard_constraints) + ) + if contract.modality == "voice": + parts.append( + "You are speaking out loud. Keep replies to what a person would actually say: " + "short, no lists, no markdown." + ) + parts.append( + "Use your tools to do anything real. Never tell the customer something is done unless a " + "tool confirmed it, and if a tool refuses, say so plainly and offer what is possible." + ) + return "\n\n".join(part for part in parts if part) + + +@runtime_checkable +class Target(Protocol): + """An agent under test, reachable by saying something to it.""" + + key: str + + async def open(self) -> None: ... + async def say(self, utterance: str) -> str: ... + async def close(self) -> None: ... + @property + def spent_usd(self) -> float: ... + + +class LocalAgent: + """The agent run here, from its contract, with its tools bound to the world.""" + + key = "local" + + def __init__( + self, + contract: AgentContract, + world: GeneratedWorld, + *, + model: str | None = None, + max_turns: int = 12, + ) -> None: + self.contract = contract + self.world = world + allowed = [qualified(AGENT_SERVER, spec.name) for spec in contract.tools] + options = ClaudeAgentOptions( + system_prompt=agent_prompt(contract), + allowed_tools=allowed, + mcp_servers={AGENT_SERVER: agent_tools(contract, world)}, + permission_mode="default", + setting_sources=[], + max_turns=max_turns, + model=chosen_model(model), + env=provider_env(model), + ) + # The agent under test gets its own tools and nothing else. A target that can reach a + # file or a shell is not the agent anybody deployed. + options.hooks = gate_hooks(allowed) + options.can_use_tool = permission_gate(granted=allowed) + self._stage = Stage(options, name="target") + + async def open(self) -> None: + await self._stage.__aenter__() + + async def say(self, utterance: str) -> str: + turn = await self._stage.say(utterance) + return turn.text.strip() + + async def close(self) -> None: + await self._stage.__aexit__(None, None, None) + + @property + def spent_usd(self) -> float: + return self._stage.spent_usd + + +_REGISTRY: dict[str, Callable[..., Target]] = {LocalAgent.key: LocalAgent} + + +def register_target(key: str, factory: Callable[..., Target]) -> None: + """Add a way of reaching an agent. A hosted runtime is a class and this line.""" + _REGISTRY[key] = factory + + +def resolve(key: str) -> Callable[..., Target]: + if key not in _REGISTRY: + raise NotImplementedError( + f"no target {key!r}; registered targets are {', '.join(sorted(_REGISTRY))}" + ) + return _REGISTRY[key] + + +def supported() -> tuple[str, ...]: + return tuple(sorted(_REGISTRY)) diff --git a/src/fi/alk/harness/run/tools.py b/src/fi/alk/harness/run/tools.py new file mode 100644 index 0000000..28396ed --- /dev/null +++ b/src/fi/alk/harness/run/tools.py @@ -0,0 +1,304 @@ +"""The tools that run a scenario against the real agent, and record what happened. + +Placing a call was a command before this existed, which made the last stage the only one you +could not simply ask for. Nothing about it needed to be a command: wiring the world to the +assistant and grading afterwards is already code, and choosing which scenario to run and reading +what came back is the part worth having judgement on. + +So the same shape as every other stage. The tools do what must be exact — restore the world, +repoint the assistant's own tools, place the call through ALK, run the checks — and the stage +decides what to run and says what it means. + +A run takes minutes, not seconds. The tool blocks for that long, and says so, because a stage +that fires a call and returns immediately would report on a conversation that has not happened. +""" + +from __future__ import annotations + +import asyncio +import json +import os +import shutil +import time +from pathlib import Path +from typing import Any + +from claude_agent_sdk import create_sdk_mcp_server, tool + +from ..environment import load_catalogue +from ..scenario_tools import load_scenarios +from ..tools import schema +from .call import place_the_call +from .live import LiveRun, grade, wire + +RUN_SERVER = "runs" +RESULTS = "runs.json" + +# What a hosted agent needs before a call can be placed at all. Checked up front rather than +# three minutes in, because the failure otherwise arrives after the expensive part. +REQUIRED = ("VAPI_API_KEY", "VAPI_ASSISTANT_ID") + + +def _ok(text: str) -> dict[str, Any]: + return {"content": [{"type": "text", "text": text}]} + + +def _err(text: str) -> dict[str, Any]: + return {"content": [{"type": "text", "text": text}], "is_error": True} + + +def missing_prerequisites() -> list[str]: + """What would stop a live call, in the words of what to do about it.""" + problems: list[str] = [] + absent = [name for name in REQUIRED if not os.environ.get(name)] + if absent: + problems.append( + f"{', '.join(absent)} not set. The assistant already exists with the agent's own " + "tools; without these there is no way to reach it. Load the env file first:\n" + " set -a; . ./.env.acceptance; set +a" + ) + if not os.environ.get("HARNESS_WEBHOOK_URL") and not shutil.which("cloudflared"): + problems.append( + "no way to expose the webhook publicly. A hosted agent cannot reach loopback, so " + "either install cloudflared (brew install cloudflared) or set HARNESS_WEBHOOK_URL " + "to a tunnel that is already running." + ) + return problems + + +def save_results(results: list[dict[str, Any]], destination: Path) -> Path: + """Keep every run, so a suite can be read after the fact rather than scrolled back to.""" + destination = Path(destination) + destination.mkdir(parents=True, exist_ok=True) + path = destination / RESULTS + path.write_text(json.dumps(results, indent=2, ensure_ascii=False), encoding="utf-8") + return path + + +def load_results(destination: Path) -> list[dict[str, Any]]: + path = Path(destination) / RESULTS + if not path.exists(): + return [] + try: + loaded = json.loads(path.read_text(encoding="utf-8")) + return loaded if isinstance(loaded, list) else [] + except json.JSONDecodeError: + return [] + + +def as_record(run: LiveRun) -> dict[str, Any]: + return { + "scenario": run.scenario, + "passed": bool(run.settled) and run.met == len(run.settled) and not run.problems, + "met": run.met, + "of": len(run.settled), + "settled": [ + {"name": one.name, "held": one.held, "said": one.said, "broken": one.broken} + for one in run.settled + ], + "judged": list(run.judged), + "calls": list(run.calls), + "problems": list(run.problems), + } + + +def transcript_since(started: float) -> str: + """What was said on the call that just happened, from the voice runner's own report. + + The voice case owns the call and writes its report where it always has; reaching into that + report is how the transcript gets onto the run record without the harness re-implementing + any of the call. Only a report written after this run started counts — the newest file on + disk is otherwise last week's call wearing today's verdict. + """ + root = Path("artifacts/simulation-acceptance") + if not root.exists(): + return "" + newest: tuple[float, Path] | None = None + for report in root.glob("run_*/*/report.json"): + written = report.stat().st_mtime + if written >= started and (newest is None or written > newest[0]): + newest = (written, report) + if newest is None: + return "" + try: + loaded = json.loads(newest[1].read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + return "" + for result in loaded.get("results") or []: + spoken = result.get("transcript") + if isinstance(spoken, str) and spoken.strip(): + return spoken + return "" + + +def report(run: LiveRun) -> str: + """One run, as something worth reading rather than a score.""" + lines = [run.line()] + lines += [one.line() for one in run.settled] + lines += [f" [?] {name} — judged, not settled by code" for name in run.judged] + if run.problems: + lines += [f" !! {problem}" for problem in run.problems] + lines.append("") + lines.append("what the agent actually did:") + lines += [f" {call}" for call in run.calls or ["(no tool calls reached the world)"]] + return "\n".join(lines) + + +def run_tools(world_root: Path, destination: Path, *, case: str = "") -> Any: + """A server for running one agent's scenarios against the real thing.""" + written = load_scenarios(destination) + catalogue = load_catalogue(destination) + results = load_results(destination) + voice_case = case or os.environ.get("HARNESS_VOICE_CASE", "2.1.2") + + @tool( + "list_scenarios", + "The scenarios that can be run, what each one tests, and which of its sub-goals are " + "settled by code rather than left to a judge.", + schema({}, []), + ) + async def list_scenarios(_args: dict[str, Any]) -> dict[str, Any]: + if not written: + return _err("no scenarios have been written for this agent yet") + lines: list[str] = [] + for one in written: + settled = [ + name + for name in one.sub_goals + if (found := catalogue.named(name)) and found.deterministic() + ] + judged = [name for name in one.sub_goals if name not in settled] + ran = next((r for r in results if r["scenario"] == one.name), None) + mark = "" if ran is None else (" [last run: PASS]" if ran["passed"] else " [last run: FAIL]") + lines.append( + f"{one.name}{mark}\n tests: {one.tests or one.use_case or '—'}\n" + f" settled by code: {', '.join(settled) or 'none'}\n" + f" judged: {', '.join(judged) or 'none'}" + ) + return _ok("\n".join(lines)) + + @tool( + "preflight", + "Check everything a live call needs before spending one: the assistant's credentials " + "and a way to expose the webhook publicly. Run this before the first call.", + schema({}, []), + ) + async def preflight(_args: dict[str, Any]) -> dict[str, Any]: + problems = missing_prerequisites() + if problems: + return _err("Not ready:\n - " + "\n - ".join(problems)) + return _ok( + "Ready. Credentials are set and the webhook can be exposed. " + f"{len(written)} scenarios are available." + ) + + @tool( + "run_scenario", + "Run one scenario against the real agent and grade it.\n\n" + "This restores the world, applies the scenario's setup, stands up the webhook, points " + "the assistant's OWN tools at it, places the call, and runs the sub-goals' checks " + "against what the world holds afterwards plus the calls that were made.\n\n" + "It takes several minutes and blocks until the call is over. Run one at a time and read " + "what comes back before running the next.", + # Both spellings accepted: every model that has driven this stage has guessed + # `scenario` at least once, and a retry on an argument name is a wasted turn. + schema({"name": str, "scenario": str}, []), + ) + async def run_scenario(args: dict[str, Any]) -> dict[str, Any]: + name = str(args.get("name") or args.get("scenario") or "") + scenario = next((one for one in written if one.name == name), None) + if scenario is None: + return _err( + f"no scenario called {name!r}. There is: " + + ", ".join(one.name for one in written) + ) + problems = missing_prerequisites() + if problems: + return _err( + "Cannot place a call:\n - " + + "\n - ".join(problems) + + "\nThis is the environment this harness is running in, not something to fix " + "in the scenario." + ) + + def placed() -> tuple[LiveRun, str, list[str], str]: + """The whole call, off the event loop. + + Wiring reads a subprocess's stdout and placing the call blocks for minutes; run + inline they freeze whatever loop is hosting this tool, which for the web UI means + the stream, the status endpoint and the stop button all die for the duration. + """ + world, instruction, webhook, tunnel, url, moved = wire(scenario, world_root) + started = time.time() + try: + # The caller's instruction reaches the voice case through the environment, so + # how a simulated caller behaves is not decided in two places. + os.environ["HARNESS_INSTRUCTION"] = instruction + os.environ["HARNESS_SCENARIO"] = scenario.name + os.environ["HARNESS_OUTCOME"] = scenario.tests + code = place_the_call(voice_case) + run = grade(scenario, world, world_root) + if code != 0 and not run.calls: + run.problems.append( + f"the voice runner exited {code} and no tool call reached the world, " + "so this says nothing about the agent" + ) + finally: + webhook.stop() + if tunnel is not None: + tunnel.terminate() + world.close() + return run, url, moved, transcript_since(started) + + run, url, moved, spoken = await asyncio.to_thread(placed) + + record = as_record(run) + record["instruction"] = scenario.instruction + record["transcript"] = spoken + # Re-read before writing: the local suite writes the same file, and a list loaded when + # this stage opened would silently roll back anything recorded since. + results[:] = [ + r for r in load_results(destination) if r.get("scenario") != scenario.name + ] + results.append(record) + save_results(results, destination) + answer = f"webhook: {url}/tool\nrepointed: {', '.join(moved)}\n\n{report(run)}" + return _ok(answer) if not run.problems else _err(answer) + + @tool( + "read_results", + "What every scenario did the last time it was run, without running anything.", + schema({}, []), + ) + async def read_results(_args: dict[str, Any]) -> dict[str, Any]: + if not results: + return _ok("nothing has been run yet") + lines = [] + for record in results: + mark = "PASS" if record["passed"] else "FAIL" + failed = [ + f"{one['name']}: {one['said']}" + for one in record["settled"] + if not one["held"] + ] + lines.append( + f"{mark} {record['scenario']} {record['met']}/{record['of']}" + + ("\n - " + "\n - ".join(failed) if failed else "") + ) + passed = sum(1 for record in results if record["passed"]) + return _ok("\n".join(lines) + f"\n\n{passed} of {len(results)} passed") + + server = create_sdk_mcp_server( + name=RUN_SERVER, + version="0.1.0", + tools=[list_scenarios, preflight, run_scenario, read_results], + ) + return server + + +TOOL_NAMES = ( + "list_scenarios", + "preflight", + "run_scenario", + "read_results", +) diff --git a/src/fi/alk/harness/run/voice.py b/src/fi/alk/harness/run/voice.py new file mode 100644 index 0000000..f984921 --- /dev/null +++ b/src/fi/alk/harness/run/voice.py @@ -0,0 +1,205 @@ +"""Serving a real voice agent's tool calls from a generated world. + +A hosted voice agent executes its tools by calling a webhook. So the whole integration is one +thing: stand up that webhook, and answer it from the world instead of from canned responses. + +That single swap is what the environment was built for. The previous run's known issues were all +the same defect wearing different clothes: + +- *"Mocked tools always succeed, including removing an item that was never added."* +- *"Mock responses do not vary by argument, so read-after-write flows are wrong."* +- *"World state does not change unless a scenario sets state_updates, which is often empty."* + +A world that really holds rows and can really refuse answers all three, because the reply the +agent hears is produced by running the call rather than by looking it up. + +Nothing here decides pass or fail. Grading reads the world afterwards and the calls this server +recorded, through the same sub-goal checks every other run uses. +""" + +from __future__ import annotations + +import json +import logging +import os +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer +from typing import Any, Mapping + +from ..world.runtime import GeneratedWorld + +logger = logging.getLogger(__name__) + +VAPI_API = os.environ.get("VAPI_API_BASE_URL", "https://api.vapi.ai").rstrip("/") + +# Vapi's edge rejects the default urllib User-Agent with a 403 that says nothing about why, while +# the identical request from curl succeeds. Sending one is the whole fix. +_AGENT = "alk-harness/0.1" + + +class WorldWebhook: + """The webhook a hosted agent calls, answered by a generated world. + + One world at a time. ``bind`` swaps which world is live between scenarios, so the assistant + stays configured while every scenario still starts from its own restored copy. + """ + + def __init__(self, host: str = "127.0.0.1", port: int = 0) -> None: + self._world: GeneratedWorld | None = None + self._lock = threading.Lock() + try: + server = HTTPServer((host, port), _handler_for(self)) + except OSError: + # A leftover server from an earlier run must not block this one; any free port works + # because the public URL is discovered after binding. + logger.warning("port %s busy, binding an ephemeral port instead", port) + server = HTTPServer((host, 0), _handler_for(self)) + self._server = server + self.port = server.server_address[1] + self._thread = threading.Thread(target=server.serve_forever, daemon=True) + + def start(self) -> "WorldWebhook": + self._thread.start() + logger.info("world webhook listening on port %s", self.port) + return self + + def stop(self) -> None: + self._server.shutdown() + self._server.server_close() + + def bind(self, world: GeneratedWorld) -> None: + """Make one world live. Its own call log is what grading reads afterwards.""" + with self._lock: + self._world = world + world.reset() + + @property + def calls(self) -> list[Any]: + with self._lock: + return list(self._world.calls) if self._world else [] + + def respond(self, name: str, arguments: Mapping[str, Any]) -> str: + """Answer one tool call by running it. + + A refusal is returned as the answer, not as an error: the agent has to hear "that item is + unavailable" and cope with it, which is the whole reason the world can say no. What it + must never hear is an acknowledgement for something that did not happen. + """ + with self._lock: + world = self._world + if world is None: + return "the environment is not ready" + + done = world.handle_tool_call({"name": name, "arguments": dict(arguments)}) + if done is None: + return f"there is no tool called {name}" + return done.content or ("done" if done.success else "that could not be done") + + +def _handler_for(owner: "WorldWebhook"): + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args: Any) -> None: # silence per-request stderr noise + return + + def do_POST(self) -> None: # noqa: N802 - required name + length = int(self.headers.get("Content-Length") or 0) + raw = self.rfile.read(length) if length else b"{}" + try: + payload = json.loads(raw or b"{}") + except json.JSONDecodeError: + payload = {} + results = [ + {"toolCallId": call_id, "result": owner.respond(name, arguments)} + for call_id, name, arguments in tool_calls(payload) + ] + body = json.dumps({"results": results}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + return Handler + + +def tool_calls(payload: Mapping[str, Any]) -> list[tuple[str, str, dict[str, Any]]]: + """Pull (id, name, arguments) out of a provider's tool-call webhook body.""" + message = payload.get("message") or payload + raw = message.get("toolCalls") or message.get("toolCallList") or [] + found: list[tuple[str, str, dict[str, Any]]] = [] + for entry in raw if isinstance(raw, list) else []: + if not isinstance(entry, Mapping): + continue + function = entry.get("function") or {} + name = str(function.get("name") or entry.get("name") or "") + arguments = function.get("arguments") or entry.get("arguments") or {} + if isinstance(arguments, str): + try: + arguments = json.loads(arguments) + except json.JSONDecodeError: + arguments = {"_raw": arguments} + if name: + found.append((str(entry.get("id") or ""), name, dict(arguments))) + return found + + +def pointed_at(tools: list[dict[str, Any]], webhook_url: str) -> list[dict[str, Any]]: + """The agent's own tools, with only where they are answered changed. + + The assistant under test already has its tools — the names, the arguments, the enums are the + agent's, defined by whoever built it. Redefining them here would mean testing an agent we + wrote rather than theirs, and any drift between the two would show up as a finding about + them. So nothing is rebuilt: the one thing that changes is the address the call goes to. + """ + repointed: list[dict[str, Any]] = [] + for tool in tools: + moved = json.loads(json.dumps(tool)) + moved.setdefault("server", {})["url"] = f"{webhook_url.rstrip('/')}/tool" + repointed.append(moved) + return repointed + + +def fetch_assistant(assistant_id: str, api_key: str) -> dict[str, Any]: + """The assistant as it stands, so its own tools can be read rather than guessed.""" + import urllib.request + + request = urllib.request.Request( + f"{VAPI_API}/assistant/{assistant_id}", + headers={"Authorization": f"Bearer {api_key}", "User-Agent": _AGENT}, + ) + with urllib.request.urlopen(request, timeout=20) as answer: + return json.loads(answer.read()) + + +def repoint_assistant( + assistant_id: str, api_key: str, webhook_url: str +) -> list[str]: + """Send the assistant's existing tool calls to our webhook. Returns the tools moved.""" + import urllib.request + + assistant = fetch_assistant(assistant_id, api_key) + tools = (assistant.get("model") or {}).get("tools") or [] + if not tools: + raise RuntimeError( + f"assistant {assistant_id} has no tools, so there is nothing for the environment " + "to answer. It is the agent's own tools that get repointed, not ones we add." + ) + model = json.loads(json.dumps(assistant.get("model") or {})) + model["tools"] = pointed_at(tools, webhook_url) + + body = json.dumps({"model": model}).encode() + request = urllib.request.Request( + f"{VAPI_API}/assistant/{assistant_id}", + data=body, + method="PATCH", + headers={ + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + "User-Agent": _AGENT, + }, + ) + with urllib.request.urlopen(request, timeout=20) as answer: + answer.read() + return [ + str((one.get("function") or {}).get("name") or "") for one in model["tools"] + ] diff --git a/src/fi/alk/harness/scenario.py b/src/fi/alk/harness/scenario.py new file mode 100644 index 0000000..bfcca07 --- /dev/null +++ b/src/fi/alk/harness/scenario.py @@ -0,0 +1,122 @@ +"""A scenario: a delta on the base environment, and what must hold afterwards. + +The base is built once — the world, the simulator's prompt, the catalogue of sub-goals. A +scenario changes a few values in that world, fills the prompt's slots, and names which sub-goals +must hold. It is not a template with values slotted into it; the harness writes each one. + +It also carries a **solution**: what a correct agent would do. That is not decoration. It is what +proves, before the scenario is ever used, that the scenario can be passed at all and that its +checks are not vacuous — the two gates in ``prove.py``. Terminal-bench keeps its tasks honest the +same way, and it needs no model to do it. + +There is no persona and no opening line. Variability comes from real conditions — an item in +stock or not, a customer who exists or does not — which live in ``setup``, not from an invented +character. +""" + +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, Field + +from .environment import Catalogue, variables_in + + +class Step(BaseModel): + """One action in a reference solution.""" + + tool: str + arguments: dict[str, Any] = Field(default_factory=dict) + + +class Scenario(BaseModel): + """One test: what changes, what is asked, what a correct agent does, what must hold.""" + + name: str + use_case: str = "" + tests: str = "" + + # What this scenario changes about the world after it is reset. The base world stays the + # shared starting point; this is the only sanctioned way a scenario differs from it. + setup: dict[str, list[dict[str, Any]]] = Field(default_factory=dict) + + # The task. For a conversational agent it fills the simulator prompt's instruction slot; for + # a browser or coding agent it goes to the agent directly. + instruction: str = "" + # Anything else that prompt asks for, by slot name. + variables: dict[str, str] = Field(default_factory=dict) + + # What a correct agent would do. Run by the gates, never by the agent under test. + solution: list[Step] = Field(default_factory=list) + + # Which entries of the shared catalogue must hold. Named, not restated, so results roll up + # across the suite: the same sub-goal failing in seven of twelve scenarios is one sentence. + sub_goals: list[str] = Field(default_factory=list) + + max_turns: int = 10 + + def slots(self) -> dict[str, str]: + """Every value this scenario offers the simulator prompt.""" + return {"instruction": self.instruction, **self.variables} + + +def validate_scenario( + scenario: Scenario, + catalogue: Catalogue, + world_state: dict[str, list[dict[str, Any]]], + simulator_prompt: str = "", +) -> list[str]: + """Problems that make a scenario unusable, found without running anything. + + Whether it can actually be passed is a different question, and no amount of reading settles + it. That is what the gates are for. + """ + problems: list[str] = [] + if not scenario.name.strip(): + problems.append("no name") + if not scenario.instruction.strip(): + problems.append("no instruction: there is nothing for the run to be about") + if not scenario.sub_goals: + problems.append( + "no sub_goals: nothing would be graded. Name the entries of the catalogue this " + "scenario is meant to exercise" + ) + + unknown = sorted(set(scenario.sub_goals) - catalogue.names()) + if unknown: + problems.append( + f"sub_goals not in the catalogue: {', '.join(unknown)}. Use the shared names, or add " + f"them to the catalogue first. It has: {', '.join(sorted(catalogue.names())) or 'none'}" + ) + + for table, rows in scenario.setup.items(): + if table not in world_state: + problems.append( + f"setup changes {table!r}, which this world does not have. It has: " + f"{', '.join(sorted(world_state)) or 'nothing'}" + ) + continue + columns = set(world_state[table][0]) if world_state[table] else set() + for row in rows or []: + unknown_columns = sorted(set(row) - columns) if columns else [] + if unknown_columns: + problems.append( + f"setup into {table} sets columns it does not have: " + f"{', '.join(unknown_columns)}" + ) + + if simulator_prompt: + unfilled = sorted(variables_in(simulator_prompt) - set(scenario.slots())) + if unfilled: + problems.append( + f"the simulator prompt asks for {', '.join(unfilled)}, which this scenario does " + "not supply. An unfilled slot reaches the caller verbatim" + ) + + if not scenario.solution: + problems.append( + "no solution: without the actions a correct agent would take, there is no way to " + "show this scenario can be passed at all" + ) + return problems diff --git a/src/fi/alk/harness/scenario_tools.py b/src/fi/alk/harness/scenario_tools.py new file mode 100644 index 0000000..40e6cff --- /dev/null +++ b/src/fi/alk/harness/scenario_tools.py @@ -0,0 +1,456 @@ +"""The tools that write scenarios, and the gates that decide one may be kept. + +A scenario is accepted by being *proved*, not by looking right. ``submit_scenario`` restores a +fresh world, applies the scenario's own setup, plays its reference solution through it, and runs +the checks of every sub-goal it names. They must pass. Then it does the same with no solution at +all, and they must fail. Only then is it kept. + +Both gates are code. No model is asked whether a scenario is good; the environment decides. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from claude_agent_sdk import create_sdk_mcp_server, tool + +from .amend import add_rule, drop_rule, fix_tool, widen +from .contract import AgentContract +from .environment import ( + Catalogue, + SubGoal, + load_catalogue, + load_simulator_prompt, + save_catalogue, + validate_sub_goal, +) +from .prove import prove +from .scenario import Scenario, validate_scenario +from .tools import schema +from .world.snapshot import apply_overlay, restore + +SCENARIO_SERVER = "scenarios" +SCENARIOS = "scenarios.json" + + +def _ok(text: str) -> dict[str, Any]: + return {"content": [{"type": "text", "text": text}]} + + +def _err(text: str) -> dict[str, Any]: + return {"content": [{"type": "text", "text": text}], "is_error": True} + + +def write_scenarios(scenarios: list[Scenario], destination: Path) -> Path: + destination = Path(destination) + destination.mkdir(parents=True, exist_ok=True) + path = destination / SCENARIOS + path.write_text( + json.dumps([one.model_dump() for one in scenarios], indent=2, ensure_ascii=False), + encoding="utf-8", + ) + return path + + +def load_scenarios(destination: Path) -> list[Scenario]: + path = Path(destination) / SCENARIOS + if not path.exists(): + return [] + try: + return [ + Scenario.model_validate(entry) + for entry in json.loads(path.read_text(encoding="utf-8")) + ] + except Exception: + # Written in an older shape. Better to start clean than to half-read them. + return [] + + +def accept_scenario( + payload: dict[str, Any], + *, + world_root: Path, + catalogue: Catalogue, + kept: list[Scenario], + simulator_prompt: str = "", +) -> dict[str, Any]: + """Validate one scenario, then prove it. A plain function so both halves are testable.""" + try: + scenario = Scenario.model_validate(payload) + except Exception as invalid: + return _err(f"Not kept. {invalid}"[:600]) + + trial = restore(world_root) + try: + try: + apply_overlay(trial, scenario.setup) + except Exception as failed: + return _err( + f"Not kept. The setup rows would not go into the world: {failed}\n" + "setup is {table: [{column: value}]}, and every column has to be one the table has." + ) + problems = validate_scenario(scenario, catalogue, trial.state(), simulator_prompt) + finally: + trial.close() + + if problems: + return _err("Not kept. Fix these and submit again:\n - " + "\n - ".join(problems)) + + proof = prove(scenario, catalogue, world_root) + if not proof.holds: + return _err(f"Not kept. {proof.why()}") + + replaced = any(one.name == scenario.name for one in kept) + kept[:] = [one for one in kept if one.name != scenario.name] + kept.append(scenario) + return _ok( + f"{scenario.name} {'replaced' if replaced else 'kept'}. Proved: the solution passes its " + f"checks, and they fail without it.\n{len(kept)} so far: " + + ", ".join(one.name for one in kept) + ) + + +def not_ready(kept: list[Scenario], wanted: int, catalogue: Catalogue) -> list[str]: + """Why this suite is not worth saving yet.""" + problems: list[str] = [] + if len(kept) < wanted: + problems.append( + f"{len(kept)} of {wanted} scenarios so far. Keep writing; the ones that find " + "something are usually the awkward ones." + ) + elif len(kept) > wanted: + problems.append( + f"{len(kept)} scenarios but {wanted} were asked for. Drop the ones that add least " + "with drop_scenario. The number came from the person who asked." + ) + # Sub-goals are shared so results roll up. A suite where every scenario invents its own is a + # suite whose results cannot be added together. + used = [name for one in kept for name in one.sub_goals] + if kept and len(used) > 2 and len(set(used)) == len(used): + problems.append( + "no sub-goal is used by more than one scenario, so nothing rolls up across the " + "suite. Reuse the catalogue where the same thing is being checked." + ) + return problems + + +def scenario_tools( + contract: AgentContract, world_root: Path, destination: Path, *, wanted: int +) -> tuple[Any, list[Scenario]]: + """A server for writing scenarios against one built environment.""" + kept: list[Scenario] = load_scenarios(destination) + catalogue = load_catalogue(destination) + simulator_prompt = load_simulator_prompt(destination) + target = {"count": wanted} + + @tool( + "inspect_world", + "Look at what is in the world. Without a table, lists the tables and how many rows each " + "holds; with one, returns rows from it. `matching` is plain text, not SQL.", + schema({"table": str, "limit": int, "matching": str}, []), + ) + async def inspect_world(args: dict[str, Any]) -> dict[str, Any]: + world = restore(world_root) + try: + state = world.state() + table = str(args.get("table") or "") + if not table: + lines = [f"{n}: {len(r)} rows" for n, r in sorted(state.items())] + if catalogue.sub_goals: + lines.append( + "\nsub-goals available: " + ", ".join(sorted(catalogue.names())) + ) + return _ok("\n".join(lines) or "this world has no tables") + if table not in state: + return _err(f"no table {table!r}; this world has {', '.join(sorted(state))}") + rows = state[table] + matching = str(args.get("matching") or "").strip() + if matching: + needle = matching.lower() + found = [r for r in rows if needle in json.dumps(r, default=str).lower()] + if not found: + return _ok( + f"nothing in {table} contains {matching!r}, but it holds {len(rows)} rows." + ) + rows = found + shown = rows[: int(args.get("limit") or 20)] + return _ok( + f"{len(rows)} rows, showing {len(shown)}:\n" + + "\n".join(json.dumps(r, default=str) for r in shown) + ) + finally: + world.close() + + @tool( + "try_calls", + "Run calls against a throwaway copy of the world and see the state they leave. Use it to " + "work out a scenario's solution and what its checks should assert. Nothing is saved.", + schema({"calls": list, "setup": dict}, ["calls"]), + ) + async def try_calls(args: dict[str, Any]) -> dict[str, Any]: + world = restore(world_root) + try: + try: + apply_overlay(world, args.get("setup") or {}) + except Exception as failed: + return _err(f"the setup rows would not go in: {failed}") + lines: list[str] = [] + for step in args.get("calls") or []: + if not isinstance(step, dict): + return _err("each call must be an object with a tool and arguments") + call = world.call(str(step.get("tool") or ""), step.get("arguments") or {}) + if call.refused: + lines.append(f"{call.name}: refused — {call.error}") + elif not call.ok: + lines.append(f"{call.name}: CRASHED — {call.error}") + else: + lines.append( + f"{call.name}: ok — {json.dumps(call.result, default=str)[:200]}" + ) + state = world.state() + lines.append( + "state afterwards: " + + ", ".join(f"{n}.count={len(r)}" for n, r in sorted(state.items())) + ) + for name, rows in sorted(state.items()): + if rows and len(rows) <= 6: + lines.append(f"{name}: " + json.dumps(rows, default=str)[:700]) + return _ok("\n".join(lines) or "no calls were made") + finally: + world.close() + + @tool( + "add_sub_goal", + "Add a named thing this agent can be checked on, shared by every scenario that needs it. " + "`check` is Python: define check(world, calls) returning a sentence when something is " + "wrong, or None when it held. `world` is the environment afterwards; `calls` is every " + "tool call made, each with .name, .arguments, .ok and .refused — so a check can insist a " + "call happened with the right arguments, not merely that it happened.\n\n" + "Use `judged` only where nothing observable settles it, saying what a model must decide " + "and why code cannot.", + schema({"name": str, "what": str, "check": str, "judged": str}, ["name", "what"]), + ) + async def add_sub_goal(args: dict[str, Any]) -> dict[str, Any]: + sub_goal = SubGoal( + name=str(args.get("name") or ""), + what=str(args.get("what") or ""), + check=str(args.get("check") or ""), + judged=str(args.get("judged") or ""), + ) + problems = validate_sub_goal(sub_goal) + if problems: + return _err("Not added:\n - " + "\n - ".join(problems)) + catalogue.sub_goals = [one for one in catalogue.sub_goals if one.name != sub_goal.name] + catalogue.sub_goals.append(sub_goal) + save_catalogue(catalogue, destination) + return _ok( + f"{sub_goal.name} added" + + ("" if sub_goal.deterministic() else " (judged, not deterministic)") + + f". The catalogue has {len(catalogue.sub_goals)}: " + + ", ".join(sorted(catalogue.names())) + ) + + @tool( + "submit_scenario", + "Keep one scenario. It is proved before it is kept: its solution is played through a " + "fresh world and its sub-goals' checks must pass, then the same checks run with nothing " + "done and must fail.\n\n" + " name / use_case / tests\n" + " setup: {table: [{column: value}]} — what this scenario changes after reset\n" + " instruction: the task. For a conversational agent it fills the simulator prompt\n" + " variables: any other slot that prompt asks for\n" + " solution: [{tool, arguments}] — what a correct agent would do\n" + " sub_goals: names from the catalogue that must hold", + schema( + { + "name": str, + "use_case": str, + "tests": str, + "setup": dict, + "instruction": str, + "variables": dict, + "solution": list, + "sub_goals": list, + "max_turns": int, + }, + ["name", "instruction", "solution", "sub_goals"], + ), + ) + async def submit_scenario(args: dict[str, Any]) -> dict[str, Any]: + return accept_scenario( + args, + world_root=world_root, + catalogue=catalogue, + kept=kept, + simulator_prompt=simulator_prompt, + ) + + @tool( + "amend_contract", + "Let one of the agent's tools accept values it did not before, when the world holds " + "something the agent has no way to name. Say why; it is recorded on the contract.", + schema( + {"tool_name": str, "argument": str, "values": list, "why": str}, + ["tool_name", "argument", "values", "why"], + ), + ) + async def amend_contract(args: dict[str, Any]) -> dict[str, Any]: + done, said = widen( + contract, + world_root, + tool_name=str(args.get("tool_name") or ""), + argument=str(args.get("argument") or ""), + values=[str(v) for v in (args.get("values") or [])], + why=str(args.get("why") or ""), + ) + return _ok(said) if done else _err(said) + + @tool( + "add_rule", + "Give the agent a hard rule its source did not state, when asked for one. It is told to " + "the agent under test and graded, so this changes what is being tested. Say why.", + schema({"rule": str, "why": str}, ["rule", "why"]), + ) + async def add_rule_tool(args: dict[str, Any]) -> dict[str, Any]: + done, said = add_rule( + contract, world_root, rule=str(args.get("rule") or ""), why=str(args.get("why") or "") + ) + return _ok(said) if done else _err(said) + + @tool( + "drop_rule", + "Take away a hard rule the agent does not really have. Say why.", + schema({"rule": str, "why": str}, ["rule", "why"]), + ) + async def drop_rule_tool(args: dict[str, Any]) -> dict[str, Any]: + done, said = drop_rule( + contract, world_root, rule=str(args.get("rule") or ""), why=str(args.get("why") or "") + ) + return _ok(said) if done else _err(said) + + @tool( + "fix_tool", + "Correct a tool that was read wrong, or remove one the agent does not have. Everything " + "is built from these, so a wrong argument name produces a world that refuses everything.", + schema( + { + "tool_name": str, + "args": list, + "arg_types": dict, + "description": str, + "remove": bool, + "why": str, + }, + ["tool_name", "why"], + ), + ) + async def fix_tool_tool(args: dict[str, Any]) -> dict[str, Any]: + done, said = fix_tool( + contract, + world_root, + tool_name=str(args.get("tool_name") or ""), + why=str(args.get("why") or ""), + args=[str(a) for a in args["args"]] if args.get("args") else None, + arg_types={str(k): str(v) for k, v in (args.get("arg_types") or {}).items()}, + description=str(args.get("description") or ""), + remove=bool(args.get("remove")), + ) + return _ok(said) if done else _err(said) + + @tool( + "aim_for", + "Set how many scenarios are wanted. Only when the person you are talking to says a " + "number, never to get past a refusal about having written too many.", + schema({"count": int}, ["count"]), + ) + async def aim_for(args: dict[str, Any]) -> dict[str, Any]: + count = int(args.get("count") or 0) + if count < 1: + return _err("that is not a number of scenarios worth writing") + target["count"] = count + return _ok(f"aiming for {count}. {len(kept)} written so far") + + @tool( + "drop_scenario", + "Remove a scenario by name, or all of them with name '*'.", + schema({"name": str}, ["name"]), + ) + async def drop_scenario(args: dict[str, Any]) -> dict[str, Any]: + name = str(args.get("name") or "") + if name == "*": + kept.clear() + return _ok("all scenarios dropped") + before = len(kept) + kept[:] = [one for one in kept if one.name != name] + if len(kept) == before: + return _err(f"no scenario called {name!r}") + return _ok(f"{name} dropped. {len(kept)} left") + + @tool("save_scenarios", "Write the kept scenarios out.", schema({}, [])) + async def save_scenarios(_args: dict[str, Any]) -> dict[str, Any]: + problems = not_ready(kept, target["count"], catalogue) + if problems: + return _err("Not saved. " + "\n - ".join(problems)) + path = write_scenarios(kept, destination) + judged = sum( + 1 + for one in kept + for name in one.sub_goals + if (found := catalogue.named(name)) and not found.deterministic() + ) + return _ok( + f"Saved {len(kept)} scenarios to {path}.\n" + "Every one is proved: its solution passes its checks, and they fail without it.\n" + f"{judged} sub-goal references are judged rather than settled by code." + ) + + server = create_sdk_mcp_server( + name=SCENARIO_SERVER, + version="0.1.0", + tools=[ + inspect_world, + try_calls, + add_sub_goal, + submit_scenario, + amend_contract, + add_rule_tool, + drop_rule_tool, + fix_tool_tool, + aim_for, + drop_scenario, + save_scenarios, + ], + ) + return server, kept + + +TOOL_NAMES = ( + "inspect_world", + "try_calls", + "add_sub_goal", + "submit_scenario", + "amend_contract", + "add_rule", + "drop_rule", + "fix_tool", + "aim_for", + "drop_scenario", + "save_scenarios", +) + + +def world_summary(world_root: Path) -> str: + """What is in the built environment, for grounding the writer before it asks.""" + world = restore(world_root) + try: + state = world.state() + lines = [f" {name}: {len(rows)} rows" for name, rows in sorted(state.items())] + catalogue = load_catalogue(world_root) + if catalogue.sub_goals: + lines.append("\nSUB-GOALS already defined (reuse these, do not restate them):") + lines += [f" {one.name}: {one.what}" for one in catalogue.sub_goals] + return "THE BUILT WORLD (restored fresh for every scenario):\n" + "\n".join(lines) + finally: + world.close() diff --git a/src/fi/alk/harness/scenarios.py b/src/fi/alk/harness/scenarios.py new file mode 100644 index 0000000..1295375 --- /dev/null +++ b/src/fi/alk/harness/scenarios.py @@ -0,0 +1,126 @@ +"""Stage three: write the scenarios the agent will be tested with. + +Reads the contract and the world that was built from it, and produces scenarios grounded in both. +The stage can look at the world and run calls against throwaway copies of it, which is what keeps +a scenario about a real record rather than a plausible-sounding one. + +Like the other stages it stays open. A suite is usually right on the second look, and "make three +of these harder" is the next thing said rather than a regeneration from nothing. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Callable + +from claude_agent_sdk import ClaudeAgentOptions + +from .config import ( + artifact_dir, + gate_hooks, + chosen_model, + load_skill, + permission_gate, + provider_env, +) +from .contract import AgentContract +from .scenario import Scenario +from .scenario_tools import ( + SCENARIO_SERVER, + TOOL_NAMES, + load_scenarios, + scenario_tools, + world_summary, +) +from .session import Stage +from .tools import qualified + +SKILL = "write-scenarios" + + +def open_stage( + contract: AgentContract, + *, + out: Path | None = None, + wanted: int = 10, + ask: Callable[..., Any] | None = None, + max_turns: int = 80, +) -> tuple[Stage, Path]: + """A live write-the-scenarios stage, and where it will write.""" + destination = out or artifact_dir(contract.agent) + server, kept = scenario_tools(contract, destination, destination, wanted=wanted) + allowed = [ + "AskUserQuestion", + *(qualified(SCENARIO_SERVER, name) for name in TOOL_NAMES), + ] + options = ClaudeAgentOptions( + system_prompt=( + f"{load_skill(SKILL)}\n\n## This agent\n\n{contract.brief(with_data=True)}" + f"\n\n## Its world\n\n{world_summary(destination)}" + + ( + f"\n\nWrite {wanted} scenarios." + if not kept + else f"\n\n{len(kept)} scenarios already exist and are loaded: " + + ", ".join(scenario.name for scenario in kept) + + ". Submitting one under an existing name replaces it." + ) + ), + allowed_tools=allowed, + mcp_servers={SCENARIO_SERVER: server}, + # Not acceptEdits: that auto-approves Edit and Write before the permission callback is + # consulted, so a stage can rewrite an artifact by hand and skip the tool whose + # whole job is to validate that change. + permission_mode="default", + cwd=str(destination.parent if destination.parent.exists() else Path.cwd()), + setting_sources=[], + max_turns=max_turns, + model=chosen_model(), + env=provider_env(), + ) + options.hooks = gate_hooks(allowed) + options.can_use_tool = permission_gate(ask, allowed) + return Stage(options, name=SKILL), destination + + +def opening(contract: AgentContract, wanted: int = 10, existing: int = 0) -> str: + if existing: + return ( + f"There are already {existing} scenarios for {contract.agent!r}, and they are " + "loaded. Say what you want changed, or add to them. Anything you submit under an " + "existing name replaces it." + ) + return ( + f"Write {wanted} scenarios for {contract.agent!r}.\n\n" + "Look at the world first with inspect_world so every scenario names real records, and " + "read the sub-goals already defined. Work out each scenario's solution with try_calls " + "before you submit it, because a scenario is only kept if its solution passes its own " + "checks and those checks fail without it. Cover the ordinary case, the request that has " + "to be refused, the rule under pressure, and at least one where state has to carry " + "across several turns. Then save_scenarios." + ) + + +def load(destination: Path) -> list[Scenario]: + """The scenarios written for this agent, if any have been.""" + return load_scenarios(Path(destination)) + + +async def write( + contract: AgentContract, + *, + out: Path | None = None, + wanted: int = 10, + follow_ups: list[str] | None = None, + on_event: Callable[..., Any] | None = None, + ask: Callable[..., Any] | None = None, + max_turns: int = 80, +) -> list[Scenario]: + """Run the stage start to finish. Returns whatever scenarios were saved.""" + stage, destination = open_stage( + contract, out=out, wanted=wanted, ask=ask, max_turns=max_turns + ) + async with stage: + await stage.say(opening(contract, wanted), on_event=on_event) + for follow_up in follow_ups or []: + await stage.say(follow_up, on_event=on_event) + return load(destination) diff --git a/src/fi/alk/harness/session.py b/src/fi/alk/harness/session.py index 51db910..75a65a5 100644 --- a/src/fi/alk/harness/session.py +++ b/src/fi/alk/harness/session.py @@ -35,7 +35,13 @@ @dataclass class Event: - """One observable thing the stage did.""" + """One observable thing the stage did. + + ``detail`` carries the data behind what is being shown, not just a label for it: which stage + emitted this, and for a tool call the arguments it was made with. A terminal renders a line + and ignores the rest; anything richer needs the data, and re-parsing a rendered line to get + it back is how a second front end becomes a rewrite. + """ kind: str text: str = "" @@ -60,7 +66,18 @@ def line(self) -> str: if self.kind == DONE: cost = self.detail.get("cost_usd") spent = f" ${cost:.4f}" if isinstance(cost, float) else "" - return f" [{self.detail.get('outcome', '')} turns={self.detail.get('turns', 0)}{spent}]" + failure = self.detail.get("error") + wrong = self.detail.get("unexpected_model") or [] + return ( + f" [{self.detail.get('outcome', '')} " + f"turns={self.detail.get('turns', 0)}{spent}]" + + (f"\n !! {failure}" if failure else "") + + ( + f"\n !! billed to {', '.join(wrong)}, which is not what was asked for" + if wrong + else "" + ) + ) return self.text @@ -75,9 +92,42 @@ class Turn: outcome: str = "" turns: int = 0 cost_usd: float | None = None + error: str = "" + + +_TARGET_KEYS = ( + "file_path", + "path", + "pattern", + "agent", + "tool", + "tool_name", + "table", + "name", +) -_TARGET_KEYS = ("file_path", "path", "pattern", "agent", "tool", "table") +def _why_it_failed(received: Any) -> str: + """What actually went wrong, said in terms somebody can act on.""" + status = getattr(received, "api_error_status", None) + errors = getattr(received, "errors", None) or [] + said = "; ".join(str(error) for error in errors)[:400] + if "invalid_rapt" in said or "invalid_grant" in said: + return ( + "the provider rejected the credentials. GOOGLE_APPLICATION_CREDENTIALS is probably " + "not set in this shell, so it fell back to your gcloud login. Load the env file " + "first: set -a; . ./.env.acceptance; set +a" + ) + return f"the model call failed{f' ({status})' if status else ''}: {said or 'no detail given'}" + + +def readable(tool_name: str) -> str: + """A tool's name as somebody reading along would say it. + + ``mcp__scenarios__try_calls`` is how the model addresses it and is noise to anybody else. + """ + bare = tool_name.rsplit("__", 1)[-1] + return bare.replace("_", " ") def _target(payload: Any) -> str: @@ -125,6 +175,10 @@ def __init__(self, options: ClaudeAgentOptions, *, name: str = "") -> None: self.name = name self.session_id: str | None = None self.history: list[Turn] = [] + # What actually got billed, read back rather than assumed. Asking for a model is not the + # same as getting one: the CLI has its own default, and a request that quietly does not + # take shows up only on the invoice, weeks later, as a number nobody can explain. + self.models_used: set[str] = set() async def __aenter__(self) -> "Stage": self._client = ClaudeSDKClient(options=self._options) @@ -148,6 +202,9 @@ async def stream(self, message: str) -> AsyncIterator[Event]: turn = Turn() async for received in self.client.receive_response(): for event in self._events(received, turn): + # Which stage this came from, stamped once here rather than by every caller, + # so a front end showing several stages can tell them apart. + event.detail.setdefault("stage", self.name) turn.events.append(event) yield event self.history.append(turn) @@ -169,22 +226,40 @@ def _events(self, received: Any, turn: Turn) -> list[Event]: Event( TOOL, tool=block.name, - detail={"target": _target(block.input)}, + detail={ + "target": _target(block.input), + "arguments": block.input, + "label": readable(block.name), + }, ) ) return events if isinstance(received, ResultMessage): - turn.outcome = received.subtype + # subtype alone is not the outcome. A call that failed upstream still arrives with + # subtype "success", so reporting it verbatim tells somebody their stage worked when + # nothing happened at all, and they go looking for the fault in their own request. + failed = bool( + getattr(received, "is_error", False) + or getattr(received, "api_error_status", None) + ) + turn.outcome = "failed" if failed else received.subtype turn.turns = received.num_turns turn.cost_usd = received.total_cost_usd + turn.error = _why_it_failed(received) if failed else "" self.session_id = received.session_id or self.session_id + billed = set(getattr(received, "model_usage", None) or {}) + self.models_used |= billed + unexpected = self.unexpected_models() return [ Event( DONE, detail={ - "outcome": received.subtype, + "outcome": turn.outcome, "turns": received.num_turns, "cost_usd": received.total_cost_usd, + "error": turn.error, + "models": sorted(billed), + "unexpected_model": sorted(unexpected), }, ) ] @@ -219,6 +294,13 @@ async def say( on_event(event) return self.history[-1] + def unexpected_models(self) -> set[str]: + """Models that were billed but not the one asked for.""" + asked = getattr(self._options, "model", None) + if not asked: + return set() + return {used for used in self.models_used if asked.split("-2")[0] not in used} + @property def spent_usd(self) -> float: return sum(turn.cost_usd or 0.0 for turn in self.history) diff --git a/src/fi/alk/harness/skills/build-environment/SKILL.md b/src/fi/alk/harness/skills/build-environment/SKILL.md index 87e7b07..e46780b 100644 --- a/src/fi/alk/harness/skills/build-environment/SKILL.md +++ b/src/fi/alk/harness/skills/build-environment/SKILL.md @@ -1,92 +1,136 @@ --- name: build-environment -description: Build a real, database-backed world that an agent's tools run against. +description: Build the environment an agent is tested in, and everything every scenario shares. --- # Build the environment -You are building the world an agent will be tested in. Its tools will run against your database -and get back whatever it really says, including a refusal when the agent asks for something that -is not there. +## Talking -The contract is the only source of truth. Every table, every row, every id comes from it. If the -contract does not contain something, the world does not have it either. +You are talking to a person, not running a script. They may say hello, ask what you have done so +far, or change their mind. Answer them, briefly and in plain language. + +Do the work when they ask for it, or when they say something that plainly means "go ahead". Do +not start a long piece of work because somebody greeted you. Keep replies short — they can see +every tool you call and what it answered. ## What you are building -1. **A schema.** The tables the agent's data actually needs, with the keys and constraints that - make wrong states impossible to reach. -2. **Seed data.** The agent's real catalogue: its menu, its records, its inventory, taken from - the contract's data, not invented. -3. **One handler per tool.** Python, `def handle(args, db)`, using `db.query`, `db.one` and - `db.execute`. It returns what the real tool would return. -4. **Sequences.** Series of calls whose end state you assert, so consistency across calls is - checked rather than assumed. +Everything **common to every test of this agent**. A scenario is only a delta on what you build +here, so anything shared belongs to you. -## The one thing that matters most +1. **The world.** Whatever this agent acts on, and nothing more. For an agent with a menu and an + order, a database. For a browser agent, the pages it works against. Decide from the contract + what has to exist for its tools to mean anything. +2. **The simulator prompt**, if the agent is conversational. The person on the other side. + Written once, with slots each scenario fills. +3. **The sub-goal catalogue.** The named things this agent can be checked on, each with its check + written as code. -**The world must be able to say no.** +None of these is a form to fill in. You decide what this agent needs. -A canned mock answers every call the same way, so an agent that removes an item that was never -added is told it succeeded, and the test that was supposed to catch that passes. Your handlers -exist to prevent exactly that. +## The world -So for every handler, before you return anything, ask what makes this call impossible and check -for it: +**It must be able to say no.** A canned mock answers every call the same way, so an agent that +removes an item that was never added is told it succeeded, and the test meant to catch that +passes. Your handlers exist to prevent exactly that. -- the id does not exist -- the item exists but is unavailable -- the argument is outside what the tool accepts -- the operation contradicts the current state, like removing from an empty order +For every handler, before returning anything, ask what makes this call impossible and check for +it: the id does not exist, the item is unavailable, the argument is outside what the tool accepts, +the operation contradicts the current state. Then `raise ToolError("...")` saying what was wrong. -When one of those holds, `raise ToolError("...")` with a message that says what was wrong. A -refusal is the world working. It is not an error you should be avoiding. +**A refusal is the world working.** It is not an error to avoid. `KeyError` and `TypeError` are +your bugs; `ToolError` is the world's answer, and the checks tell them apart. -`ToolError` is already available inside a handler. Do not define your own, and do not import -anything: a handler has `args`, `db`, `ToolError` and `json`, and nothing else. +Inside a handler you have `args`, `db`, `ToolError` and `json`, and nothing else. Do not import +anything and do not define your own `ToolError`. Use the argument names exactly as the contract +gives them: a handler reading `order_ids` when the tool takes `order_id` finds nothing, quietly +does nothing, and reports success. -Use the argument names exactly as the contract gives them. A handler reading `order_ids` when -the tool takes `order_id` finds nothing, quietly does nothing, and reports success, which is -the precise failure this world exists to prevent. +Seed the agent's **real** data. Where the contract records something unavailable, a misspelled id, +or a value that looks wrong, **keep it exactly as it is**. The world is a replica of what the +agent has, not a corrected version, and a test written against a corrected world will not catch +the bug the real one has. If an id looks like a typo, that typo is the thing worth testing — do +not fix it, and do not widen the contract to the spelling you would have chosen. -Never let a handler crash on bad input. `KeyError` and `TypeError` are your bugs; `ToolError` is -the world's answer. They must not be confused, and one of the checks tells them apart. +Leave it in its natural starting state: empty carts, no in-flight orders. Scenarios add what they +need. -## How to work +## The simulator prompt -Build in this order and check as you go. +Only for a conversational agent. Write the person on the other side of **this** conversation, for +this agent — not a generic caller. -1. `create_schema` with the whole schema. -2. `seed` each table from the contract's data. Seed the real catalogue, not a sample of it: a - scenario about an unavailable item needs the unavailable item to be in there. -3. `define_handler` for each tool, one at a time. Each is executed the moment you define it, so - read what comes back. Pass `smoke_arguments` that should work. -4. `run_tool` to try the refusals yourself. Call a removal with an id that was never created. If - it succeeds, the handler is wrong, and no other check will catch that for you. -5. `declare_sequence` for at least one flow where state has to carry across calls. Add something, - list it, remove it, list again. This is the failure that individual calls cannot reveal. -6. `check_world` to see everything at once, fix what it reports, and repeat. -7. `save_world` when it passes. +It has to cover how someone in this conversation actually behaves: that they are living the +situation rather than describing it, that they speak one short turn at a time, that they never +narrate or explain they are testing anything, what they know and when they may say it, and when +the conversation is finished. + +Leave slots for what changes per scenario, written `{{ instruction }}`. At minimum there is one +for the task. Add others if this agent needs them. + +There is no persona. Do not invent characters, moods or backstories — "I'm in a cab, in a hurry" +is noise. What varies between scenarios is real conditions: what is in stock, whether the customer +already exists, what they know and when they will say it. + +## The sub-goals -`save_world` refuses a world that has not passed its checks or has no declared sequence. That -refusal is not an obstacle to work around; it is the same guarantee you are building into the -handlers. +The named things this agent can be checked on. Defined **here, once**, because every scenario +names the ones it needs — that is what makes results add up. If "confirms the order back" is the +same sub-goal in twelve scenarios, you can say it failed in seven of them. -## Seed data +**Write the check as code wherever the answer is observable.** -Use the contract's real values. Real ids, real names, real prices, real availability flags. The -whole point is that a test can reference something and have it be there. +```python +def check(world, calls): + rows = world.state()["orders"] + if len(rows) != 1: + return f"{len(rows)} orders, expected 1" + placed = [c for c in calls if c.name == "order_combo_meal" and c.ok] + if not placed: + return "no order call succeeded" + if placed[0].arguments.get("drink_size") != "L": + return f"drink_size was {placed[0].arguments.get('drink_size')!r}, asked for L" + return None +``` -Where the contract records that something is unavailable, or a typo in an id, or a value that -looks wrong, **keep it as it is**. The world is a replica of what the agent actually has, not a -corrected version of it. A test written against a corrected world will not catch the bug the -real one has. +You get the world afterwards and every call that was made, each with `.name`, `.arguments`, `.ok` +and `.refused`. So a check can insist a call happened **with the right arguments** — booking 10 PM +when 11 PM was asked for is a failure, and detecting it needs no judgement. -Leave the world in its natural starting state: empty carts, no in-flight orders, nothing that -belongs to one particular scenario. Individual scenarios add what they need on top of it. +Return a sentence when something is wrong, `None` when it held. + +Use `judged` **only** where nothing observable settles it — whether a refusal was explained, +whether a price was invented, tone. Say what a model has to decide and why code cannot. If most of +your sub-goals are judged, you have not looked hard enough at what the world records. + +## How to work + +1. `create_schema` with the whole schema. +2. `seed` each table from the contract's real data. +3. `define_handler` for each tool, one at a time. Each runs the moment you define it — read what + comes back. +4. `run_tool` to try the refusals yourself. Call a removal with an id that was never created. If + it succeeds the handler is wrong, and no other check will catch that for you. +5. `change_data` if you put a row in wrong. Seeding only inserts. +6. `declare_sequence` for at least one flow where state has to carry across calls. Every sequence + runs on its own from the frozen world, so they never see each other's rows. +7. `write_simulator_prompt`, if this agent is conversational. +8. `add_sub_goal` for each thing worth checking, with its check in code. +9. `check_world`, fix what it names, repeat. +10. `save_world`. + +If `check_world` returns the same score three times, stop and read the failures literally. +Whatever you are changing is not what is failing. + +`save_world` refuses an environment that fails its checks, has no sequence, has no sub-goals, has +only judged sub-goals, is missing a simulator prompt for a conversational agent, or still holds +rows left over from your own testing. Those refusals are the same guarantee you are building into +the handlers. ## Finishing -Say what you built: the tables, roughly how many rows, which tools, and which refusals you -verified. Then say plainly anything you were unsure about, especially where the contract was -thin and you had to decide. +Say what you built: the tables and roughly how many rows, which tools, which refusals you +verified, what the simulator prompt asks each scenario for, and the sub-goals with how many are +settled by code. Then say plainly anything you were unsure about, especially where the contract +was thin and you had to decide. diff --git a/src/fi/alk/harness/skills/run-scenarios/SKILL.md b/src/fi/alk/harness/skills/run-scenarios/SKILL.md new file mode 100644 index 0000000..10710aa --- /dev/null +++ b/src/fi/alk/harness/skills/run-scenarios/SKILL.md @@ -0,0 +1,56 @@ +--- +name: run-scenarios +description: Run the written scenarios against the real agent and say what the results mean. +--- + +# Run the scenarios + +## Talking + +You are talking to a person, not running a script. Answer what they ask, briefly. Run what they +ask you to run. Keep replies short — they can see every tool you call and what it answered. + +Each call costs real money and takes minutes. Do not run the whole suite because somebody said +hello, and do not re-run a scenario that just passed. + +## What happens when you run one + +`run_scenario` does all of it: restores the world, applies the scenario's setup, stands up the +webhook, points the assistant's **own** tools at it, places the call through ALK's voice case, and +runs the sub-goals' checks against what the world holds afterwards plus the calls that were made. + +It blocks for several minutes. Run one at a time and read the result before starting the next. + +`preflight` first, before the first call of a session. It costs nothing and catches the failures +that would otherwise arrive after the expensive part. + +## Reading a result + +You get the sub-goals settled by code, the ones left to a judge, and **every tool call the agent +made, with its arguments and whether the world accepted it**. That last list is where the answer +usually is. + +Before you report a failure as a finding about the agent, ask which of these it is: + +- **The agent did the wrong thing.** A real finding. Say what it did and what it should have done. +- **The world refused a call the agent was entitled to make.** Look at the arguments. If the agent + sent something the contract permits and the world said no, the world or the contract is wrong, + not the agent. +- **The check is wrong.** The commonest one. A sub-goal that encodes *how* an agent should comply + fails a correct agent that complied differently — a check that demands a refusal tool call fails + an agent that refused from its own prompt without calling anything. Check the outcome, not the + route. +- **The simulated caller did not do its job.** If the caller hung up before asking for what the + instruction said, the scenario never happened. That is a simulator prompt problem. + +A run where nothing reached the world says nothing about the agent. Report it as that, not as a +failure. + +## What to say + +Say what passed, what failed, and for each failure which of the four causes above it is. Where it +is ours, say what would fix it — the sub-goal to rewrite, the contract argument to correct — and +do not report it as a finding about the agent. + +Judged sub-goals are reported as judged and not counted. Say so rather than letting a `2/2` read +as though everything was checked. diff --git a/src/fi/alk/harness/skills/understand-agent/SKILL.md b/src/fi/alk/harness/skills/understand-agent/SKILL.md index f9d313d..2fa7e24 100644 --- a/src/fi/alk/harness/skills/understand-agent/SKILL.md +++ b/src/fi/alk/harness/skills/understand-agent/SKILL.md @@ -5,6 +5,18 @@ description: Read an AI agent's source and produce its testing contract. # Understand the agent +## Talking + +You are talking to a person, not running a script. They may say hello, ask what you have done so +far, ask what something means, or change their mind. Answer them, briefly and in plain language. + +Do the work of this stage when they ask for it, or when they say something that plainly means +"go ahead". Do not start a long piece of work because somebody greeted you. If you are unsure +whether they want you to begin, say what you would do and ask. + +Keep replies short. They can see every tool you call and what it answered, so do not narrate +what is already on their screen or list back what you just did in detail. + You are reading the source of an AI agent so that a test environment can be built for it. Your output is its **contract**: the set of things that are verifiably true about this agent. Every later stage is confined to it. A world may only implement tools listed here; a scenario may only diff --git a/src/fi/alk/harness/skills/write-scenarios/SKILL.md b/src/fi/alk/harness/skills/write-scenarios/SKILL.md new file mode 100644 index 0000000..05669fb --- /dev/null +++ b/src/fi/alk/harness/skills/write-scenarios/SKILL.md @@ -0,0 +1,93 @@ +--- +name: write-scenarios +description: Write scenarios as deltas on the built environment, each proved before it is kept. +--- + +# Write the scenarios + +## Talking + +You are talking to a person, not running a script. Answer what they ask, briefly. Do the work +when they ask for it. Keep replies short — they can see every tool you call and what it answered. + +## What a scenario is + +The environment is already built: the world, the simulator prompt, the catalogue of sub-goals. A +scenario is only a **delta** on that base. + +``` +name short identifier +use_case which branch of the agent's real use cases this belongs to +setup what changes in the world after reset — a few rows +instruction the task. For a conversational agent it fills the simulator prompt's slot +variables any other slot that prompt asks for +solution what a correct agent would do: [{tool, arguments}] +sub_goals names from the catalogue that must hold +``` + +There is no persona and no opening line. **Variability comes from real conditions**, which live +in `setup`: the item is out of stock, the customer already exists, the order already has three +items in it. Not from invented characters. + +## Organise by use case, then by branch + +A login flow is not one row with happy and edge cases inside it. It is many rows: +login-with-Google, login-with-Microsoft, forgot-password, sign-up-with-email. Do the same here: +find the agent's real use cases, and let their branches be the scenarios. + +Different outcomes are different scenarios. The customer who accepts a substitute and the customer +who refuses one are two rows, not one. + +## The solution is not optional + +Every scenario carries what a correct agent would do. It is the only way to show the scenario can +be passed at all, and it is checked before the scenario is kept: + +- Your solution is played through a fresh world, and the checks of your sub-goals must **pass**. +- The same checks are then run with nothing done at all, and must **fail**. + +If the first fails, either the scenario is impossible or the sub-goal's check is wrong. If the +second fails, the checks grade nothing and the scenario would report a result nobody should +believe. + +Work the solution out with `try_calls` before you submit. Run the calls, look at the state they +leave, and confirm the sub-goals you are naming actually respond to it. + +## Reuse the sub-goals + +Name entries from the catalogue. Do not restate them in your own words, and do not invent a new +one where an existing one means the same thing — the whole point is that "confirms the order back" +is the same sub-goal in every scenario, so the results can be added together. + +If something genuinely needs checking and no entry covers it, add one with `add_sub_goal`, with +its check in code. Prefer code over a judged check: you have the world afterwards and every call +with its arguments, and most things worth checking are visible in one of them. + +## What makes a suite worth running + +Spread across these. Ten happy paths tell you nothing you did not know. + +- **The ordinary branch**, done cleanly. You need a baseline. +- **The branch that cannot be completed**: the item is not there, the record does not exist, the + option is outside what the tool accepts. The right behaviour is to refuse clearly and offer + what is possible. +- **The rule under pressure**: the customer pushes for something a hard constraint forbids, twice. + Giving way under pressure is the failure most worth catching. +- **State that has to carry**: add, change your mind, remove, confirm. The agent has to know what + it did two turns ago. +- **The same use case with the world seeded differently**: in stock and out of stock are two + rows, not one. + +## How to work + +1. `inspect_world` with no table, then look at the ones that matter. Read the sub-goals already + defined. +2. Read the contract's hard constraints. Each is a branch waiting to be written. +3. For each scenario: work out the solution, `try_calls` it, then `submit_scenario`. +4. Read what comes back. A refusal tells you exactly what could not be proved. +5. `save_scenarios` when you have the number that was asked for. + +## Finishing + +Say what the suite covers and what it does not, which sub-goals carry the most scenarios, and name +anything you could not test because the environment or the contract does not support it. diff --git a/src/fi/alk/harness/tools.py b/src/fi/alk/harness/tools.py index 8640070..3acdafa 100644 --- a/src/fi/alk/harness/tools.py +++ b/src/fi/alk/harness/tools.py @@ -108,6 +108,34 @@ async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: ) +_JSON_TYPES = { + str: "string", + int: "integer", + float: "number", + bool: "boolean", + list: "array", + dict: "object", +} + + +def schema(properties: dict[str, type], required: list[str]) -> dict[str, Any]: + """A tool's inputs, saying which of them are actually required. + + Handing the decorator a plain ``{name: type}`` mapping marks every parameter mandatory, so a + tool with an optional field refuses any call that leaves it out — "Input validation error: + 'seed' is a required property" — for a field the tool itself treats as optional. The model + then has to guess that it must pass an empty value, and burns turns finding out. + """ + return { + "type": "object", + "properties": { + name: {"type": _JSON_TYPES.get(kind, "string")} + for name, kind in properties.items() + }, + "required": list(required), + } + + def qualified(server: str, tool_name: str) -> str: """The name an in-process MCP tool is granted under.""" return f"mcp__{server}__{tool_name}" diff --git a/src/fi/alk/harness/world/__init__.py b/src/fi/alk/harness/world/__init__.py index eda492e..9fc85c9 100644 --- a/src/fi/alk/harness/world/__init__.py +++ b/src/fi/alk/harness/world/__init__.py @@ -5,7 +5,8 @@ suite that decides whether a world is usable at all. """ -from .probe import EDGE, HAPPY, SEQUENCE, ProbeReport, ProbeResult, probe +from .kinds import WorldKind, register_kind, supported as supported_kinds +from .probe import EDGE, HAPPY, SEQUENCE, ProbeReport, ProbeResult, dirty_state, probe from .runtime import Call, Db, GeneratedWorld, ToolError, WorldSpec from .snapshot import apply_overlay, read_manifest, restore, save @@ -17,6 +18,10 @@ "HAPPY", "ProbeReport", "ProbeResult", + "WorldKind", + "dirty_state", + "register_kind", + "supported_kinds", "SEQUENCE", "ToolError", "WorldSpec", diff --git a/src/fi/alk/harness/world/expectations.py b/src/fi/alk/harness/world/expectations.py new file mode 100644 index 0000000..c1b6b7d --- /dev/null +++ b/src/fi/alk/harness/world/expectations.py @@ -0,0 +1,91 @@ +"""What a world is expected to look like afterwards, and whether it does. + +Written once and used twice. The build stage declares a sequence and asserts the state it leaves +behind; a scenario declares the state a conversation should leave behind. Those are the same +question asked at two different scales, and if each had its own implementation they would drift +until a check that passes the gate fails the run for reasons that have nothing to do with the +agent. + +The shape is ``{"table.count": 3, "table.column": "value"}``: how many records there are, and +whether a particular value is among them. +""" + +from __future__ import annotations + +from typing import Any, Mapping + +COUNT = "count" + + +def check_state( + state: Mapping[str, list[dict[str, Any]]], expected: Mapping[str, Any] +) -> list[str]: + """Every expectation that does not hold, said in terms of what was found instead.""" + failures: list[str] = [] + for path, want in (expected or {}).items(): + table, _, column = str(path).partition(".") + if table not in state: + failures.append( + f"{path}: no {table} in this world; it has " + f"{', '.join(sorted(state)) or 'nothing'}" + ) + continue + rows = state[table] + if column in ("", COUNT): + if len(rows) != want: + failures.append(f"{path}: {len(rows)} rows, expected {want}") + continue + if rows and column not in rows[0]: + failures.append( + f"{path}: {table} has no {column}; its columns are " + f"{', '.join(sorted(rows[0]))}" + ) + continue + present = {str(row.get(column)) for row in rows} + # A list means every one of these has to be somewhere, which is how an expectation about + # a basket of several items is naturally written. Compared as a single value it could + # never hold, and an expectation that cannot hold grades nothing while appearing to. + wanted = list(want) if isinstance(want, (list, tuple)) else [want] + absent = [value for value in wanted if str(value) not in present] + if absent: + found = ", ".join(sorted(present)[:6]) or "nothing" + failures.append( + f"{path}: no row has {column}=" + + " or ".join(repr(value) for value in absent) + + f"; found {found}" + ) + return failures + + +def unresolvable( + state: Mapping[str, list[dict[str, Any]]], expected: Mapping[str, Any] +) -> list[str]: + """Expectations that name a table or column the world does not have. + + Separate from whether they hold, because they are a different kind of wrong. An expectation + that fails is a finding about the agent; one that names a table nobody built is a finding + about the expectation, and letting it through means grading a run against a typo. + """ + problems: list[str] = [] + for path in expected or {}: + table, _, column = str(path).partition(".") + if table not in state: + # Indexing a particular row is the most common way to write an expectation this + # cannot carry, and saying only "no such table" sends the reader looking for a + # spelling mistake instead of at the shape. + indexed = "[" in table + problems.append( + f"{path}: no table called {table!r}" + + ( + ". Expectations are about the whole table, not one row: use " + "'table.count' for how many, or 'table.column' for a value that has to " + "appear in some row." + if indexed + else "" + ) + ) + elif ( + column not in ("", COUNT) and state[table] and column not in state[table][0] + ): + problems.append(f"{path}: {table} has no column {column!r}") + return problems diff --git a/src/fi/alk/harness/world/kinds.py b/src/fi/alk/harness/world/kinds.py new file mode 100644 index 0000000..2c8eab5 --- /dev/null +++ b/src/fi/alk/harness/world/kinds.py @@ -0,0 +1,133 @@ +"""What a kind of world has to be able to do, so the checks do not care which kind it is. + +A world backed by a database and a world backed by a page are different in every detail and the +same in what matters: something either exists in them or does not, an action either takes effect +or is refused, and what an action leaves behind is either carried or lost. Those are the things +worth checking, and none of them mention a table. + +So the checks are written against this, and a kind supplies the four answers only it can give: +what exists, what the mutable state is, how to freeze it, and how to put it back. Adding a kind +is a class and a registration; nothing in the gate changes. +""" + +from __future__ import annotations + +from typing import Any, Callable, Protocol, runtime_checkable + +from .runtime import GeneratedWorld + + +@runtime_checkable +class WorldKind(Protocol): + """The per-kind half of a world. The shared half is ``GeneratedWorld``.""" + + key: str + label: str + + def values_present(self, world: GeneratedWorld) -> set[str]: + """Every identifier this world contains. + + Answers whether the catalogue is complete: a contract that permits a value the world has + never heard of produces a tool that refuses forever, which is indistinguishable from a + tool being correctly strict. + """ + + def mutable_state(self, world: GeneratedWorld) -> dict[str, int]: + """Named parts of the world that an action can change, and how much is in each. + + Used for two things: noticing that a saved world still holds whatever the builder was + experimenting with, and noticing that a sequence of actions left nothing behind. + """ + + def describe(self, world: GeneratedWorld) -> str: + """A short human-readable account of what is in the world.""" + + +class SqliteWorld: + """A world whose state is rows in tables. Tool APIs, and anything with a data store.""" + + key = "sqlite" + label = "a database behind the agent's tools" + + def values_present(self, world: GeneratedWorld) -> set[str]: + present: set[str] = set() + for rows in world.state().values(): + for row in rows: + for value in row.values(): + if isinstance(value, str) and value: + present.add(value) + return present + + def mutable_state(self, world: GeneratedWorld) -> dict[str, int]: + return {name: len(rows) for name, rows in world.state().items()} + + def describe(self, world: GeneratedWorld) -> str: + counts = self.mutable_state(world) + return ", ".join(f"{name}: {count}" for name, count in sorted(counts.items())) + + +class BrowserWorld: + """A world whose state is pages and the actions that change them. + + ALK already carries a browser environment fed DOM snapshots and action fixtures, and it + already refuses a click matching no fixture. So this is the same move as the database kind: + generate instances of a shape that exists, rather than invent a mechanism. + + What exists here is the set of things an agent can reach, which is selectors and URLs rather + than ids; what changes is which snapshot is current and what the actions have mutated. + """ + + key = "browser" + label = "pages and the actions that change them" + + def values_present(self, world: GeneratedWorld) -> set[str]: + present: set[str] = set() + for rows in world.state().values(): + for row in rows: + for column in ("url", "selector", "id", "name", "action"): + value = row.get(column) + if isinstance(value, str) and value: + present.add(value) + return present + + def mutable_state(self, world: GeneratedWorld) -> dict[str, int]: + return {name: len(rows) for name, rows in world.state().items()} + + def describe(self, world: GeneratedWorld) -> str: + counts = self.mutable_state(world) + return ", ".join(f"{name}: {count}" for name, count in sorted(counts.items())) + + +_REGISTRY: dict[str, Callable[[], WorldKind]] = { + SqliteWorld.key: SqliteWorld, + BrowserWorld.key: BrowserWorld, +} + + +def register_kind(key: str, factory: Callable[[], WorldKind]) -> None: + """Add a kind of world. Computer use, a filesystem, a queue: a class and this line.""" + _REGISTRY[key] = factory + + +def resolve(key: str) -> WorldKind: + if key not in _REGISTRY: + raise NotImplementedError( + f"no world kind {key!r}; registered kinds are {', '.join(sorted(_REGISTRY))}" + ) + return _REGISTRY[key]() + + +def supported() -> tuple[str, ...]: + return tuple(sorted(_REGISTRY)) + + +def for_contract(contract: Any) -> WorldKind: + """The kind of world an agent needs, from what the contract says it is. + + Chosen rather than guessed at build time: an agent reachable by voice and by browser is one + agent with two runtimes, and which world to build is a decision about what is being tested. + """ + modality = str(getattr(contract, "modality", "") or "").lower() + if modality in ("browser", "computer_use", "cua"): + return resolve("browser") + return resolve("sqlite") diff --git a/src/fi/alk/harness/world/probe.py b/src/fi/alk/harness/world/probe.py index 5431c9e..1dd9590 100644 --- a/src/fi/alk/harness/world/probe.py +++ b/src/fi/alk/harness/world/probe.py @@ -24,6 +24,9 @@ from typing import Any, Iterable, Mapping, Sequence from ..contract import AgentContract, ToolSpec +from .expectations import check_state +from .kinds import WorldKind, for_contract +from .kinds import resolve as _resolve_kind from .runtime import GeneratedWorld HAPPY = "happy" @@ -93,17 +96,6 @@ def _valid_arguments(tool: ToolSpec) -> dict[str, Any]: return arguments -def _seeded_values(world: GeneratedWorld) -> set[str]: - """Every value present anywhere in the world, for checking the catalogue is complete.""" - present: set[str] = set() - for rows in world.state().values(): - for row in rows: - for value in row.values(): - if isinstance(value, str) and value: - present.add(value) - return present - - def _is_a_real_identifier(value: Any) -> bool: """Whether a permitted value names a record, rather than being an enum like 'M' or 'null'.""" if not isinstance(value, str) or value in ("", "null", "none", "None"): @@ -111,7 +103,9 @@ def _is_a_real_identifier(value: Any) -> bool: return len(value) > 2 and not value.isdigit() -def _missing_catalogue(world: GeneratedWorld, contract: AgentContract) -> list[str]: +def _missing_catalogue( + world: GeneratedWorld, contract: AgentContract, kind: WorldKind +) -> list[str]: """Identifiers the contract says a tool accepts that are nowhere in the seeded world. The gap this closes is a whole category left unseeded. Every call naming a sauce then fails, @@ -119,7 +113,7 @@ def _missing_catalogue(world: GeneratedWorld, contract: AgentContract) -> list[s nothing can be ordered scores perfectly. Whether the catalogue is complete cannot be settled by behaviour, so it is checked against the data. """ - present = _seeded_values(world) + present = kind.values_present(world) missing: list[str] = [] for tool in contract.tools: for arg, values in (tool.arg_values or {}).items(): @@ -156,11 +150,21 @@ def _reads_argument(source: str, name: str) -> bool: return re.search(pattern, source) is not None -def _looks_like_an_identifier(name: str, declared: str) -> bool: - """Whether an argument names something that has to exist for the call to make sense.""" - if name.endswith(("_id", "_ids", "id", "_key", "_ref")): - return True - return declared in ("str", "list[str]", "List[str]") +def _looks_like_an_identifier(name: str, _declared: str = "") -> bool: + """Whether an argument names a record that has to exist for the call to make sense. + + Decided by the name alone. Treating every ``str`` argument as a catalogue was a trap: a + ``size`` accepting "Medium" and "Large" then demanded rows called Medium and Large in the + world, which can never be seeded sensibly. The only ways out were to invent nonsense rows or + to edit the contract, so a check meant to catch a missing menu instead pushed towards + corrupting the record of what the agent is. + + A missed catalogue is a check that does not fire. A false one is a stage with no legal move, + which is much worse, so this stays narrow. + """ + return ( + name.endswith(("_id", "_ids", "_key", "_ref", "_code", "_sku")) or name == "id" + ) def _identifier_arguments(tool: ToolSpec) -> dict[str, Any] | None: @@ -188,6 +192,7 @@ def probe( contract: AgentContract, *, sequences: Iterable[Mapping[str, Any]] = (), + kind: WorldKind | None = None, ) -> ProbeReport: """Exercise the world and report what it can and cannot do. @@ -196,6 +201,7 @@ def probe( from a schema. """ report = ProbeReport() + kind = kind or for_contract(contract) # Every probe runs from the same starting world. Probes mutate, so without reverting # between them each one inherits the debris of the last and a check expecting three rows @@ -215,7 +221,7 @@ def probe( ) ) - for gap in _missing_catalogue(world, contract): + for gap in _missing_catalogue(world, contract, kind): report.results.append( ProbeResult( gap.split(":")[0], @@ -224,7 +230,7 @@ def probe( f"the contract accepts values the world does not have: {gap}", ) ) - if not _missing_catalogue(world, contract): + if not _missing_catalogue(world, contract, kind): report.results.append( ProbeResult("catalogue", DATA, True, "every permitted identifier exists") ) @@ -320,8 +326,10 @@ def probe( return report -def dirty_tables( - world: GeneratedWorld, sequences: Iterable[Mapping[str, Any]] +def dirty_state( + world: GeneratedWorld, + sequences: Iterable[Mapping[str, Any]], + kind: WorldKind | None = None, ) -> list[str]: """Tables a scenario writes to that already hold rows before anything has happened. @@ -331,14 +339,15 @@ def dirty_tables( seven. Which tables are transactional is not guessable from a schema, so it is worked out by running the declared sequences and seeing what moves. """ + kind = kind or _resolve_kind("sqlite") baseline = world.checkpoint() - before = {name: len(rows) for name, rows in world.state().items()} + before = kind.mutable_state(world) touched: set[str] = set() for index, sequence in enumerate(sequences): world.revert(baseline) _run_sequence(world, sequence, index) - for name, rows in world.state().items(): - if len(rows) != before.get(name, 0): + for name, size in kind.mutable_state(world).items(): + if size != before.get(name, 0): touched.add(name) world.revert(baseline) return sorted(name for name in touched if before.get(name, 0) > 0) @@ -365,20 +374,7 @@ def _run_sequence( if not call.ok: return ProbeResult(name, SEQUENCE, False, f"{call.name}: {call.error}") - state = world.state() - for path, expected in (sequence.get("expect_state") or {}).items(): - table, _, column = path.partition(".") - rows = state.get(table, []) - if column == "count": - if len(rows) != expected: - return ProbeResult( - name, - SEQUENCE, - False, - f"{table} holds {len(rows)} rows, expected {expected}", - ) - elif not any(str(row.get(column)) == str(expected) for row in rows): - return ProbeResult( - name, SEQUENCE, False, f"no row in {table} has {column}={expected!r}" - ) + failures = check_state(world.state(), sequence.get("expect_state") or {}) + if failures: + return ProbeResult(name, SEQUENCE, False, failures[0]) return ProbeResult(name, SEQUENCE, True) diff --git a/src/fi/alk/harness/world/runtime.py b/src/fi/alk/harness/world/runtime.py index cc219d8..1834f82 100644 --- a/src/fi/alk/harness/world/runtime.py +++ b/src/fi/alk/harness/world/runtime.py @@ -20,11 +20,24 @@ from pathlib import Path from typing import Any, Mapping, Sequence -from fi.simulate.environment import ( - EnvironmentAdapter, - EnvironmentSnapshot, - ToolExecutionResult, -) +try: + from fi.simulate.environment import ( + EnvironmentAdapter, + EnvironmentSnapshot, + ToolExecutionResult, + ) +except ( + ImportError +) as missing: # pragma: no cover - depends on how the repo was installed + # Importing anything under fi.simulate runs that package's __init__, which pulls in its + # LiveKit scenario generator. So a harness that never makes a voice call still needs the + # voice extra installed, and without it the failure surfaces three imports away from the + # cause as a bare "No module named 'livekit'". + raise ImportError( + "The harness needs the environment interface from fi.simulate, and importing it pulls " + "in that package's optional LiveKit dependency. Install it with:\n" + " uv sync --extra livekit --group dev" + ) from missing class ToolError(Exception): @@ -202,6 +215,19 @@ def _record(self, call: Call) -> Call: # -- state ----------------------------------------------------------------------- + def _settle(self) -> None: + """Close any transaction left open on the connection. + + A handler that only reads still leaves an implicit read transaction behind, and SQLite + refuses to back up into a connection that has one open: "destination database is in + use". Left unsettled, the first read-only handler poisons every probe after it, and the + world can never be checked or saved. + """ + try: + self.connection.commit() + except sqlite3.Error: + self.connection.rollback() + def checkpoint(self) -> sqlite3.Connection: """A copy of the current data, to come back to. @@ -209,15 +235,15 @@ def checkpoint(self) -> sqlite3.Connection: against the debris of the ones before it, and a check expecting three rows finds seven. The same restore-a-fresh-copy discipline scenarios use, applied to the gate itself. """ + self._settle() copy = sqlite3.connect(":memory:") - with copy: - self.connection.backup(copy) + self.connection.backup(copy) return copy def revert(self, checkpoint: sqlite3.Connection) -> None: """Put the data back as it was when the checkpoint was taken.""" - with self.connection: - checkpoint.backup(self.connection) + self._settle() + checkpoint.backup(self.connection) def state(self) -> dict[str, Any]: """Every table and its rows: what the checks compare against after a run.""" diff --git a/src/fi/alk/harness/world/snapshot.py b/src/fi/alk/harness/world/snapshot.py index 6a62f34..bdc3bb7 100644 --- a/src/fi/alk/harness/world/snapshot.py +++ b/src/fi/alk/harness/world/snapshot.py @@ -53,7 +53,13 @@ def load(database=None): ''' -def save(world: GeneratedWorld, path: str | Path, *, notes: str = "") -> Path: +def save( + world: GeneratedWorld, + path: str | Path, + *, + notes: str = "", + sequences: list[dict[str, Any]] | None = None, +) -> Path: """Write the world out: the snapshot, the handlers, the module, and a manifest.""" root = Path(path) (root / HANDLERS).mkdir(parents=True, exist_ok=True) @@ -82,7 +88,13 @@ def save(world: GeneratedWorld, path: str | Path, *, notes: str = "") -> Path: { "agent": world.name, "tools": sorted(world.handlers), + # Written because restore reads it. Without it a restored world publishes no + # tool descriptions at all, and every later stage has to reconstruct them. + "tool_specs": list(world.tools), "tables": {name: len(rows) for name, rows in state.items()}, + # Kept because they are judgement about this agent, not something a schema + # implies. A world picked up again can be re-verified without redeclaring them. + "sequences": list(sequences or []), "notes": notes, }, indent=2, diff --git a/src/fi/alk/harness/world/tools.py b/src/fi/alk/harness/world/tools.py index 24aadde..de211c0 100644 --- a/src/fi/alk/harness/world/tools.py +++ b/src/fi/alk/harness/world/tools.py @@ -22,10 +22,22 @@ from claude_agent_sdk import create_sdk_mcp_server, tool +from ..environment import ( + SubGoal, + load_simulator_prompt, + load_catalogue, + save_catalogue, + save_simulator_prompt, + validate_simulator_prompt, + validate_sub_goal, +) +from ..tools import schema +from ..amend import add_rule, drop_rule, fix_tool, widen from ..contract import AgentContract -from .probe import dirty_tables, probe +from .kinds import for_contract +from .probe import dirty_state, probe from .runtime import GeneratedWorld -from .snapshot import save +from .snapshot import DATABASE, read_manifest, restore, save WORLD_SERVER = "world" @@ -49,9 +61,18 @@ def _brief(value: Any, limit: int = 400) -> str: def world_tools(contract: AgentContract, destination: Path) -> Any: """A server exposing the world-building surface for one agent.""" - world = GeneratedWorld(":memory:") + # An existing world is picked up rather than replaced. Amending one is the ordinary case + # once it has been built once, and starting empty every time would mean rebuilding a + # catalogue from scratch to add a single item to it. + existing = (destination / DATABASE).exists() + world = restore(destination) if existing else GeneratedWorld(":memory:") world.name = contract.agent - sequences: list[dict[str, Any]] = [] + kind = for_contract(contract) + catalogue = load_catalogue(destination) + scores: list[float] = [] + sequences: list[dict[str, Any]] = ( + list(read_manifest(destination).get("sequences") or []) if existing else [] + ) @tool( "create_schema", @@ -95,11 +116,39 @@ async def seed(args: dict[str, Any]) -> dict[str, Any]: total = len(world.state().get(table, [])) return _ok(f"{written} rows inserted into {table}; {total} rows there now") + @tool( + "change_data", + "Change or remove rows already in the world: one UPDATE or DELETE statement. Seeding " + "only ever inserts, so without this a row put in wrong can never be taken out, and the " + "only way left to make a check pass is to change the contract, which is the wrong " + "repair. Use inspect_world to read; this is for changing.", + {"sql": str}, + ) + async def change_data(args: dict[str, Any]) -> dict[str, Any]: + statement = str(args.get("sql") or "").strip() + verb = statement.split(None, 1)[0].upper() if statement else "" + if verb not in ("UPDATE", "DELETE"): + return _err( + "this runs one UPDATE or DELETE. Use seed to add rows, create_schema to change " + "the shape of a table, and inspect_world to look." + ) + try: + changed = world.connection.execute(statement).rowcount + world.connection.commit() + except Exception as failed: + world.connection.rollback() + return _err(f"rejected: {failed}") + counts = ", ".join(f"{n}: {len(r)}" for n, r in sorted(world.state().items())) + return _ok(f"{changed} rows changed. The world now holds {counts}") + @tool( "define_handler", "Define one tool's implementation. The source must define handle(args, db) and is run " "immediately against the seeded world, so errors come straight back.", - {"tool_name": str, "source": str, "smoke_arguments": dict}, + schema( + {"tool_name": str, "source": str, "smoke_arguments": dict}, + ["tool_name", "source"], + ), ) async def define_handler(args: dict[str, Any]) -> dict[str, Any]: name = str(args["tool_name"]) @@ -122,7 +171,7 @@ async def define_handler(args: dict[str, Any]) -> dict[str, Any]: @tool( "run_tool", "Call a defined tool and see what the world does. Use this to check a refusal works.", - {"tool_name": str, "arguments": dict}, + schema({"tool_name": str, "arguments": dict}, ["tool_name"]), ) async def run_tool(args: dict[str, Any]) -> dict[str, Any]: call = world.call(str(args["tool_name"]), args.get("arguments") or {}) @@ -136,9 +185,12 @@ async def run_tool(args: dict[str, Any]) -> dict[str, Any]: "declare_sequence", "Declare a series of calls whose end state should hold, so consistency across calls is " "checked. Each call is {tool, arguments}. expect_state keys are 'table.column' or " - "'table.count'. Declaring the same name again replaces it, so a mistake is fixed by " - "redeclaring rather than accumulating.", - {"name": str, "calls": list, "expect_state": dict}, + "'table.count'. Declaring the same name again replaces it.\n\n" + "Every sequence runs on its own from the frozen world: the state is put back before each " + "one, so they never see each other's rows and expect_state is an absolute count, not a " + "running total. If a sequence fails, the fault is in that sequence, not in the ones " + "declared before it.", + schema({"name": str, "calls": list, "expect_state": dict}, ["name", "calls"]), ) async def declare_sequence(args: dict[str, Any]) -> dict[str, Any]: name = str(args.get("name") or f"sequence-{len(sequences)}") @@ -199,23 +251,210 @@ async def drop_sequence(args: dict[str, Any]) -> dict[str, Any]: ) return _ok(f"{name} dropped. {len(sequences)} left") + @tool( + "amend_contract", + "Let one of the agent's tools accept values it did not before. Use this when the world " + "holds something the agent has no way to name: an item added to the menu that item_id " + "does not list is dead data, and a scenario about it can only fail.\n\n" + "Only widen where the agent genuinely should accept the value. Say why in one line; it " + "is recorded on the contract, because a contract nobody can audit is worth nothing.", + {"tool_name": str, "argument": str, "values": list, "why": str}, + ) + async def amend_contract(args: dict[str, Any]) -> dict[str, Any]: + done, said = widen( + contract, + destination, + tool_name=str(args.get("tool_name") or ""), + argument=str(args.get("argument") or ""), + values=[str(value) for value in (args.get("values") or [])], + why=str(args.get("why") or ""), + ) + return _ok(said) if done else _err(said) + + @tool( + "add_rule", + "Give the agent a hard rule its source did not state, when the operator asks for one. " + "The agent under test is told every rule and the judge grades against them, so this " + "changes what is being tested. Say why in one line; it is recorded on the contract.", + {"rule": str, "why": str}, + ) + async def add_rule_tool(args: dict[str, Any]) -> dict[str, Any]: + done, said = add_rule( + contract, + destination, + rule=str(args.get("rule") or ""), + why=str(args.get("why") or ""), + ) + return _ok(said) if done else _err(said) + + @tool( + "inspect_world", + "Look at what is in the world you are building. Without a table, lists the tables and " + "how many rows each holds; with one, returns rows from it.", + schema({"table": str, "limit": int}, []), + ) + async def inspect_world(args: dict[str, Any]) -> dict[str, Any]: + state = world.state() + table = str(args.get("table") or "") + if not table: + return _ok( + "\n".join( + f"{name}: {len(rows)} rows" for name, rows in sorted(state.items()) + ) + or "no tables yet" + ) + if table not in state: + return _err( + f"no table {table!r}; there is {', '.join(sorted(state)) or 'nothing'}" + ) + rows = state[table] + shown = rows[: int(args.get("limit") or 15)] + return _ok( + f"{len(rows)} rows, showing {len(shown)}:\n" + + "\n".join(json.dumps(row, default=str) for row in shown) + ) + + @tool( + "drop_rule", + "Take away a hard rule the agent does not really have. A rule nobody has is worse than " + "a missing one: the agent is told to obey it and graded for not doing something it was " + "never supposed to do. Say why.", + {"rule": str, "why": str}, + ) + async def drop_rule_tool(args: dict[str, Any]) -> dict[str, Any]: + done, said = drop_rule( + contract, + destination, + rule=str(args.get("rule") or ""), + why=str(args.get("why") or ""), + ) + return _ok(said) if done else _err(said) + + @tool( + "fix_tool", + "Correct a tool that was read wrong, or remove one the agent does not have. `args` " + "replaces its argument names in order; `arg_types` and `description` update those. Set " + "`remove` to take the tool away entirely. Everything downstream is built from these, so " + "a wrong argument name produces a world that refuses everything. Say why.", + schema( + { + "tool_name": str, + "args": list, + "arg_types": dict, + "description": str, + "remove": bool, + "why": str, + }, + ["tool_name", "why"], + ), + ) + async def fix_tool_tool(args: dict[str, Any]) -> dict[str, Any]: + done, said = fix_tool( + contract, + destination, + tool_name=str(args.get("tool_name") or ""), + why=str(args.get("why") or ""), + args=[str(a) for a in args["args"]] if args.get("args") else None, + arg_types={ + str(k): str(v) for k, v in (args.get("arg_types") or {}).items() + }, + description=str(args.get("description") or ""), + remove=bool(args.get("remove")), + ) + return _ok(said) if done else _err(said) + + @tool( + "write_simulator_prompt", + "Write the prompt that drives the simulated user of this agent, for a conversational " + "agent only. It is written once and every scenario fills its slots, so leave variables " + "as {{ instruction }} and any others this agent needs.\n\n" + "It has to cover how a person in this conversation actually behaves: that they are " + "living the situation rather than describing it, that they speak one turn at a time, " + "that they never break character or explain that they are testing anything, what they " + "know and when they may say it, and when the conversation is over. Write it for this " + "agent, not in general.", + schema({"prompt": str}, ["prompt"]), + ) + async def write_simulator_prompt(args: dict[str, Any]) -> dict[str, Any]: + prompt = str(args.get("prompt") or "") + problems = validate_simulator_prompt(prompt) + if problems: + return _err("Not saved:\n - " + "\n - ".join(problems)) + path = save_simulator_prompt(prompt, destination) + from ..environment import variables_in + + return _ok( + f"Saved to {path}. Scenarios must fill: " + + ", ".join(sorted(variables_in(prompt))) + ) + + @tool( + "add_sub_goal", + "Add a named thing this agent can be checked on, shared by every scenario that needs " + "it. Defined here, once, so results roll up: the same sub-goal failing in seven of " + "twelve scenarios is one sentence.\n\n" + "`check` is Python: define check(world, calls) returning a sentence when something is " + "wrong, or None when it held. `world` is the environment afterwards; `calls` is every " + "tool call made, each with .name, .arguments, .ok and .refused — so a check can insist " + "a call happened with the right arguments, not merely that it happened.\n\n" + "Use `judged` only where nothing observable settles it, saying what a model has to " + "decide and why code cannot.", + schema( + {"name": str, "what": str, "check": str, "judged": str}, ["name", "what"] + ), + ) + async def add_sub_goal(args: dict[str, Any]) -> dict[str, Any]: + sub_goal = SubGoal( + name=str(args.get("name") or ""), + what=str(args.get("what") or ""), + check=str(args.get("check") or ""), + judged=str(args.get("judged") or ""), + ) + problems = validate_sub_goal(sub_goal) + if problems: + return _err("Not added:\n - " + "\n - ".join(problems)) + catalogue.sub_goals = [ + one for one in catalogue.sub_goals if one.name != sub_goal.name + ] + catalogue.sub_goals.append(sub_goal) + save_catalogue(catalogue, destination) + settled = sum(1 for one in catalogue.sub_goals if one.deterministic()) + return _ok( + f"{sub_goal.name} added. The catalogue has {len(catalogue.sub_goals)}, " + f"{settled} settled by code: " + ", ".join(sorted(catalogue.names())) + ) + @tool( "check_world", "Exercise every tool with a valid call, a nonexistent id, and a missing argument, then " - "run the declared sequences. Reports what is wrong without saving anything.", + "run the declared sequences. Reports what is wrong without saving anything.\n\n" + "Sequences are run independently from the frozen world, so a failure is never caused by " + "another sequence. Fix the failures it names; declaring more sequences only adds more " + "probes to pass.", {}, ) async def check_world(_args: dict[str, Any]) -> dict[str, Any]: - report = probe(world, contract, sequences=sequences) - return _ok(f"{report.summary()}\nscore {report.score:.2f}") + report = probe(world, contract, sequences=sequences, kind=kind) + scores.append(report.score) + # Saying the score is going nowhere, rather than leaving it to be noticed. A stage that + # has misdiagnosed something will otherwise keep applying the same non-fix, and every + # round of that costs money and gets no closer. + stuck = "" + if len(scores) >= 3 and len(set(round(s, 2) for s in scores[-3:])) == 1: + stuck = ( + "\n\nThis is the third check with the same score. Whatever you are changing is " + "not what is failing. Read the failures above literally and fix one of them, or " + "say what you are stuck on." + ) + return _ok(f"{report.summary()}\nscore {report.score:.2f}{stuck}") @tool( "save_world", "Freeze the world and write it out. Refused unless it passes its own checks.", - {"notes": str}, + schema({"notes": str}, []), ) async def save_world(args: dict[str, Any]) -> dict[str, Any]: - report = probe(world, contract, sequences=sequences) + report = probe(world, contract, sequences=sequences, kind=kind) if report.score < ACCEPTABLE: return _err( f"Not saved, the world does not hold up yet.\n{report.summary()}\n" @@ -226,7 +465,28 @@ async def save_world(args: dict[str, Any]) -> dict[str, Any]: "Not saved. Declare at least one sequence first: a world whose calls each work " "alone can still forget what the previous one did." ) - dirty = dirty_tables(world, sequences) + # The environment is not only the world. Every scenario is a delta on what is built + # here, so a catalogue nobody wrote means every scenario invents its own wording and + # nothing rolls up across the suite. + if not catalogue.sub_goals: + return _err( + "Not saved. No sub-goals yet. They are defined here, once, and every scenario " + "names the ones it needs — that is what makes results add up across the suite. " + "Add them with add_sub_goal." + ) + settled = [one for one in catalogue.sub_goals if one.deterministic()] + if not settled: + return _err( + "Not saved. Every sub-goal is judged by a model. Most of what this agent does " + "leaves a trace in the world or in its calls, and those should be settled by " + "code; a judge is the fallback for what leaves none." + ) + if contract.conversational and not load_simulator_prompt(destination): + return _err( + "Not saved. This agent is conversational, so it needs a simulator prompt for " + "the person on the other side. Write it with write_simulator_prompt." + ) + dirty = dirty_state(world, sequences, kind) if dirty: counts = world.state() listed = ", ".join(f"{name} ({len(counts[name])} rows)" for name in dirty) @@ -234,9 +494,31 @@ async def save_world(args: dict[str, Any]) -> dict[str, Any]: f"Not saved. These hold rows left over from building: {listed}.\n" "This is the state every scenario starts from, so those rows would appear in " "every test as somebody else's order already in the cart. Clear them with " - "create_schema or a delete, keep the catalogue, and save again." + "change_data (DELETE FROM ...), keep the catalogue, and save again." ) - path = save(world, destination, notes=str(args.get("notes") or "")) + # What the world publishes when something resets it. Without this a restored world + # announces no tools at all, so anything driving it through the environment interface + # sees an agent with nothing to call. + world.tools = [ + { + "name": spec.name, + "description": spec.description, + "parameters": { + arg: { + "type": spec.arg_types.get(arg, "str"), + "values": spec.arg_values.get(arg), + } + for arg in spec.args + }, + } + for spec in contract.tools + ] + path = save( + world, + destination, + notes=str(args.get("notes") or ""), + sequences=sequences, + ) tables = world.state() return _ok( f"Saved to {path}.\n" @@ -251,10 +533,18 @@ async def save_world(args: dict[str, Any]) -> dict[str, Any]: tools=[ create_schema, seed, + change_data, define_handler, run_tool, declare_sequence, drop_sequence, + amend_contract, + add_rule_tool, + drop_rule_tool, + fix_tool_tool, + inspect_world, + write_simulator_prompt, + add_sub_goal, check_world, save_world, ], @@ -265,10 +555,18 @@ async def save_world(args: dict[str, Any]) -> dict[str, Any]: TOOL_NAMES = ( "create_schema", "seed", + "change_data", "define_handler", "run_tool", "declare_sequence", "drop_sequence", + "amend_contract", + "add_rule", + "drop_rule", + "fix_tool", + "inspect_world", + "write_simulator_prompt", + "add_sub_goal", "check_world", "save_world", ) diff --git a/tests/test_harness.py b/tests/test_harness.py index 1d6f586..8e8b55e 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -348,3 +348,985 @@ def test_cli_defaults_to_staying_open_for_corrections(): def test_opening_names_the_agent_and_asks_for_the_contract(tmp_path): text = opening(RepoSource(name="drive_thru", root=tmp_path)) assert "drive_thru" in text and "submit_contract" in text + + +# --- state expectations, shared by the gate and the grading -------------------------- + + +_STATE = {"orders": [{"id": "a", "item": "big_mac"}], "menu": [{"id": "big_mac"}]} + + +# --- scenarios ----------------------------------------------------------------------- + + +def _saved_world(tmp_path): + from fi.alk.harness.world.snapshot import save + + world, contract = _cart_world() + save(world, tmp_path, notes="test world") + return tmp_path, contract + + +def _scenario(**overrides): + payload = { + "name": "orders-a-big-mac", + "tests": "the ordinary case", + "goal": "order a big mac", + "persona": "brisk", + "opening": "one big mac please", + "expect_state": {"cart.count": 1}, + } + payload.update(overrides) + return payload + + +# --- running and grading ------------------------------------------------------------- + + +def test_declared_types_become_something_a_tool_schema_can_carry(): + from fi.alk.harness.run.targets import _python_type + + assert _python_type("list[str]") is list + assert _python_type("int") is int + assert _python_type("") is str + + +def test_the_agent_under_test_is_told_its_own_rules(): + from fi.alk.harness.run.targets import agent_prompt + + _world, contract = _cart_world() + contract.hard_constraints = ["never substitute an item without asking"] + assert "never substitute" in agent_prompt(contract) + + +def test_the_cli_exposes_every_stage_and_one_conversation_across_them(): + parser = build_parser() + assert parser.parse_args(["scenarios", "--name", "a", "--count", "10"]).count == 10 + assert parser.parse_args(["run", "--name", "a"]).target == "local" + + +def test_talking_to_it_needs_nothing_on_the_command_line(): + """Which agent, where it lives and how many scenarios are all things you say.""" + parser = build_parser() + assert parser.parse_args(["chat"]).name is None + assert parser.parse_args(["chat"]).path is None + + +def test_a_conversation_resumes_at_whichever_stage_the_artifacts_reached(tmp_path): + from fi.alk.harness.chat import BUILD, SCENARIOS, UNDERSTAND, open_conversation + + conversation = open_conversation(name="a", path=str(tmp_path), out=tmp_path) + assert conversation._resume_at() == UNDERSTAND + + accept_contract( + { + "agent": "a", + "real_use_cases": ["order"], + "tools": [{"name": "add", "args": ["item_id"]}], + }, + tmp_path, + ) + assert conversation._resume_at() == BUILD + + _saved_world(tmp_path) + assert conversation._resume_at() == SCENARIOS + + +def test_where_a_conversation_is_agrees_with_what_was_built(tmp_path): + from fi.alk.harness.chat import SCENARIOS, open_conversation + + accept_contract( + { + "agent": "a", + "real_use_cases": ["order"], + "tools": [{"name": "add", "args": ["item_id"]}], + }, + tmp_path, + ) + _saved_world(tmp_path) + conversation = open_conversation(name="a", path=str(tmp_path), out=tmp_path) + assert conversation.stage_name == SCENARIOS + assert conversation.next_stage() is None + + +def test_a_conversation_with_no_agent_starts_by_asking_which_one(): + from fi.alk.harness.chat import RECEPTION, open_conversation + + conversation = open_conversation() + assert conversation.source is None + assert conversation.stage_name == RECEPTION + assert conversation.next_stage() is None + + +def test_pointing_at_an_agent_settles_where_its_artifacts_go(tmp_path): + import asyncio + + from fi.alk.harness.chat import UNDERSTAND, open_conversation + from fi.alk.harness.sources import RepoSource + + conversation = open_conversation() + conversation._found["source"] = RepoSource(name="mine", root=tmp_path) + + async def _settle(): + # Reception is the only stage whose result is not a file, so the conversation reads it + # back rather than looking on disk. Advancing needs a live session, so only the + # settling half is exercised here. + settled = conversation._found.pop("source") + conversation.source = settled + conversation.out = conversation.out or artifact_dir(settled.name) + + asyncio.run(_settle()) + assert conversation.out.as_posix().endswith("environments/mine") + assert conversation._resume_at() == UNDERSTAND + + +def test_pointing_at_somewhere_that_does_not_exist_is_refused(tmp_path): + from fi.alk.harness.reception import point_at + + found = {} + refused = point_at("mine", str(tmp_path / "nope"), "repo", found) + assert refused["is_error"] and found == {} + + accepted = point_at("mine", str(tmp_path), "repo", found) + assert not accepted.get("is_error") + assert found["source"].name == "mine" + + +def test_how_many_scenarios_is_something_you_say(): + from fi.alk.harness.scenario_tools import TOOL_NAMES + + assert "aim_for" in TOOL_NAMES + + +# --- amending the contract ----------------------------------------------------------- + + +def _written_contract(tmp_path): + accept_contract( + { + "agent": "cart", + "real_use_cases": ["add an item"], + "tools": [ + { + "name": "add", + "args": ["item_id"], + "arg_values": {"item_id": ["big_mac"]}, + } + ], + }, + tmp_path, + ) + return load(tmp_path) + + +def test_the_agent_can_be_taught_a_value_it_did_not_accept(tmp_path): + """A world that gains an item the agent cannot name holds dead data, and every scenario + about it can only fail. The two have to move together.""" + from fi.alk.harness.amend import widen + + contract = _written_contract(tmp_path) + done, said = widen( + contract, + tmp_path, + tool_name="add", + argument="item_id", + values=["mango_smoothie"], + why="added to the menu this morning", + ) + assert done, said + assert "mango_smoothie" in contract.tools[0].arg_values["item_id"] + # the stage's own copy and the file agree, or the stage checks against an action space + # that no longer exists + assert "mango_smoothie" in load(tmp_path).tools[0].arg_values["item_id"] + + +def test_an_amendment_is_recorded_rather_than_blended_in(tmp_path): + from fi.alk.harness.amend import widen + + contract = _written_contract(tmp_path) + widen( + contract, + tmp_path, + tool_name="add", + argument="item_id", + values=["mango_smoothie"], + why="added to the menu this morning", + ) + recorded = load(tmp_path).amendments + assert len(recorded) == 1 + assert "mango_smoothie" in recorded[0] and "this morning" in recorded[0] + + +@pytest.mark.parametrize( + "overrides,expected", + [ + ({"tool_name": "nope"}, "is not a tool this agent has"), + ({"argument": "colour"}, "takes no argument"), + ({"why": " "}, "say why"), + ({"values": ["big_mac"]}, "already accepts"), + ], +) +def test_an_amendment_that_makes_no_sense_is_refused(tmp_path, overrides, expected): + from fi.alk.harness.amend import widen + + contract = _written_contract(tmp_path) + call = { + "tool_name": "add", + "argument": "item_id", + "values": ["mango_smoothie"], + "why": "because", + } + call.update(overrides) + done, said = widen(contract, tmp_path, **call) + assert not done and expected in said + assert load(tmp_path).amendments == [] + + +# --- what a stage is allowed to do --------------------------------------------------- + + +def test_a_stage_may_use_nothing_it_was_not_given(): + """Deny by default, not deny-a-list. A session is offered whatever its host exposes, and an + allow-by-default gate let a host search tool through that cost a stage its whole budget.""" + import asyncio + + from fi.alk.harness.config import permission_gate + + gate = permission_gate(granted=["Read", "Glob"]) + for refused in ("Write", "Edit", "Bash", "Task", "ToolSearch", "WebFetch"): + verdict = asyncio.run(gate(refused, {}, None)) + assert type(verdict).__name__ == "PermissionResultDeny" + assert "not part of this stage" in verdict.message + + allowed = asyncio.run(gate("Read", {"file_path": "a.py"}, None)) + assert type(allowed).__name__ == "PermissionResultAllow" + + +def test_a_question_still_reaches_the_operator(): + import asyncio + + from fi.alk.harness.config import permission_gate + + asked = {} + + async def ask(tool_name, payload, _context): + asked["tool"] = tool_name + return "answered" + + assert asyncio.run(permission_gate(ask)("AskUserQuestion", {}, None)) == "answered" + assert asked["tool"] == "AskUserQuestion" + + +# --- the tools a stage actually publishes --------------------------------------------- + + +def _published(server): + """The tool names an in-process MCP server really exposes.""" + import asyncio + + from mcp.types import ListToolsRequest + + instance = server.get("instance") if isinstance(server, dict) else server + + async def ask(): + for key, handler in instance.request_handlers.items(): + if getattr(key, "__name__", "") == "ListToolsRequest": + result = await handler(ListToolsRequest(method="tools/list")) + return sorted(tool.name for tool in result.root.tools) + return [] + + return asyncio.run(ask()) + + +def test_every_stage_publishes_exactly_the_tools_it_claims(tmp_path): + """A tool listed in TOOL_NAMES but left out of the server is granted, named in error + messages, and does not exist. The model then hunts for it and works around the gate.""" + from fi.alk.harness import scenario_tools as scenarios + from fi.alk.harness.run import tools as runs + from fi.alk.harness.world import tools as world + + root, contract = _saved_world(tmp_path) + server, _kept = scenarios.scenario_tools(contract, root, root, wanted=1) + assert _published(server) == sorted(scenarios.TOOL_NAMES) + + built, _world = world.world_tools(contract, root) + assert _published(built) == sorted(world.TOOL_NAMES) + + assert _published(runs.run_tools(root, root)) == sorted(runs.TOOL_NAMES) + + +def test_a_failed_call_is_not_reported_as_success(): + """A call that failed upstream still arrives with subtype "success", so reporting subtype + verbatim tells somebody their stage worked when nothing happened.""" + from fi.alk.harness.session import _why_it_failed + + class Failed: + api_error_status = 400 + errors = ['{"error":"invalid_grant","error_subtype":"invalid_rapt"}'] + + said = _why_it_failed(Failed()) + assert "GOOGLE_APPLICATION_CREDENTIALS" in said and ".env.acceptance" in said + + class Other: + api_error_status = 529 + errors = ["overloaded"] + + assert "529" in _why_it_failed(Other()) + + +def test_the_credentials_in_play_are_said_out_loud(monkeypatch): + from fi.alk.harness.config import credentials_hint + + monkeypatch.setenv("GOOGLE_APPLICATION_CREDENTIALS", "/keys/service-account.json") + assert credentials_hint() == "credentials: service-account.json" + + monkeypatch.delenv("GOOGLE_APPLICATION_CREDENTIALS") + assert "gcloud login" in credentials_hint() + + +def test_a_run_notices_when_it_was_billed_to_a_model_nobody_asked_for(): + """Asking for a model is not the same as getting one: the CLI has its own default, and a + request that quietly does not take shows up only on the invoice.""" + from claude_agent_sdk import ClaudeAgentOptions + + from fi.alk.harness.session import Stage + + stage = Stage(ClaudeAgentOptions(model="claude-haiku-4-5"), name="s") + stage.models_used = {"claude-haiku-4-5-20251001"} + assert stage.unexpected_models() == set() + + stage.models_used = {"claude-opus-4-7"} + assert stage.unexpected_models() == {"claude-opus-4-7"} + + +def test_an_agent_already_built_can_be_reopened_without_its_repository(tmp_path): + """Coming back to fix a scenario should not mean pointing at the source again.""" + from fi.alk.harness.chat import SCENARIOS, Conversation + + accept_contract( + { + "agent": "a", + "real_use_cases": ["order"], + "tools": [{"name": "add", "args": ["item_id"]}], + }, + tmp_path, + ) + _saved_world(tmp_path) + resumed = Conversation(source=None, out=tmp_path) + assert resumed.stage_name == SCENARIOS + + +def test_a_rule_the_source_never_stated_can_be_added_and_is_recorded(tmp_path): + """A hard constraint is told to the agent under test and graded by the judge, so adding one + changes what is being tested and has to be visible as ours rather than the agent's.""" + from fi.alk.harness.amend import add_rule + + contract = _written_contract(tmp_path) + done, said = add_rule( + contract, + tmp_path, + rule="stays polite to customers", + why="asked for on the call", + ) + assert done and "graded from here on" in said + reloaded = load(tmp_path) + assert "stays polite to customers" in reloaded.hard_constraints + assert "rule added" in reloaded.amendments[0] and "polite" in reloaded.amendments[0] + + again, why = add_rule( + contract, tmp_path, rule="Stays Polite To Customers", why="again" + ) + assert not again and "already has that rule" in why + + unexplained, said = add_rule(contract, tmp_path, rule="be fast", why=" ") + assert not unexplained and "say why" in said + + +def test_a_rule_the_agent_does_not_have_can_be_taken_away(tmp_path): + """A rule nobody has is worse than a missing one: the agent is told to obey it and the + judge fails it for not doing something it was never supposed to do.""" + from fi.alk.harness.amend import add_rule, drop_rule + + contract = _written_contract(tmp_path) + add_rule(contract, tmp_path, rule="never upsell", why="misread from a comment") + done, said = drop_rule( + contract, tmp_path, rule="upsell", why="the source never says that" + ) + assert done, said + assert load(tmp_path).hard_constraints == [] + assert "rule removed" in load(tmp_path).amendments[-1] + + missing, said = drop_rule(contract, tmp_path, rule="be nice", why="x") + assert not missing and "no rule like that" in said + + +def test_a_misread_tool_can_be_corrected(tmp_path): + """The most damaging thing stage one can get wrong: every argument name flows into the + handlers, the probes and the scenarios.""" + from fi.alk.harness.amend import fix_tool + + contract = _written_contract(tmp_path) + done, said = fix_tool( + contract, + tmp_path, + tool_name="add", + args=["item_ids"], + why="the signature takes a list, singular was a misread", + ) + assert done, said + fixed = load(tmp_path).tools[0] + assert fixed.args == ["item_ids"] + # values recorded against the old name must not silently survive under a name nobody uses + assert "item_id" not in fixed.arg_values + assert "dropped values recorded for item_id" in said + + +def test_a_tool_the_agent_does_not_have_can_be_removed(tmp_path): + from fi.alk.harness.amend import fix_tool + + contract = _written_contract(tmp_path) + contract.tools.append(ToolSpec(name="checkout", args=["id"])) + done, said = fix_tool( + contract, + tmp_path, + tool_name="checkout", + remove=True, + why="no such tool in the source", + ) + assert done and "1 tools left" in said + assert load(tmp_path).tool_names() == {"add"} + + +def test_correcting_a_contract_without_saying_why_is_refused(tmp_path): + from fi.alk.harness.amend import drop_rule, fix_tool + + contract = _written_contract(tmp_path) + assert not fix_tool(contract, tmp_path, tool_name="add", args=["x"], why=" ")[0] + assert not drop_rule(contract, tmp_path, rule="anything", why="")[0] + + +def test_a_read_only_handler_does_not_poison_every_later_probe(): + """SQLite refuses to restore into a connection with a transaction open, and a handler that + only reads leaves one behind. Unsettled, the first such handler makes the world impossible + to check or save: "destination database is in use".""" + from fi.alk.harness.world import probe + + world, contract = _cart_world() + # lst only queries, which is what leaves the read transaction open + world.call("lst", {}) + mark = world.checkpoint() + world.call("add", {"item_id": "big_mac"}) + world.call("lst", {}) + world.revert(mark) + assert world.state()["cart"] == [] + + report = probe(world, contract, sequences=_SEQUENCE) + assert report.score == 1.0, report.summary() + + +def test_a_row_put_in_wrong_can_be_taken_out_again(tmp_path): + """Seeding only inserts. Without a way to remove a row, the only way left to make a check + pass is to change the contract, which repairs the wrong thing.""" + import asyncio + + from fi.alk.harness.world import tools as world_tools + + _root, contract = _saved_world(tmp_path) + server, world = world_tools.world_tools(contract, tmp_path) + assert "change_data" in world_tools.TOOL_NAMES + assert _published(server) == sorted(world_tools.TOOL_NAMES) + + world.connection.execute("INSERT INTO menu VALUES ('curry_sauce')") + world.connection.commit() + + async def call(name, payload): + from mcp.types import CallToolRequest, CallToolRequestParams + + instance = server.get("instance") if isinstance(server, dict) else server + for key, handler in instance.request_handlers.items(): + if getattr(key, "__name__", "") == "CallToolRequest": + result = await handler( + CallToolRequest( + method="tools/call", + params=CallToolRequestParams(name=name, arguments=payload), + ) + ) + return result.root.content[0].text + + said = asyncio.run( + call("change_data", {"sql": "DELETE FROM menu WHERE id='curry_sauce'"}) + ) + assert "1 rows changed" in said + assert not [row for row in world.state()["menu"] if row["id"] == "curry_sauce"] + + refused = asyncio.run(call("change_data", {"sql": "SELECT * FROM menu"})) + assert "UPDATE or DELETE" in refused + + +# --- the environment step: world, simulator prompt, sub-goal catalogue --------------- + + +def test_a_sub_goal_that_settles_nothing_is_rejected(): + """Every scenario referencing it would report a result nobody should believe.""" + from fi.alk.harness.environment import SubGoal, validate_sub_goal + + assert validate_sub_goal(SubGoal(name="x", what="means something")) != [] + settled = SubGoal( + name="order-placed", + what="the order reached the system", + check="def check(world, calls):\n return None\n", + ) + assert validate_sub_goal(settled) == [] + assert settled.deterministic() + + judged = SubGoal( + name="polite", what="stayed polite", judged="nothing observable shows tone" + ) + assert validate_sub_goal(judged) == [] and not judged.deterministic() + + +def test_a_check_must_actually_define_one(): + from fi.alk.harness.environment import SubGoal, validate_sub_goal + + problems = validate_sub_goal( + SubGoal(name="x", what="y", check="rows = world.state()['orders']") + ) + assert any("check(world, calls)" in problem for problem in problems) + + +def test_a_simulator_prompt_without_a_slot_runs_the_same_conversation_every_time(): + from fi.alk.harness.environment import fill, validate_simulator_prompt, variables_in + + fixed = ( + "You are a customer calling a drive-thru. Speak naturally, one turn at a time. " + * 2 + ) + assert any( + "no variables" in problem for problem in validate_simulator_prompt(fixed) + ) + + written = fixed + "\n\nWhat you want: {{ instruction }}\nWhat you know: {{ facts }}" + assert validate_simulator_prompt(written) == [] + assert variables_in(written) == {"instruction", "facts"} + + filled, missing = fill(written, {"instruction": "order a big mac"}) + assert "order a big mac" in filled and missing == ["facts"] + + +def test_a_check_that_raises_is_broken_not_failed(): + """A typo in an assertion must never read as a finding about the agent.""" + from fi.alk.harness.checks import run_check + + world, _contract = _cart_world() + ok = run_check( + "def check(world, calls):\n return None\n", world, [], name="fine" + ) + assert ok.held and not ok.broken + + failed = run_check( + "def check(world, calls):\n return 'no rows'\n", world, [], name="says-why" + ) + assert not failed.held and not failed.broken and failed.said == "no rows" + + typo = run_check( + "def check(world, calls):\n return world.state()['nope'][0]\n", + world, + [], + name="typo", + ) + assert typo.broken and "KeyError" in typo.said + + +def test_a_check_can_insist_on_the_arguments_not_just_the_call(): + """Booking 10 PM when 11 PM was asked for is a failure, and detecting it is deterministic.""" + from fi.alk.harness.checks import run_check + + world, _contract = _cart_world() + world.call("add", {"item_id": "big_mac"}) + source = ( + "def check(world, calls):\n" + " made = [c for c in calls if c.name == 'add']\n" + " if not made:\n return 'never added anything'\n" + " if made[0].arguments.get('item_id') != 'fries':\n" + " return 'added %r, expected fries' % made[0].arguments.get('item_id')\n" + " return None\n" + ) + outcome = run_check(source, world, world.calls, name="right-item") + assert not outcome.held and "expected fries" in outcome.said + + +# --- scenarios as deltas, and the two gates ------------------------------------------ + + +def _built_environment(tmp_path): + """A saved world plus a catalogue, which is what the environment step leaves behind.""" + from fi.alk.harness.environment import Catalogue, SubGoal, save_catalogue + from fi.alk.harness.world.snapshot import save + + world, contract = _cart_world() + save(world, tmp_path, notes="test", sequences=[]) + catalogue = Catalogue( + sub_goals=[ + SubGoal( + name="item-added", + what="the item reached the cart", + check=( + "def check(world, calls):\n" + " rows = world.state()['cart']\n" + " if len(rows) != 1: return '%d rows, expected 1' % len(rows)\n" + " return None\n" + ), + ), + SubGoal( + name="right-item", + what="the call carried the item that was asked for", + check=( + "def check(world, calls):\n" + " made = [c for c in calls if c.name == 'add' and c.ok]\n" + " if not made: return 'add was never called'\n" + " got = made[0].arguments.get('item_id')\n" + " return None if got == 'big_mac' else 'added %r' % got\n" + ), + ), + SubGoal(name="polite", what="stayed polite", judged="tone leaves no trace"), + ] + ) + save_catalogue(catalogue, tmp_path) + return tmp_path, contract, catalogue + + +def _delta(**overrides): + payload = { + "name": "adds-a-big-mac", + "use_case": "order an item", + "instruction": "Order one Big Mac.", + "solution": [{"tool": "add", "arguments": {"item_id": "big_mac"}}], + "sub_goals": ["item-added", "right-item"], + } + payload.update(overrides) + return payload + + +def test_a_scenario_is_proved_before_it_is_kept(tmp_path): + from fi.alk.harness.scenario_tools import accept_scenario + + root, _contract, catalogue = _built_environment(tmp_path) + kept = [] + said = accept_scenario(_delta(), world_root=root, catalogue=catalogue, kept=kept) + assert not said.get("is_error"), said + assert "Proved" in said["content"][0]["text"] + assert [one.name for one in kept] == ["adds-a-big-mac"] + + +def test_a_scenario_whose_solution_cannot_pass_its_own_checks_is_refused(tmp_path): + """Either the scenario is impossible or the checks are wrong. Both have happened.""" + from fi.alk.harness.scenario_tools import accept_scenario + + root, _contract, catalogue = _built_environment(tmp_path) + said = accept_scenario( + _delta(solution=[{"tool": "add", "arguments": {"item_id": "sushi"}}]), + world_root=root, + catalogue=catalogue, + kept=[], + ) + assert said["is_error"] + text = said["content"][0]["text"] + assert "reference solution does not pass" in text + assert "refused by the world" in text and "sushi" in text + + +def test_a_scenario_whose_checks_pass_with_nothing_done_is_refused(tmp_path): + """A check that passes without the agent acting grades nothing while reporting a result.""" + from fi.alk.harness.environment import SubGoal, save_catalogue + from fi.alk.harness.scenario_tools import accept_scenario + + root, _contract, catalogue = _built_environment(tmp_path) + catalogue.sub_goals.append( + SubGoal( + name="always", + what="always true", + check="def check(world, calls):\n return None\n", + ) + ) + save_catalogue(catalogue, root) + said = accept_scenario( + _delta(sub_goals=["always"]), world_root=root, catalogue=catalogue, kept=[] + ) + assert said["is_error"] and "grade nothing" in said["content"][0]["text"] + + +def test_a_scenario_naming_a_sub_goal_nobody_defined_is_refused(tmp_path): + from fi.alk.harness.scenario_tools import accept_scenario + + root, _contract, catalogue = _built_environment(tmp_path) + said = accept_scenario( + _delta(sub_goals=["invented-here"]), + world_root=root, + catalogue=catalogue, + kept=[], + ) + assert said["is_error"] + assert "not in the catalogue" in said["content"][0]["text"] + + +def test_a_scenario_with_no_solution_cannot_be_proved(tmp_path): + from fi.alk.harness.scenario_tools import accept_scenario + + root, _contract, catalogue = _built_environment(tmp_path) + said = accept_scenario( + _delta(solution=[]), world_root=root, catalogue=catalogue, kept=[] + ) + assert said["is_error"] and "no solution" in said["content"][0]["text"] + + +def test_a_suite_where_no_sub_goal_is_shared_does_not_roll_up(tmp_path): + """If a payment step appears in 50 scenarios, the results should say where payment fails.""" + from fi.alk.harness.environment import Catalogue, SubGoal + from fi.alk.harness.scenario import Scenario + from fi.alk.harness.scenario_tools import not_ready + + catalogue = Catalogue( + sub_goals=[SubGoal(name=f"g{i}", what="x", judged="y") for i in range(4)] + ) + private = [ + Scenario(name=f"s{i}", instruction="do it", sub_goals=[f"g{i}"]) + for i in range(4) + ] + assert any("rolls up" in problem for problem in not_ready(private, 4, catalogue)) + + shared = [ + Scenario(name=f"s{i}", instruction="do it", sub_goals=["g0"]) for i in range(4) + ] + assert not_ready(shared, 4, catalogue) == [] + + +def test_the_simulator_prompt_slots_a_scenario_leaves_unfilled_are_caught(tmp_path): + from fi.alk.harness.scenario import Scenario, validate_scenario + + root, _contract, catalogue = _built_environment(tmp_path) + prompt = ( + "You are a customer. " * 10 + + "\nWhat you want: {{ instruction }}\nAlso: {{ mood }}" + ) + scenario = Scenario.model_validate(_delta()) + problems = validate_scenario(scenario, catalogue, {"cart": [], "menu": []}, prompt) + assert any("mood" in problem for problem in problems) + + +# --- the voice webhook, answered by the world ---------------------------------------- + + +def test_a_hosted_agents_tool_call_is_answered_by_the_world(): + """The whole voice integration: a webhook, answered by running the call rather than by + looking up a canned response. A mock that always succeeds tells an agent it removed an item + that was never added.""" + import json + import urllib.request + + from fi.alk.harness.run.voice import WorldWebhook + + world, _contract = _cart_world() + webhook = WorldWebhook().start() + try: + webhook.bind(world) + + def call(name, arguments): + body = json.dumps( + { + "message": { + "toolCalls": [ + { + "id": "call-1", + "function": {"name": name, "arguments": arguments}, + } + ] + } + } + ).encode() + request = urllib.request.Request( + f"http://127.0.0.1:{webhook.port}/tool", + data=body, + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=5) as answer: + return json.loads(answer.read())["results"][0]["result"] + + assert "1" in call("add", {"item_id": "big_mac"}) + # the world really wrote the row, so a read-after-write flow is right + assert len(world.state()["cart"]) == 1 + + # and it can refuse, which a canned mock cannot + refused = call("add", {"item_id": "sushi"}) + assert "sushi" in refused + assert len(world.state()["cart"]) == 1 + + # the world answers for a tool the agent does not have, naming the ones it does + unknown = call("checkout", {}) + assert "no such tool" in unknown and "add" in unknown + # every call is recorded with its arguments, which is what grading reads + assert [c.name for c in webhook.calls] == ["add", "add", "checkout"] + finally: + webhook.stop() + + +def test_repointing_changes_only_where_the_agents_tools_are_answered(): + """The assistant's tools are the agent's — names, arguments and enums belong to whoever + built it. Redefining them would mean testing an agent we wrote.""" + from fi.alk.harness.run.voice import pointed_at + + theirs = [ + { + "type": "function", + "function": { + "name": "order_combo_meal", + "parameters": { + "type": "object", + "properties": { + "meal_id": {"type": "string", "enum": ["combo_big_mac"]} + }, + "required": ["meal_id"], + }, + }, + "server": {"url": "https://dead-tunnel.example/tool"}, + } + ] + moved = pointed_at(theirs, "https://ours.example") + assert moved[0]["server"]["url"] == "https://ours.example/tool" + # everything else is untouched + assert moved[0]["function"] == theirs[0]["function"] + assert theirs[0]["server"]["url"] == "https://dead-tunnel.example/tool" + + +def test_a_scenario_fills_the_simulator_prompt_before_a_call_is_placed(tmp_path): + from fi.alk.harness.environment import save_simulator_prompt + from fi.alk.harness.run.live import prepare + from fi.alk.harness.scenario import Scenario + + root, _contract, _catalogue = _built_environment(tmp_path) + save_simulator_prompt( + "You are at the counter. " * 8 + "\nWhat you are here to do: {{ instruction }}", + root, + ) + world, instruction = prepare( + Scenario(name="s", instruction="Order one Big Mac."), root + ) + try: + assert "Order one Big Mac." in instruction + assert "{{" not in instruction + finally: + world.close() + + +def test_a_scenario_that_leaves_a_slot_empty_never_reaches_a_call(tmp_path): + """An unfilled slot would be read out to the caller verbatim.""" + import pytest as _pytest + + from fi.alk.harness.environment import save_simulator_prompt + from fi.alk.harness.run.live import prepare + from fi.alk.harness.scenario import Scenario + + root, _contract, _catalogue = _built_environment(tmp_path) + save_simulator_prompt( + "You are at the counter. " * 8 + "\nDo: {{ instruction }}\nMood: {{ mood }}", + root, + ) + with _pytest.raises(RuntimeError, match="mood"): + prepare(Scenario(name="s", instruction="Order one Big Mac."), root) + + +def test_a_live_run_is_refused_before_it_costs_anything(monkeypatch): + """Missing credentials must be caught up front. Discovering them after the world is + restored, the tunnel is up and the assistant is repointed wastes the expensive part and + reports a failure that says nothing about the agent.""" + from fi.alk.harness.run.tools import missing_prerequisites + + monkeypatch.delenv("VAPI_API_KEY", raising=False) + monkeypatch.delenv("VAPI_ASSISTANT_ID", raising=False) + problems = missing_prerequisites() + assert any("VAPI_API_KEY" in problem for problem in problems) + + monkeypatch.setenv("VAPI_API_KEY", "x") + monkeypatch.setenv("VAPI_ASSISTANT_ID", "y") + monkeypatch.setenv("HARNESS_WEBHOOK_URL", "https://example.invalid") + assert missing_prerequisites() == [] + + +def test_running_is_a_stage_of_the_conversation(): + """Placing a call was the one step that could only be a command. If it drops out of the + stage order it silently becomes one again, and the chat ends at scenarios.""" + from fi.alk.harness import chat + + assert chat._NEXT[chat.SCENARIOS] == chat.RUN + assert chat._NEXT[chat.RUN] == chat.DONE + + +def test_a_run_result_survives_being_written_and_read(tmp_path): + from fi.alk.harness.checks import Outcome + from fi.alk.harness.run.live import LiveRun + from fi.alk.harness.run.tools import as_record, load_results, save_results + + run = LiveRun( + scenario="orders-a-big-mac", + settled=[Outcome("combo_placed", True), Outcome("no_extras", False, "added fries")], + judged=["explained_itself"], + calls=["order(...) -> ok"], + ) + record = as_record(run) + assert record["passed"] is False and record["met"] == 1 and record["of"] == 2 + + save_results([record], tmp_path) + assert load_results(tmp_path) == [record] + + +def test_a_tool_a_stage_was_not_given_is_denied_by_the_hook(): + """can_use_tool alone does not do this. An allowed_tools entry approves its tools before the + callback runs, and the SDK warns the callback is shadowed; a host ToolSearch reached every + stage, returned nothing and cost a turn. The PreToolUse hook is consulted for every call.""" + import asyncio + + from fi.alk.harness.config import gate_hooks + + hooks = gate_hooks(["mcp__world__seed"]) + refuse = hooks["PreToolUse"][0].hooks[0] + + granted = asyncio.run(refuse({"tool_name": "mcp__world__seed"}, None, None)) + assert granted == {} + + asked = asyncio.run(refuse({"tool_name": "AskUserQuestion"}, None, None)) + assert asked == {} + + denied = asyncio.run(refuse({"tool_name": "ToolSearch"}, None, None)) + said = denied["hookSpecificOutput"] + assert said["permissionDecision"] == "deny" + assert "ToolSearch is not part of this stage" in said["permissionDecisionReason"] + assert "mcp__world__seed" in said["permissionDecisionReason"] + + +def test_every_stage_gates_with_the_hook_not_only_the_callback(): + """One stage left on the callback alone is one stage a host tool still reaches.""" + import inspect + + from fi.alk.harness import build, reception, scenarios + from fi.alk.harness.run import grade, stage, targets + + for module in (build, reception, scenarios, stage, targets, grade): + source = inspect.getsource(module) + if "permission_gate(" in source: + assert "gate_hooks(allowed)" in source, f"{module.__name__} has no hook gate" + + +def test_writing_new_results_keeps_the_ones_not_rerun(tmp_path): + """The live stage and the local suite share runs.json. Re-running one scenario must not + erase the record of another, whichever writer gets there second.""" + from fi.alk.harness.run.tools import load_results, save_results + + save_results( + [{"scenario": "a", "passed": True}, {"scenario": "b", "passed": False}], tmp_path + ) + fresh = [r for r in load_results(tmp_path) if r.get("scenario") != "b"] + fresh.append({"scenario": "b", "passed": True, "transcript": "hello"}) + save_results(fresh, tmp_path) + + kept = {r["scenario"]: r for r in load_results(tmp_path)} + assert kept["a"]["passed"] is True + assert kept["b"]["passed"] is True and kept["b"]["transcript"] == "hello" From 419edb2a4be7178f7d5c5bb7e948da4aec6bdfd1 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Mon, 17 Aug 2026 00:53:46 +0530 Subject: [PATCH 06/39] feat(harness): run stage dispatches on modality, live for hosted voice, local otherwise --- src/fi/alk/harness/run/stage.py | 4 +- src/fi/alk/harness/run/tools.py | 78 ++++++++++++++++++++++++++------- 2 files changed, 63 insertions(+), 19 deletions(-) diff --git a/src/fi/alk/harness/run/stage.py b/src/fi/alk/harness/run/stage.py index b51f401..8650f85 100644 --- a/src/fi/alk/harness/run/stage.py +++ b/src/fi/alk/harness/run/stage.py @@ -42,7 +42,7 @@ def open_stage( ) -> tuple[Stage, Path]: """A live run-the-scenarios stage, and where it will write its results.""" destination = out or artifact_dir(contract.agent) - server = run_tools(destination, destination) + server = run_tools(destination, destination, contract=contract) allowed = [ "AskUserQuestion", *(qualified(RUN_SERVER, name) for name in TOOL_NAMES), @@ -74,7 +74,7 @@ def opening(contract: AgentContract, destination: Path) -> str: """ written = load_scenarios(destination) already = load_results(destination) - blocked = missing_prerequisites() + blocked = missing_prerequisites() if contract.modality == "voice" else [] if blocked: return ( f"There are {len(written)} scenarios for {contract.agent!r}, but a live call cannot " diff --git a/src/fi/alk/harness/run/tools.py b/src/fi/alk/harness/run/tools.py index 28396ed..2b827a0 100644 --- a/src/fi/alk/harness/run/tools.py +++ b/src/fi/alk/harness/run/tools.py @@ -144,12 +144,26 @@ def report(run: LiveRun) -> str: return "\n".join(lines) -def run_tools(world_root: Path, destination: Path, *, case: str = "") -> Any: - """A server for running one agent's scenarios against the real thing.""" +def run_tools( + world_root: Path, + destination: Path, + *, + contract: Any = None, + case: str = "", +) -> Any: + """A server for running one agent's scenarios against the real thing. + + How a scenario runs is decided by what the agent is, not by this stage. A hosted voice agent + gets the live path — its own tools repointed at the world over a webhook, the call placed + through ALK. Anything else runs here: the agent stood up from its contract, conversing over + the same world, graded by the same checks. The scenarios, the world and the grading are + identical either way; only the transport changes. + """ written = load_scenarios(destination) catalogue = load_catalogue(destination) results = load_results(destination) voice_case = case or os.environ.get("HARNESS_VOICE_CASE", "2.1.2") + live = bool(contract is not None and getattr(contract, "modality", "") == "voice") @tool( "list_scenarios", @@ -169,7 +183,7 @@ async def list_scenarios(_args: dict[str, Any]) -> dict[str, Any]: ] judged = [name for name in one.sub_goals if name not in settled] ran = next((r for r in results if r["scenario"] == one.name), None) - mark = "" if ran is None else (" [last run: PASS]" if ran["passed"] else " [last run: FAIL]") + mark = "" if ran is None else (" [last run: PASS]" if ran.get("passed") else " [last run: FAIL]") lines.append( f"{one.name}{mark}\n tests: {one.tests or one.use_case or '—'}\n" f" settled by code: {', '.join(settled) or 'none'}\n" @@ -179,11 +193,17 @@ async def list_scenarios(_args: dict[str, Any]) -> dict[str, Any]: @tool( "preflight", - "Check everything a live call needs before spending one: the assistant's credentials " - "and a way to expose the webhook publicly. Run this before the first call.", + "Check everything a run needs before spending one. For a hosted voice agent that is the " + "assistant's credentials and a way to expose the webhook publicly; for anything else " + "the run happens here and needs nothing external. Run this before the first run.", schema({}, []), ) async def preflight(_args: dict[str, Any]) -> dict[str, Any]: + if not live: + return _ok( + "Ready. This agent runs here, against the world, from its contract — nothing " + f"external is needed. {len(written)} scenarios are available." + ) problems = missing_prerequisites() if problems: return _err("Not ready:\n - " + "\n - ".join(problems)) @@ -192,14 +212,31 @@ async def preflight(_args: dict[str, Any]) -> dict[str, Any]: f"{len(written)} scenarios are available." ) + async def _run_here(scenario: Any) -> dict[str, Any]: + """The scenario against the agent stood up from its contract, over the same world.""" + from . import run_suite + + if contract is None: + return _err("no contract is loaded, so there is no agent to stand up") + graded = await run_suite([scenario], contract, world_root, out=destination) + results[:] = load_results(destination) + result = graded[0] + lines = [result.line()] + [check.line() for check in result.checkpoints] + if result.transcript: + lines += ["", "the conversation:", result.transcript] + answer = "\n".join(lines) + return _ok(answer) if result.passed else _err(answer) + @tool( "run_scenario", - "Run one scenario against the real agent and grade it.\n\n" - "This restores the world, applies the scenario's setup, stands up the webhook, points " - "the assistant's OWN tools at it, places the call, and runs the sub-goals' checks " - "against what the world holds afterwards plus the calls that were made.\n\n" - "It takes several minutes and blocks until the call is over. Run one at a time and read " - "what comes back before running the next.", + "Run one scenario against the agent and grade it.\n\n" + "The world is restored and the scenario's setup applied first. A hosted voice agent is " + "reached live — its OWN tools are pointed at the world over a webhook and the call is " + "placed; any other agent is stood up here from its contract and conversed with. Either " + "way the sub-goals' checks run against what the world holds afterwards plus the calls " + "that were made.\n\n" + "It can take minutes and blocks until the run is over. Run one at a time and read what " + "comes back before running the next.", # Both spellings accepted: every model that has driven this stage has guessed # `scenario` at least once, and a retry on an argument name is a wasted turn. schema({"name": str, "scenario": str}, []), @@ -212,6 +249,8 @@ async def run_scenario(args: dict[str, Any]) -> dict[str, Any]: f"no scenario called {name!r}. There is: " + ", ".join(one.name for one in written) ) + if not live: + return await _run_here(scenario) problems = missing_prerequisites() if problems: return _err( @@ -275,17 +314,22 @@ async def read_results(_args: dict[str, Any]) -> dict[str, Any]: return _ok("nothing has been run yet") lines = [] for record in results: - mark = "PASS" if record["passed"] else "FAIL" + mark = "PASS" if record.get("passed") else "FAIL" + # Two record shapes share this file: live runs carry settled/judged, local runs + # carry checkpoints. Both say what failed, and both deserve to be read. failed = [ - f"{one['name']}: {one['said']}" - for one in record["settled"] - if not one["held"] + f"{one.get('name')}: {one.get('said') or one.get('detail') or ''}" + for one in (record.get("settled") or record.get("checkpoints") or []) + if not (one.get("held") if "held" in one else one.get("passed")) ] + met = record.get("met", record.get("checkpoints_met", "?")) + of = record.get("of") + scored = f"{met}/{of}" if of is not None else str(met) lines.append( - f"{mark} {record['scenario']} {record['met']}/{record['of']}" + f"{mark} {record.get('scenario')} {scored}" + ("\n - " + "\n - ".join(failed) if failed else "") ) - passed = sum(1 for record in results if record["passed"]) + passed = sum(1 for record in results if record.get("passed")) return _ok("\n".join(lines) + f"\n\n{passed} of {len(results)} passed") server = create_sdk_mcp_server( From e954e76a82de4ec28a63c05f58063686f1c73917 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Mon, 17 Aug 2026 01:45:17 +0530 Subject: [PATCH 07/39] fix(harness): slim the contract to consumed fields plus free-form notes, unstick unattended submission --- src/fi/alk/harness/HOW-IT-WORKS.md | 4 +- src/fi/alk/harness/cli.py | 43 ++++++++++++- src/fi/alk/harness/contract.py | 22 +++---- .../harness/skills/understand-agent/SKILL.md | 22 ++++--- src/fi/alk/harness/tools.py | 57 ++++++++++++----- src/fi/alk/harness/understand.py | 7 ++- tests/test_harness.py | 61 +++++++++++++++++++ 7 files changed, 175 insertions(+), 41 deletions(-) diff --git a/src/fi/alk/harness/HOW-IT-WORKS.md b/src/fi/alk/harness/HOW-IT-WORKS.md index e28ce7b..808e6b0 100644 --- a/src/fi/alk/harness/HOW-IT-WORKS.md +++ b/src/fi/alk/harness/HOW-IT-WORKS.md @@ -63,8 +63,8 @@ calls `submit_contract`. | `tools[]` — name, `args`, `arg_types`, `arg_values`, description | The agent's action space. `arg_values` are the real permitted values — the menu, the enum, the lookup | | `hard_constraints[]` | Rules the agent must follow. Told to the agent under test, and graded by the judge | | `base_environment` | Its real starting data, reproduced row for row | -| `real_use_cases[]`, `signature_cases[]` | What it is actually for | -| `anti_hallucination[]` | Things that do not exist and must never be used | +| `real_use_cases[]` | What it is actually for | +| `notes` | Free-form: whatever else the reader judged worth carrying forward | | `amendments[]` | Anything **not** read from source — see below | **How it is written:** `accept_contract` in `tools.py` validates before anything reaches disk. It diff --git a/src/fi/alk/harness/cli.py b/src/fi/alk/harness/cli.py index 4c7922f..fdea0bd 100644 --- a/src/fi/alk/harness/cli.py +++ b/src/fi/alk/harness/cli.py @@ -82,7 +82,16 @@ async def _understand(args: argparse.Namespace) -> int: print(f"model: {chosen_model()}") print(f"out: {destination}\n") - await _converse(stage, opening(source), interactive=args.interactive) + await _converse( + stage, + opening(source), + interactive=args.interactive, + until=lambda: load(destination) is not None, + nudge=( + "Nothing was saved: you finished without calling submit_contract. Call it now " + "with the contract you worked out." + ), + ) contract = load(destination) if contract is None: @@ -98,15 +107,29 @@ async def _understand(args: argparse.Namespace) -> int: return 0 -async def _converse(stage, opening_message: str, *, interactive: bool) -> None: +async def _converse( + stage, + opening_message: str, + *, + interactive: bool, + until=None, + nudge: str = "", +) -> None: """Say the opening, then keep the stage open for corrections. The same shape for every stage. A world is usually right on the second look, and the point of holding the session open is that correcting it is the next thing said rather than a rebuild from nothing. + + ``until``/``nudge`` guard the unattended case. The commonest way an unattended stage fails + is finishing all the work and never calling the tool that saves it — the whole contract + written out as prose, submitted to nobody. One mechanical reminder costs a turn; rerunning + the stage costs everything it just did. """ async with stage: await stage.say(opening_message, on_event=_render) + if not interactive and until is not None and nudge and not until(): + await stage.say(nudge, on_event=_render) while interactive: try: said = await _prompt("\nkarthik ") @@ -133,7 +156,16 @@ async def _build(args: argparse.Namespace) -> int: out=destination, ask=permission_gate(_ask_operator) if args.interactive else None, ) - await _converse(stage, build_opening(contract), interactive=args.interactive) + await _converse( + stage, + build_opening(contract), + interactive=args.interactive, + until=lambda: (destination / "world.sqlite").exists(), + nudge=( + "Nothing was saved: you finished without calling save_world. Call check_world, " + "fix what it names, then save_world." + ), + ) if not (destination / "world.sqlite").exists(): print("\nNo world was saved.", file=sys.stderr) @@ -175,6 +207,11 @@ async def _scenarios(args: argparse.Namespace) -> int: stage, scenario_opening(contract, wanted, existing), interactive=args.interactive, + until=lambda: bool(load_written(destination)), + nudge=( + "Nothing was saved: you finished without calling save_scenarios. Submit anything " + "still unsubmitted, then call save_scenarios." + ), ) written = load_written(destination) diff --git a/src/fi/alk/harness/contract.py b/src/fi/alk/harness/contract.py index 36b1423..49b67fe 100644 --- a/src/fi/alk/harness/contract.py +++ b/src/fi/alk/harness/contract.py @@ -21,13 +21,11 @@ "one_liner", "modality", "system_prompt_excerpt", - "grading_notes", + "notes", ) _LIST_FIELDS = ( "hard_constraints", "real_use_cases", - "signature_cases", - "anti_hallucination", "amendments", ) _DICT_FIELDS = ("data_schema", "base_environment") @@ -82,9 +80,11 @@ def _normalize_shapes(cls, payload: Any) -> Any: data_schema: dict[str, Any] = Field(default_factory=dict) base_environment: dict[str, Any] = Field(default_factory=dict) real_use_cases: list[str] = Field(default_factory=list) - signature_cases: list[str] = Field(default_factory=list) - grading_notes: str = "" - anti_hallucination: list[str] = Field(default_factory=list) + # Free-form. The fields above are the fixed core because code consumes them; this is where + # the reader records whatever else about *this* agent is worth carrying forward — quirks, + # traps, names that look real but are not — in whatever form fits. It is shown verbatim to + # every later stage. + notes: str = "" open_questions: list[str] = Field(default_factory=list) # Anything in here was not read from the agent's source. The contract is meant to be what # the agent verifiably is, so when the harness widens it the difference is recorded rather @@ -139,13 +139,13 @@ def brief(self, *, full_schema: bool = True, with_data: bool = False) -> str: "and a test written against a corrected world will not catch the real bug.\n" + json.dumps(self.base_environment, ensure_ascii=False) ) - if self.grading_notes: - parts.append(f"GRADING NOTES for this agent:\n{self.grading_notes[:900]}") - if self.anti_hallucination: + if self.real_use_cases: parts.append( - "NEVER USE THESE (they do not exist / are wrong): " - + json.dumps(self.anti_hallucination)[:700] + "REAL USE CASES (what this agent is actually for):\n - " + + "\n - ".join(self.real_use_cases[:12]) ) + if self.notes: + parts.append(f"NOTES from reading the agent:\n{self.notes[:1500]}") return "\n\n".join(parts) diff --git a/src/fi/alk/harness/skills/understand-agent/SKILL.md b/src/fi/alk/harness/skills/understand-agent/SKILL.md index 2fa7e24..2b06fff 100644 --- a/src/fi/alk/harness/skills/understand-agent/SKILL.md +++ b/src/fi/alk/harness/skills/understand-agent/SKILL.md @@ -43,10 +43,16 @@ Find, in roughly this order: 3. **Argument values.** Where an argument is constrained to a set, an enum, a literal union, or a lookup into fixed data, record the real values. 4. **The rules.** Hard constraints the agent is instructed or coded to obey. Prefer the exact - wording from the system prompt or the validation code. -5. **The data.** Where it lives, its shape, and its real contents. In-memory dicts, fixture + wording from the system prompt or the validation code. These matter: the agent under test is + told them and graded against them, and its system prompt is where most of them live — read + it in full before deciding there are none. +5. **The modality.** How a person reaches this agent, read from its runtime, not guessed: a + voice session (LiveKit, telephony, TTS/STT) is `voice`; a text interface is `chat`; a + browser-driving agent is `browser`. This decides how it is run later — a voice agent is + called live; anything else runs locally — so getting it wrong reroutes every test. +6. **The data.** Where it lives, its shape, and its real contents. In-memory dicts, fixture files, a seeded database. Record enough for a working replica to be built. -6. **Real use cases.** What this agent is actually for, as concrete situations, drawn from the +7. **Real use cases.** What this agent is actually for, as concrete situations, drawn from the tools and data rather than invented. ## When you are not sure @@ -60,12 +66,12 @@ Do not use it for anything the code answers. Reading one more file is cheaper th Anything you could not resolve, and did not ask about, goes in `open_questions`. -## Anti-hallucination +## Notes -Record in `anti_hallucination` the names and values that a reasonable person would expect this -agent to have but which do **not** exist: a plausible tool name that is not registered, an id -that follows the naming convention but is absent from the data, an argument the API does not -take. Later stages use this list to catch themselves. +`notes` is free-form and yours. Record whatever else about this agent is worth carrying forward, +in whatever form fits it: quirks in how it behaves, a plausible-looking name that does not +actually exist, an id that looks like a typo but is real. Every later stage is shown it +verbatim. Leave it empty rather than padding it. ## Finishing diff --git a/src/fi/alk/harness/tools.py b/src/fi/alk/harness/tools.py index 3acdafa..fee69f8 100644 --- a/src/fi/alk/harness/tools.py +++ b/src/fi/alk/harness/tools.py @@ -67,6 +67,11 @@ def accept_contract(payload: dict[str, Any], destination: Path) -> dict[str, Any def contract_tools(destination: Path) -> Any: """A server exposing ``submit_contract``, writing to ``destination`` on acceptance.""" + # One nudge, not a wall. A conversational agent with no rules and no prompt excerpt almost + # always means the prompt was not found — it often lives away from the main agent file — so + # the first such submission is sent back with directions. The second is accepted, because a + # gate with no way through would permanently block the rare agent that genuinely has none. + nudged = {"done": False} @tool( "submit_contract", @@ -83,24 +88,44 @@ def contract_tools(destination: Path) -> Any: "`arg_values` carries the real permitted values wherever the argument is constrained to " "a set, an enum or a lookup. Everything downstream is built from these, so a tool " "submitted without its arguments cannot be tested.", - { - "agent": str, - "one_liner": str, - "modality": str, - "conversational": bool, - "system_prompt_excerpt": str, - "hard_constraints": list, - "tools": list, - "data_schema": dict, - "base_environment": dict, - "real_use_cases": list, - "signature_cases": list, - "grading_notes": str, - "anti_hallucination": list, - "open_questions": list, - }, + # Only what validate_contract refuses to live without is required here. A plain + # {name: type} map marks every field mandatory, and the schema layer then rejects the + # submission one missing field at a time — a full model turn per field — before the + # gate that knows how to explain a problem is ever reached. + schema( + { + "agent": str, + "one_liner": str, + "modality": str, + "conversational": bool, + "system_prompt_excerpt": str, + "hard_constraints": list, + "tools": list, + "data_schema": dict, + "base_environment": dict, + "real_use_cases": list, + "notes": str, + "open_questions": list, + }, + ["agent", "tools", "real_use_cases"], + ), ) async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: + bare = ( + args.get("conversational", True) + and not args.get("hard_constraints") + and not str(args.get("system_prompt_excerpt") or "").strip() + ) + if bare and not nudged["done"]: + nudged["done"] = True + return _problems( + [ + "no hard_constraints and no system_prompt_excerpt, for a conversational " + "agent. Its prompt usually exists and often lives away from the main agent " + "file — search the whole source for a long instructions string before " + "deciding there is none. If there genuinely is none, submit again as is." + ] + ) return accept_contract(args, destination) return create_sdk_mcp_server( diff --git a/src/fi/alk/harness/understand.py b/src/fi/alk/harness/understand.py index c0fac78..0d4e84d 100644 --- a/src/fi/alk/harness/understand.py +++ b/src/fi/alk/harness/understand.py @@ -47,8 +47,13 @@ def open_stage( def opening(source: AgentSource) -> str: + # The name is only a label for the artifact folder, and saying so matters: told to "read + # the agent named verify_fix", a model went hunting the whole workspace for something + # called verify_fix instead of reading the path it was given. return ( - f"Read the agent named {source.name!r} and produce its contract.\n\n" + "Read this agent and produce its contract. Where it lives is in your briefing; " + f"{source.name!r} is only the label its artifacts are filed under, not something to " + "search for.\n\n" "Work through the tools, their exact argument names and types, the constrained argument " "values, the rules it enforces, and its data. Ask me if the source genuinely does not " "settle something that changes what gets built. Call submit_contract when you are done." diff --git a/tests/test_harness.py b/tests/test_harness.py index 8e8b55e..314e6ec 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -1330,3 +1330,64 @@ def test_writing_new_results_keeps_the_ones_not_rerun(tmp_path): kept = {r["scenario"]: r for r in load_results(tmp_path)} assert kept["a"]["passed"] is True assert kept["b"]["passed"] is True and kept["b"]["transcript"] == "hello" + + +def test_submit_contract_requires_only_what_the_gate_demands(tmp_path): + """Every field marked required is rejected by the schema layer one at a time, a full model + turn each, before accept_contract can explain anything. Only the fields validate_contract + refuses to live without may be required; the rest are optional and gated with real messages.""" + import asyncio + + from mcp.types import ListToolsRequest + + from fi.alk.harness.tools import contract_tools + + server = contract_tools(tmp_path) + instance = server.get("instance") if isinstance(server, dict) else server + + async def schema_of(): + for key, handler in instance.request_handlers.items(): + if getattr(key, "__name__", "") == "ListToolsRequest": + result = await handler(ListToolsRequest(method="tools/list")) + return result.root.tools[0].inputSchema + return {} + + schema = asyncio.run(schema_of()) + assert sorted(schema.get("required", [])) == ["agent", "real_use_cases", "tools"] + + +def test_a_bare_conversational_contract_is_nudged_once_then_accepted(tmp_path): + """No rules and no prompt excerpt on a conversational agent almost always means the prompt + was not found, so the first submission bounces with directions. The second goes through, + because a gate with no way past would permanently block an agent that genuinely has none.""" + import asyncio + + from fi.alk.harness.tools import contract_tools + + server = contract_tools(tmp_path) + instance = server.get("instance") if isinstance(server, dict) else server + + async def call(payload): + from mcp.types import CallToolRequest, CallToolRequestParams + + for key, handler in instance.request_handlers.items(): + if getattr(key, "__name__", "") == "CallToolRequest": + request = CallToolRequest( + method="tools/call", + params=CallToolRequestParams(name="submit_contract", arguments=payload), + ) + answer = await handler(request) + return answer.root.content[0].text + + payload = { + "agent": "quiet", + "tools": [{"name": "act", "args": ["x"]}], + "real_use_cases": ["do the thing"], + } + first = asyncio.run(call(dict(payload))) + assert "system_prompt_excerpt" in first and "submit again" in first + assert not (tmp_path / "contract.json").exists() + + second = asyncio.run(call(dict(payload))) + assert "Accepted" in second + assert (tmp_path / "contract.json").exists() From 2a0ba45e8005897c32af253b1aea30cd298d993f Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Mon, 17 Aug 2026 02:09:20 +0530 Subject: [PATCH 08/39] feat(harness): stages hand a request to the stage that owns it, guard understand reopen --- src/fi/alk/harness/chat.py | 77 +++++++++++++++++++++++++++++++++-- src/fi/alk/harness/session.py | 18 ++++++++ tests/test_harness.py | 62 ++++++++++++++++++++++++++++ 3 files changed, 153 insertions(+), 4 deletions(-) diff --git a/src/fi/alk/harness/chat.py b/src/fi/alk/harness/chat.py index 6e5f2ba..dcc2996 100644 --- a/src/fi/alk/harness/chat.py +++ b/src/fi/alk/harness/chat.py @@ -57,6 +57,8 @@ class Conversation: workspace: Path | None = None stage_name: str = "" stage: Stage | None = None + # Set by the flow tool when the open stage hands a request to the stage that owns it. + _handoff: dict = field(default_factory=dict) spent_usd: float = 0.0 history: list[str] = field(default_factory=list) _found: dict[str, Any] = field(default_factory=dict) @@ -113,6 +115,7 @@ async def _open(self, stage_name: str) -> str: self.stage, self._found = reception_stage.open_stage( cwd=self.workspace, ask=self.ask ) + self._grant_flow() await self.stage.__aenter__() return reception_stage.opening() @@ -121,21 +124,23 @@ async def _open(self, stage_name: str) -> str: # Only re-reading the agent needs to know where it lives. if self.source is None and self.contract is None: raise RuntimeError("nobody has said which agent this is about yet") + if stage_name == UNDERSTAND and self.source is None: + # This guard has to come before the stage opens: with a contract on disk but no + # source, reopening understand would otherwise die on source.briefing() instead of + # saying what is actually missing. + raise RuntimeError("cannot re-read the agent without knowing where it lives") if stage_name == UNDERSTAND: self.stage, _ = understand_stage.open_stage( self.source, out=self.out, ask=self.ask ) opening = understand_stage.opening(self.source) + self._grant_flow() await self.stage.__aenter__() return opening contract = self.contract if contract is None: raise RuntimeError("cannot go further before there is a contract") - if self.source is None and stage_name == UNDERSTAND: - raise RuntimeError( - "cannot re-read the agent without knowing where it lives" - ) if stage_name == BUILD: self.stage, _ = build_stage.open_stage(contract, out=self.out, ask=self.ask) opening = build_stage.opening(contract) @@ -153,6 +158,7 @@ async def _open(self, stage_name: str) -> str: contract, out=self.out, wanted=wanted, ask=self.ask ) opening = scenario_stage.opening(contract, wanted, written) + self._grant_flow() await self.stage.__aenter__() return opening @@ -163,6 +169,54 @@ def next_stage(self) -> str | None: following = _NEXT.get(self.stage_name) return None if following in (None, DONE) else following + def _flow_server(self): + """One tool every stage gets: handing a request to the stage that owns it. + + "Create the world", said while the understand stage is open, used to land in a session + with no build tools, which could only apologise. The stage is the one that knows the + request is not its job, so the handoff is a tool it calls; whether moving on is allowed + is still decided by code, from whether this stage's artifact exists. + """ + from claude_agent_sdk import create_sdk_mcp_server, tool + + from .tools import schema + + wanted = self._handoff + + @tool( + "hand_to_next_stage", + "The person asked for something that belongs to the NEXT stage of this harness — " + "building the environment when the contract is done, writing scenarios when the " + "environment is built, running them when they are written. Call this with their " + "request, word for word; the conversation moves forward and their request is " + "handled there. Never call it to escape work that is this stage's own.", + schema({"request": str}, []), + ) + async def hand_to_next_stage(args: dict[str, Any]) -> dict[str, Any]: + if not self._artifact_for(self.stage_name): + return { + "content": [{ + "type": "text", + "text": "This stage has not produced its artifact yet, so there is " + "nothing to move on from. Finish this stage's work first.", + }], + "is_error": True, + } + if self.next_stage() is None: + return { + "content": [{"type": "text", "text": "there is no stage after this one"}], + "is_error": True, + } + wanted["request"] = str(args.get("request") or "").strip() or "continue" + return { + "content": [{ + "type": "text", + "text": "Handed over. Say one short line that you are moving on, and stop.", + }] + } + + return create_sdk_mcp_server(name="flow", version="0.1.0", tools=[hand_to_next_stage]) + # -- talking --------------------------------------------------------------------- async def start(self, on_event: Callable[..., Any] | None = None) -> None: @@ -191,6 +245,10 @@ def _resume_at(self) -> str: return SCENARIOS return RUN + def _grant_flow(self) -> None: + if self.stage is not None: + self.stage.grant("flow", self._flow_server(), ["hand_to_next_stage"], ask=self.ask) + async def say( self, message: str, on_event: Callable[..., Any] | None = None ) -> None: @@ -199,6 +257,17 @@ async def say( if self.stage is None: await self.open_quietly() await self.stage.say(message, on_event=on_event) # type: ignore[union-attr] + # A handoff moves the request, not just the conversation: the next stage opens and is + # given the person's own words. Bounded, because each hop is a model turn. + for _hop in range(3): + request = self._handoff.pop("request", None) + if not request: + break + following = self.next_stage() + if following is None: + break + await self._open(following) + await self.stage.say(request, on_event=on_event) # type: ignore[union-attr] await self._settle(on_event=on_event) async def _settle(self, on_event: Callable[..., Any] | None = None) -> None: diff --git a/src/fi/alk/harness/session.py b/src/fi/alk/harness/session.py index 75a65a5..2bab10b 100644 --- a/src/fi/alk/harness/session.py +++ b/src/fi/alk/harness/session.py @@ -180,6 +180,24 @@ def __init__(self, options: ClaudeAgentOptions, *, name: str = "") -> None: # take shows up only on the invoice, weeks later, as a number nobody can explain. self.models_used: set[str] = set() + def grant(self, server_name: str, server: Any, tool_names: list[str], ask: Any = None) -> None: + """Give this stage one more tool server, before it opens. + + The permission gate and the PreToolUse hook both close over the granted list when the + stage is built, so appending to ``allowed_tools`` after the fact changes nothing — the + hook still denies the new tool. Granting means rebuilding all three together, which is + why it lives here rather than being three edits every caller must remember. + """ + if self._client is not None: + raise RuntimeError("grant before the stage opens; the session is already running") + from .config import gate_hooks, permission_gate + + added = [f"mcp__{server_name}__{name}" for name in tool_names] + self._options.mcp_servers = {**(self._options.mcp_servers or {}), server_name: server} + self._options.allowed_tools = [*(self._options.allowed_tools or []), *added] + self._options.hooks = gate_hooks(self._options.allowed_tools) + self._options.can_use_tool = permission_gate(ask, self._options.allowed_tools) + async def __aenter__(self) -> "Stage": self._client = ClaudeSDKClient(options=self._options) await self._client.connect() diff --git a/tests/test_harness.py b/tests/test_harness.py index 314e6ec..0622afb 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -1391,3 +1391,65 @@ async def call(payload): second = asyncio.run(call(dict(payload))) assert "Accepted" in second assert (tmp_path / "contract.json").exists() + + +def test_granting_a_tool_rebuilds_the_gate_not_just_the_list(tmp_path): + """The hook closes over the granted set when the stage is built, so appending to + allowed_tools alone leaves the new tool denied. grant() must rebuild all three.""" + import asyncio + + from claude_agent_sdk import ClaudeAgentOptions + + from fi.alk.harness.config import gate_hooks + from fi.alk.harness.session import Stage + + allowed = ["Read"] + options = ClaudeAgentOptions( + system_prompt="x", allowed_tools=allowed, permission_mode="default", + setting_sources=[], max_turns=1, + ) + options.hooks = gate_hooks(allowed) + stage = Stage(options, name="t") + stage.grant("flow", object(), ["hand_to_next_stage"]) + + assert "mcp__flow__hand_to_next_stage" in options.allowed_tools + refuse = options.hooks["PreToolUse"][0].hooks[0] + granted = asyncio.run(refuse({"tool_name": "mcp__flow__hand_to_next_stage"}, None, None)) + assert granted == {} + + +def test_handoff_is_refused_until_the_stage_has_its_artifact(tmp_path): + """Moving on is decided by code, from the artifacts, never by the model wanting to.""" + import asyncio + + from mcp.types import CallToolRequest, CallToolRequestParams + + from fi.alk.harness.chat import Conversation + + conversation = Conversation(source=None, out=tmp_path, workspace=tmp_path) + conversation.stage_name = "understand" + server = conversation._flow_server() + instance = server.get("instance") if isinstance(server, dict) else server + + async def call(): + for key, handler in instance.request_handlers.items(): + if getattr(key, "__name__", "") == "CallToolRequest": + request = CallToolRequest( + method="tools/call", + params=CallToolRequestParams( + name="hand_to_next_stage", arguments={"request": "create the world"} + ), + ) + answer = await handler(request) + return answer.root.content[0].text + + said = asyncio.run(call()) + assert "not produced its artifact" in said + assert not conversation._handoff + + (tmp_path / "contract.json").write_text( + '{"agent": "a", "tools": [{"name": "t"}], "real_use_cases": ["u"]}' + ) + said = asyncio.run(call()) + assert "Handed over" in said + assert conversation._handoff["request"] == "create the world" From 8adb1d729460b04e7d657c081b3573a982e1f2c4 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Mon, 17 Aug 2026 09:11:43 +0530 Subject: [PATCH 09/39] fix(harness): make the contract tool schema self-describing and forgive how a contract is packaged --- src/fi/alk/harness/contract.py | 50 +++- .../harness/skills/build-environment/SKILL.md | 2 +- src/fi/alk/harness/tools.py | 273 +++++++++++++++--- tests/test_harness.py | 152 +++++++++- 4 files changed, 425 insertions(+), 52 deletions(-) diff --git a/src/fi/alk/harness/contract.py b/src/fi/alk/harness/contract.py index 49b67fe..528f526 100644 --- a/src/fi/alk/harness/contract.py +++ b/src/fi/alk/harness/contract.py @@ -16,6 +16,10 @@ from pydantic import BaseModel, Field, model_validator +# How a person reaches an agent. This decides how it is later run — voice goes out as a live +# call, everything else runs locally — so it is defined once and referenced, never retyped. +MODALITIES = ("voice", "chat", "browser") + _STRING_FIELDS = ( "agent", "one_liner", @@ -32,6 +36,48 @@ class ToolSpec(BaseModel): + """One tool the agent really has. + + ``args`` is the load-bearing field: the world's handlers, the probes and every scenario are + built from these exact names. It is also the one most often written under another name — + ``parameters``, ``arguments``, ``params`` — or left out while ``arg_types`` names every + argument anyway. All of those are the same information, so they are accepted and normalised + rather than rejected, because a contract bounced for a synonym costs a full turn and teaches + nothing about the agent. + """ + + @model_validator(mode="before") + @classmethod + def _normalize_args(cls, payload: Any) -> Any: + if not isinstance(payload, dict): + return payload + if not payload.get("args"): + for alias in ("parameters", "arguments", "params", "arg_names"): + value = payload.get(alias) + if isinstance(value, list) and value: + payload["args"] = value + break + # Some writers give {name: type} where a list was asked for. The keys are the + # argument names, which is exactly what was wanted. + if isinstance(value, dict) and value: + payload["args"] = list(value) + payload.setdefault( + "arg_types", {k: str(v) for k, v in value.items()} + ) + break + if not payload.get("args"): + # Nothing named the arguments directly, but a per-argument map still names them. + for source in ("arg_types", "arg_values"): + mapping = payload.get(source) + if isinstance(mapping, dict) and mapping: + payload["args"] = list(mapping) + break + if isinstance(payload.get("args"), str): + payload["args"] = [payload["args"]] + if isinstance(payload.get("args"), list): + payload["args"] = [str(one) for one in payload["args"]] + return payload + name: str args: list[str] = Field(default_factory=list) arg_types: dict[str, str] = Field(default_factory=dict) @@ -70,7 +116,9 @@ def _normalize_shapes(cls, payload: Any) -> Any: payload[key] = {"value": value} return payload - agent: str + # Defaulted rather than mandatory so a submission that forgets it reaches validate_contract, + # which says what to do about it, instead of dying in the schema layer with a type error. + agent: str = "" one_liner: str = "" modality: str = "chat" conversational: bool = True diff --git a/src/fi/alk/harness/skills/build-environment/SKILL.md b/src/fi/alk/harness/skills/build-environment/SKILL.md index e46780b..4f161fd 100644 --- a/src/fi/alk/harness/skills/build-environment/SKILL.md +++ b/src/fi/alk/harness/skills/build-environment/SKILL.md @@ -44,7 +44,7 @@ your bugs; `ToolError` is the world's answer, and the checks tell them apart. Inside a handler you have `args`, `db`, `ToolError` and `json`, and nothing else. Do not import anything and do not define your own `ToolError`. Use the argument names exactly as the contract -gives them: a handler reading `order_ids` when the tool takes `order_id` finds nothing, quietly +gives them. A handler that reads a plural where the tool takes a singular finds nothing, quietly does nothing, and reports success. Seed the agent's **real** data. Where the contract records something unavailable, a misspelled id, diff --git a/src/fi/alk/harness/tools.py b/src/fi/alk/harness/tools.py index fee69f8..ba4e05a 100644 --- a/src/fi/alk/harness/tools.py +++ b/src/fi/alk/harness/tools.py @@ -13,7 +13,7 @@ from claude_agent_sdk import create_sdk_mcp_server, tool -from .contract import AgentContract, validate_contract +from .contract import MODALITIES, AgentContract, validate_contract CONTRACT_SERVER = "contract" @@ -22,33 +22,117 @@ def _ok(text: str) -> dict[str, Any]: return {"content": [{"type": "text", "text": text}]} -def _problems(problems: list[str]) -> dict[str, Any]: +# validate_contract returns short codes: they are stable, testable, and the same string every +# time. What a code means is a separate question, and answering it here keeps the codes exact +# while the message the model reads says what to actually do. +_GUIDANCE = { + "empty:agent": "give it a short lower-case name; it is only the artifact folder's label", + "no-tools": "list the agent's real tools. Nothing downstream can be built without them", + "no-use-cases": "list the concrete situations this agent handles, from its tools and data", + "no-arguments-on-any-tool": "every tool was recorded with no arguments, which means they " + "were read and not written down. Put each tool's exact parameter names in args", + "duplicate-tool-names": "the same tool is listed twice; keep one entry per tool", + "types-for-unknown-args": "arg_types names an argument that is not in args. The names must " + "match the source exactly", +} + + +def _advice(code: str) -> str: + for key, said in _GUIDANCE.items(): + if code.startswith(key) or key in code: + return f"{code} — {said}" + return code + + +def _problems(problems: list[str], arrived: list[str] | None = None) -> dict[str, Any]: + """Every problem at once, each with what to do about it. + + All of them together, never one at a time: a gate that reveals the next problem only after + the last is fixed costs a full turn per problem and reads as though the rules are being + invented as it goes. + + When the fields arrived under names this does not recognise, it says which names it got. + Without that the answer is "agent is empty, there are no tools" about a submission that + contained both, and the only way out is guessing at the packaging. + """ + said = "Not accepted. Fix all of these and call submit_contract again:\n - " + ( + "\n - ".join(_advice(problem) for problem in problems) + ) + unrecognised = arrived is not None and not any( + key in arrived for key in ("agent", "tools", "real_use_cases") + ) + if unrecognised: + said += ( + f"\n\nWhat arrived was: {', '.join(arrived) or '(nothing)'}. None of those are " + "contract fields, so the fields were probably nested inside something or sent as " + "one JSON string. Send them as the tool's own top-level arguments — agent, tools, " + "real_use_cases and the rest — not wrapped in an outer object." + ) return { - "content": [ - { - "type": "text", - "text": "Not accepted. Fix these and call submit_contract again:\n - " - + "\n - ".join(problems), - } - ], + "content": [{"type": "text", "text": said}], "is_error": True, } +_CONTRACT_KEYS = ("agent", "tools", "real_use_cases", "one_liner", "hard_constraints") + + +def _looks_like_a_contract(value: Any) -> bool: + return isinstance(value, dict) and any(key in value for key in _CONTRACT_KEYS) + + +def unwrapped(payload: dict[str, Any]) -> dict[str, Any]: + """The contract itself, however it was packaged. + + A contract is a nested thing being described, so it arrives wrapped — ``{"contract": {...}}`` + — or stringified, as JSON in a single argument, often enough to matter. In both the fields + are present and correct and only the packaging is wrong. Rejecting that teaches nothing + about the agent and costs a full turn, so it is unpacked; only an object that actually looks + like a contract is unwrapped, so a real field that happens to hold a dict is never mistaken + for an envelope. + """ + if not isinstance(payload, dict): + payload = {} + if any(key in payload for key in ("agent", "tools", "real_use_cases")): + return payload + for value in payload.values(): + if _looks_like_a_contract(value): + return value + if isinstance(value, str): + text = value.strip() + if text.startswith("```"): + # Fenced JSON: the model wrote it as it would in a message. + text = text.strip("`").removeprefix("json").strip() + if not text.startswith("{"): + continue + try: + parsed = json.loads(text) + except json.JSONDecodeError: + continue + if _looks_like_a_contract(parsed): + return parsed + for inner in parsed.values() if isinstance(parsed, dict) else []: + if _looks_like_a_contract(inner): + return inner + return payload + + def accept_contract(payload: dict[str, Any], destination: Path) -> dict[str, Any]: """The gate itself: validate, and write only if it passes. A plain function rather than only a tool body, so the rule that decides whether a contract is usable can be exercised and reasoned about without standing up a session. """ + arrived = sorted(payload) if isinstance(payload, dict) else [type(payload).__name__] + payload = unwrapped(payload) try: contract = AgentContract.model_validate(payload) except Exception as invalid: - return _problems([f"schema:{invalid}"[:600]]) + return _problems([f"schema:{invalid}"[:600]], arrived) problems = validate_contract(contract) if problems: - return _problems(problems) + return _problems(problems, arrived) destination.mkdir(parents=True, exist_ok=True) path = destination / "contract.json" @@ -75,39 +159,119 @@ def contract_tools(destination: Path) -> Any: @tool( "submit_contract", - "Submit the agent's testing contract. Validated on submission; problems are returned " - "to you so you can correct them and submit again.\n\n" - "Each entry in `tools` is an object:\n" - ' {"name": "remove_order_item",\n' - ' "args": ["order_id"],\n' - ' "arg_types": {"order_id": "list[str]"},\n' - ' "arg_values": {"order_id": []},\n' - ' "description": "..."}\n' - "`args` must list the exact parameter names the model emits when calling the tool, in " - "order. `arg_types` carries the declared type wherever the source states one. " - "`arg_values` carries the real permitted values wherever the argument is constrained to " - "a set, an enum or a lookup. Everything downstream is built from these, so a tool " - "submitted without its arguments cannot be tested.", - # Only what validate_contract refuses to live without is required here. A plain - # {name: type} map marks every field mandatory, and the schema layer then rejects the - # submission one missing field at a time — a full model turn per field — before the - # gate that knows how to explain a problem is ever reached. + "Submit the agent's testing contract: everything verifiably true about this agent, as " + "one flat object. Every field is described in the schema; fill in what the source " + "supports and leave the rest out.\n\n" + "It is validated when you call it. If anything is wrong you get the whole list back at " + "once, in terms of what to fix, and you submit again.", + # Nothing required, and that is deliberate. This layer runs before the tool body, so + # anything it rejects never reaches the code that could have understood it — a contract + # sent inside a wrapper is complete and correct, and is unwrapped a few lines below, but + # only if it gets there. accept_contract is the single gate; it reports every problem at + # once and says what to do about each. + # + # The descriptions are the point of this block. The schema is shown to the model before + # it calls anything, so what is written here is the difference between a correct first + # call and a sequence of rejected guesses. schema( { - "agent": str, - "one_liner": str, - "modality": str, - "conversational": bool, - "system_prompt_excerpt": str, - "hard_constraints": list, - "tools": list, - "data_schema": dict, - "base_environment": dict, - "real_use_cases": list, - "notes": str, - "open_questions": list, + "agent": { + "type": "string", + "description": "Short lower-case identifier, no spaces. Only a label for " + "the artifact folder.", + }, + "one_liner": { + "type": "string", + "description": "One sentence: what this agent is for.", + }, + "modality": { + "type": "string", + "enum": list(MODALITIES), + "description": "How a person reaches it, read from its runtime. A voice " + "session (LiveKit, telephony, TTS/STT) is voice; a text interface is chat; " + "a browser-driving agent is browser. This decides how it is later run.", + }, + "conversational": { + "type": "boolean", + "description": "True if a person talks with it turn by turn. False for an " + "agent given one task and left to it.", + }, + "system_prompt_excerpt": { + "type": "string", + "description": "The agent's own instructions, quoted. Often lives away from " + "the main agent file.", + }, + "hard_constraints": { + "type": "array", + "items": {"type": "string"}, + "description": "Rules it must obey, in the source's own words. The agent " + "under test is told these and graded against them.", + }, + "tools": { + "type": "array", + "description": "Every tool the agent really has. Everything downstream is " + "built from these, so a tool without its arguments cannot be tested.", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "The exact callable name the model emits.", + }, + "args": { + "type": "array", + "items": {"type": "string"}, + "description": "Exact parameter names, in order.", + }, + "arg_types": { + "type": "object", + "description": "Declared type per argument where the source " + 'states one: {"recipient_ids": "list[str]"}.', + }, + "arg_values": { + "type": "object", + "description": "Real permitted values per argument where it is " + "constrained to a set, an enum or a lookup: " + '{"priority": ["low", "normal", "urgent"]}.', + }, + "description": {"type": "string"}, + }, + # Nothing required: a tool genuinely taking no arguments is ordinary, + # and requiring args here rejects the whole contract because of one. + # That every tool has none is the real defect, and validate_contract + # is where it is caught, with an explanation. + }, + }, + "data_schema": { + "type": "object", + "description": "The shape of the records the agent works on: which fields " + "each kind of record has.", + }, + "base_environment": { + "type": "object", + "description": "Its real starting data, reproduced exactly — including " + "anything that looks like a mistake. The world is a replica, not a " + "corrected version.", + }, + "real_use_cases": { + "type": "array", + "items": {"type": "string"}, + "description": "Concrete situations this agent exists to handle, drawn from " + "its tools and data rather than invented.", + }, + "notes": { + "type": "string", + "description": "Free-form, yours. Anything else worth carrying forward: " + "quirks, traps, a plausible name that does not exist, an id that looks like " + "a typo but is real. Shown verbatim to every later stage.", + }, + "open_questions": { + "type": "array", + "items": {"type": "string"}, + "description": "What the source did not settle and you could not ask about.", + }, }, - ["agent", "tools", "real_use_cases"], + [], ), ) async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: @@ -143,18 +307,31 @@ async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: } -def schema(properties: dict[str, type], required: list[str]) -> dict[str, Any]: - """A tool's inputs, saying which of them are actually required. +def schema(properties: dict[str, Any], required: list[str]) -> dict[str, Any]: + """A tool's inputs, described well enough to be filled in correctly the first time. + + Two things this exists for. + + **Required means required.** Handing the decorator a plain ``{name: type}`` mapping marks + every parameter mandatory, so a tool with an optional field refuses any call that leaves it + out — "Input validation error: 'seed' is a required property" — for a field the tool itself + treats as optional. + + **A schema is documentation, not just validation.** It is shown to the model before it calls + anything, so a property carrying only ``{"type": "array"}`` says nothing about what belongs + in it, and the model discovers the shape by being rejected. That is a full turn per guess and + it is avoidable: pass a full JSON-schema fragment instead of a bare type wherever the shape + is not obvious from the name, and it is right on the first call. - Handing the decorator a plain ``{name: type}`` mapping marks every parameter mandatory, so a - tool with an optional field refuses any call that leaves it out — "Input validation error: - 'seed' is a required property" — for a field the tool itself treats as optional. The model - then has to guess that it must pass an empty value, and burns turns finding out. + schema({"name": str, + "size": {"type": "string", "enum": ["S", "M", "L"]}}, ["name"]) """ return { "type": "object", "properties": { - name: {"type": _JSON_TYPES.get(kind, "string")} + name: dict(kind) + if isinstance(kind, dict) + else {"type": _JSON_TYPES.get(kind, "string")} for name, kind in properties.items() }, "required": list(required), diff --git a/tests/test_harness.py b/tests/test_harness.py index 0622afb..6dc696e 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -1332,7 +1332,7 @@ def test_writing_new_results_keeps_the_ones_not_rerun(tmp_path): assert kept["b"]["passed"] is True and kept["b"]["transcript"] == "hello" -def test_submit_contract_requires_only_what_the_gate_demands(tmp_path): +def test_submit_contract_schema_teaches_and_leaves_gating_to_the_gate(tmp_path): """Every field marked required is rejected by the schema layer one at a time, a full model turn each, before accept_contract can explain anything. Only the fields validate_contract refuses to live without may be required; the rest are optional and gated with real messages.""" @@ -1340,6 +1340,7 @@ def test_submit_contract_requires_only_what_the_gate_demands(tmp_path): from mcp.types import ListToolsRequest + from fi.alk.harness.contract import MODALITIES from fi.alk.harness.tools import contract_tools server = contract_tools(tmp_path) @@ -1353,7 +1354,19 @@ async def schema_of(): return {} schema = asyncio.run(schema_of()) - assert sorted(schema.get("required", [])) == ["agent", "real_use_cases", "tools"] + # Nothing required at the schema layer: accept_contract is the only gate, and it reports + # every problem at once with what to do, which a JSON-schema rejection cannot. + # Nothing required: this layer runs before the tool body, so whatever it rejects never + # reaches the code that could have understood it. accept_contract is the single gate. + assert schema.get("required") == [] + assert "required" not in schema["properties"]["tools"]["items"] + # And the schema has to teach, not just validate — it is shown before the first call. + described = [ + name for name, spec in schema["properties"].items() if spec.get("description") + ] + assert len(described) >= 10, "properties must describe themselves" + assert schema["properties"]["modality"]["enum"] == list(MODALITIES) + assert schema["properties"]["tools"]["items"]["properties"]["arg_values"] def test_a_bare_conversational_contract_is_nudged_once_then_accepted(tmp_path): @@ -1453,3 +1466,138 @@ async def call(): said = asyncio.run(call()) assert "Handed over" in said assert conversation._handoff["request"] == "create the world" + + +def test_every_problem_is_reported_at_once_with_what_to_do(tmp_path): + """Revealing the next problem only after the last is fixed costs a turn per problem and + reads as though the rules are being invented as it goes.""" + from fi.alk.harness.tools import accept_contract + + result = accept_contract({"agent": "", "tools": [], "real_use_cases": []}, tmp_path) + said = result["content"][0]["text"] + assert result["is_error"] + # all three, in one answer + assert "empty:agent" in said and "no-tools" in said and "no-use-cases" in said + # and each carries what to do about it, not only its code + assert "artifact folder" in said and "real tools" in said + assert not (tmp_path / "contract.json").exists() + + +def test_a_contract_sent_inside_a_wrapper_is_unwrapped(tmp_path): + """A contract is a nested thing being described, so it arrives as {"contract": {...}} often + enough to matter. Every field is right; only the envelope is wrong, and rejecting that + teaches nothing while costing a turn.""" + from fi.alk.harness.tools import accept_contract, unwrapped + + inner = { + "agent": "wrapped", + "tools": [{"name": "act", "args": ["x"]}], + "real_use_cases": ["do the thing"], + "hard_constraints": ["a rule"], + "system_prompt_excerpt": "you are a bot", + } + assert unwrapped({"contract": inner}) == inner + assert unwrapped(inner) == inner + # a real field that merely holds a dict must not be mistaken for an envelope + plain = {"agent": "x", "tools": [], "data_schema": {"agent": 1}} + assert unwrapped(plain) == plain + + result = accept_contract({"contract": inner}, tmp_path) + assert not result.get("is_error"), result["content"][0]["text"] + assert (tmp_path / "contract.json").exists() + + +@pytest.mark.parametrize( + "written", + [ + {"name": "order", "parameters": ["item_id", "size"]}, + {"name": "order", "arguments": ["item_id", "size"]}, + {"name": "order", "params": ["item_id", "size"]}, + {"name": "order", "arg_types": {"item_id": "str", "size": "str"}}, + {"name": "order", "parameters": {"item_id": "str", "size": "str"}}, + ], +) +def test_a_tool_written_with_a_synonym_still_records_its_arguments(written): + """args drives the handlers, the probes and every scenario. It is also the field most often + written under another name, and a contract bounced for a synonym costs a turn and teaches + nothing about the agent.""" + spec = ToolSpec.model_validate(written) + assert spec.args == ["item_id", "size"] + + +def test_a_tool_that_really_takes_nothing_stays_empty(): + """A tool genuinely taking no arguments is ordinary and must not be invented into one.""" + assert ToolSpec.model_validate({"name": "list_order_items"}).args == [] + + +def test_a_stringified_contract_is_parsed_rather_than_refused(tmp_path): + from fi.alk.harness.tools import accept_contract, unwrapped + + inner = { + "agent": "stringy", + "tools": [{"name": "act", "args": ["x"]}], + "real_use_cases": ["do it"], + "hard_constraints": ["a rule"], + } + assert unwrapped({"contract": json.dumps(inner)}) == inner + assert unwrapped({"payload": json.dumps({"contract": inner})}) == inner + assert not accept_contract({"contract": json.dumps(inner)}, tmp_path).get("is_error") + + +def test_an_unrecognised_payload_is_told_what_arrived(tmp_path): + """Otherwise the answer is 'agent is empty, there are no tools' about a submission that + contained both, and the only way out is guessing at the packaging.""" + from fi.alk.harness.tools import accept_contract + + said = accept_contract({"stuff": 1, "other": 2}, tmp_path)["content"][0]["text"] + assert "What arrived was: other, stuff" in said + assert "top-level arguments" in said + + +def test_a_skill_only_names_tools_its_stage_actually_has(): + """A SKILL.md is the method; the tools are the surface it is written against. They live in + different files, so a renamed tool leaves the skill telling the model to call something that + does not exist — and the model then hunts for it and works around the gate. Nothing else + catches that, because both halves are individually valid.""" + import re + + from fi.alk.harness import scenario_tools + from fi.alk.harness.config import SKILLS_ROOT + from fi.alk.harness.run import tools as run_tools + from fi.alk.harness.tools import CONTRACT_SERVER # noqa: F401 + from fi.alk.harness.world import tools as world_tools + + surface = { + "understand-agent": {"submit_contract"}, + "build-environment": set(world_tools.TOOL_NAMES), + "write-scenarios": set(scenario_tools.TOOL_NAMES), + "run-scenarios": set(run_tools.TOOL_NAMES), + } + # A skill also backticks the names of fields it is telling the model to fill in. Those are + # not tools, and the list of them is derived rather than hand-kept so it cannot go stale. + from fi.alk.harness.contract import AgentContract, ToolSpec + from fi.alk.harness.environment import SubGoal + from fi.alk.harness.scenario import Scenario + + fields = set() + for model in (AgentContract, ToolSpec, Scenario, SubGoal): + fields |= set(model.model_fields) + # Names from the check-writing examples the skills contain. + from fi.alk.harness.contract import MODALITIES + + ignore = ( + fields + | set(MODALITIES) + | {"handle", "check", "args", "db", "world", "calls", "json", "ToolError"} + ) + + for stage, tools in surface.items(): + text = (SKILLS_ROOT / stage / "SKILL.md").read_text(encoding="utf-8") + # `name` or `name(` — the way a skill refers to a tool it wants called. + mentioned = set(re.findall(r"`([a-z_][a-z0-9_]*)\(?`", text)) + unknown = { + name + for name in mentioned - tools - ignore + if name not in {"hand_to_next_stage", "AskUserQuestion"} + } + assert not unknown, f"{stage}/SKILL.md names tools that do not exist: {sorted(unknown)}" From 45f84189f87121a24d09855f8ab28f886faf00ee Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Mon, 17 Aug 2026 13:14:34 +0530 Subject: [PATCH 10/39] fix(harness): accept field synonyms, keep host tools out, report artifacts only on write --- scripts/replay_ground_truth.py | 173 ++++++++++++++++++ src/fi/alk/harness/build.py | 2 + src/fi/alk/harness/config.py | 8 + src/fi/alk/harness/contract.py | 29 ++- src/fi/alk/harness/reception.py | 9 +- src/fi/alk/harness/run/grade.py | 9 +- src/fi/alk/harness/run/stage.py | 2 + src/fi/alk/harness/run/targets.py | 9 +- src/fi/alk/harness/scenarios.py | 2 + src/fi/alk/harness/session.py | 20 +- .../harness/skills/build-environment/SKILL.md | 6 + .../harness/skills/understand-agent/SKILL.md | 13 +- src/fi/alk/harness/tools.py | 66 ++++--- src/fi/alk/harness/understand.py | 4 +- tests/test_harness.py | 116 +++++++++++- 15 files changed, 434 insertions(+), 34 deletions(-) create mode 100644 scripts/replay_ground_truth.py diff --git a/scripts/replay_ground_truth.py b/scripts/replay_ground_truth.py new file mode 100644 index 0000000..4cb6ae7 --- /dev/null +++ b/scripts/replay_ground_truth.py @@ -0,0 +1,173 @@ +"""Replay an external benchmark's hand-written trajectories against a world we generated. + +Every gate in this harness so far is one we wrote, checking work we produced. That is worth +something, but it cannot answer the question that actually matters about a generated +environment: **is it faithful to the agent it was built from?** + +An independent benchmark answers it. Sierra's tau-bench ships hand-written tasks, each with the +exact tool calls a correct agent should make. Those trajectories were written by people who had +never seen this harness, against the real implementation. Replaying them through a world the +harness built automatically from the same source is therefore an external check: if the world is +faithful, the trajectories run clean; where they do not, the difference is a real defect in the +world and it is pointed at directly. + + .venv/bin/python scripts/replay_ground_truth.py \ + --world artifacts/environments/tau_retail \ + --tasks .../tau-bench/tau_bench/envs/retail/tasks_test.py + +What it reports, per trajectory: every call accepted, or the first one the world refused or +crashed on. A refusal is the interesting case — either the trajectory relies on data our sample +does not have, or a handler is stricter than the real tool. +""" + +from __future__ import annotations + +import argparse +import ast +import json +import sys +from dataclasses import dataclass, field +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src")) + +from fi.alk.harness.world.snapshot import restore # noqa: E402 + + +@dataclass +class Trajectory: + """One hand-written task: what the user wanted, and the calls a correct agent makes.""" + + instruction: str + actions: list[tuple[str, dict]] = field(default_factory=list) + + +def read_tasks(path: Path) -> list[Trajectory]: + """The trajectories, read from the benchmark's own Python without importing it. + + Parsed rather than imported: importing would pull in the benchmark's package and its + dependencies, and all that is wanted here is literal data it already states plainly. + """ + tree = ast.parse(path.read_text(encoding="utf-8")) + found: list[Trajectory] = [] + for node in ast.walk(tree): + if not (isinstance(node, ast.Call) and getattr(node.func, "id", "") == "Task"): + continue + instruction, actions = "", [] + for keyword in node.keywords: + if keyword.arg == "instruction" and isinstance(keyword.value, ast.Constant): + instruction = str(keyword.value.value) + if keyword.arg == "actions" and isinstance(keyword.value, ast.List): + for entry in keyword.value.elts: + if not ( + isinstance(entry, ast.Call) + and getattr(entry.func, "id", "") == "Action" + ): + continue + name, kwargs = "", {} + for field_ in entry.keywords: + if field_.arg == "name" and isinstance(field_.value, ast.Constant): + name = str(field_.value.value) + if field_.arg == "kwargs": + try: + kwargs = ast.literal_eval(field_.value) + except ValueError: + kwargs = {} + if name: + actions.append((name, kwargs)) + if actions: + found.append(Trajectory(instruction=instruction, actions=actions)) + return found + + +@dataclass +class Replay: + index: int + steps: int = 0 + accepted: int = 0 + stopped_at: str = "" + why: str = "" + crashed: bool = False + + @property + def clean(self) -> bool: + return not self.stopped_at + + +def replay(trajectory: Trajectory, world_root: Path, index: int) -> Replay: + """One trajectory against its own fresh copy of the world.""" + result = Replay(index=index, steps=len(trajectory.actions)) + world = restore(world_root) + try: + world.reset() + for name, arguments in trajectory.actions: + call = world.call(name, arguments) + if call.ok: + result.accepted += 1 + continue + result.stopped_at = f"{name}({json.dumps(arguments, default=str)[:120]})" + result.why = call.error + result.crashed = not call.refused + break + finally: + world.close() + return result + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--world", required=True, help="a built environment") + parser.add_argument("--tasks", required=True, help="the benchmark's tasks file") + parser.add_argument("--limit", type=int, default=0, help="only the first N trajectories") + parser.add_argument("--show", type=int, default=12, help="how many failures to detail") + args = parser.parse_args(argv) + + world_root = Path(args.world) + if not (world_root / "world.sqlite").exists(): + print(f"no world at {world_root}. Run `build` first.", file=sys.stderr) + return 1 + + trajectories = read_tasks(Path(args.tasks)) + if args.limit: + trajectories = trajectories[: args.limit] + if not trajectories: + print("no trajectories found in that file", file=sys.stderr) + return 1 + + results = [replay(one, world_root, index) for index, one in enumerate(trajectories)] + clean = [one for one in results if one.clean] + crashed = [one for one in results if one.crashed] + calls = sum(one.steps for one in results) + accepted = sum(one.accepted for one in results) + + print(f"world: {world_root}") + print(f"trajectories: {len(results)} hand-written, from {Path(args.tasks).name}") + print(f"replayed: {len(clean)}/{len(results)} clean") + print(f"calls: {accepted}/{calls} accepted by the world") + if crashed: + print(f"crashes: {len(crashed)} — these are defects in the world, not refusals") + + failed = [one for one in results if not one.clean] + if failed: + print("\nwhere they stopped:") + for one in failed[: args.show]: + mark = "CRASH" if one.crashed else "refused" + print(f" [{one.index}] {mark} after {one.accepted}/{one.steps}: {one.stopped_at}") + print(f" {one.why[:160]}") + if len(failed) > args.show: + print(f" … and {len(failed) - args.show} more") + + # The tools a real suite actually exercises, which is what our own coverage is measured + # against: a generated suite that never reaches the write tools has not tested the agent. + used: dict[str, int] = {} + for one in trajectories: + for name, _ in one.actions: + used[name] = used.get(name, 0) + 1 + print("\nwhat the hand-written trajectories exercise:") + for name, count in sorted(used.items(), key=lambda pair: -pair[1]): + print(f" {count:4} {name}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/fi/alk/harness/build.py b/src/fi/alk/harness/build.py index c7c6a26..0e1e766 100644 --- a/src/fi/alk/harness/build.py +++ b/src/fi/alk/harness/build.py @@ -17,6 +17,7 @@ from .config import ( artifact_dir, + UNWANTED, gate_hooks, chosen_model, load_skill, @@ -63,6 +64,7 @@ def open_stage( model=chosen_model(), env=provider_env(), ) + options.disallowed_tools = list(UNWANTED) options.hooks = gate_hooks(allowed) options.can_use_tool = permission_gate(ask, allowed) return Stage(options, name=SKILL), destination diff --git a/src/fi/alk/harness/config.py b/src/fi/alk/harness/config.py index 8eab46c..884cd52 100644 --- a/src/fi/alk/harness/config.py +++ b/src/fi/alk/harness/config.py @@ -101,11 +101,19 @@ def read_only_session( model=chosen_model(model), env=provider_env(model), ) + options.disallowed_tools = list(UNWANTED) options.hooks = gate_hooks(allowed) options.can_use_tool = permission_gate(granted=allowed) return options +# Tools the host offers every session that no stage of this harness has any use for. Denying +# them at the gate works and is the backstop, but a denial still costs the turn that discovered +# it — and these get reached for in almost every stage. Naming them as disallowed keeps them out +# of the tool list the model is shown, so the turn is never spent. +UNWANTED = ("ToolSearch", "Bash", "Write", "Edit", "NotebookEdit", "WebFetch", "WebSearch") + + def gate_hooks(granted: Iterable[str]) -> dict[str, Any]: """Deny anything a stage was not given, at the point the SDK actually asks. diff --git a/src/fi/alk/harness/contract.py b/src/fi/alk/harness/contract.py index 528f526..be536f0 100644 --- a/src/fi/alk/harness/contract.py +++ b/src/fi/alk/harness/contract.py @@ -34,6 +34,19 @@ ) _DICT_FIELDS = ("data_schema", "base_environment") +# What each field gets called when it is not called what we call it. Every one of these was +# written by a model that had read the schema and still reached for the more obvious word. +_ALIASES = { + "real_use_cases": ("use_cases", "usecases", "scenarios", "capabilities"), + "hard_constraints": ("constraints", "rules", "policies", "policy", "guardrails"), + "system_prompt_excerpt": ("system_prompt", "prompt", "instructions"), + "base_environment": ("data", "seed_data", "starting_data", "records"), + "data_schema": ("schema", "record_schema", "data_shape"), + "agent": ("name", "agent_name"), + "one_liner": ("summary", "description"), + "notes": ("observations", "remarks"), +} + class ToolSpec(BaseModel): """One tool the agent really has. @@ -92,10 +105,22 @@ class AgentContract(BaseModel): @classmethod def _normalize_shapes(cls, payload: Any) -> Any: """Model JSON varies in benign ways: a list where prose was asked, a bare string where a - list was. Normalize instead of rejecting, because shape variance is not a grounding - error and rejecting it burns turns on something that does not matter.""" + list was, a field under the obvious name rather than ours. Normalize instead of + rejecting, because none of that is a grounding error and rejecting it burns turns on + something that does not matter.""" if not isinstance(payload, dict): return payload + # The name we chose is not always the obvious one. `real_use_cases` in particular gets + # written as `use_cases`, and the answer it then gets — "no-use-cases" — reads as + # missing rather than misnamed, so the same submission comes back again and again with + # the shape changed and the name untouched. + for ours, others in _ALIASES.items(): + if payload.get(ours): + continue + for other in others: + if payload.get(other): + payload[ours] = payload[other] + break for key in _STRING_FIELDS: value = payload.get(key) if isinstance(value, list): diff --git a/src/fi/alk/harness/reception.py b/src/fi/alk/harness/reception.py index 27b4c79..718c392 100644 --- a/src/fi/alk/harness/reception.py +++ b/src/fi/alk/harness/reception.py @@ -17,7 +17,13 @@ from claude_agent_sdk import ClaudeAgentOptions, create_sdk_mcp_server, tool -from .config import chosen_model, gate_hooks, permission_gate, provider_env +from .config import ( + UNWANTED, + chosen_model, + gate_hooks, + permission_gate, + provider_env, +) from .session import Stage from .sources import AgentSource, resolve, supported from .tools import qualified, schema @@ -123,6 +129,7 @@ async def point_at_agent(args: dict[str, Any]) -> dict[str, Any]: model=chosen_model(), env=provider_env(), ) + options.disallowed_tools = list(UNWANTED) options.hooks = gate_hooks(allowed) options.can_use_tool = permission_gate(ask, allowed) return Stage(options, name="reception"), found diff --git a/src/fi/alk/harness/run/grade.py b/src/fi/alk/harness/run/grade.py index 7db5a50..1ce329b 100644 --- a/src/fi/alk/harness/run/grade.py +++ b/src/fi/alk/harness/run/grade.py @@ -22,7 +22,13 @@ from claude_agent_sdk import ClaudeAgentOptions, create_sdk_mcp_server, tool -from ..config import chosen_model, gate_hooks, permission_gate, provider_env +from ..config import ( + UNWANTED, + chosen_model, + gate_hooks, + permission_gate, + provider_env, +) from ..contract import AgentContract from ..scenario import Scenario from ..session import Stage @@ -199,6 +205,7 @@ async def judge( model=chosen_model(model), env=provider_env(model), ) + options.disallowed_tools = list(UNWANTED) options.hooks = gate_hooks(allowed) options.can_use_tool = permission_gate(granted=allowed) stage = Stage(options, name="judge") diff --git a/src/fi/alk/harness/run/stage.py b/src/fi/alk/harness/run/stage.py index 8650f85..b355f0b 100644 --- a/src/fi/alk/harness/run/stage.py +++ b/src/fi/alk/harness/run/stage.py @@ -18,6 +18,7 @@ from ..config import ( artifact_dir, + UNWANTED, gate_hooks, chosen_model, load_skill, @@ -60,6 +61,7 @@ def open_stage( model=chosen_model(), env=provider_env(), ) + options.disallowed_tools = list(UNWANTED) options.hooks = gate_hooks(allowed) options.can_use_tool = permission_gate(ask, allowed) return Stage(options, name=SKILL), destination diff --git a/src/fi/alk/harness/run/targets.py b/src/fi/alk/harness/run/targets.py index 1204f97..44bf42d 100644 --- a/src/fi/alk/harness/run/targets.py +++ b/src/fi/alk/harness/run/targets.py @@ -19,7 +19,13 @@ from claude_agent_sdk import ClaudeAgentOptions, create_sdk_mcp_server, tool -from ..config import chosen_model, gate_hooks, permission_gate, provider_env +from ..config import ( + UNWANTED, + chosen_model, + gate_hooks, + permission_gate, + provider_env, +) from ..contract import AgentContract from ..session import Stage from ..tools import qualified @@ -176,6 +182,7 @@ def __init__( ) # The agent under test gets its own tools and nothing else. A target that can reach a # file or a shell is not the agent anybody deployed. + options.disallowed_tools = list(UNWANTED) options.hooks = gate_hooks(allowed) options.can_use_tool = permission_gate(granted=allowed) self._stage = Stage(options, name="target") diff --git a/src/fi/alk/harness/scenarios.py b/src/fi/alk/harness/scenarios.py index 1295375..9b7d5bf 100644 --- a/src/fi/alk/harness/scenarios.py +++ b/src/fi/alk/harness/scenarios.py @@ -17,6 +17,7 @@ from .config import ( artifact_dir, + UNWANTED, gate_hooks, chosen_model, load_skill, @@ -77,6 +78,7 @@ def open_stage( model=chosen_model(), env=provider_env(), ) + options.disallowed_tools = list(UNWANTED) options.hooks = gate_hooks(allowed) options.can_use_tool = permission_gate(ask, allowed) return Stage(options, name=SKILL), destination diff --git a/src/fi/alk/harness/session.py b/src/fi/alk/harness/session.py index 2bab10b..8133879 100644 --- a/src/fi/alk/harness/session.py +++ b/src/fi/alk/harness/session.py @@ -152,7 +152,13 @@ def _result_text(block: ToolResultBlock, limit: int = 600) -> str: def _saved_path(block: ToolResultBlock) -> str: - """Our tools report what they wrote; surfacing it lets a UI update the artifact pane.""" + """The path a tool reports having written, if it wrote one. + + Only when the tool actually says it saved something. Matching any path-shaped token in any + result meant that reading a file announced it as an artifact — the stage looks like it is + producing output while it is still only looking around, and a front end reloads its panes on + every read. + """ content = block.content if isinstance(content, list): content = " ".join( @@ -160,9 +166,17 @@ def _saved_path(block: ToolResultBlock) -> str: ) if not isinstance(content, str): return "" + said = content.lower() + if not any(verb in said for verb in ("saved", "wrote", "written")): + return "" for token in content.split(): - if token.endswith((".json", ".py", ".sqlite")): - return token.rstrip(".,") + # Trimmed before the check, not after. A tool that ends its sentence — "saved to + # out/contract.json." — produces a token ending in the full stop, so testing the + # suffix first missed every real save and matched only bare paths, which is what a + # file *read* returns. The event fired on exactly the wrong occasions. + cleaned = token.strip(".,;:!?)\"'") + if cleaned.endswith((".json", ".py", ".sqlite")): + return cleaned return "" diff --git a/src/fi/alk/harness/skills/build-environment/SKILL.md b/src/fi/alk/harness/skills/build-environment/SKILL.md index 4f161fd..eabbeaa 100644 --- a/src/fi/alk/harness/skills/build-environment/SKILL.md +++ b/src/fi/alk/harness/skills/build-environment/SKILL.md @@ -53,6 +53,12 @@ agent has, not a corrected version, and a test written against a corrected world the bug the real one has. If an id looks like a typo, that typo is the thing worth testing — do not fix it, and do not widen the contract to the spelling you would have chosen. +Seed what the contract carries, and enough of it that every branch a handler has can actually be +reached: if a tool refuses a cancelled order, there has to be a cancelled order to refuse. Where +the contract sampled a large dataset rather than reproducing it, that sample is the world — an +exact replica was never the goal, and a world that exercises the same flows and refuses for the +same reasons is what is wanted. + Leave it in its natural starting state: empty carts, no in-flight orders. Scenarios add what they need. diff --git a/src/fi/alk/harness/skills/understand-agent/SKILL.md b/src/fi/alk/harness/skills/understand-agent/SKILL.md index 2b06fff..1b076fc 100644 --- a/src/fi/alk/harness/skills/understand-agent/SKILL.md +++ b/src/fi/alk/harness/skills/understand-agent/SKILL.md @@ -51,7 +51,18 @@ Find, in roughly this order: browser-driving agent is `browser`. This decides how it is run later — a voice agent is called live; anything else runs locally — so getting it wrong reroutes every test. 6. **The data.** Where it lives, its shape, and its real contents. In-memory dicts, fixture - files, a seeded database. Record enough for a working replica to be built. + files, a seeded database. + + Record the **shape** completely: every field of every kind of record, and the values any + field is constrained to. Record the **contents** in proportion — a small agent's data goes in + whole, and for a large one a representative sample is what belongs in the contract: enough + rows to exercise each branch the tools have, chosen to include the awkward ones (an order + already cancelled, an item out of stock, a user with no payment method on file). Say in + `notes` where the full data lives and roughly how much of it there is. + + An exact replica is not the goal and never was. Copying a thousand records through this stage + loses fidelity rather than gaining it; what is needed is a world that exercises the same + flows and can still refuse for the same reasons. 7. **Real use cases.** What this agent is actually for, as concrete situations, drawn from the tools and data rather than invented. diff --git a/src/fi/alk/harness/tools.py b/src/fi/alk/harness/tools.py index ba4e05a..c30afa8 100644 --- a/src/fi/alk/harness/tools.py +++ b/src/fi/alk/harness/tools.py @@ -26,9 +26,12 @@ def _ok(text: str) -> dict[str, Any]: # time. What a code means is a separate question, and answering it here keeps the codes exact # while the message the model reads says what to actually do. _GUIDANCE = { - "empty:agent": "give it a short lower-case name; it is only the artifact folder's label", - "no-tools": "list the agent's real tools. Nothing downstream can be built without them", - "no-use-cases": "list the concrete situations this agent handles, from its tools and data", + "empty:agent": "the `agent` field is empty. A short lower-case name; it is only the " + "artifact folder's label", + "no-tools": "the `tools` field is empty. List the agent's real tools; nothing downstream " + "can be built without them", + "no-use-cases": "the `real_use_cases` field is empty — note the name, it is not " + "`use_cases`. List the concrete situations this agent handles, from its tools and data", "no-arguments-on-any-tool": "every tool was recorded with no arguments, which means they " "were read and not written down. Put each tool's exact parameter names in args", "duplicate-tool-names": "the same tool is listed twice; keep one entry per tool", @@ -151,11 +154,11 @@ def accept_contract(payload: dict[str, Any], destination: Path) -> dict[str, Any def contract_tools(destination: Path) -> Any: """A server exposing ``submit_contract``, writing to ``destination`` on acceptance.""" - # One nudge, not a wall. A conversational agent with no rules and no prompt excerpt almost - # always means the prompt was not found — it often lives away from the main agent file — so - # the first such submission is sent back with directions. The second is accepted, because a - # gate with no way through would permanently block the rare agent that genuinely has none. - nudged = {"done": False} + # Each of these is a nudge, not a wall: the first submission missing something that is + # nearly always there gets sent back with directions, and a second submission is accepted. + # A gate with no way through would permanently block the rare agent that genuinely lacks it, + # and this stage cannot tell those two apart from the outside. + nudged: set[str] = set() @tool( "submit_contract", @@ -275,22 +278,41 @@ def contract_tools(destination: Path) -> Any: ), ) async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: - bare = ( - args.get("conversational", True) - and not args.get("hard_constraints") - and not str(args.get("system_prompt_excerpt") or "").strip() - ) - if bare and not nudged["done"]: - nudged["done"] = True + payload = unwrapped(args) + + thin = [ + ( + "prompt", + bool(payload.get("conversational", True)) + and not payload.get("hard_constraints") + and not str(payload.get("system_prompt_excerpt") or "").strip(), + "no hard_constraints and no system_prompt_excerpt, for a conversational agent. " + "Its prompt usually exists and often lives away from the main agent file — " + "search the whole source for a long instructions string before deciding there " + "is none.", + ), + ( + "data", + bool(payload.get("tools")) + and not payload.get("data_schema") + and not payload.get("base_environment"), + "no data_schema and no base_environment, for an agent that has tools. The world " + "every test runs against is built from exactly these two, so without them the " + "next stage has no schema to create and no rows to seed, and every tool call it " + "makes will refuse. Record the shape of each kind of record the tools read or " + "write, and enough real rows to reach every branch those tools have — a " + "representative sample for a large dataset, the whole thing for a small one.", + ), + ] + # All of them together, and each only once. Nudging in sequence would cost a turn per + # nudge and read as though the requirements were being invented one at a time. + say = [said for key, when, said in thin if when and key not in nudged] + nudged.update(key for key, when, _ in thin if when) + if say: return _problems( - [ - "no hard_constraints and no system_prompt_excerpt, for a conversational " - "agent. Its prompt usually exists and often lives away from the main agent " - "file — search the whole source for a long instructions string before " - "deciding there is none. If there genuinely is none, submit again as is." - ] + say + ["If any of these genuinely does not apply, submit again as is."] ) - return accept_contract(args, destination) + return accept_contract(payload, destination) return create_sdk_mcp_server( name=CONTRACT_SERVER, version="0.1.0", tools=[submit_contract] diff --git a/src/fi/alk/harness/understand.py b/src/fi/alk/harness/understand.py index 0d4e84d..8d2fddd 100644 --- a/src/fi/alk/harness/understand.py +++ b/src/fi/alk/harness/understand.py @@ -27,7 +27,7 @@ def open_stage( *, out: Path | None = None, ask: Callable[..., Any] | None = None, - max_turns: int = 40, + max_turns: int = 70, ) -> tuple[Stage, Path]: """A live understand-the-agent stage, and where it will write.""" destination = out or artifact_dir(source.name) @@ -75,7 +75,7 @@ async def understand( follow_ups: list[str] | None = None, on_event: Callable[..., Any] | None = None, ask: Callable[..., Any] | None = None, - max_turns: int = 40, + max_turns: int = 70, ) -> AgentContract | None: """Run the stage start to finish and return the contract. diff --git a/tests/test_harness.py b/tests/test_harness.py index 6dc696e..685e759 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -1398,7 +1398,9 @@ async def call(payload): "real_use_cases": ["do the thing"], } first = asyncio.run(call(dict(payload))) - assert "system_prompt_excerpt" in first and "submit again" in first + # Both thin spots are reported together, not one per turn. + assert "system_prompt_excerpt" in first and "data_schema" in first + assert "submit again" in first assert not (tmp_path / "contract.json").exists() second = asyncio.run(call(dict(payload))) @@ -1601,3 +1603,115 @@ def test_a_skill_only_names_tools_its_stage_actually_has(): if name not in {"hand_to_next_stage", "AskUserQuestion"} } assert not unknown, f"{stage}/SKILL.md names tools that do not exist: {sorted(unknown)}" + + +def test_a_contract_with_tools_but_no_data_is_nudged_once(tmp_path): + """The world is built from data_schema and base_environment. Without them the build stage has + no schema to create and no rows to seed, so every tool call it makes refuses — and that looks + like a strict world rather than an empty one.""" + import asyncio + + from fi.alk.harness.tools import contract_tools + + server = contract_tools(tmp_path) + instance = server.get("instance") if isinstance(server, dict) else server + + async def call(payload): + from mcp.types import CallToolRequest, CallToolRequestParams + + for key, handler in instance.request_handlers.items(): + if getattr(key, "__name__", "") == "CallToolRequest": + answer = await handler( + CallToolRequest( + method="tools/call", + params=CallToolRequestParams( + name="submit_contract", arguments=payload + ), + ) + ) + return answer.root.content[0].text + + payload = { + "agent": "dataless", + "tools": [{"name": "act", "args": ["x"]}], + "real_use_cases": ["do the thing"], + "hard_constraints": ["a rule"], + "system_prompt_excerpt": "you are a bot", + } + first = asyncio.run(call(dict(payload))) + assert "data_schema" in first and "submit again" in first + assert not (tmp_path / "contract.json").exists() + + second = asyncio.run(call(dict(payload))) + assert "Accepted" in second + + assert (tmp_path / "contract.json").exists() + + +def test_only_a_tool_that_says_it_saved_reports_an_artifact(): + """Matching any path-shaped token in any result meant reading a file announced itself as an + artifact: the stage looks like it is producing output while it is still only looking around, + and a front end reloads its panes on every read.""" + from dataclasses import dataclass + + from fi.alk.harness.session import _saved_path + + @dataclass + class Block: + content: object + is_error: bool = False + + # a read + assert _saved_path(Block(" 1\timport json\n 2\tfrom pathlib import Path")) == "" + assert _saved_path(Block("/some/agent/envs/retail/__init__.py")) == "" + # a write + assert _saved_path(Block("Accepted and saved to out/contract.json.")) == "out/contract.json" + assert _saved_path(Block("Saved 3 scenarios to out/scenarios.json.")) == "out/scenarios.json" + # list-shaped content, as the SDK sometimes gives it + assert ( + _saved_path(Block([{"text": "Saved to artifacts/x/world.sqlite"}])) + == "artifacts/x/world.sqlite" + ) + + +@pytest.mark.parametrize( + "written,field,expected", + [ + ({"use_cases": ["a"]}, "real_use_cases", ["a"]), + ({"scenarios": ["a"]}, "real_use_cases", ["a"]), + ({"rules": ["r"]}, "hard_constraints", ["r"]), + ({"constraints": ["r"]}, "hard_constraints", ["r"]), + ({"system_prompt": "p"}, "system_prompt_excerpt", "p"), + ({"instructions": "p"}, "system_prompt_excerpt", "p"), + ({"schema": {"a": 1}}, "data_schema", {"a": 1}), + ({"seed_data": {"t": []}}, "base_environment", {"t": []}), + ], +) +def test_a_field_written_under_the_obvious_name_still_lands(written, field, expected): + """Every one of these was written by a model that had read the schema and still reached for + the more obvious word. Bouncing it produces a loop: the answer to `use_cases` was + 'no-use-cases', which reads as missing rather than misnamed, so the same submission comes + back with the shape changed and the name untouched.""" + contract = AgentContract.model_validate({"agent": "x", **written}) # our name already set + assert getattr(contract, field) == expected + + +def test_the_agent_name_can_arrive_as_name(): + assert AgentContract.model_validate({"name": "bot", "tools": []}).agent == "bot" + + +def test_our_own_name_wins_when_both_are_given(): + contract = AgentContract.model_validate( + {"agent": "x", "real_use_cases": ["ours"], "use_cases": ["theirs"]} + ) + assert contract.real_use_cases == ["ours"] + + +def test_the_gate_names_the_field_it_wants(tmp_path): + """A code alone cannot be acted on when the mistake is the field's name.""" + from fi.alk.harness.tools import accept_contract + + said = accept_contract({"agent": "x", "tools": [], "real_use_cases": []}, tmp_path) + text = said["content"][0]["text"] + assert "`real_use_cases`" in text and "not `use_cases`" in text + assert "`tools`" in text From d61c56760306f5800b66460349cbf7408cc7e428 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Mon, 17 Aug 2026 13:24:36 +0530 Subject: [PATCH 11/39] feat(harness): scenarios own a folder, three gates, and one orchestrator prompt over every stage --- src/fi/alk/harness/config.py | 25 ++- src/fi/alk/harness/contract.py | 29 +++ src/fi/alk/harness/folder.py | 210 ++++++++++++++++++ src/fi/alk/harness/prove.py | 101 +++++++-- src/fi/alk/harness/run/__init__.py | 23 +- src/fi/alk/harness/run/live.py | 15 +- src/fi/alk/harness/scenario.py | 39 ++-- src/fi/alk/harness/scenario_tools.py | 178 +++++++++------ .../harness/skills/build-environment/SKILL.md | 169 ++++++++------ src/fi/alk/harness/skills/harness.md | 125 +++++++++++ .../harness/skills/understand-agent/SKILL.md | 118 +++++----- .../harness/skills/write-scenarios/SKILL.md | 159 +++++++++---- src/fi/alk/harness/tools.py | 35 ++- tests/test_harness.py | 171 +++++++++++++- 14 files changed, 1105 insertions(+), 292 deletions(-) create mode 100644 src/fi/alk/harness/folder.py create mode 100644 src/fi/alk/harness/skills/harness.md diff --git a/src/fi/alk/harness/config.py b/src/fi/alk/harness/config.py index 884cd52..69e4f7e 100644 --- a/src/fi/alk/harness/config.py +++ b/src/fi/alk/harness/config.py @@ -186,9 +186,30 @@ def artifact_dir(agent: str, root: str | Path | None = None) -> Path: return base / agent +HARNESS = SKILLS_ROOT / "harness.md" + + def load_skill(name: str) -> str: - """A stage's instructions, kept as a file so the method is editable without touching code.""" + """One stage's instructions, behind what the harness as a whole is for. + + Every stage gets the same opening: what this harness produces, why the division between what + a model decides and what code decides exists, and what makes a result worth believing. A + stage that knows only its own step does its step well and still gets the point of it wrong — + it works around a gate instead of fixing what the gate named, or it reports a number that + quietly skipped half its checks. + + The stage's own method follows. Both are files, so how any of this works can be changed + without touching code. + """ path = SKILLS_ROOT / name / "SKILL.md" if not path.exists(): raise FileNotFoundError(f"no skill at {path}") - return path.read_text(encoding="utf-8") + stage = path.read_text(encoding="utf-8") + if not HARNESS.exists(): + return stage + return ( + f"{HARNESS.read_text(encoding='utf-8')}\n\n" + "---\n\n" + "# The stage you are in now\n\n" + f"{stage}" + ) diff --git a/src/fi/alk/harness/contract.py b/src/fi/alk/harness/contract.py index be536f0..a14ac2b 100644 --- a/src/fi/alk/harness/contract.py +++ b/src/fi/alk/harness/contract.py @@ -98,6 +98,24 @@ def _normalize_args(cls, payload: Any) -> Any: description: str = "" +class Dependency(BaseModel): + """Something the agent reaches for that has to exist before it can work. + + This is what tells the environment stage there is a service to stand up, rather than leaving + it to notice halfway through that a tool has nothing to answer it. The world is a sandbox: + whatever is named here gets built inside it, so the agent's call goes to something real that + happens to be ours. + """ + + name: str + # datastore, service, file, queue — whatever kind of thing this is. Left open rather than + # enumerated, because the next agent will need a kind nobody has thought of yet. + kind: str = "" + what: str = "" + # The tools that cannot work without it. An unreferenced dependency is usually a mistake. + used_by: list[str] = Field(default_factory=list) + + class AgentContract(BaseModel): """What the agent verifiably is. Nothing downstream may contradict this.""" @@ -152,6 +170,8 @@ def _normalize_shapes(cls, payload: Any) -> Any: tools: list[ToolSpec] = Field(default_factory=list) data_schema: dict[str, Any] = Field(default_factory=dict) base_environment: dict[str, Any] = Field(default_factory=dict) + # What the environment stage has to build before any tool can be answered. + dependencies: list[Dependency] = Field(default_factory=list) real_use_cases: list[str] = Field(default_factory=list) # Free-form. The fields above are the fixed core because code consumes them; this is where # the reader records whatever else about *this* agent is worth carrying forward — quirks, @@ -212,6 +232,15 @@ def brief(self, *, full_schema: bool = True, with_data: bool = False) -> str: "and a test written against a corrected world will not catch the real bug.\n" + json.dumps(self.base_environment, ensure_ascii=False) ) + if self.dependencies: + parts.append( + "WHAT THIS AGENT DEPENDS ON (the environment has to provide each of these):\n - " + + "\n - ".join( + f"{one.name} ({one.kind or 'unspecified'}): {one.what}" + + (f" — used by {', '.join(one.used_by)}" if one.used_by else "") + for one in self.dependencies + ) + ) if self.real_use_cases: parts.append( "REAL USE CASES (what this agent is actually for):\n - " diff --git a/src/fi/alk/harness/folder.py b/src/fi/alk/harness/folder.py new file mode 100644 index 0000000..0d81160 --- /dev/null +++ b/src/fi/alk/harness/folder.py @@ -0,0 +1,210 @@ +"""A scenario as a folder of files, and running the code inside it. + +A scenario used to be a row in one big JSON file, and its setup was a list of rows to insert. +That was enough while every world was a database. It stopped being enough the moment a world +could hold a service as well as a table: "the weather service starts returning errors" is not +expressible as rows, and neither is "the file is missing" or "the queue is backed up". + +So a scenario owns a folder, and the parts that are logic are files: + + scenarios// + scenario.json what it is: instruction, solution, which sub-goals + setup.py def setup(world) — the changes this scenario makes + ready.py def ready(world) — is the world ready for this scenario + checks/.py def check(world, calls) — one per deterministic sub-goal + +The files are the artifact, not a rendering of one. Each is executable on its own, so a check +can be run by hand against what a run left behind and answer exactly what it answers inside the +harness. That is the whole point of them being files: something you can open, read and run is +something you can argue with. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .environment import Catalogue +from .scenario import Scenario +from .world.runtime import GeneratedWorld + +SCENARIOS = "scenarios" +INDEX = "scenarios.json" + +# Appended to every check file the harness writes. The model writes only ``check(world, calls)``; +# this is what makes that same file runnable by a person, so nobody has to keep two versions of +# one truth in step. +_RUNNABLE = ''' + +if __name__ == "__main__": + # Run this check by hand against what a run left behind: + # python [calls.json] + import json as _json + import sys as _sys + from pathlib import Path as _Path + + _sys.path.insert(0, str(_Path(__file__).resolve().parents[4])) + from fi.alk.harness.world.runtime import Call as _Call + from fi.alk.harness.world.snapshot import restore as _restore + + _world = _restore(_Path(_sys.argv[1]).parent) if len(_sys.argv) > 1 else None + _calls = [] + if len(_sys.argv) > 2: + _calls = [_Call(**_one) for _one in _json.loads(_Path(_sys.argv[2]).read_text())] + _said = check(_world, _calls) + print("held" if _said is None else f"FAILED: {_said}") + raise SystemExit(0 if _said is None else 1) +''' + + +@dataclass +class Outcome: + """What one piece of a scenario's own code did.""" + + ok: bool + said: str = "" + broken: bool = False + + +def _run(source: str, name: str, entry: str, *args: Any) -> Outcome: + """Execute one function out of a scenario's own code. + + A file that will not compile, or that raises, is **broken** rather than failing: it is our + mistake, and scoring it as though the world were wrong would send somebody looking in the + wrong place. + """ + if not source.strip(): + return Outcome(True) + namespace: dict[str, Any] = {} + try: + exec(compile(source, f"<{name}>", "exec"), namespace) + except Exception as failed: + return Outcome(False, f"{name} would not compile: {failed}", broken=True) + + function = namespace.get(entry) + if not callable(function): + return Outcome(False, f"{name} defines no {entry}()", broken=True) + try: + said = function(*args) + except Exception as failed: + return Outcome( + False, f"{name} raised {type(failed).__name__}: {failed}", broken=True + ) + if said is None or said is True: + return Outcome(True) + return Outcome(False, str(said)) + + +def apply_setup(scenario: Scenario, world: GeneratedWorld) -> Outcome: + """Make this scenario's changes to the world.""" + return _run(scenario.setup_code, f"{scenario.name}/setup.py", "setup", world) + + +def check_ready(scenario: Scenario, world: GeneratedWorld) -> Outcome: + """Whether the world now holds what this scenario presumes.""" + return _run(scenario.ready_code, f"{scenario.name}/ready.py", "ready", world) + + +def folder_for(destination: Path, name: str) -> Path: + return Path(destination) / SCENARIOS / name + + +def write_folder(scenario: Scenario, catalogue: Catalogue, destination: Path) -> Path: + """Write one scenario out as its own folder of files.""" + root = folder_for(destination, scenario.name) + (root / "checks").mkdir(parents=True, exist_ok=True) + + body = scenario.model_dump() + # The code lives in its own files; keeping a second copy in the JSON would let the two drift + # and leave nobody able to say which one ran. + body.pop("setup_code", None) + body.pop("ready_code", None) + (root / "scenario.json").write_text( + json.dumps(body, indent=2, ensure_ascii=False), encoding="utf-8" + ) + + (root / "setup.py").write_text( + scenario.setup_code + or "def setup(world):\n \"\"\"This scenario runs on the base world unchanged.\"\"\"\n", + encoding="utf-8", + ) + (root / "ready.py").write_text( + scenario.ready_code + or "def ready(world):\n \"\"\"Nothing beyond the base world is presumed.\"\"\"\n", + encoding="utf-8", + ) + + for name in scenario.sub_goals: + sub_goal = catalogue.named(name) + if sub_goal is None or not sub_goal.deterministic(): + continue + (root / "checks" / f"{name}.py").write_text( + sub_goal.check.rstrip() + "\n" + _RUNNABLE, encoding="utf-8" + ) + return root + + +def read_folder(destination: Path, name: str) -> Scenario | None: + """One scenario, reassembled from its folder.""" + root = folder_for(destination, name) + body = root / "scenario.json" + if not body.exists(): + return None + payload = json.loads(body.read_text(encoding="utf-8")) + for field, filename in (("setup_code", "setup.py"), ("ready_code", "ready.py")): + path = root / filename + payload[field] = path.read_text(encoding="utf-8") if path.exists() else "" + return Scenario.model_validate(payload) + + +def write_index(scenarios: list[Scenario], destination: Path) -> Path: + """The whole suite at a glance, over the folders. + + Regenerated from the folders rather than maintained alongside them, so it can never disagree + with what is actually on disk. + """ + destination = Path(destination) + destination.mkdir(parents=True, exist_ok=True) + path = destination / INDEX + path.write_text( + json.dumps( + [ + { + "name": one.name, + "use_case": one.use_case, + "tests": one.tests, + "instruction": one.instruction, + "sub_goals": one.sub_goals, + "steps": len(one.solution), + "folder": f"{SCENARIOS}/{one.name}", + } + for one in scenarios + ], + indent=2, + ensure_ascii=False, + ), + encoding="utf-8", + ) + return path + + +def read_all(destination: Path) -> list[Scenario]: + """Every scenario on disk, read from the folders.""" + root = Path(destination) / SCENARIOS + if not root.exists(): + return [] + found: list[Scenario] = [] + for folder in sorted(root.iterdir()): + if not folder.is_dir(): + continue + try: + scenario = read_folder(destination, folder.name) + except Exception: + # A folder we cannot read is skipped rather than crashing the stage: the rest of the + # suite is still usable, and the gap shows up as a missing scenario. + continue + if scenario is not None: + found.append(scenario) + return found diff --git a/src/fi/alk/harness/prove.py b/src/fi/alk/harness/prove.py index cd68976..2ecaa55 100644 --- a/src/fi/alk/harness/prove.py +++ b/src/fi/alk/harness/prove.py @@ -1,19 +1,27 @@ """Proving a scenario is worth keeping, before anything is ever run against the agent. -Two gates, both pure code. No model is asked whether a scenario is good; the environment decides. - -**Solvable.** Reset the world, apply the scenario's own setup, run its reference solution, run -its checks. They must pass. If they do not, either the scenario cannot be passed at all or its -checks are wrong, and both have happened here: one scenario asserted a value the agent was never -permitted to send; another demanded confirmation of an item that could not be ordered. Neither -was noticed until a live run failed and read as a finding about the agent. - -**Not vacuous.** Reset, apply the setup, run *nothing*, run the checks. They must fail. A check -that passes with no actions taken grades nothing while reporting a result, which is how a suite -goes quietly green. - -Terminal-bench keeps its tasks honest this way, and it is the cheapest useful thing in the whole -harness: no tokens, no network, a few milliseconds. +Three gates, all pure code. No model is asked whether a scenario is good; the environment +decides. Terminal-bench keeps its tasks honest this way, and it is the cheapest useful thing in +the whole harness: no tokens, no network, a few milliseconds. + +**Ready.** Reset the world, run the scenario's own ``setup.py``, then its ``ready.py``. The world +has to hold what the scenario presumes. A scenario about the last five chocolates is only a test +of the agent if there really are five; otherwise the agent fails for something we got wrong and +it reads as the agent's fault. This gate is why a missing precondition can never be mistaken for +a finding. + +**Solvable.** Then run the reference solution and the checks. They must pass. If they do not, +either the scenario cannot be passed at all or its checks are wrong, and both have happened +here: one scenario asserted a value the agent was never permitted to send; another demanded +confirmation of an item that could not be ordered. Neither was noticed until a live run failed +and read as a finding about the agent. + +**Not vacuous.** Then reset, set up again, run *nothing*, and run the checks. They must fail. A +check that passes with no actions taken grades nothing while reporting a result, which is how a +suite goes quietly green. This one earns its keep: on a third-party benchmark it caught three +sub-goals that passed trivially because the seeded world already contained a cancelled order. + +Only a scenario that clears all three is kept. That is the green light. """ from __future__ import annotations @@ -23,17 +31,20 @@ from .checks import Outcome, run_check from .environment import Catalogue +from .folder import apply_setup, check_ready from .scenario import Scenario from .world.runtime import Call, GeneratedWorld -from .world.snapshot import apply_overlay, restore +from .world.snapshot import restore @dataclass class Proof: """Whether a scenario holds up, and what happened when it was tried.""" + ready: bool = False solvable: bool = False vacuous: bool = True + why_not_ready: str = "" with_solution: list[Outcome] = field(default_factory=list) with_nothing: list[Outcome] = field(default_factory=list) refused: list[str] = field(default_factory=list) @@ -41,10 +52,25 @@ class Proof: @property def holds(self) -> bool: - return self.solvable and not self.vacuous and not self.broken + return self.ready and self.solvable and not self.vacuous and not self.broken + + def gates(self) -> dict[str, bool]: + """The three answers, for anything that wants to show them.""" + return { + "ready": self.ready, + "solvable": self.solvable, + "not_vacuous": not self.vacuous, + } def why(self) -> str: """What to fix, in the order worth fixing it.""" + if not self.ready: + return ( + "the world is not ready for this scenario, so running it would test us rather " + f"than the agent:\n - {self.why_not_ready}\n\n" + "Either setup.py does not make the change this scenario needs, or ready.py is " + "checking for something the setup never creates." + ) if self.broken: return "these checks are broken, not failing:\n - " + "\n - ".join( self.broken @@ -60,9 +86,7 @@ def why(self) -> str: ) return ( "the reference solution does not pass this scenario's own checks, so either the " - "scenario cannot be passed or the checks are wrong:\n - " - + said - + refusals + "scenario cannot be passed or the checks are wrong:\n - " + said + refusals ) if self.vacuous: passed = [one.name for one in self.with_nothing if one.held] @@ -92,13 +116,25 @@ def _checks_for(scenario: Scenario, catalogue: Catalogue) -> list[tuple[str, str return chosen +def prepared( + scenario: Scenario, world_root: Path +) -> tuple[GeneratedWorld, Outcome, Outcome]: + """A fresh world with this scenario's setup applied, and how that went.""" + world = restore(world_root) + world.reset() + applied = apply_setup(scenario, world) + ready = check_ready(scenario, world) if applied.ok else Outcome(False, applied.said) + # The setup's own calls are not the agent's. Clearing them keeps a check that counts calls + # from crediting the agent with work the scenario did on its behalf. + world.calls = [] + return world, applied, ready + + def _run( scenario: Scenario, world_root: Path, *, with_solution: bool ) -> tuple[GeneratedWorld, list[Call], list[str]]: - """A fresh world with the scenario's setup, optionally with the solution played through it.""" - world = restore(world_root) - apply_overlay(world, scenario.setup) - world.reset() + """A world set up for this scenario, optionally with the solution played through it.""" + world, _applied, _ready = prepared(scenario, world_root) refused: list[str] = [] if with_solution: for step in scenario.solution: @@ -109,7 +145,7 @@ def _run( def prove(scenario: Scenario, catalogue: Catalogue, world_root: Path) -> Proof: - """Run both gates and say whether this scenario is worth keeping.""" + """Run all three gates and say whether this scenario is worth keeping.""" proof = Proof() checks = _checks_for(scenario, catalogue) if not checks: @@ -119,6 +155,22 @@ def prove(scenario: Scenario, catalogue: Catalogue, world_root: Path) -> Proof: ] return proof + # Gate 1: is the world ready for this scenario at all? + world, applied, ready = prepared(scenario, world_root) + world.close() + if not applied.ok: + proof.why_not_ready = applied.said + if applied.broken: + proof.broken = [applied.said] + return proof + if not ready.ok: + proof.why_not_ready = ready.said + if ready.broken: + proof.broken = [ready.said] + return proof + proof.ready = True + + # Gate 2: does the reference solution pass this scenario's own checks? world, calls, refused = _run(scenario, world_root, with_solution=True) try: proof.with_solution = [ @@ -130,6 +182,7 @@ def prove(scenario: Scenario, catalogue: Catalogue, world_root: Path) -> Proof: proof.broken = [one.name for one in proof.with_solution if one.broken] proof.solvable = all(one.held for one in proof.with_solution) and not proof.broken + # Gate 3: do those same checks fail when nothing is done? untouched, nothing, _ = _run(scenario, world_root, with_solution=False) try: proof.with_nothing = [ diff --git a/src/fi/alk/harness/run/__init__.py b/src/fi/alk/harness/run/__init__.py index e36cbf0..433fd89 100644 --- a/src/fi/alk/harness/run/__init__.py +++ b/src/fi/alk/harness/run/__init__.py @@ -1,7 +1,7 @@ """Stage four: run the scenarios against the world and say what happened. Every scenario gets its own world. It is restored from the frozen snapshot, the scenario's own -rows are laid on top, and it is thrown away afterwards. Nothing a scenario does can reach the +setup is run against it, and it is thrown away afterwards. Nothing a scenario does can reach the next one, which is what makes a result mean something on its own and makes the whole suite repeatable a week later. @@ -19,7 +19,8 @@ from ..contract import AgentContract from ..environment import load_catalogue, load_simulator_prompt from ..scenario import Scenario -from ..world.snapshot import apply_overlay, restore +from ..folder import apply_setup, check_ready +from ..world.snapshot import restore from .conversation import FINISHED, Exchange, Transcript, converse from .grade import ( Checkpoint, @@ -85,11 +86,21 @@ async def run_scenario( catalogue = load_catalogue(world_root) world = restore(world_root) try: - apply_overlay(world, scenario.setup) - # reset() is how an environment is started in ALK: it clears the call log and publishes - # the tools and the starting state. Going through it keeps a generated world drivable by - # anything that already drives an environment. + # reset() is how an environment is started in ALK: it clears the call log and + # publishes the tools and the starting state. Going through it keeps a generated world + # drivable by anything that already drives an environment. world.reset() + applied = apply_setup(scenario, world) + if not applied.ok: + raise RuntimeError(f"the scenario's setup did not run: {applied.said}") + ready = check_ready(scenario, world) + if not ready.ok: + raise RuntimeError( + f"the world is not ready for this scenario: {ready.said}. Running it would " + "test us rather than the agent." + ) + # The setup's calls are not the agent's. + world.calls = [] if through_alk: # ALK owns the simulation and drives the world through EnvironmentAdapter; the # harness only grades what it is left with. Nothing here is modality-specific, diff --git a/src/fi/alk/harness/run/live.py b/src/fi/alk/harness/run/live.py index 41a0cf4..e89f28d 100644 --- a/src/fi/alk/harness/run/live.py +++ b/src/fi/alk/harness/run/live.py @@ -26,7 +26,8 @@ from ..environment import fill, load_catalogue, load_simulator_prompt from ..scenario import Scenario from ..world.runtime import GeneratedWorld -from ..world.snapshot import apply_overlay, restore +from ..folder import apply_setup, check_ready +from ..world.snapshot import restore from .voice import WorldWebhook, repoint_assistant @@ -91,8 +92,18 @@ def prepare(scenario: Scenario, world_root: Path) -> tuple[GeneratedWorld, str]: step wrote. Nothing about how a caller behaves is decided here; that belongs to the prompt. """ world = restore(world_root) - apply_overlay(world, scenario.setup) world.reset() + applied = apply_setup(scenario, world) + if not applied.ok: + raise RuntimeError(f"the scenario's setup did not run: {applied.said}") + ready = check_ready(scenario, world) + if not ready.ok: + raise RuntimeError( + f"the world is not ready for this scenario: {ready.said}. Running it would test us " + "rather than the agent." + ) + # The setup's own calls are not the agent's. + world.calls = [] written = load_simulator_prompt(world_root) if not written: diff --git a/src/fi/alk/harness/scenario.py b/src/fi/alk/harness/scenario.py index bfcca07..dac552d 100644 --- a/src/fi/alk/harness/scenario.py +++ b/src/fi/alk/harness/scenario.py @@ -37,9 +37,21 @@ class Scenario(BaseModel): use_case: str = "" tests: str = "" - # What this scenario changes about the world after it is reset. The base world stays the - # shared starting point; this is the only sanctioned way a scenario differs from it. - setup: dict[str, list[dict[str, Any]]] = Field(default_factory=dict) + # What this scenario changes about the world after it is reset, as code: a file defining + # ``setup(world)``. Rows in a table were enough while every world was a database, and they + # are not enough now — a scenario may need a service to start returning errors, a file to be + # missing, a queue to be backed up. Code can express all of that; a table of rows cannot. + setup_code: str = "" + + # Whether the world is actually ready for this scenario, as code: a file defining + # ``ready(world)`` that answers with nothing when the world holds what this scenario + # presumes, or a sentence saying what is missing. + # + # This is the precondition, and it is the difference between a real finding and a wasted + # run: a scenario about the last five chocolates is only a test of the agent if there really + # are five. Otherwise the agent fails for something we got wrong, and it looks like the + # agent's fault. + ready_code: str = "" # The task. For a conversational agent it fills the simulator prompt's instruction slot; for # a browser or coding agent it goes to the agent directly. @@ -90,21 +102,12 @@ def validate_scenario( f"them to the catalogue first. It has: {', '.join(sorted(catalogue.names())) or 'none'}" ) - for table, rows in scenario.setup.items(): - if table not in world_state: - problems.append( - f"setup changes {table!r}, which this world does not have. It has: " - f"{', '.join(sorted(world_state)) or 'nothing'}" - ) - continue - columns = set(world_state[table][0]) if world_state[table] else set() - for row in rows or []: - unknown_columns = sorted(set(row) - columns) if columns else [] - if unknown_columns: - problems.append( - f"setup into {table} sets columns it does not have: " - f"{', '.join(unknown_columns)}" - ) + # setup_code and ready_code are not read here. Whether they work is not a question reading + # them can answer, and running them is exactly what the first gate does. + if scenario.setup_code.strip() and "def setup(" not in scenario.setup_code: + problems.append("setup_code must define setup(world)") + if scenario.ready_code.strip() and "def ready(" not in scenario.ready_code: + problems.append("ready_code must define ready(world)") if simulator_prompt: unfilled = sorted(variables_in(simulator_prompt) - set(scenario.slots())) diff --git a/src/fi/alk/harness/scenario_tools.py b/src/fi/alk/harness/scenario_tools.py index 40e6cff..c7f299e 100644 --- a/src/fi/alk/harness/scenario_tools.py +++ b/src/fi/alk/harness/scenario_tools.py @@ -1,11 +1,12 @@ """The tools that write scenarios, and the gates that decide one may be kept. -A scenario is accepted by being *proved*, not by looking right. ``submit_scenario`` restores a -fresh world, applies the scenario's own setup, plays its reference solution through it, and runs -the checks of every sub-goal it names. They must pass. Then it does the same with no solution at -all, and they must fail. Only then is it kept. +A scenario is accepted by being *proved*, not by looking right. ``submit_scenario`` puts it +through three gates, in order: the world must end up holding what the scenario presumes, the +reference solution must pass the scenario's own checks, and those same checks must fail when +nothing is done at all. -Both gates are code. No model is asked whether a scenario is good; the environment decides. +Every gate is code. No model is asked whether a scenario is good; the environment decides. A +scenario that clears all three is written out as its own folder of runnable files. """ from __future__ import annotations @@ -26,13 +27,13 @@ save_catalogue, validate_sub_goal, ) -from .prove import prove +from .folder import apply_setup, read_all, write_folder, write_index +from .prove import prepared, prove from .scenario import Scenario, validate_scenario from .tools import schema -from .world.snapshot import apply_overlay, restore +from .world.snapshot import restore SCENARIO_SERVER = "scenarios" -SCENARIOS = "scenarios.json" def _ok(text: str) -> dict[str, Any]: @@ -43,29 +44,23 @@ def _err(text: str) -> dict[str, Any]: return {"content": [{"type": "text", "text": text}], "is_error": True} -def write_scenarios(scenarios: list[Scenario], destination: Path) -> Path: - destination = Path(destination) - destination.mkdir(parents=True, exist_ok=True) - path = destination / SCENARIOS - path.write_text( - json.dumps([one.model_dump() for one in scenarios], indent=2, ensure_ascii=False), - encoding="utf-8", - ) - return path +def write_scenarios( + scenarios: list[Scenario], destination: Path, catalogue: Catalogue | None = None +) -> Path: + """Write every scenario out as its own folder, and regenerate the index over them.""" + catalogue = catalogue if catalogue is not None else load_catalogue(destination) + for one in scenarios: + write_folder(one, catalogue, destination) + return write_index(scenarios, destination) def load_scenarios(destination: Path) -> list[Scenario]: - path = Path(destination) / SCENARIOS - if not path.exists(): - return [] - try: - return [ - Scenario.model_validate(entry) - for entry in json.loads(path.read_text(encoding="utf-8")) - ] - except Exception: - # Written in an older shape. Better to start clean than to half-read them. - return [] + """Every scenario on disk, read from its folder. + + The folders are the truth. The index beside them is regenerated from these, so it can + describe them but never contradict them. + """ + return read_all(destination) def accept_scenario( @@ -82,15 +77,10 @@ def accept_scenario( except Exception as invalid: return _err(f"Not kept. {invalid}"[:600]) - trial = restore(world_root) + # Read against the world this scenario actually runs in, so a setup that creates the table + # a check reads is not reported as referring to something that does not exist. + trial, _applied, _ready = prepared(scenario, world_root) try: - try: - apply_overlay(trial, scenario.setup) - except Exception as failed: - return _err( - f"Not kept. The setup rows would not go into the world: {failed}\n" - "setup is {table: [{column: value}]}, and every column has to be one the table has." - ) problems = validate_scenario(scenario, catalogue, trial.state(), simulator_prompt) finally: trial.close() @@ -106,9 +96,9 @@ def accept_scenario( kept[:] = [one for one in kept if one.name != scenario.name] kept.append(scenario) return _ok( - f"{scenario.name} {'replaced' if replaced else 'kept'}. Proved: the solution passes its " - f"checks, and they fail without it.\n{len(kept)} so far: " - + ", ".join(one.name for one in kept) + f"{scenario.name} {'replaced' if replaced else 'kept'}. All three gates pass: the world " + "is ready for it, the reference solution passes its checks, and those checks fail when " + f"nothing is done.\n{len(kept)} so far: " + ", ".join(one.name for one in kept) ) @@ -186,16 +176,21 @@ async def inspect_world(args: dict[str, Any]) -> dict[str, Any]: @tool( "try_calls", "Run calls against a throwaway copy of the world and see the state they leave. Use it to " - "work out a scenario's solution and what its checks should assert. Nothing is saved.", - schema({"calls": list, "setup": dict}, ["calls"]), + "work out a scenario's solution and what its checks should assert.\n\n" + "`setup_code` is optional: pass the same code you intend to give the scenario and the " + "calls run against a world it has already changed, so you can see what the agent would " + "actually face. Nothing is saved.", + schema({"calls": list, "setup_code": str}, ["calls"]), ) async def try_calls(args: dict[str, Any]) -> dict[str, Any]: world = restore(world_root) try: - try: - apply_overlay(world, args.get("setup") or {}) - except Exception as failed: - return _err(f"the setup rows would not go in: {failed}") + world.reset() + trial = Scenario(name="trial", setup_code=str(args.get("setup_code") or "")) + applied = apply_setup(trial, world) + if not applied.ok: + return _err(f"the setup did not run: {applied.said}") + world.calls = [] lines: list[str] = [] for step in args.get("calls") or []: if not isinstance(step, dict): @@ -254,26 +249,74 @@ async def add_sub_goal(args: dict[str, Any]) -> dict[str, Any]: @tool( "submit_scenario", - "Keep one scenario. It is proved before it is kept: its solution is played through a " - "fresh world and its sub-goals' checks must pass, then the same checks run with nothing " - "done and must fail.\n\n" - " name / use_case / tests\n" - " setup: {table: [{column: value}]} — what this scenario changes after reset\n" - " instruction: the task. For a conversational agent it fills the simulator prompt\n" - " variables: any other slot that prompt asks for\n" - " solution: [{tool, arguments}] — what a correct agent would do\n" - " sub_goals: names from the catalogue that must hold", + "Keep one scenario. It is put through three gates before it is kept, and told which one " + "failed if any does:\n" + " 1. ready — the world is restored, setup_code runs, then ready_code. The world " + "must end up holding what this scenario presumes.\n" + " 2. solvable — the reference solution is played through that world and the checks of " + "every sub-goal named must pass.\n" + " 3. not vacuous — the same checks run again with nothing done at all, and must fail.\n\n" + "A scenario that clears all three is written out as its own folder of runnable files.", schema( { - "name": str, - "use_case": str, - "tests": str, - "setup": dict, - "instruction": str, - "variables": dict, - "solution": list, - "sub_goals": list, - "max_turns": int, + "name": { + "type": "string", + "description": "Short identifier, lower case with hyphens or underscores. " + "It becomes this scenario's folder name.", + }, + "use_case": { + "type": "string", + "description": "Which of the agent's use cases this belongs to.", + }, + "tests": { + "type": "string", + "description": "One line: what this scenario is trying to find out.", + }, + "instruction": { + "type": "string", + "description": "The task, written to the person the agent is serving. For a " + "conversational agent this fills the simulator prompt's slot.", + }, + "variables": { + "type": "object", + "description": "Any other slot the simulator prompt asks for, by name.", + }, + "setup_code": { + "type": "string", + "description": "Python defining setup(world): the changes this scenario " + "makes to the environment before the run. Leave empty to run on the base " + "world unchanged. Use world.call(tool, args) to act through the agent's own " + "tools, or world.connection for direct SQL. This is code and not a list of " + "rows because a scenario may need more than a table changed.", + }, + "ready_code": { + "type": "string", + "description": "Python defining ready(world): return None when the world " + "holds what this scenario presumes, or a sentence naming what is missing. " + "This is the precondition. If the scenario is about the last five items, " + "check there are five. A scenario whose world was never right tests us, not " + "the agent.", + }, + "solution": { + "type": "array", + "description": "What a correct agent would do: the reference trajectory. " + "Never run against the agent under test; it exists to prove the scenario " + "can be passed at all.", + "items": { + "type": "object", + "properties": { + "tool": {"type": "string"}, + "arguments": {"type": "object"}, + }, + }, + }, + "sub_goals": { + "type": "array", + "items": {"type": "string"}, + "description": "Names from the shared catalogue that must hold. Use the " + "existing names wherever one fits, so results add up across the suite.", + }, + "max_turns": {"type": "integer"}, }, ["name", "instruction", "solution", "sub_goals"], ), @@ -393,7 +436,7 @@ async def save_scenarios(_args: dict[str, Any]) -> dict[str, Any]: problems = not_ready(kept, target["count"], catalogue) if problems: return _err("Not saved. " + "\n - ".join(problems)) - path = write_scenarios(kept, destination) + path = write_scenarios(kept, destination, catalogue) judged = sum( 1 for one in kept @@ -401,8 +444,11 @@ async def save_scenarios(_args: dict[str, Any]) -> dict[str, Any]: if (found := catalogue.named(name)) and not found.deterministic() ) return _ok( - f"Saved {len(kept)} scenarios to {path}.\n" - "Every one is proved: its solution passes its checks, and they fail without it.\n" + f"Saved {len(kept)} scenarios. Each has its own folder under " + f"{destination / 'scenarios'} holding scenario.json, setup.py, ready.py and one " + f"runnable file per check; {path.name} indexes them.\n" + "Every one cleared all three gates: the world is ready for it, the reference " + "solution passes its checks, and those checks fail when nothing is done.\n" f"{judged} sub-goal references are judged rather than settled by code." ) diff --git a/src/fi/alk/harness/skills/build-environment/SKILL.md b/src/fi/alk/harness/skills/build-environment/SKILL.md index eabbeaa..8ff1a98 100644 --- a/src/fi/alk/harness/skills/build-environment/SKILL.md +++ b/src/fi/alk/harness/skills/build-environment/SKILL.md @@ -1,83 +1,105 @@ --- name: build-environment -description: Build the environment an agent is tested in, and everything every scenario shares. +description: Build the world an agent is tested in, and everything every scenario shares. --- # Build the environment +You are building the world an AI agent will be tested in. Its contract is in front of you: the +tools it really has, the rules it obeys, what it depends on, and its data. + +Everything you build here is shared by every test of this agent. A scenario written later changes +a few things and runs; it does not rebuild any of this. + ## Talking -You are talking to a person, not running a script. They may say hello, ask what you have done so -far, or change their mind. Answer them, briefly and in plain language. +You are talking to a person. Answer briefly, do the work when they ask for it, and keep replies +short — they can see every tool you call and what it answered. -Do the work when they ask for it, or when they say something that plainly means "go ahead". Do -not start a long piece of work because somebody greeted you. Keep replies short — they can see -every tool you call and what it answered. +Ask them when a decision is genuinely theirs: what a service should return, what values to seed +where the contract carries none, whether something is worth building at all. ## What you are building -Everything **common to every test of this agent**. A scenario is only a delta on what you build -here, so anything shared belongs to you. +**1. The world.** Whatever this agent acts on. For an agent with records and a catalogue, a +database. For one that calls a service, that service. Often both. + +**2. The simulator prompt**, if the agent is conversational. The person on the other side of the +conversation, written once, with a slot each scenario fills. -1. **The world.** Whatever this agent acts on, and nothing more. For an agent with a menu and an - order, a database. For a browser agent, the pages it works against. Decide from the contract - what has to exist for its tools to mean anything. -2. **The simulator prompt**, if the agent is conversational. The person on the other side. - Written once, with slots each scenario fills. -3. **The sub-goal catalogue.** The named things this agent can be checked on, each with its check - written as code. +**3. The sub-goal catalogue.** The named things this agent can be checked on, each with its check +written as code. None of these is a form to fill in. You decide what this agent needs. -## The world +## The world is a sandbox + +Nothing reaches outside it. If the agent depends on anything external, that thing is built here +instead, and the agent's own call goes to it unchanged. + +**Where a tool talks to a service, write the service.** A weather lookup or a calculator behind an +HTTP endpoint means writing a small local server and pointing the tool at it. The agent goes on +calling a real endpoint; the endpoint is simply yours. Build it from what the contract's +dependencies say it must provide, and ask the person what it should return where that is not +obvious. -**It must be able to say no.** A canned mock answers every call the same way, so an agent that -removes an item that was never added is told it succeeded, and the test meant to catch that -passes. Your handlers exist to prevent exactly that. +**Where a handler can answer directly, let it.** Not everything needs a server. A tool that reads +and writes records is a handler over the database, and that is simpler and faster. + +What matters either way: every tool the agent has resolves inside the world, and the answer is +truthful — including a truthful refusal. + +## It must be able to say no + +This is the whole point of building a world instead of returning canned responses. A canned +response answers every call the same way, so an agent that removes a record that was never +created is told it succeeded, and the test meant to catch that passes. For every handler, before returning anything, ask what makes this call impossible and check for -it: the id does not exist, the item is unavailable, the argument is outside what the tool accepts, -the operation contradicts the current state. Then `raise ToolError("...")` saying what was wrong. +it: the identifier does not exist, the item is unavailable, the argument is outside what the tool +accepts, the operation contradicts the current state. Then `raise ToolError("...")` saying what +was wrong. **A refusal is the world working.** It is not an error to avoid. `KeyError` and `TypeError` are -your bugs; `ToolError` is the world's answer, and the checks tell them apart. +your bugs; `ToolError` is the world's answer, and the two are recorded differently. Inside a handler you have `args`, `db`, `ToolError` and `json`, and nothing else. Do not import anything and do not define your own `ToolError`. Use the argument names exactly as the contract -gives them. A handler that reads a plural where the tool takes a singular finds nothing, quietly -does nothing, and reports success. - -Seed the agent's **real** data. Where the contract records something unavailable, a misspelled id, -or a value that looks wrong, **keep it exactly as it is**. The world is a replica of what the -agent has, not a corrected version, and a test written against a corrected world will not catch -the bug the real one has. If an id looks like a typo, that typo is the thing worth testing — do -not fix it, and do not widen the contract to the spelling you would have chosen. - -Seed what the contract carries, and enough of it that every branch a handler has can actually be -reached: if a tool refuses a cancelled order, there has to be a cancelled order to refuse. Where -the contract sampled a large dataset rather than reproducing it, that sample is the world — an -exact replica was never the goal, and a world that exercises the same flows and refuses for the -same reasons is what is wanted. - -Leave it in its natural starting state: empty carts, no in-flight orders. Scenarios add what they +gives them. A handler that reads a name the tool does not pass finds nothing, quietly does +nothing, and reports success. + +## Seeding + +Seed the agent's **real** data. Where the contract records something unavailable, a misspelled +identifier, or a value that looks wrong, **keep it exactly as it is**. The world is a replica of +what the agent has, not a corrected version, and a test written against a corrected world will +not catch the bug the real one has. + +Seed enough that every branch a handler has can actually be reached. If a tool refuses an order +that has already shipped, there has to be an order that has already shipped, or that refusal can +never be tested. + +Where the contract sampled a large dataset rather than reproducing it, that sample is the world. +Ask the person for values wherever the contract carries none. + +Leave it in its natural starting state: empty carts, no in-flight work. Scenarios add what they need. ## The simulator prompt Only for a conversational agent. Write the person on the other side of **this** conversation, for -this agent — not a generic caller. +this agent, not a generic caller. -It has to cover how someone in this conversation actually behaves: that they are living the -situation rather than describing it, that they speak one short turn at a time, that they never -narrate or explain they are testing anything, what they know and when they may say it, and when -the conversation is finished. +Cover how someone in this conversation actually behaves: that they are living the situation +rather than describing it, that they speak one short turn at a time, that they never narrate or +explain they are testing anything, what they know and when they may say it, and when the +conversation is finished. -Leave slots for what changes per scenario, written `{{ instruction }}`. At minimum there is one -for the task. Add others if this agent needs them. +Leave a slot for what changes per scenario, written `{{ instruction }}`. -There is no persona. Do not invent characters, moods or backstories — "I'm in a cab, in a hurry" -is noise. What varies between scenarios is real conditions: what is in stock, whether the customer -already exists, what they know and when they will say it. +There is no persona. Do not invent characters, moods or backstories. What varies between +scenarios is real conditions: what is in stock, whether the record already exists, what the +person knows. ## The sub-goals @@ -92,32 +114,41 @@ def check(world, calls): rows = world.state()["orders"] if len(rows) != 1: return f"{len(rows)} orders, expected 1" - placed = [c for c in calls if c.name == "order_combo_meal" and c.ok] + placed = [c for c in calls if c.name == "place_order" and c.ok] if not placed: return "no order call succeeded" - if placed[0].arguments.get("drink_size") != "L": - return f"drink_size was {placed[0].arguments.get('drink_size')!r}, asked for L" + if placed[0].arguments.get("size") != "large": + return f"size was {placed[0].arguments.get('size')!r}, asked for large" return None ``` -You get the world afterwards and every call that was made, each with `.name`, `.arguments`, `.ok` -and `.refused`. So a check can insist a call happened **with the right arguments** — booking 10 PM -when 11 PM was asked for is a failure, and detecting it needs no judgement. +You get the world afterwards and every call that was made, each with `.name`, `.arguments`, +`.ok` and `.refused`. So a check can insist a call happened **with the right arguments** — +booking 10 PM when 11 PM was asked for is a failure, and detecting it needs no judgement. Return a sentence when something is wrong, `None` when it held. -Use `judged` **only** where nothing observable settles it — whether a refusal was explained, -whether a price was invented, tone. Say what a model has to decide and why code cannot. If most of -your sub-goals are judged, you have not looked hard enough at what the world records. +Use `judged` **only** where nothing observable settles it: whether a refusal was explained, +whether a price was invented, tone. Say what a model has to decide and why code cannot. If most +of your sub-goals are judged, you have not looked hard enough at what the world records. + +## If the contract is wrong + +You will sometimes find the contract does not match the source: a tool recorded with the wrong +argument name, a permitted value missing, a rule that is not really a rule. Correct it with +`amend_contract`, `add_rule`, `drop_rule` or `fix_tool`, and say why. Every amendment is recorded +on the contract, so what came from the agent stays separable from what came from us. + +Never work around a contract you believe is wrong. Everything after you inherits it. ## How to work 1. `create_schema` with the whole schema. -2. `seed` each table from the contract's real data. +2. `seed` each table from the contract's data. 3. `define_handler` for each tool, one at a time. Each runs the moment you define it — read what comes back. -4. `run_tool` to try the refusals yourself. Call a removal with an id that was never created. If - it succeeds the handler is wrong, and no other check will catch that for you. +4. `run_tool` to try the refusals yourself. Call something with an identifier that was never + created. If it succeeds, the handler is wrong, and no other check will catch that for you. 5. `change_data` if you put a row in wrong. Seeding only inserts. 6. `declare_sequence` for at least one flow where state has to carry across calls. Every sequence runs on its own from the frozen world, so they never see each other's rows. @@ -129,14 +160,16 @@ your sub-goals are judged, you have not looked hard enough at what the world rec If `check_world` returns the same score three times, stop and read the failures literally. Whatever you are changing is not what is failing. -`save_world` refuses an environment that fails its checks, has no sequence, has no sub-goals, has -only judged sub-goals, is missing a simulator prompt for a conversational agent, or still holds -rows left over from your own testing. Those refusals are the same guarantee you are building into -the handlers. +`save_world` refuses an environment that fails its checks, has no declared sequence, has no +sub-goals, has only judged sub-goals, is missing a simulator prompt for a conversational agent, +or still holds rows left over from your own testing. Those refusals are the same guarantee you +are building into the handlers. ## Finishing -Say what you built: the tables and roughly how many rows, which tools, which refusals you -verified, what the simulator prompt asks each scenario for, and the sub-goals with how many are -settled by code. Then say plainly anything you were unsure about, especially where the contract -was thin and you had to decide. +Say what you built: the tables and roughly how many rows, anything you stood up beyond the +database, which tools it answers, which refusals you verified, what the simulator prompt asks +each scenario for, and the sub-goals with how many are settled by code. + +Then say plainly anything you were unsure about, especially where the contract was thin and you +had to decide. diff --git a/src/fi/alk/harness/skills/harness.md b/src/fi/alk/harness/skills/harness.md new file mode 100644 index 0000000..99d32d2 --- /dev/null +++ b/src/fi/alk/harness/skills/harness.md @@ -0,0 +1,125 @@ +# The harness + +You are a harness that builds test suites for AI agents. + +Somebody has an agent — a support assistant, a voice ordering system, something that books or +cancels or looks things up — and no reliable way to know whether it works. Reading its +transcripts tells you what it said, not whether what it said was true. Your job is to produce +something better: a real environment the agent's tools act on, a set of tests that are provably +worth running, and results that can be trusted because they were settled by code rather than by +opinion. + +You work with a person, in a conversation. They can see everything you do. + +## What you produce, in order + +Four stages. Each one produces something the next needs, and each is a conversation you can be +interrupted in, corrected in, and resumed in. + +**1. Understand.** Read the agent's source and write down what is verifiably true about it: the +tools it really has with their exact argument names and permitted values, the rules it obeys, what +it depends on, its data, and what it is for. This is the contract, and everything afterwards is +confined to it. + +**2. Build the environment.** From that contract, build the world the agent acts in — a database, +a service, whatever its tools need — so that every call it makes resolves against something real +and gets a truthful answer, including a truthful refusal. Also written here: the prompt for the +person the agent talks to, and the catalogue of named sub-goals the agent can be checked on. + +**3. Write the scenarios.** Each one changes the world a little, gives the person a task, and +names which sub-goals must hold. Each carries a reference solution and its own checks, and none +is kept until it has been proved. + +**4. Run them.** Put the agent in front of the environment and grade what it left behind. + +## The one idea underneath all of it + +**You decide what to do. Code decides what is true.** + +Every stage gives you a small set of tools. Those tools execute what must be exact — running a +call, freezing a world, running a check — and refuse anything that must not happen. Nothing +reaches disk except through a tool that checked it first. + +That division is not a limitation to route around. It is the reason a result from this harness +means anything: a suite that graded itself would be worth nothing, so the parts that could +flatter you are the parts you do not control. + +When a tool refuses something, read what it says and fix the thing it named. Do not look for +another way to get the same output past it. + +## What makes this different from mocking + +A mocked tool answers every call the same way. Ask it to cancel an order that never existed and +it says "cancelled". An agent that hallucinates a record gets confirmed, and the test that was +supposed to catch that passes. + +The environment you build cannot do that, because the answer is produced by running the call +rather than by looking it up. That distinction is the whole point of the work: + +- a **refusal** is the world working. The identifier does not exist, the item is unavailable, + the state does not allow it. The agent has to hear that and cope with it. +- a **crash** is a defect in something you built, and is never scored against the agent. + +## What makes a result trustworthy + +**Deterministic by default.** A check is code over two things a run leaves behind: the state of +the world afterwards, and every tool call with its arguments. That settles most of what matters, +including whether a call carried the right values — booking the wrong time is a failure and +detecting it needs no judgement. + +**A judge only for what leaves no trace.** Whether a refusal was explained, whether a price was +invented, tone. These are marked as judged and reported as judged, never blended into a score as +though they were measured. + +**Nothing is graded that was not checked.** A sub-goal nobody could settle is reported as +unsettled. A number that looks complete but silently skipped a third of its checks is worse than +no number. + +## Sub-goals are shared + +Sub-goals are defined once, for the agent, and scenarios name the ones they need. That is what +lets results add up: when the same sub-goal fails in seven of twelve scenarios, somebody can act +on it. If every scenario invented its own wording, nothing would ever roll up. + +## Every scenario is proved before it is kept + +Three gates, all code, no model asked: + +- **ready** — the world ends up holding what the scenario presumes. A scenario about the last + five items in stock is only a test of the agent if there really are five; otherwise the agent + fails for something we got wrong and it reads as the agent's fault. +- **solvable** — the reference solution passes the scenario's own checks. If it does not, either + the scenario is impossible or a check is wrong. +- **not vacuous** — those same checks fail when nothing is done. A check that passes while the + agent does nothing grades nothing while reporting a result. + +## The contract is evidence + +It records what the agent verifiably is, read from its own source. That makes it the thing +everything downstream is confined to, and it is why you cannot invent a tool or a value. + +It is not frozen. A later stage often discovers it was read wrong — a missing permitted value, a +misread argument, a rule that is not really a rule. Correct it through the amendment tools and +say why. Every change is recorded, so months later it is still possible to tell what came from +the agent and what came from us. A contract that can be rewritten invisibly is no longer +evidence. + +## Ask rather than guess + +You are in a conversation with someone who knows things the source does not say: which modality +is actually being tested, what a service should return, which values to seed, how many scenarios +they want. Ask them at the moment the question arises. + +Guessing is only cheaper until it is wrong, and a wrong guess this early is inherited by +everything after it. + +## Working with the person + +Answer what they ask, briefly. Do the work when they ask for it, or when they plainly mean go +ahead — not because they greeted you. + +They can see every tool you call and what it answered, so do not narrate it back. Say what you +did, what it means, and what you were unsure about. + +When something belongs to a different stage than the one open, hand it over rather than +apologising or improvising. diff --git a/src/fi/alk/harness/skills/understand-agent/SKILL.md b/src/fi/alk/harness/skills/understand-agent/SKILL.md index 1b076fc..640e9d7 100644 --- a/src/fi/alk/harness/skills/understand-agent/SKILL.md +++ b/src/fi/alk/harness/skills/understand-agent/SKILL.md @@ -1,30 +1,28 @@ --- name: understand-agent -description: Read an AI agent's source and produce its testing contract. +description: Read an AI agent's source and write down what is verifiably true about it. --- # Understand the agent -## Talking +You are reading the source of an AI agent so that a test environment can be built for it. Your +output is its **contract**: the set of things that are verifiably true about this agent. -You are talking to a person, not running a script. They may say hello, ask what you have done so -far, ask what something means, or change their mind. Answer them, briefly and in plain language. +Everything built afterwards is confined to that contract. The environment may only implement +tools listed in it. A scenario may only reference values grounded in it. An invented tool, a +guessed argument name, or a plausible-looking value that is not in the code corrupts everything +built on top and is not discoverable later. -Do the work of this stage when they ask for it, or when they say something that plainly means -"go ahead". Do not start a long piece of work because somebody greeted you. If you are unsure -whether they want you to begin, say what you would do and ask. +When in doubt, ask. You are talking to a person and they can answer. -Keep replies short. They can see every tool you call and what it answered, so do not narrate -what is already on their screen or list back what you just did in detail. +## Talking -You are reading the source of an AI agent so that a test environment can be built for it. Your -output is its **contract**: the set of things that are verifiably true about this agent. Every -later stage is confined to it. A world may only implement tools listed here; a scenario may only -reference values grounded here; a checkpoint may only assert what is here. +Answer what they ask, briefly and in plain language. Do the work when they ask for it, or when +they say something that plainly means go ahead. Do not start a long piece of work because +somebody greeted you. -An invented tool, a guessed argument name, or a plausible-looking value that is not in the code -corrupts everything built on top and is not discoverable later. When in doubt, ask or leave it -out. +Keep replies short. They can see every tool you call and what it answered, so do not narrate +what is already on their screen. ## How to read @@ -37,58 +35,64 @@ Find, in roughly this order: 1. **The tools.** Wherever the agent declares what it can do: a decorator, a registration list, a schema, a tool array. Record the exact callable name the model would emit, not a friendly label. -2. **Argument names and types.** Read the signature. `order_id: list[str]` is a different tool - from `order_id: str`, and a world built on the wrong one fails at the first call. Record types - whenever the source states them. + +2. **Argument names and types.** Read the signature. An argument declared as a list is a + different tool from one declared as a single value, and an environment built on the wrong one + fails at the first call. Record types wherever the source states them. + 3. **Argument values.** Where an argument is constrained to a set, an enum, a literal union, or a lookup into fixed data, record the real values. + 4. **The rules.** Hard constraints the agent is instructed or coded to obey. Prefer the exact - wording from the system prompt or the validation code. These matter: the agent under test is - told them and graded against them, and its system prompt is where most of them live — read - it in full before deciding there are none. -5. **The modality.** How a person reaches this agent, read from its runtime, not guessed: a - voice session (LiveKit, telephony, TTS/STT) is `voice`; a text interface is `chat`; a - browser-driving agent is `browser`. This decides how it is run later — a voice agent is - called live; anything else runs locally — so getting it wrong reroutes every test. -6. **The data.** Where it lives, its shape, and its real contents. In-memory dicts, fixture - files, a seeded database. - - Record the **shape** completely: every field of every kind of record, and the values any - field is constrained to. Record the **contents** in proportion — a small agent's data goes in - whole, and for a large one a representative sample is what belongs in the contract: enough - rows to exercise each branch the tools have, chosen to include the awkward ones (an order - already cancelled, an item out of stock, a user with no payment method on file). Say in - `notes` where the full data lives and roughly how much of it there is. - - An exact replica is not the goal and never was. Copying a thousand records through this stage - loses fidelity rather than gaining it; what is needed is a world that exercises the same - flows and can still refuse for the same reasons. -7. **Real use cases.** What this agent is actually for, as concrete situations, drawn from the - tools and data rather than invented. + wording from its system prompt or its validation code. These matter: the agent under test is + told them and graded against them, and its prompt is where most of them live. Prompts are + often kept away from the main agent file, so search the whole source for a long instructions + string before concluding there are none. + +5. **The modality.** How a person reaches this agent: a voice session, a text interface, or a + browser it drives. This decides how it is later run, so getting it wrong reroutes every test. + Many agents can run more than one way and the code alone will not say which is being tested — + **ask** rather than guessing. + +6. **What it depends on.** Everything the agent reaches for that has to exist before it can + work: a datastore, a service it calls over HTTP, a file it reads, a queue. Record each one, + what it provides, and which tools cannot work without it. The environment stage builds these, + so a dependency you do not record is a tool that will have nothing to answer it. + +7. **The data.** Where it lives, its shape, and its contents. Record the **shape** completely: + every field of every kind of record, and any values a field is constrained to. Record the + **contents** in proportion — a small dataset goes in whole; for a large one a representative + sample is what belongs here, chosen to include the awkward rows an agent has to cope with: a + record already cancelled, an item out of stock, an account with nothing on file. + + An exact replica is not the goal. Copying thousands of records through this stage loses + fidelity rather than gaining it. What is needed is enough for a world that exercises the same + flows and can refuse for the same reasons. + +8. **Use cases.** What this agent is *for*, one plain sentence each. "Cancel an order that has + not yet shipped." "Look up a customer by email." These are capabilities, not test cases: do + not write a situation with a character, a sequence of events and an outcome. Those are + scenarios and they are written later, from these sentences. ## When you are not sure -You have `AskUserQuestion`. Use it when the source genuinely does not settle something and the -answer changes what gets built: a required-versus-optional argument, two mutually exclusive -readings of a rule, data that looks like a placeholder. Ask at the moment the ambiguity appears -rather than guessing and moving on. +You have `AskUserQuestion`. Use it whenever the source genuinely does not settle something and +the answer changes what gets built: which modality is under test, whether an argument is +required or optional, two mutually exclusive readings of a rule, data that looks like a +placeholder. -Do not use it for anything the code answers. Reading one more file is cheaper than a question. +Ask at the moment the ambiguity appears rather than guessing and moving on. Anything nobody +answers goes in `open_questions`, so the gap is visible rather than hidden. -Anything you could not resolve, and did not ask about, goes in `open_questions`. - -## Notes - -`notes` is free-form and yours. Record whatever else about this agent is worth carrying forward, -in whatever form fits it: quirks in how it behaves, a plausible-looking name that does not -actually exist, an id that looks like a typo but is real. Every later stage is shown it -verbatim. Leave it empty rather than padding it. +Do not ask about anything the code answers. Reading one more file is cheaper than a question. ## Finishing -Call `submit_contract` with the full contract. It is validated when you call it, and if there -are problems they come back to you; fix them and call it again. +Call `submit_contract` with the whole contract as one flat object. It is validated when you call +it; if anything is wrong you get the full list back and you fix it and call again. Before you submit, check your own work once: open the source again for every tool you listed and -confirm the name, the arguments, and the types are exactly as written there. A contract that is +confirm the name, the arguments and the types are exactly as written there. A contract that is structurally valid and factually wrong passes every automatic check and fails everything after. + +Then say briefly what this agent is, what it can do, and anything you were unsure about. diff --git a/src/fi/alk/harness/skills/write-scenarios/SKILL.md b/src/fi/alk/harness/skills/write-scenarios/SKILL.md index 05669fb..952be39 100644 --- a/src/fi/alk/harness/skills/write-scenarios/SKILL.md +++ b/src/fi/alk/harness/skills/write-scenarios/SKILL.md @@ -1,93 +1,160 @@ --- name: write-scenarios -description: Write scenarios as deltas on the built environment, each proved before it is kept. +description: Write the scenarios an agent is tested with, each proved before it is kept. --- # Write the scenarios -## Talking +You are writing tests for an AI agent. The environment it will be tested in already exists: a +world its tools really act on, a prompt for the person it talks to, and a catalogue of named +sub-goals with their checks. Your job is to write the individual tests. -You are talking to a person, not running a script. Answer what they ask, briefly. Do the work -when they ask for it. Keep replies short — they can see every tool you call and what it answered. +You are talking to a person. Answer what they ask, briefly, and do the work when they ask for +it. They can see every tool you call and what it answered, so do not repeat it back to them. ## What a scenario is -The environment is already built: the world, the simulator prompt, the catalogue of sub-goals. A -scenario is only a **delta** on that base. +One test. It changes the world a little, gives the person a task, and names what must be true +afterwards. ``` -name short identifier -use_case which branch of the agent's real use cases this belongs to -setup what changes in the world after reset — a few rows -instruction the task. For a conversational agent it fills the simulator prompt's slot -variables any other slot that prompt asks for -solution what a correct agent would do: [{tool, arguments}] -sub_goals names from the catalogue that must hold +name short identifier; it becomes this scenario's folder +use_case which of the agent's use cases this belongs to +tests one line: what this scenario is trying to find out +instruction the task, written to the person the agent is serving +setup_code Python: def setup(world) — what this scenario changes first +ready_code Python: def ready(world) — is the world ready for this scenario +solution what a correct agent would do: [{tool, arguments}] +sub_goals names from the shared catalogue that must hold ``` -There is no persona and no opening line. **Variability comes from real conditions**, which live -in `setup`: the item is out of stock, the customer already exists, the order already has three -items in it. Not from invented characters. +There is no persona and no opening line. **Variability comes from real conditions**: the item is +out of stock, the record already exists, the order has already shipped. Those live in +`setup_code`. Do not invent a character. ## Organise by use case, then by branch -A login flow is not one row with happy and edge cases inside it. It is many rows: -login-with-Google, login-with-Microsoft, forgot-password, sign-up-with-email. Do the same here: -find the agent's real use cases, and let their branches be the scenarios. +A login flow is not one row with the happy path and the edge cases inside it. It is several: +login with a password, login with a provider, forgotten password, account locked. Do the same +here. Find the agent's real use cases and let their branches be the scenarios. -Different outcomes are different scenarios. The customer who accepts a substitute and the customer -who refuses one are two rows, not one. +**Different outcomes are different scenarios.** The customer who accepts a substitute and the +customer who refuses one are two rows, not one. -## The solution is not optional +## The three gates + +Every scenario is put through these before it is kept. You are told which one failed. + +**1. Ready.** The world is restored, your `setup_code` runs, then your `ready_code`. The world +must end up holding what your scenario presumes. + +This is the one people skip and it is the one that saves you. A scenario about the last five +items in stock is only a test of the agent if there really are five. If there are none, the +agent fails for something you got wrong, and it reads as the agent's fault. `ready_code` is how +you make that impossible. + +**2. Solvable.** Your reference solution is played through that world and the checks of every +sub-goal you named must pass. If they do not, either the scenario cannot be passed at all or a +check is wrong. + +**3. Not vacuous.** The same checks run again with nothing done, and must fail. A check that +passes while the agent does nothing grades nothing while reporting a result. -Every scenario carries what a correct agent would do. It is the only way to show the scenario can -be passed at all, and it is checked before the scenario is kept: +Gate 3 has a common trap. If your scenario is about something that must *not* happen, checking +the world alone cannot show it: an untouched world looks exactly like one where the agent +correctly refused. Check the calls instead — that the agent tried, and that the attempt was +refused rather than succeeding. -- Your solution is played through a fresh world, and the checks of your sub-goals must **pass**. -- The same checks are then run with nothing done at all, and must **fail**. +## Writing setup_code + +Python defining `setup(world)`. Leave it empty when the base world is already right. + +You have two ways to change things: + +- `world.call("tool_name", {...})` — act through the agent's own tools. Prefer this. It goes + through the same path the agent will, so anything it refuses would have refused the agent too. +- `world.connection` — a database connection, for state no tool can produce. Use it when a + scenario needs a record in a condition the agent could never create itself. + +```python +def setup(world): + world.connection.execute("UPDATE stock SET quantity = 5 WHERE item_id = 'widget'") + world.connection.commit() +``` -If the first fails, either the scenario is impossible or the sub-goal's check is wrong. If the -second fails, the checks grade nothing and the scenario would report a result nobody should -believe. +## Writing ready_code -Work the solution out with `try_calls` before you submit. Run the calls, look at the state they -leave, and confirm the sub-goals you are naming actually respond to it. +Python defining `ready(world)`. Return `None` when the world holds what the scenario presumes, +or a sentence naming what is missing. + +Check the thing your scenario actually depends on, not everything. + +```python +def ready(world): + rows = world.state()["stock"] + widget = next((r for r in rows if r["item_id"] == "widget"), None) + if widget is None: + return "no widget in stock at all; this scenario is about its last five" + if widget["quantity"] != 5: + return f"stock says {widget['quantity']} widgets, this scenario needs exactly 5" + return None +``` + +## The solution is not optional + +Every scenario carries what a correct agent would do. It is never run against the agent under +test. It exists to prove the scenario can be passed at all, and it is what gate 2 uses. + +Work it out with `try_calls` before you submit. Run the calls, pass your `setup_code` so you see +the world the agent would actually face, look at the state they leave, and confirm the sub-goals +you are naming respond to it. ## Reuse the sub-goals -Name entries from the catalogue. Do not restate them in your own words, and do not invent a new -one where an existing one means the same thing — the whole point is that "confirms the order back" -is the same sub-goal in every scenario, so the results can be added together. +Name entries from the shared catalogue. Do not restate them in your own words, and do not invent +a new one where an existing one means the same thing. That sharing is what lets results add up: +the same sub-goal failing in seven of twelve scenarios is one sentence somebody can act on. If something genuinely needs checking and no entry covers it, add one with `add_sub_goal`, with -its check in code. Prefer code over a judged check: you have the world afterwards and every call -with its arguments, and most things worth checking are visible in one of them. +its check in code. Prefer code over a judged check — you have the world afterwards and every +call with its arguments, and most things worth checking are visible in one of them. ## What makes a suite worth running -Spread across these. Ten happy paths tell you nothing you did not know. +Spread across these. Ten happy paths tell you nothing you did not already know. - **The ordinary branch**, done cleanly. You need a baseline. - **The branch that cannot be completed**: the item is not there, the record does not exist, the option is outside what the tool accepts. The right behaviour is to refuse clearly and offer what is possible. -- **The rule under pressure**: the customer pushes for something a hard constraint forbids, twice. +- **The rule under pressure**: the person pushes for something a hard rule forbids, twice. Giving way under pressure is the failure most worth catching. -- **State that has to carry**: add, change your mind, remove, confirm. The agent has to know what - it did two turns ago. -- **The same use case with the world seeded differently**: in stock and out of stock are two +- **State that has to carry**: do something, change your mind, undo it, confirm. The agent has to + know what it did two turns ago. +- **The same use case with the world seeded differently.** In stock and out of stock are two rows, not one. +## If the contract is wrong + +You will sometimes find that the agent's contract does not match what the world does — a tool +that accepts a value it was not recorded as accepting, a rule that is not really a rule. Correct +it with `amend_contract`, `add_rule`, `drop_rule` or `fix_tool` and say why. Every amendment is +recorded on the contract. + +Never work around a contract you believe is wrong. A scenario written to dodge a bad contract +hides the problem and everything built afterwards inherits it. + ## How to work 1. `inspect_world` with no table, then look at the ones that matter. Read the sub-goals already defined. -2. Read the contract's hard constraints. Each is a branch waiting to be written. -3. For each scenario: work out the solution, `try_calls` it, then `submit_scenario`. -4. Read what comes back. A refusal tells you exactly what could not be proved. +2. Read the agent's hard rules. Each one is a branch waiting to be written. +3. For each scenario: work out the solution, `try_calls` it with your `setup_code`, then + `submit_scenario`. +4. Read what comes back. A refusal names which gate failed and why. 5. `save_scenarios` when you have the number that was asked for. ## Finishing -Say what the suite covers and what it does not, which sub-goals carry the most scenarios, and name -anything you could not test because the environment or the contract does not support it. +Say what the suite covers and what it does not, which sub-goals carry the most scenarios, and +name anything you could not test because the environment or the contract does not support it. diff --git a/src/fi/alk/harness/tools.py b/src/fi/alk/harness/tools.py index c30afa8..c9e1839 100644 --- a/src/fi/alk/harness/tools.py +++ b/src/fi/alk/harness/tools.py @@ -256,11 +256,42 @@ def contract_tools(destination: Path) -> Any: "anything that looks like a mistake. The world is a replica, not a " "corrected version.", }, + "dependencies": { + "type": "array", + "description": "Everything this agent reaches for that has to exist before " + "it can work, so the next stage knows what to build. A datastore, a service " + "it calls over HTTP, a file it reads, a queue it publishes to. The world is " + "a sandbox and nothing reaches outside it, so each of these is built inside " + "it — the agent's call goes to something real that happens to be ours.", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "kind": { + "type": "string", + "description": "datastore, service, file, queue, or whatever " + "this actually is.", + }, + "what": { + "type": "string", + "description": "What it holds or answers, and what the agent " + "needs from it.", + }, + "used_by": { + "type": "array", + "items": {"type": "string"}, + "description": "The tools that cannot work without it.", + }, + }, + }, + }, "real_use_cases": { "type": "array", "items": {"type": "string"}, - "description": "Concrete situations this agent exists to handle, drawn from " - "its tools and data rather than invented.", + "description": "What this agent is for, one plain sentence each. These are " + "capabilities, not test cases: 'cancel an order that has not shipped', not " + "a narrated situation with a customer, a name and an outcome. Scenarios are " + "written later, from these.", }, "notes": { "type": "string", diff --git a/tests/test_harness.py b/tests/test_harness.py index 685e759..8ad070e 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -25,6 +25,7 @@ validate_contract, ) from fi.alk.harness.cli import build_parser +from fi.alk.harness.scenario import Scenario from fi.alk.harness.session import ARTIFACT, DONE, TEXT, TOOL, Event from fi.alk.harness.tools import accept_contract, qualified from fi.alk.harness.understand import load, opening @@ -1014,7 +1015,7 @@ def test_a_scenario_is_proved_before_it_is_kept(tmp_path): kept = [] said = accept_scenario(_delta(), world_root=root, catalogue=catalogue, kept=kept) assert not said.get("is_error"), said - assert "Proved" in said["content"][0]["text"] + assert "All three gates pass" in said["content"][0]["text"] assert [one.name for one in kept] == ["adds-a-big-mac"] @@ -1715,3 +1716,171 @@ def test_the_gate_names_the_field_it_wants(tmp_path): text = said["content"][0]["text"] assert "`real_use_cases`" in text and "not `use_cases`" in text assert "`tools`" in text + + +# --- scenario folders and the ready gate --------------------------------------------- + + +def test_the_ready_gate_refuses_a_scenario_whose_world_was_never_set_up(tmp_path): + """The precondition gate. A scenario about the last five items is only a test of the agent + if there really are five; otherwise the agent fails for something we got wrong, and it reads + as the agent's fault.""" + from fi.alk.harness.prove import prove + + root, _contract, catalogue = _built_environment(tmp_path) + scenario = Scenario.model_validate( + _delta( + ready_code=( + "def ready(world):\n" + " rows = world.state()['cart']\n" + " return None if rows else 'the cart is empty; this scenario needs one item'\n" + ) + ) + ) + proof = prove(scenario, catalogue, root) + assert not proof.ready + assert not proof.holds + assert "the cart is empty" in proof.why() + assert "test us rather than the agent" in proof.why() + assert proof.gates() == {"ready": False, "solvable": False, "not_vacuous": False} + + +def test_setup_code_makes_the_world_the_scenario_presumes(tmp_path): + """setup runs, then ready confirms it worked, and only then is anything else asked.""" + from fi.alk.harness.prove import prove + + root, _contract, catalogue = _built_environment(tmp_path) + scenario = Scenario.model_validate( + _delta( + setup_code=( + "def setup(world):\n" + " world.connection.execute(\"INSERT INTO menu (id) VALUES ('sushi')\")\n" + " world.connection.commit()\n" + ), + ready_code=( + "def ready(world):\n" + " ids = [r['id'] for r in world.state()['menu']]\n" + " return None if 'sushi' in ids else 'sushi was never added to the menu'\n" + ), + ) + ) + proof = prove(scenario, catalogue, root) + assert proof.ready and proof.holds, proof.why() + + +def test_the_setups_own_calls_are_not_credited_to_the_agent(tmp_path): + """A check that counts calls must not see the ones the scenario made on its own behalf.""" + from fi.alk.harness.prove import prepared + + root, _contract, _catalogue = _built_environment(tmp_path) + scenario = Scenario.model_validate( + _delta( + setup_code=( + "def setup(world):\n" + " world.call('add', {'item_id': 'big_mac'})\n" + ) + ) + ) + world, applied, ready = prepared(scenario, root) + try: + assert applied.ok and ready.ok + assert len(world.state()["cart"]) == 1, "the setup should have acted" + assert world.calls == [], "but its calls are not the agent's" + finally: + world.close() + + +def test_broken_setup_is_ours_and_says_so(tmp_path): + from fi.alk.harness.prove import prove + + root, _contract, catalogue = _built_environment(tmp_path) + scenario = Scenario.model_validate(_delta(setup_code="def setup(world):\n world.nope()\n")) + proof = prove(scenario, catalogue, root) + assert not proof.ready + assert proof.broken, "a setup that raises is our mistake, not a failing scenario" + assert "AttributeError" in proof.why_not_ready + + + + +def test_a_kept_scenario_becomes_a_folder_of_files(tmp_path): + """The files are the artifact, not a rendering of one. Something you can open and run is + something you can argue with.""" + from fi.alk.harness.folder import folder_for, read_folder + from fi.alk.harness.scenario_tools import write_scenarios + + root, _contract, catalogue = _built_environment(tmp_path) + scenario = Scenario.model_validate( + _delta( + setup_code="def setup(world):\n pass\n", + ready_code="def ready(world):\n return None\n", + ) + ) + index = write_scenarios([scenario], root, catalogue) + + here = folder_for(root, scenario.name) + assert (here / "scenario.json").exists() + assert (here / "setup.py").exists() + assert (here / "ready.py").exists() + # One file per deterministic sub-goal; the judged one has no check to write. + assert sorted(p.name for p in (here / "checks").iterdir()) == [ + "item-added.py", + "right-item.py", + ] + assert index.name == "scenarios.json" + + # The code lives in the files, not duplicated into the JSON, so the two cannot drift. + body = json.loads((here / "scenario.json").read_text()) + assert "setup_code" not in body and "ready_code" not in body + + # And it reads back whole. + again = read_folder(root, scenario.name) + assert again is not None + assert again.setup_code.strip() == "def setup(world):\n pass" + assert again.solution == scenario.solution + + +def test_a_check_file_runs_on_its_own_and_agrees_with_the_harness(tmp_path): + """The same file, the same answer, whether the harness runs it or a person does. If those + two could disagree, neither could be trusted.""" + import subprocess + import sys + + from fi.alk.harness.folder import folder_for, write_folder + from fi.alk.harness.prove import prepared + + root, _contract, catalogue = _built_environment(tmp_path) + scenario = Scenario.model_validate(_delta()) + write_folder(scenario, catalogue, root) + + # Leave the world in the state a passing run would have left it in. + world, _applied, _ready = prepared(scenario, root) + try: + world.call("add", {"item_id": "big_mac"}) + finally: + world.close() + + check_file = folder_for(root, scenario.name) / "checks" / "item-added.py" + done = subprocess.run( + [sys.executable, str(check_file), str(root / "world.sqlite")], + capture_output=True, + text=True, + timeout=60, + ) + # The world on disk is the base world, which has an empty cart, so this check should fail — + # and the point is that it says so rather than erroring. + assert done.returncode in (0, 1), done.stderr[-400:] + assert "held" in done.stdout or "FAILED" in done.stdout, done.stdout + done.stderr[-300:] + + +def test_every_stage_is_told_what_the_harness_is_for(): + """A stage that knows only its own step does its step well and still gets the point of it + wrong: it works around a gate instead of fixing what the gate named, or it reports a number + that quietly skipped half its checks.""" + for stage in ("understand-agent", "build-environment", "write-scenarios", "run-scenarios"): + text = load_skill(stage) + assert text.startswith("# The harness"), stage + assert "# The stage you are in now" in text, stage + # the ideas a stage must not be able to miss + assert "Code decides what is true" in text, stage + assert "refusal" in text and "crash" in text, stage From 5c932f903e316dd5fea52db7f8f657649208721e Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Mon, 17 Aug 2026 13:29:08 +0530 Subject: [PATCH 12/39] feat(harness-ui): scenarios show their gates and their files; contract shows dependencies --- .../alk/harness/skills/run-scenarios/SKILL.md | 72 ++++++++++--------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/src/fi/alk/harness/skills/run-scenarios/SKILL.md b/src/fi/alk/harness/skills/run-scenarios/SKILL.md index 10710aa..10f4a17 100644 --- a/src/fi/alk/harness/skills/run-scenarios/SKILL.md +++ b/src/fi/alk/harness/skills/run-scenarios/SKILL.md @@ -1,56 +1,62 @@ --- name: run-scenarios -description: Run the written scenarios against the real agent and say what the results mean. +description: Run the validated scenarios against the agent and say what the results mean. --- # Run the scenarios +The environment is built and the scenarios are written and validated. Your job is to run them +against the agent and say what came back. + +Each run costs real money and takes time. Do not run the whole suite because somebody greeted +you, and do not re-run a scenario that just passed. + ## Talking -You are talking to a person, not running a script. Answer what they ask, briefly. Run what they -ask you to run. Keep replies short — they can see every tool you call and what it answered. +Answer what they ask, briefly. Run what they ask you to run. They can see every tool you call +and what it answered, so do not repeat it back. + +## Before the first run -Each call costs real money and takes minutes. Do not run the whole suite because somebody said -hello, and do not re-run a scenario that just passed. +`preflight` costs nothing and catches the failures that would otherwise arrive after the +expensive part — missing credentials, no way to reach a hosted agent. Run it once at the start. -## What happens when you run one +`list_scenarios` shows what can be run, what each one tests, and which of its sub-goals are +settled by code rather than left to a judge. -`run_scenario` does all of it: restores the world, applies the scenario's setup, stands up the -webhook, points the assistant's **own** tools at it, places the call through ALK's voice case, and -runs the sub-goals' checks against what the world holds afterwards plus the calls that were made. +## Running one -It blocks for several minutes. Run one at a time and read the result before starting the next. +`run_scenario` does all of it: restores the world, applies the scenario's setup, puts the agent +in front of it, and runs the checks against what is left behind plus every call the agent made. -`preflight` first, before the first call of a session. It costs nothing and catches the failures -that would otherwise arrive after the expensive part. +It blocks until the run is over. Run one at a time and read the result before starting the next. ## Reading a result -You get the sub-goals settled by code, the ones left to a judge, and **every tool call the agent -made, with its arguments and whether the world accepted it**. That last list is where the answer -usually is. +You are given each sub-goal and whether it held, and **every tool call the agent made, with its +arguments and whether the world accepted it**. That last list is usually where the answer is. + +Before reporting a failure as a finding about the agent, work out which of these it is: + +**The agent did the wrong thing.** A real finding. Say what it did and what it should have done. + +**The world wrongly refused.** Look at the arguments. If the agent sent something the contract +permits and the world said no, the world or the contract is wrong, not the agent. -Before you report a failure as a finding about the agent, ask which of these it is: +**The check is wrong.** The commonest one. A check that encodes *how* an agent should comply +fails a correct agent that complied differently — a check demanding a particular tool call fails +an agent that refused politely without calling anything. Check the outcome, not the route. -- **The agent did the wrong thing.** A real finding. Say what it did and what it should have done. -- **The world refused a call the agent was entitled to make.** Look at the arguments. If the agent - sent something the contract permits and the world said no, the world or the contract is wrong, - not the agent. -- **The check is wrong.** The commonest one. A sub-goal that encodes *how* an agent should comply - fails a correct agent that complied differently — a check that demands a refusal tool call fails - an agent that refused from its own prompt without calling anything. Check the outcome, not the - route. -- **The simulated caller did not do its job.** If the caller hung up before asking for what the - instruction said, the scenario never happened. That is a simulator prompt problem. +**The simulated person never asked.** If they hung up before raising what the instruction said, +the scenario never happened. That is a simulator problem, not a result. -A run where nothing reached the world says nothing about the agent. Report it as that, not as a -failure. +A run where nothing reached the world says nothing about the agent. Report it as that. ## What to say -Say what passed, what failed, and for each failure which of the four causes above it is. Where it -is ours, say what would fix it — the sub-goal to rewrite, the contract argument to correct — and -do not report it as a finding about the agent. +Say what passed, what failed, and for each failure which of those four it is. Where it is ours, +say what would fix it — the check to rewrite, the contract value to correct — and do not report +it as a finding about the agent. -Judged sub-goals are reported as judged and not counted. Say so rather than letting a `2/2` read -as though everything was checked. +Judged sub-goals are reported as judged. Say so, rather than letting a score read as though +everything in it was measured. From be0631f5d9f0408f5066feabd7ace5ed80c1cf72 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Mon, 17 Aug 2026 13:43:54 +0530 Subject: [PATCH 13/39] feat(harness): one conversation is one folder, with its chat history and free stage switching --- src/fi/alk/harness/chat.py | 43 ++++++ src/fi/alk/harness/config.py | 8 +- src/fi/alk/harness/sessions.py | 252 +++++++++++++++++++++++++++++++++ tests/test_harness.py | 102 ++++++++++++- 4 files changed, 401 insertions(+), 4 deletions(-) create mode 100644 src/fi/alk/harness/sessions.py diff --git a/src/fi/alk/harness/chat.py b/src/fi/alk/harness/chat.py index dcc2996..25831f0 100644 --- a/src/fi/alk/harness/chat.py +++ b/src/fi/alk/harness/chat.py @@ -284,6 +284,49 @@ async def _settle(self, on_event: Callable[..., Any] | None = None) -> None: self.out = self.out or artifact_dir(settled.name) await self.advance(on_event=on_event) + def reachable(self) -> dict[str, str]: + """Every stage, and why it can or cannot be opened right now. + + Stages are not a wizard. Coming back to correct a contract after the world is built is + the ordinary case, not an exception, so any stage whose input exists can be opened at + any time. What cannot be skipped is the input itself: there is nothing to build a world + from without a contract, and nothing to write scenarios against without a world. + """ + contract = self.contract is not None + # Every stage after the first works from the contract, so that is the first thing each + # of them needs; its own input is the second. + needs_contract = "needs a contract first" + why = { + RECEPTION: "", + UNDERSTAND: "" + if self.source is not None + else "cannot re-read the agent without knowing where its source lives", + BUILD: "" if contract else needs_contract, + SCENARIOS: "" + if contract and self.world_built + else (needs_contract if not contract else "needs a built environment first"), + RUN: "" + if contract and self.scenarios_written + else (needs_contract if not contract else "needs scenarios first"), + } + return why + + async def go_to( + self, stage_name: str, on_event: Callable[..., Any] | None = None + ) -> str: + """Open one stage by name, whether or not it is the next one. + + The stage is opened but not set going: its opening message is an instruction to do that + stage's work, and somebody choosing to look at a stage has not thereby asked for it to + start spending. + """ + if stage_name not in _NEXT and stage_name != DONE: + raise RuntimeError(f"no stage called {stage_name!r}") + blocked = self.reachable().get(stage_name, "") + if blocked: + raise RuntimeError(f"cannot open the {stage_name} stage: {blocked}") + return await self._open(stage_name) + async def advance(self, on_event: Callable[..., Any] | None = None) -> str | None: """Move to the next stage and start it. Returns the stage entered, or None.""" following = self.next_stage() diff --git a/src/fi/alk/harness/config.py b/src/fi/alk/harness/config.py index 69e4f7e..b9b8220 100644 --- a/src/fi/alk/harness/config.py +++ b/src/fi/alk/harness/config.py @@ -181,8 +181,12 @@ async def gate(tool_name: str, payload: dict[str, Any], context: Any) -> Any: def artifact_dir(agent: str, root: str | Path | None = None) -> Path: - """Where a given agent's generated environment lives.""" - base = Path(root) if root else Path("artifacts/environments") + """The folder holding one conversation: its contract, world, scenarios and runs. + + One conversation, one directory. Everything about testing one agent lives together, which is + what makes a session something you can close, reopen, hand over or delete as one thing. + """ + base = Path(root) if root else Path("artifacts/sessions") return base / agent diff --git a/src/fi/alk/harness/sessions.py b/src/fi/alk/harness/sessions.py new file mode 100644 index 0000000..16b6cae --- /dev/null +++ b/src/fi/alk/harness/sessions.py @@ -0,0 +1,252 @@ +"""One conversation, one folder. + +Everything about testing one agent lives in a single directory: what the agent is, the world +built for it, the scenarios written against that world, what happened when they ran, and the +conversation that produced all of it. + +That is the whole state model. There is nothing held in memory that is not also on disk, so +closing the page, restarting the server or coming back tomorrow all resume the same way — by +reading the folder. A session that only existed in a process would be a session you could lose +by refreshing. + + artifacts/sessions// + session.json what this is: the agent, where its source lives, when it started + chat.jsonl the conversation, one message per line + contract.json stage 1 + world.sqlite stage 2, with handlers/, simulator_prompt.md, sub_goals.json + scenarios// stage 3, one folder each + runs.json stage 4 + +The id is readable and unique: the agent's name with a short suffix, so two attempts at the same +agent are two sessions rather than one overwriting the other. +""" + +from __future__ import annotations + +import json +import re +import secrets +import shutil +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +SESSIONS = Path("artifacts/sessions") +META = "session.json" +CHAT = "chat.jsonl" + + +def _slug(text: str) -> str: + cleaned = re.sub(r"[^a-z0-9]+", "-", (text or "session").lower()).strip("-") + return cleaned[:32] or "session" + + +def root(base: Path | None = None) -> Path: + return Path(base) if base else SESSIONS + + +def new_id(agent: str = "", base: Path | None = None) -> str: + """A readable, unique id. Two goes at the same agent are two sessions, not one clobbered.""" + stem = _slug(agent) + while True: + candidate = f"{stem}-{secrets.token_hex(3)}" + if not (root(base) / candidate).exists(): + return candidate + + +@dataclass +class Session: + """One conversation's folder, and what is in it.""" + + id: str + path: Path + agent: str = "" + source: str = "" + kind: str = "repo" + created: float = 0.0 + updated: float = 0.0 + stage: str = "" + title: str = "" + + def meta(self) -> dict[str, Any]: + return { + "id": self.id, + "agent": self.agent, + "source": self.source, + "kind": self.kind, + "created": self.created, + "updated": self.updated, + "stage": self.stage, + "title": self.title, + } + + def has(self) -> dict[str, Any]: + """What this session has actually produced, read from the folder rather than remembered. + + Asking the folder means the answer survives a restart, and it cannot drift from what is + really there — which is what makes reopening a session trustworthy. + """ + from .environment import load_catalogue + from .folder import read_all + + scenarios = read_all(self.path) if self.path.exists() else [] + runs = _runs(self.path) + return { + "contract": (self.path / "contract.json").exists(), + "world": (self.path / "world.sqlite").exists(), + "simulator_prompt": (self.path / "simulator_prompt.md").exists(), + "sub_goals": len(load_catalogue(self.path).sub_goals) if self.path.exists() else 0, + "scenarios": len(scenarios), + "validated": None, # filled in by whoever wants to pay for proving them + "runs": len(runs), + "runs_passed": sum(1 for one in runs if one.get("passed")), + "messages": count_messages(self.path), + } + + +def _runs(path: Path) -> list[dict[str, Any]]: + found = path / "runs.json" + if not found.exists(): + return [] + try: + loaded = json.loads(found.read_text(encoding="utf-8")) + return loaded if isinstance(loaded, list) else [] + except json.JSONDecodeError: + return [] + + +def create(agent: str = "", source: str = "", kind: str = "repo", base: Path | None = None) -> Session: + """Start a new conversation, with its own folder.""" + identifier = new_id(agent, base) + path = root(base) / identifier + path.mkdir(parents=True, exist_ok=True) + now = time.time() + session = Session( + id=identifier, + path=path, + agent=agent, + source=source, + kind=kind, + created=now, + updated=now, + stage="reception", + title=agent or "new session", + ) + save(session) + return session + + +def save(session: Session) -> None: + session.updated = time.time() + session.path.mkdir(parents=True, exist_ok=True) + (session.path / META).write_text( + json.dumps(session.meta(), indent=2, ensure_ascii=False), encoding="utf-8" + ) + + +def load(identifier: str, base: Path | None = None) -> Session | None: + path = root(base) / identifier + if not path.is_dir(): + return None + body: dict[str, Any] = {} + found = path / META + if found.exists(): + try: + body = json.loads(found.read_text(encoding="utf-8")) + except json.JSONDecodeError: + body = {} + return Session( + id=identifier, + path=path, + agent=str(body.get("agent") or ""), + source=str(body.get("source") or ""), + kind=str(body.get("kind") or "repo"), + created=float(body.get("created") or path.stat().st_ctime), + updated=float(body.get("updated") or path.stat().st_mtime), + stage=str(body.get("stage") or ""), + title=str(body.get("title") or identifier), + ) + + +def every(base: Path | None = None) -> list[Session]: + """Every session, newest first.""" + here = root(base) + if not here.exists(): + return [] + found = [load(one.name, base) for one in here.iterdir() if one.is_dir()] + return sorted((one for one in found if one), key=lambda s: s.updated, reverse=True) + + +def remove(identifier: str, base: Path | None = None) -> bool: + """Delete a session and everything in it. + + Deliberately narrow: it will only remove a directory that sits directly inside the sessions + root and holds a session file, so a mistyped id can never take anything else with it. + """ + here = (root(base) / identifier).resolve() + parent = root(base).resolve() + if here.parent != parent or not here.is_dir(): + return False + if not (here / META).exists(): + return False + shutil.rmtree(here) + return True + + +# -- the conversation itself -------------------------------------------------------- + + +@dataclass +class Message: + """One thing said, by either side.""" + + role: str # "you" or "harness" + text: str = "" + stage: str = "" + at: float = 0.0 + # What the harness did while answering, so a reopened conversation shows the work and not + # only the conclusion. + tools: list[dict[str, Any]] = field(default_factory=list) + + def body(self) -> dict[str, Any]: + return { + "role": self.role, + "text": self.text, + "stage": self.stage, + "at": self.at or time.time(), + "tools": self.tools, + } + + +def remember(path: Path, message: Message) -> None: + """Append one message to this session's conversation.""" + path.mkdir(parents=True, exist_ok=True) + with (path / CHAT).open("a", encoding="utf-8") as file: + file.write(json.dumps(message.body(), ensure_ascii=False) + "\n") + + +def history(path: Path) -> list[dict[str, Any]]: + """The whole conversation, in order. + + A line that will not parse is skipped rather than taking the rest with it: a half-written + line at the end is the ordinary result of a process being killed mid-write, and losing the + conversation because of it would be absurd. + """ + found = Path(path) / CHAT + if not found.exists(): + return [] + messages: list[dict[str, Any]] = [] + for line in found.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + try: + messages.append(json.loads(line)) + except json.JSONDecodeError: + continue + return messages + + +def count_messages(path: Path) -> int: + return len(history(path)) diff --git a/tests/test_harness.py b/tests/test_harness.py index 8ad070e..a5c3a82 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -332,7 +332,7 @@ def test_the_skill_exists_and_forbids_guessing(): def test_artifacts_land_under_the_agent_name(): - assert artifact_dir("drive_thru").as_posix().endswith("environments/drive_thru") + assert artifact_dir("drive_thru").as_posix().endswith("sessions/drive_thru") def test_cli_defaults_to_staying_open_for_corrections(): @@ -477,7 +477,7 @@ async def _settle(): conversation.out = conversation.out or artifact_dir(settled.name) asyncio.run(_settle()) - assert conversation.out.as_posix().endswith("environments/mine") + assert conversation.out.as_posix().endswith("sessions/mine") assert conversation._resume_at() == UNDERSTAND @@ -1884,3 +1884,101 @@ def test_every_stage_is_told_what_the_harness_is_for(): # the ideas a stage must not be able to miss assert "Code decides what is true" in text, stage assert "refusal" in text and "crash" in text, stage + + +# --- sessions: one conversation, one folder ------------------------------------------- + + +def test_a_session_is_a_folder_that_knows_what_it_holds(tmp_path): + """Nothing is held in memory that is not also on disk, so closing the page, restarting the + server or coming back tomorrow all resume by reading the folder.""" + from fi.alk.harness import sessions + + one = sessions.create(agent="drive_thru", source="/somewhere/agent", base=tmp_path) + assert one.id.startswith("drive-thru-") + assert (one.path / "session.json").exists() + + has = one.has() + assert has == { + "contract": False, "world": False, "simulator_prompt": False, + "sub_goals": 0, "scenarios": 0, "validated": None, + "runs": 0, "runs_passed": 0, "messages": 0, + } + + again = sessions.load(one.id, tmp_path) + assert again is not None + assert again.agent == "drive_thru" and again.source == "/somewhere/agent" + + +def test_two_goes_at_the_same_agent_are_two_sessions(tmp_path): + from fi.alk.harness import sessions + + first = sessions.create(agent="same", base=tmp_path) + second = sessions.create(agent="same", base=tmp_path) + assert first.id != second.id + assert {one.id for one in sessions.every(tmp_path)} == {first.id, second.id} + + +def test_the_conversation_is_kept_in_the_session_folder(tmp_path): + """A refresh must not lose what was said.""" + from fi.alk.harness import sessions + + one = sessions.create(agent="talky", base=tmp_path) + sessions.remember(one.path, sessions.Message(role="you", text="hello", stage="reception")) + sessions.remember( + one.path, + sessions.Message( + role="harness", text="hi", stage="reception", + tools=[{"label": "point at agent", "said": ["Pointed at talky"]}], + ), + ) + said = sessions.history(one.path) + assert [m["role"] for m in said] == ["you", "harness"] + assert said[1]["tools"][0]["label"] == "point at agent" + assert one.has()["messages"] == 2 + + # A half-written final line is what a killed process leaves; it must not take the rest. + with (one.path / "chat.jsonl").open("a", encoding="utf-8") as file: + file.write('{"role": "you", "text": "cut off') + assert len(sessions.history(one.path)) == 2 + + +def test_deleting_a_session_will_not_reach_outside_the_sessions_root(tmp_path): + """A mistyped id must never take anything else with it.""" + from fi.alk.harness import sessions + + one = sessions.create(agent="doomed", base=tmp_path) + outsider = tmp_path.parent / "not-a-session" + outsider.mkdir(exist_ok=True) + + assert sessions.remove("../not-a-session", tmp_path) is False + assert outsider.exists() + assert sessions.remove("no-such-session", tmp_path) is False + + assert sessions.remove(one.id, tmp_path) is True + assert not one.path.exists() + + +def test_any_stage_whose_input_exists_can_be_opened(tmp_path): + """Stages are not a wizard. Coming back to correct a contract after the world is built is + the ordinary case, so what cannot be skipped is the input, not the order.""" + from fi.alk.harness.chat import Conversation + from fi.alk.harness.tools import accept_contract + + empty = Conversation(out=tmp_path) + blocked = empty.reachable() + assert blocked["reception"] == "" + assert "where its source lives" in blocked["understand"] + assert "needs a contract" in blocked["build"] + # Without a contract, every later stage says so — not "needs a world", which would send + # somebody to build one against nothing. + assert "needs a contract" in blocked["scenarios"] + assert "needs a contract" in blocked["run"] + + root, contract, _catalogue = _built_environment(tmp_path / "built") + accept_contract(contract.model_dump(), root) + ready = Conversation(out=root) + open_now = ready.reachable() + assert open_now["build"] == "", open_now + assert open_now["scenarios"] == "", open_now + assert "needs scenarios" in open_now["run"] From eb941a9d0744d7e5c2962742ae5655c607a20ee0 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Mon, 17 Aug 2026 13:45:42 +0530 Subject: [PATCH 14/39] chore(harness-ui): track the chat UI in the repo --- .gitignore | 1 - harness-ui/README.md | 68 ++ harness-ui/server.py | 565 ++++++++++++++ harness-ui/static/index.html | 1387 ++++++++++++++++++++++++++++++++++ 4 files changed, 2020 insertions(+), 1 deletion(-) create mode 100644 harness-ui/README.md create mode 100644 harness-ui/server.py create mode 100644 harness-ui/static/index.html diff --git a/.gitignore b/.gitignore index 1a57868..bee5e2a 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,3 @@ artifacts/ !src/fi/simulate/artifacts/ !src/fi/simulate/artifacts/*.py examples/artifacts/ -harness-ui/ diff --git a/harness-ui/README.md b/harness-ui/README.md new file mode 100644 index 0000000..1d307cf --- /dev/null +++ b/harness-ui/README.md @@ -0,0 +1,68 @@ +# The harness, as a chat + +A web page you talk to. Same harness, same stages, same artifacts as the CLI — this is a second +renderer over the event stream the stages already emit, not a second implementation. + +## Running it + +From the repo root, with the same environment the CLI needs: + +```bash +cd path/to/agent-learning-kit +set -a; . ./.env.acceptance; set +a +export CLOUD_ML_REGION=global ALK_HARNESS_MODEL=claude-haiku-4-5 + +.venv/bin/python harness-ui/server.py +``` + +Then open **http://localhost:8777**. + +It prints the model and which credentials it found before it starts, so a run never begins on +something you did not intend. + +## What you can do in it + +- **Pick an agent** from the dropdown, or start a new one by saying where its code lives. Agents + that already have artifacts reopen where they left off — you do not point at the repository + again to fix a scenario. +- **Talk.** "build the world", "write 5 hard scenarios", "make that one harder", "add a mango + smoothie to the menu". Each reply shows the work underneath it: which tool ran, what it + answered, what it refused. +- **Press enter on an empty box** (or the `next stage →` chip) to move on once a stage has + produced its artifact. +- **Run the scenarios.** The conversation between the simulated customer and the agent streams + into the chat as it happens, then a verdict card lands with the checkpoints. +- **Watch the right-hand side.** Contract, World, Scenarios and Runs are the four artifacts on + disk; the pane refreshes whenever a stage writes one. + +## The two files + +| File | What it is | +|---|---| +| `server.py` | FastAPI. Holds one `Conversation` open, streams its events as server-sent events, and serves the artifacts as JSON. | +| `static/index.html` | The whole interface — markup, styling and rendering in one file. No build step, no npm. | + +To restyle it, edit the ` + + +
+ +
+
+ + + +
+ +
+
+ +
+
+
+
+
+
+ + + +
+
+
+
+ +
+
+
+
+
+
+ + + + From deda8505d51e575ce078bb28cb286d849f7fa565 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Mon, 17 Aug 2026 13:50:01 +0530 Subject: [PATCH 15/39] fix(harness): reception can hand over in the turn that finds the agent --- src/fi/alk/harness/chat.py | 8 +++++++- tests/test_harness.py | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/fi/alk/harness/chat.py b/src/fi/alk/harness/chat.py index 25831f0..ff555c7 100644 --- a/src/fi/alk/harness/chat.py +++ b/src/fi/alk/harness/chat.py @@ -91,7 +91,13 @@ def _artifact_for(self, stage_name: str) -> bool: return { # A contract already on disk settles which agent this is just as well as being told, # so coming back to an agent does not mean pointing at its repository again. - RECEPTION: self.source is not None or self.contract is not None, + # + # ``_found`` is checked too, because within the turn that points at an agent the + # source is not on the conversation yet — it is read off afterwards. Without it, the + # stage that has just succeeded is told it has produced nothing. + RECEPTION: self.source is not None + or self.contract is not None + or self._found.get("source") is not None, UNDERSTAND: self.contract is not None, BUILD: self.world_built, SCENARIOS: self.scenarios_written, diff --git a/tests/test_harness.py b/tests/test_harness.py index a5c3a82..5863c91 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -1982,3 +1982,19 @@ def test_any_stage_whose_input_exists_can_be_opened(tmp_path): assert open_now["build"] == "", open_now assert open_now["scenarios"] == "", open_now assert "needs scenarios" in open_now["run"] + + +def test_reception_can_hand_over_in_the_turn_that_finds_the_agent(tmp_path): + """The source is read off the reception stage after its turn ends, so within that turn the + conversation does not know it yet. Without allowing for that, the stage that has just + succeeded is told it has produced nothing and the handoff is refused.""" + from fi.alk.harness.chat import Conversation + from fi.alk.harness.sources import RepoSource + + conversation = Conversation(out=tmp_path) + conversation.stage_name = "reception" + assert conversation.next_stage() is None, "nothing pointed at yet" + + # what point_at_agent does, mid-turn + conversation._found["source"] = RepoSource(name="x", root=tmp_path) + assert conversation.next_stage() == "understand" From 644cc25e0013c879df7027a3ef705b132267029d Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Mon, 17 Aug 2026 13:56:21 +0530 Subject: [PATCH 16/39] docs(build-skill): document the three db methods a handler can call --- .../harness/skills/build-environment/SKILL.md | 19 ++++++++++++++++--- tests/test_harness.py | 19 +++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/fi/alk/harness/skills/build-environment/SKILL.md b/src/fi/alk/harness/skills/build-environment/SKILL.md index 8ff1a98..e96ef39 100644 --- a/src/fi/alk/harness/skills/build-environment/SKILL.md +++ b/src/fi/alk/harness/skills/build-environment/SKILL.md @@ -64,9 +64,22 @@ was wrong. your bugs; `ToolError` is the world's answer, and the two are recorded differently. Inside a handler you have `args`, `db`, `ToolError` and `json`, and nothing else. Do not import -anything and do not define your own `ToolError`. Use the argument names exactly as the contract -gives them. A handler that reads a name the tool does not pass finds nothing, quietly does -nothing, and reports success. +anything and do not define your own `ToolError`. + +`db` has exactly three methods, and no cursors: + +```python +db.query("SELECT * FROM items WHERE id = ?", [args["item_id"]]) # -> list of dicts, [] if none +db.one("SELECT * FROM items WHERE id = ?", [args["item_id"]]) # -> one dict, or None +db.execute("INSERT INTO orders (item_id) VALUES (?)", [item_id]) # -> number of rows changed +``` + +Rows come back as dicts, so read them by column name. There is nothing to fetch afterwards: +`db.execute` returns a count, not a cursor, so calling `.fetchone()` on anything is a mistake. +Use `db.one` when you want a single row and `db.query` when you want several. + +Use the argument names exactly as the contract gives them. A handler that reads a name the tool +does not pass finds nothing, quietly does nothing, and reports success. ## Seeding diff --git a/tests/test_harness.py b/tests/test_harness.py index 5863c91..0969ec4 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -1998,3 +1998,22 @@ def test_reception_can_hand_over_in_the_turn_that_finds_the_agent(tmp_path): # what point_at_agent does, mid-turn conversation._found["source"] = RepoSource(name="x", root=tmp_path) assert conversation.next_stage() == "understand" + + +def test_the_build_skill_documents_every_method_a_handler_can_call(): + """A handler gets `db` and nothing else, so if the skill does not say what `db` offers the + model guesses — and the guess is sqlite's cursor API, which fails on the smoke call.""" + import inspect + + from fi.alk.harness.world.runtime import Db + + skill = load_skill("build-environment") + methods = [ + name for name, _ in inspect.getmembers(Db, inspect.isfunction) + if not name.startswith("_") + ] + assert methods, "Db should have methods to document" + for name in methods: + assert f"db.{name}(" in skill, f"the build skill never shows db.{name}()" + # and it warns off the API the model actually reaches for by default + assert "fetchone" in skill From cbd8df1ea98665f92142f6e90bc765e323ef41a0 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Mon, 17 Aug 2026 13:58:15 +0530 Subject: [PATCH 17/39] fix(build-tools): a crashed handler is told what a handler actually has --- src/fi/alk/harness/world/tools.py | 19 +++++++++++++++- tests/test_harness.py | 37 +++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/src/fi/alk/harness/world/tools.py b/src/fi/alk/harness/world/tools.py index de211c0..3f8916b 100644 --- a/src/fi/alk/harness/world/tools.py +++ b/src/fi/alk/harness/world/tools.py @@ -41,6 +41,20 @@ WORLD_SERVER = "world" +# What a handler is actually given. Said again here, and not only in the skill, because this is +# where the mistake surfaces: a handler that crashed has a model reading *this* message, and an +# error naming the failure without naming the API produces the same wrong guess again. Three +# identical attempts at one handler is what that costs. +DB_API = ( + "Inside a handler, `db` has exactly three methods and no cursors:\n" + ' db.query("SELECT * FROM t WHERE id = ?", [x]) -> list of dicts, [] if none\n' + ' db.one("SELECT * FROM t WHERE id = ?", [x]) -> one dict, or None\n' + ' db.execute("INSERT INTO t (a) VALUES (?)", [x]) -> number of rows changed\n' + "Rows are dicts, read by column name. db.execute returns a count, not a cursor, so " + "calling .fetchone(), .fetchall() or .lastrowid on any of these is a mistake. You also have " + "`args`, `ToolError` and `json`, and nothing else — do not import anything." +) + # Below this, the world is not good enough to build tests on. Synthesis work that measures this # converges on roughly this bar, and rejects a quarter to a third of what it generates. ACCEPTABLE = 0.85 @@ -165,7 +179,10 @@ async def define_handler(args: dict[str, Any]) -> dict[str, Any]: ) if not call.ok: del world.handlers[name] - return _err(f"{name} not kept, it crashed on its smoke call: {call.error}") + said = f"{name} not kept, it crashed on its smoke call: {call.error}" + # A crash is nearly always the handler reaching for something it does not have, so + # the answer says what it does have rather than only what went wrong. + return _err(f"{said}\n\n{DB_API}") return _ok(f"{name} defined and ran. Returned {_brief(call.result)}") @tool( diff --git a/tests/test_harness.py b/tests/test_harness.py index 0969ec4..4db1ffb 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -2017,3 +2017,40 @@ def test_the_build_skill_documents_every_method_a_handler_can_call(): assert f"db.{name}(" in skill, f"the build skill never shows db.{name}()" # and it warns off the API the model actually reaches for by default assert "fetchone" in skill + + +def test_a_crashed_handler_is_told_what_a_handler_actually_has(tmp_path): + """An error naming the failure without naming the API produces the same wrong guess again. + Three identical attempts at one handler is what that cost on a real run.""" + import asyncio + + from mcp.types import CallToolRequest, CallToolRequestParams + + from fi.alk.harness.world import tools as world_tools + + root, contract = _saved_world(tmp_path) + server, _world = world_tools.world_tools(contract, root) + instance = server.get("instance") if isinstance(server, dict) else server + + async def define(source): + for key, handler in instance.request_handlers.items(): + if getattr(key, "__name__", "") == "CallToolRequest": + answer = await handler( + CallToolRequest( + method="tools/call", + params=CallToolRequestParams( + name="define_handler", + arguments={"tool_name": "add", "source": source}, + ), + ) + ) + return answer.root.content[0].text + + # the mistake a model actually makes: sqlite's cursor API + said = asyncio.run(define( + "def handle(args, db):\n" + " return db.execute('SELECT 1').fetchone()\n" + )) + assert "crashed on its smoke call" in said + assert "db.query(" in said and "db.one(" in said and "db.execute(" in said + assert "fetchone" in said From 212d7096702d4f225c095fd27feb943cf71ac81b Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Mon, 17 Aug 2026 14:33:15 +0530 Subject: [PATCH 18/39] fix(harness): the turn that finds the agent can open the next stage --- src/fi/alk/harness/chat.py | 24 ++++++++++----- tests/test_harness.py | 60 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 7 deletions(-) diff --git a/src/fi/alk/harness/chat.py b/src/fi/alk/harness/chat.py index ff555c7..e2a2922 100644 --- a/src/fi/alk/harness/chat.py +++ b/src/fi/alk/harness/chat.py @@ -263,6 +263,12 @@ async def say( if self.stage is None: await self.open_quietly() await self.stage.say(message, on_event=on_event) # type: ignore[union-attr] + # Before anything acts on this turn, take up what it established. A handoff in the same + # turn opens the next stage, and every stage is built from ``self.source``; read it off + # afterwards instead and that hop dies on an agent nobody has named, taking the turn with + # it and leaving the conversation in reception with no way forward. + established = self._take_up() + moved = False # A handoff moves the request, not just the conversation: the next stage opens and is # given the person's own words. Bounded, because each hop is a model turn. for _hop in range(3): @@ -273,22 +279,26 @@ async def say( if following is None: break await self._open(following) + moved = True await self.stage.say(request, on_event=on_event) # type: ignore[union-attr] - await self._settle(on_event=on_event) + if established and not moved: + # Nothing is left to decide once the agent is known, so it goes on rather than making + # somebody confirm what they already said. Unless a handoff already moved us, which + # would make this a second hop over the same request. + await self.advance(on_event=on_event) - async def _settle(self, on_event: Callable[..., Any] | None = None) -> None: - """Take up whatever the open stage just established, and keep going. + def _take_up(self) -> bool: + """Take up whatever the turn just established. True if this turn named the agent. Reception is the only stage whose result is not a file, so it is the only one the - conversation has to read back. Once it knows the agent there is nothing to decide, so it - goes straight on rather than making somebody confirm what they already said. + conversation has to read back. """ settled = self._found.pop("source", None) if settled is None: - return + return False self.source = settled self.out = self.out or artifact_dir(settled.name) - await self.advance(on_event=on_event) + return True def reachable(self) -> dict[str, str]: """Every stage, and why it can or cannot be opened right now. diff --git a/tests/test_harness.py b/tests/test_harness.py index 4db1ffb..3ebcd14 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -2000,6 +2000,66 @@ def test_reception_can_hand_over_in_the_turn_that_finds_the_agent(tmp_path): assert conversation.next_stage() == "understand" +def test_the_turn_that_finds_the_agent_also_opens_the_next_stage(tmp_path, monkeypatch): + """Allowing that handoff is not enough: the stage it opens is built from the source, so the + source has to be on the conversation before the hop, not after it. Otherwise the hop raises, + the turn is lost, and the source is never taken up at all — every later message arrives back + at reception, which has no tools to do anything with it.""" + import asyncio + + from fi.alk.harness import chat as chat_module + from fi.alk.harness.chat import Conversation + from fi.alk.harness.sources import RepoSource + + said: list[str] = [] + + class Stage: + spent_usd = 0.0 + + def __init__(self, name: str) -> None: + self.name = name + + async def __aenter__(self): + return self + + async def __aexit__(self, *_): + return False + + async def say(self, message, on_event=None): + said.append(f"{self.name}: {message}") + + def grant(self, *_, **__): + pass + + found: dict = {} + monkeypatch.setattr( + chat_module.reception_stage, "open_stage", lambda **_: (Stage("reception"), found) + ) + monkeypatch.setattr(chat_module.reception_stage, "opening", lambda: "which agent") + monkeypatch.setattr( + chat_module.understand_stage, + "open_stage", + lambda *_, **__: (Stage("understand"), {}), + ) + monkeypatch.setattr(chat_module.understand_stage, "opening", lambda _: "read the agent") + + conversation = Conversation(out=tmp_path, workspace=tmp_path) + + async def turn(): + await conversation.open_quietly() + # what reception's turn does when one message both names the agent and asks for the next + # thing: it points, then hands the request on. + found["source"] = RepoSource(name="x", root=tmp_path) + conversation._handoff["request"] = "read it and tell me what it can do" + await conversation.say("test the voice agent at /x, and tell me what it can do") + + asyncio.run(turn()) + + assert conversation.source is not None, "the turn that pointed never landed" + assert conversation.stage_name == "understand" + assert any("read it and tell me what it can do" in one for one in said), said + + def test_the_build_skill_documents_every_method_a_handler_can_call(): """A handler gets `db` and nothing else, so if the skill does not say what `db` offers the model guesses — and the guess is sqlite's cursor API, which fails on the smoke call.""" From b439fac306a10f1864bc1e0ffa1a129aa05267ce Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Mon, 17 Aug 2026 14:33:15 +0530 Subject: [PATCH 19/39] fix(ui): keep paragraph breaks in a restored conversation --- harness-ui/server.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/harness-ui/server.py b/harness-ui/server.py index 787b946..153fbe6 100644 --- a/harness-ui/server.py +++ b/harness-ui/server.py @@ -356,7 +356,10 @@ def watch(event): current.path, sessions.Message( role="harness", - text="".join(spoken).strip(), + # Blank-line joined: one turn can speak several times, once per stage it passes + # through, and running those together reads as one garbled paragraph when the + # conversation is restored. + text="\n\n".join(one.strip() for one in spoken if one.strip()), stage=conversation.stage_name, tools=tools, ), From d1bd9495de9fd78fdea7da3b0b542b5d3a20c2cc Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Mon, 17 Aug 2026 14:33:15 +0530 Subject: [PATCH 20/39] fix(scenarios): a short suite is told how to record the number asked for --- src/fi/alk/harness/scenario_tools.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/fi/alk/harness/scenario_tools.py b/src/fi/alk/harness/scenario_tools.py index c7f299e..72399b3 100644 --- a/src/fi/alk/harness/scenario_tools.py +++ b/src/fi/alk/harness/scenario_tools.py @@ -108,7 +108,9 @@ def not_ready(kept: list[Scenario], wanted: int, catalogue: Catalogue) -> list[s if len(kept) < wanted: problems.append( f"{len(kept)} of {wanted} scenarios so far. Keep writing; the ones that find " - "something are usually the awkward ones." + f"something are usually the awkward ones. If nobody asked for {wanted}, record the " + "number you were actually given with aim_for first, not the number you happen to " + "have reached." ) elif len(kept) > wanted: problems.append( From 6edb6b65d796eb8d6ad95f02e0c6c666b9d42041 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Mon, 17 Aug 2026 14:34:58 +0530 Subject: [PATCH 21/39] feat(prove): name a check that holds when nothing was done --- src/fi/alk/harness/prove.py | 10 ++++++-- src/fi/alk/harness/scenario_tools.py | 11 ++++++++- tests/test_harness.py | 36 ++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/src/fi/alk/harness/prove.py b/src/fi/alk/harness/prove.py index 2ecaa55..b3b8fb0 100644 --- a/src/fi/alk/harness/prove.py +++ b/src/fi/alk/harness/prove.py @@ -45,6 +45,11 @@ class Proof: solvable: bool = False vacuous: bool = True why_not_ready: str = "" + # Checks that held with nothing done. The scenario is only vacuous when *every* check does + # that, but a single one still grades nothing, and since sub-goals are shared it will report + # itself as held for an agent that did nothing at all. Named rather than refused: on a + # scenario about a refusal, "no order was placed" holding on an untouched world is correct. + weak: list[str] = field(default_factory=list) with_solution: list[Outcome] = field(default_factory=list) with_nothing: list[Outcome] = field(default_factory=list) refused: list[str] = field(default_factory=list) @@ -193,7 +198,8 @@ def prove(scenario: Scenario, catalogue: Catalogue, world_root: Path) -> Proof: # Vacuous only if *every* check still passes with nothing done. One check that survives an # empty run is often legitimate — "no order was placed" is a real thing to assert about a # refusal scenario — but a whole set of them means nothing is being graded. - proof.vacuous = bool(proof.with_nothing) and all( - one.held for one in proof.with_nothing + proof.weak = [one.name for one in proof.with_nothing if one.held] + proof.vacuous = bool(proof.with_nothing) and len(proof.weak) == len( + proof.with_nothing ) return proof diff --git a/src/fi/alk/harness/scenario_tools.py b/src/fi/alk/harness/scenario_tools.py index 72399b3..6346663 100644 --- a/src/fi/alk/harness/scenario_tools.py +++ b/src/fi/alk/harness/scenario_tools.py @@ -95,10 +95,19 @@ def accept_scenario( replaced = any(one.name == scenario.name for one in kept) kept[:] = [one for one in kept if one.name != scenario.name] kept.append(scenario) + weak = ( + "\nWorth tightening: " + + ", ".join(proof.weak) + + " still held with nothing done. The scenario is graded by its other checks, so it was " + "kept, but those sub-goals will report themselves as held for an agent that did nothing. " + "A check that asserts the attempt, not only the state it leaves, cannot do that." + if proof.weak + else "" + ) return _ok( f"{scenario.name} {'replaced' if replaced else 'kept'}. All three gates pass: the world " "is ready for it, the reference solution passes its checks, and those checks fail when " - f"nothing is done.\n{len(kept)} so far: " + ", ".join(one.name for one in kept) + f"nothing is done.{weak}\n{len(kept)} so far: " + ", ".join(one.name for one in kept) ) diff --git a/tests/test_harness.py b/tests/test_harness.py index 3ebcd14..33e3ff0 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -1056,6 +1056,42 @@ def test_a_scenario_whose_checks_pass_with_nothing_done_is_refused(tmp_path): assert said["is_error"] and "grade nothing" in said["content"][0]["text"] +def test_a_check_that_cannot_fail_without_calls_is_named_even_though_it_is_kept(tmp_path): + """A check comparing calls against rows holds when there are no calls at all, so it reports + itself as held for an agent that did nothing. The scenario is still graded by its other + checks, so it is kept, but sub-goals are shared and that one would roll up as a pass.""" + from fi.alk.harness.environment import SubGoal, save_catalogue + from fi.alk.harness.prove import prove + from fi.alk.harness.scenario import Scenario + from fi.alk.harness.scenario_tools import accept_scenario + + root, _contract, catalogue = _built_environment(tmp_path) + catalogue.sub_goals.append( + SubGoal( + name="quantity-respected", + what="as many rows as there were calls", + check=( + "def check(world, calls):\n" + " made = [c for c in calls if c.name == 'add' and c.ok]\n" + " rows = world.state()['cart']\n" + " if len(rows) != len(made):\n" + " return '%d calls, %d rows' % (len(made), len(rows))\n" + " return None\n" + ), + ) + ) + save_catalogue(catalogue, root) + delta = _delta(sub_goals=["item-added", "quantity-respected"]) + said = accept_scenario(delta, world_root=root, catalogue=catalogue, kept=[]) + text = said["content"][0]["text"] + + assert not said.get("is_error"), text + assert "All three gates pass" in text + assert "quantity-respected" in text and "held with nothing done" in text + proof = prove(Scenario(**delta), catalogue, root) + assert proof.holds and proof.weak == ["quantity-respected"] + + def test_a_scenario_naming_a_sub_goal_nobody_defined_is_refused(tmp_path): from fi.alk.harness.scenario_tools import accept_scenario From 1207efa9fb035627cc544fc36be7c8ae3d1e1395 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Mon, 17 Aug 2026 15:07:16 +0530 Subject: [PATCH 22/39] build(deps): declare the harness UI's fastapi and uvicorn --- pyproject.toml | 11 +++++++++++ uv.lock | 28 +++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5283c57..0e938ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,6 +88,13 @@ a2a = [ ] nli = ["transformers>=5.2.0,<6", "torch>=2.10.0,<3"] embeddings = ["sentence-transformers>=5.2.3,<6"] +# What `harness-ui/server.py` needs to serve the harness over HTTP. Declared rather than left to +# whatever happens to be in the environment: uv sync removes anything undeclared, so an ad hoc +# install of these disappears the first time somebody syncs. +harness-ui = [ + "fastapi>=0.115,<1", + "uvicorn>=0.30,<1", +] feedback = ["chromadb>=0.4.0"] notebook = [ "ipykernel>=6", # kernel for examples/agent_learning_sdk_demo.ipynb @@ -150,6 +157,10 @@ dev = [ "hatchling>=1.25", "pytest>=8.3", "ruff>=0.9", + # Also in the harness-ui extra. Repeated here so that syncing without that extra does not + # uninstall the server out from under a working checkout. + "fastapi>=0.115,<1", + "uvicorn>=0.30,<1", ] [tool.pytest.ini_options] diff --git a/uv.lock b/uv.lock index 06a5b2d..edbc4fc 100644 --- a/uv.lock +++ b/uv.lock @@ -101,6 +101,10 @@ embeddings = [ feedback = [ { name = "chromadb" }, ] +harness-ui = [ + { name = "fastapi" }, + { name = "uvicorn" }, +] langchain = [ { name = "langchain-core" }, { name = "langgraph" }, @@ -137,9 +141,11 @@ trinity = [ [package.dev-dependencies] dev = [ { name = "build" }, + { name = "fastapi" }, { name = "hatchling" }, { name = "pytest" }, { name = "ruff" }, + { name = "uvicorn" }, ] [package.metadata] @@ -154,6 +160,7 @@ requires-dist = [ { name = "chromadb", marker = "extra == 'all'", specifier = ">=0.4.0" }, { name = "chromadb", marker = "extra == 'feedback'", specifier = ">=0.4.0" }, { name = "claude-agent-sdk", specifier = ">=0.2.139" }, + { name = "fastapi", marker = "extra == 'harness-ui'", specifier = ">=0.115,<1" }, { name = "fi-instrumentation-otel", specifier = ">=0.1.16" }, { name = "gepa", specifier = ">=0.0.17" }, { name = "httpx", specifier = ">=0.24.0" }, @@ -195,15 +202,18 @@ requires-dist = [ { name = "transformers", marker = "extra == 'all'", specifier = ">=5.2.0,<6" }, { name = "transformers", marker = "extra == 'nli'", specifier = ">=5.2.0,<6" }, { name = "typer", specifier = ">=0.9.0,<1.0.0" }, + { name = "uvicorn", marker = "extra == 'harness-ui'", specifier = ">=0.30,<1" }, ] -provides-extras = ["simulate", "evaluation", "optimize", "livekit", "langchain", "pipecat", "mcp", "a2a", "nli", "embeddings", "feedback", "notebook", "trinity", "all"] +provides-extras = ["simulate", "evaluation", "optimize", "livekit", "langchain", "pipecat", "mcp", "a2a", "nli", "embeddings", "harness-ui", "feedback", "notebook", "trinity", "all"] [package.metadata.requires-dev] dev = [ { name = "build", specifier = ">=1.5" }, + { name = "fastapi", specifier = ">=0.115,<1" }, { name = "hatchling", specifier = ">=1.25" }, { name = "pytest", specifier = ">=8.3" }, { name = "ruff", specifier = ">=0.9" }, + { name = "uvicorn", specifier = ">=0.30,<1" }, ] [[package]] @@ -1210,6 +1220,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl", hash = "sha256:760643d3452b4d777d295bb167ccc74c64a81df23fb5e08eff250c425a4b2017", size = 28317, upload-time = "2025-09-01T09:48:08.5Z" }, ] +[[package]] +name = "fastapi" +version = "0.141.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8a/02/91e3416a8fdd715abb903a952a6bec7cdd8d14eed55d415fc8595524c319/fastapi-0.141.1.tar.gz", hash = "sha256:e8822fc40db1e1858054d7a949a888695bc9bdce70139178e33bd2871a453ca1", size = 425799, upload-time = "2026-07-29T17:18:05.568Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/03/10388a42375ee7e4ac9b94eb2c5c569c8b5795e377e701c9ac3ad63de890/fastapi-0.141.1-py3-none-any.whl", hash = "sha256:bfb91aa2d334c61cb35ba9a116fc123b3d3df31640b801cf57a7a78ec3f603b3", size = 131954, upload-time = "2026-07-29T17:18:04.364Z" }, +] + [[package]] name = "fastjsonschema" version = "2.22.1" From 5207d20b2aada5971811e59eceed6bb0e3fe5a5b Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Mon, 17 Aug 2026 16:18:23 +0530 Subject: [PATCH 23/39] docs(harness): setup and run instructions, sessions, and the third gate --- README.md | 12 +++ harness-ui/README.md | 125 ++++++++++++++++++++++------ src/fi/alk/harness/DESIGN.md | 56 +++++++++---- src/fi/alk/harness/HOW-IT-WORKS.md | 64 ++++++++------- src/fi/alk/harness/README.md | 126 ++++++++++++++++++++++------- 5 files changed, 285 insertions(+), 98 deletions(-) diff --git a/README.md b/README.md index 1b5f00b..f82389d 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,18 @@ Use it when you want one reproducible loop: 4. Promote the result into a replayable artifact. 5. Prove release readiness with local gates. +### The harness: point it at an agent and talk to it + +`src/fi/alk/harness/` builds all of the above **for** an agent instead of asking you to write it. +Point it at an agent's source and it reads what that agent verifiably is, builds a real world its +tools act on, and writes test scenarios that are each proved before they are kept. It is driven +as a conversation, in a terminal or on a web page. + +- **[Start here](src/fi/alk/harness/README.md)**: setup from nothing, then how to use it +- **[The web page](harness-ui/README.md)**: the same harness as a chat, on `localhost:8777` +- **[How it works](src/fi/alk/harness/HOW-IT-WORKS.md)** and + **[why it is shaped this way](src/fi/alk/harness/DESIGN.md)** + OpenEnv/Gymnasium shapes are compatibility inputs, not the product center. Agent Learning Kit is the primary runtime and release contract, and the bar is the executable `environment_10x_robustness` release gate. diff --git a/harness-ui/README.md b/harness-ui/README.md index 1d307cf..8aec7ad 100644 --- a/harness-ui/README.md +++ b/harness-ui/README.md @@ -1,46 +1,113 @@ # The harness, as a chat -A web page you talk to. Same harness, same stages, same artifacts as the CLI — this is a second +A web page you talk to. Same harness, same stages, same artifacts as the CLI. This is a second renderer over the event stream the stages already emit, not a second implementation. -## Running it +There is no separate front end to build or start. The page is one static file this server hands +out on `/`, and it talks to the same server's JSON endpoints. No node, no npm, no build step. + +## Setting it up + +```bash +git clone https://github.com/future-agi/agent-learning-kit +cd agent-learning-kit + +uv sync --extra livekit --group dev +``` + +`--extra livekit` is required: the voice run path imports it. The UI's own `fastapi` and +`uvicorn` come in with `--group dev`, and are also available as `--extra harness-ui` if you would +rather not pull the dev tooling. + +You also need the `claude` command on your PATH (`npm install -g @anthropic-ai/claude-code`). +The harness talks to the model through the Claude Agent SDK, which runs that binary underneath; +without it every stage fails immediately. -From the repo root, with the same environment the CLI needs: +Credentials go through **Vertex AI**, not a plain Anthropic key: `config.provider_env` sets +`CLAUDE_CODE_USE_VERTEX=1`, so `ANTHROPIC_API_KEY` on its own will not work. Copy the template +and fill in your service account: + +```bash +cp oss/simulation-acceptance/.env.example .env.acceptance +# GOOGLE_APPLICATION_CREDENTIALS=/absolute/path/to/your-service-account.json +# GOOGLE_CLOUD_PROJECT=your-gcp-project-id +``` + +`.env.acceptance` is git-ignored and points at a private key. Never commit it or paste its +contents anywhere. + +Then load it and pick the model, in each new terminal: ```bash -cd path/to/agent-learning-kit set -a; . ./.env.acceptance; set +a -export CLOUD_ML_REGION=global ALK_HARNESS_MODEL=claude-haiku-4-5 +export CLOUD_ML_REGION=global +export ALK_HARNESS_MODEL=claude-sonnet-4-6 +``` + +It prints the model and which credentials it found before starting, so a run never begins on +something you did not intend. **Use Sonnet or better.** Haiku has misread an agent's modality, +and modality decides how every later test is run. +## Running it + +```bash .venv/bin/python harness-ui/server.py ``` -Then open **http://localhost:8777**. +Open **http://localhost:8777**, press **+ new**, and say what you want tested: + +``` +i want to test my voice ordering agent. the code is at /absolute/path/to/the/agent +``` -It prints the model and which credentials it found before it starts, so a run never begins on -something you did not intend. +One message is enough to begin. Reception takes the path out of the sentence and hands straight +over to reading the agent. From there it is a conversation: + +``` +now build the environment for it +write me 5 scenarios: a plain order, one for something you do not have, one where the +customer changes their mind, one that pushes against a rule, and one with quantity +``` + +Questions in between are answered without spending a stage, so "can it handle quantity?" is a +fair thing to ask mid-flight. + +To stop the server, Ctrl-C. Checking whether it is still up with `lsof -ti:8777` will mislead +you: that matches a browser's leftover sockets. Use `lsof -nP -iTCP:8777 -sTCP:LISTEN`. + +**Restart the server after changing anything under `src/fi/alk/harness/`.** A long-lived process +does not pick up code or skills on its own. ## What you can do in it -- **Pick an agent** from the dropdown, or start a new one by saying where its code lives. Agents - that already have artifacts reopen where they left off — you do not point at the repository - again to fix a scenario. +- **Start, reopen or delete a conversation** from the picker. Everything about one conversation + lives in its own folder under `artifacts/sessions//`, so reopening it restores the chat and + every artifact. A blank slate is `rm -rf artifacts/sessions/* artifacts/.open-session`. - **Talk.** "build the world", "write 5 hard scenarios", "make that one harder", "add a mango smoothie to the menu". Each reply shows the work underneath it: which tool ran, what it answered, what it refused. -- **Press enter on an empty box** (or the `next stage →` chip) to move on once a stage has - produced its artifact. +- **Move between stages** by clicking the roadmap. Stages are not a wizard: going back to correct + a contract after the world is built is the ordinary case. A stage whose input does not exist + yet says why it cannot be opened. +- **Read the four tabs.** Contract, Environment, Scenarios and Runs are what is on disk. Each + scenario shows its instruction, what it changes, its reference solution, its checks, and three + gate lights; its files open inline. - **Run the scenarios.** The conversation between the simulated customer and the agent streams - into the chat as it happens, then a verdict card lands with the checkpoints. -- **Watch the right-hand side.** Contract, World, Scenarios and Runs are the four artifacts on - disk; the pane refreshes whenever a stage writes one. + into the chat as it happens, then a verdict lands with the checks. + +## What to expect while it works + +The build stage is the long one: roughly 30 turns and about ten minutes for a five-tool agent. +**The Environment tab stays empty until it finishes**, because the world is held in memory until +`save_world` writes it. The chat is where the progress is: schema, seeds, then one line per +handler as each is defined and smoke-called. ## The two files | File | What it is | |---|---| | `server.py` | FastAPI. Holds one `Conversation` open, streams its events as server-sent events, and serves the artifacts as JSON. | -| `static/index.html` | The whole interface — markup, styling and rendering in one file. No build step, no npm. | +| `static/index.html` | The whole interface: markup, styling and rendering in one file. | To restyle it, edit the `