From 420267caa8703ec3a192a764b78b6f4f3ddbbe87 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 13 Aug 2026 22:04:19 +0530 Subject: [PATCH 01/55] feat(generation): local-first scenario generation harness (explore, generate, verify loops) --- src/fi/alk/generation/README.md | 71 +++++++ src/fi/alk/generation/__init__.py | 43 ++++ src/fi/alk/generation/__main__.py | 3 + src/fi/alk/generation/cli.py | 64 ++++++ src/fi/alk/generation/contract.py | 140 +++++++++++++ src/fi/alk/generation/emit.py | 252 +++++++++++++++++++++++ src/fi/alk/generation/explorer.py | 301 ++++++++++++++++++++++++++++ src/fi/alk/generation/llm.py | 260 ++++++++++++++++++++++++ src/fi/alk/generation/pipeline.py | 293 +++++++++++++++++++++++++++ src/fi/alk/generation/prompts.py | 287 ++++++++++++++++++++++++++ src/fi/alk/generation/sources.py | 145 ++++++++++++++ src/fi/alk/generation/validators.py | 174 ++++++++++++++++ tests/test_generation_pipeline.py | 225 +++++++++++++++++++++ 13 files changed, 2258 insertions(+) create mode 100644 src/fi/alk/generation/README.md create mode 100644 src/fi/alk/generation/__init__.py create mode 100644 src/fi/alk/generation/__main__.py create mode 100644 src/fi/alk/generation/cli.py create mode 100644 src/fi/alk/generation/contract.py create mode 100644 src/fi/alk/generation/emit.py create mode 100644 src/fi/alk/generation/explorer.py create mode 100644 src/fi/alk/generation/llm.py create mode 100644 src/fi/alk/generation/pipeline.py create mode 100644 src/fi/alk/generation/prompts.py create mode 100644 src/fi/alk/generation/sources.py create mode 100644 src/fi/alk/generation/validators.py create mode 100644 tests/test_generation_pipeline.py diff --git a/src/fi/alk/generation/README.md b/src/fi/alk/generation/README.md new file mode 100644 index 0000000..78bb208 --- /dev/null +++ b/src/fi/alk/generation/README.md @@ -0,0 +1,71 @@ +# fi.alk.generation + +Local-first scenario generation: point at an agent, get a reviewed set of runnable test scenarios. + +```bash +python -m fi.alk.generation --repo /path/to/agent --n 20 --out artifacts/scenarios +``` + +## What it produces + +For each scenario, one record with three strictly separated parts: + +- **(A) agent input** - what the simulated user is told (situation, goal, facts revealed only when + asked). Never contains the answer or hidden state. +- **(B) environment** - seed state plus per-tool mock responses (`static_fixture` tier, the one the + runtime executes today). +- **(C) hidden checks** - sub-goals drawn from a shared per-agent catalog, each with a checkpoint + that asserts the right end state or the right tool call with the right arguments. Deterministic + where possible; judge only where the world is not inspectable. + +Emitted artifacts: `scenarios/*.json` (rich records), `alk/*.json` (typed `fi.simulate` `Scenario` +objects with `goal` / `verification` / `constraints` populated), `subgoal_catalog.json`, +`report.md` (coverage + verdicts), `usage.json` (tokens and USD). + +## The pipeline + +``` +AgentSource ──▶ evidence blob ──▶ CONTRACT ──▶ sub-goal catalog ──▶ rows (use-case ▸ branch) + (LLM+validate) (LLM+validate) (LLM, round loop, dedup) + │ per row + materialize ─▶ validate ─▶ critic + ▲ │(problems) + └── repair ◀───┘ max 2 + │ accepted + emit +``` + +Generation is a loop over prompts plus deterministic validators, not an agent framework. The LLM does +semantics; plain code does structure, dedup, and grounding checks; nothing hardcodes a domain. + +## Design rules + +1. **Grounding is a contract, not a vibe.** Everything the model writes must use interfaces the + extracted `AgentContract` actually lists (exact tool and argument names). Violations are caught + by validators, not by hoping. +2. **Rows are the agent's real use-cases and their branches.** Distinct outcomes are distinct rows. + No happy/edge/adversarial buckets, no infra rows, no forced personas. +3. **Sub-goals are shared.** A per-agent catalog is derived once; scenarios reference catalog names + so results roll up across scenarios (where does payment fail, across all 50 rows). +4. **Checkpoints assert the right arguments.** Asked for 11 PM, a 10 PM booking must fail. A check + is `deterministic: true` only when it carries an executable definition. +5. **Extensible by registry, not by edit.** New agent connections implement `AgentSource` (three + members) and register; new modalities add one entry to `AGENT_INPUT_BY_MODALITY`; the LLM is a + two-method protocol with the model string as config. + +## Extending + +| Want | Do | +|---|---| +| New agent connection (Vapi, Retell, platform id) | implement `AgentSource`, `@register_source("vapi")` | +| New modality (computer-use, code, ...) | add an `AGENT_INPUT_BY_MODALITY` entry; contract `modality` is open vocabulary | +| Different model | `--model vertex_ai/gemini-2.5-pro` or any litellm string; `LLMClient` is a protocol for non-litellm backends | +| Different budget | `--budget-usd 5` (hard stop, raises `BudgetExceeded`) | +| Stricter or looser QA | critic threshold and retry counts are `GenerationConfig` fields | + +## Boundaries honored + +This package lives on the studio side of the one-way rule: it imports `fi.simulate.simulation.models` +and never the reverse. It emits typed `Scenario` objects; running them is the simulation runtime's +job. Secrets are environment variables only (`GOOGLE_APPLICATION_CREDENTIALS`); nothing is written +into specs. diff --git a/src/fi/alk/generation/__init__.py b/src/fi/alk/generation/__init__.py new file mode 100644 index 0000000..244e278 --- /dev/null +++ b/src/fi/alk/generation/__init__.py @@ -0,0 +1,43 @@ +"""Local-first scenario generation: point at an agent, get reviewed, checkable test scenarios.""" + +from .contract import AgentContract, ToolSpec, extract_contract, validate_contract +from .emit import smoke_manifest, to_alk_scenario, write_outputs +from .llm import AuthFailed, BudgetExceeded, FakeLLMClient, LiteLLMClient, LLMClient, Usage +from .pipeline import GenerationConfig, GenerationResult, generate +from .sources import ( + AgentEvidence, + AgentSource, + RepoFolderSource, + register_source, + resolve_source, + source_registry, +) +from .validators import banned_tokens, repair_hint, validate_scenario + +__all__ = [ + "AgentContract", + "AgentEvidence", + "AgentSource", + "AuthFailed", + "BudgetExceeded", + "FakeLLMClient", + "GenerationConfig", + "GenerationResult", + "LLMClient", + "LiteLLMClient", + "RepoFolderSource", + "ToolSpec", + "Usage", + "banned_tokens", + "extract_contract", + "generate", + "register_source", + "repair_hint", + "resolve_source", + "smoke_manifest", + "source_registry", + "to_alk_scenario", + "validate_contract", + "validate_scenario", + "write_outputs", +] diff --git a/src/fi/alk/generation/__main__.py b/src/fi/alk/generation/__main__.py new file mode 100644 index 0000000..eb53e2f --- /dev/null +++ b/src/fi/alk/generation/__main__.py @@ -0,0 +1,3 @@ +from .cli import main + +raise SystemExit(main()) diff --git a/src/fi/alk/generation/cli.py b/src/fi/alk/generation/cli.py new file mode 100644 index 0000000..65f615d --- /dev/null +++ b/src/fi/alk/generation/cli.py @@ -0,0 +1,64 @@ +"""CLI: ``python -m fi.alk.generation --repo /path/to/agent --n 20 --out artifacts/scenarios``.""" + +from __future__ import annotations + +import argparse +import json +import logging +import sys + +from .llm import DEFAULT_MODEL, LiteLLMClient +from .pipeline import GenerationConfig, generate +from .sources import resolve_source + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="python -m fi.alk.generation", + description="Generate grounded, checkable test scenarios for an agent.", + ) + parser.add_argument("--source", default="repo", help="agent connection kind (default: repo)") + parser.add_argument("--repo", help="path to the agent's repository folder (repo source)") + parser.add_argument("--n", type=int, default=20, help="target number of scenarios") + parser.add_argument("--model", default=DEFAULT_MODEL, help="litellm model string") + parser.add_argument("--budget-usd", type=float, default=2.0, help="hard spend ceiling for this run") + parser.add_argument("--out", default="artifacts/generated-scenarios", help="output directory") + parser.add_argument("--no-critic", action="store_true", help="skip the QA review pass") + parser.add_argument("--verbose", action="store_true") + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + logging.basicConfig( + level=logging.INFO if args.verbose else logging.WARNING, + format="%(levelname)s %(name)s %(message)s", + ) + source_kwargs = {} + if args.source == "repo": + if not args.repo: + print("--repo is required for the repo source", file=sys.stderr) + return 2 + source_kwargs["path"] = args.repo + source = resolve_source(args.source, **source_kwargs) + llm = LiteLLMClient(model=args.model, budget_usd=args.budget_usd) + config = GenerationConfig(n=args.n, critic_enabled=not args.no_critic, out_dir=args.out) + + result = generate(source, llm, config) + print( + json.dumps( + { + "agent": result.contract.agent, + "scenarios": len(result.records), + "rejected": len(result.rejected), + "out": args.out, + "usage": result.usage, + }, + indent=2, + ) + ) + return 0 if result.records else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/fi/alk/generation/contract.py b/src/fi/alk/generation/contract.py new file mode 100644 index 0000000..e4a6bcb --- /dev/null +++ b/src/fi/alk/generation/contract.py @@ -0,0 +1,140 @@ +"""The agent CONTRACT: the code-verified ground truth every later prompt is confined to. + +Extraction hands the evidence blob to the model once and validates the result structurally. The +contract is the anti-hallucination device: rows, scenarios, and checks may only reference the tools, +arguments, entities, and constraints listed here, and the validators enforce that. +""" + +from __future__ import annotations + +import json +from typing import Any + +from pydantic import BaseModel, Field + +from .llm import LLMClient + + +class ToolSpec(BaseModel): + name: str + args: list[str] = Field(default_factory=list) + arg_values: dict[str, Any] = Field(default_factory=dict) + description: str = "" + + +class AgentContract(BaseModel): + """What the agent verifiably is. Nothing downstream may contradict this.""" + + agent: str + one_liner: str = "" + modality: str = "chat" + conversational: bool = True + system_prompt_excerpt: str = "" + hard_constraints: list[str] = Field(default_factory=list) + tools: list[ToolSpec] = Field(default_factory=list) + data_schema: dict[str, Any] = Field(default_factory=dict) + base_environment: dict[str, Any] = Field(default_factory=dict) + real_use_cases: list[str] = Field(default_factory=list) + signature_cases: list[str] = Field(default_factory=list) + grading_notes: str = "" + anti_hallucination: list[str] = Field(default_factory=list) + + def tool_names(self) -> set[str]: + return {tool.name for tool in self.tools} + + def brief(self, *, full_schema: bool = True) -> str: + """The grounding block handed to the model on every downstream call.""" + lines: list[str] = [] + for tool in self.tools: + values = f" [values: {json.dumps(tool.arg_values)[:300]}]" if tool.arg_values else "" + lines.append(f" - {tool.name}({', '.join(tool.args)}){values} : {tool.description[:140]}") + parts = [ + f"AGENT: {self.agent} - {self.one_liner}", + f"MODALITY: {self.modality}", + "REAL TOOLS (use ONLY these, with these exact arg names):\n" + ("\n".join(lines) or " (none)"), + ] + if self.hard_constraints: + parts.append( + "HARD CONSTRAINTS the agent MUST follow (checks must never contradict these):\n - " + + "\n - ".join(self.hard_constraints[:14]) + ) + if self.data_schema and full_schema: + parts.append( + "REAL DATA / SCHEMA (ground every value and id in this; never invent):\n" + + json.dumps(self.data_schema)[:2400] + ) + if self.grading_notes: + parts.append(f"GRADING NOTES (how checks must be written for THIS agent):\n{self.grading_notes[:900]}") + if self.anti_hallucination: + parts.append( + "NEVER USE THESE (they do not exist / are wrong): " + + json.dumps(self.anti_hallucination)[:700] + ) + return "\n\n".join(parts) + + +_EXTRACT_SYSTEM = """You are a senior engineer reading an AI agent's actual source to write its \ +testing CONTRACT. Everything you output must be verifiably present in the provided material; when \ +unsure, leave a field empty rather than guess. Exact identifiers matter: tool names, argument names, \ +enum values, and entity ids must be copied character for character.""" + +_EXTRACT_USER = """Read this agent's repository material and return its CONTRACT as JSON. + +{evidence} + +Return JSON with exactly these keys: +- agent: short name +- one_liner: what the agent does, one sentence +- modality: one of voice | chat | browser | code | data_sql | research | computer_use | other +- conversational: true if a user talks to it across turns (voice/chat), else false +- system_prompt_excerpt: the most behavior-defining 10-20 lines of its instructions, verbatim +- hard_constraints: rules its instructions or code enforce (refusals, required elicitation, limits), \ +each one line, verbatim-grounded +- tools: [{{name, args (exact parameter names), arg_values (enums / valid ids per arg, from code or \ +data), description}}] - only tools that exist in the code +- data_schema: the real data model it operates over (menus, tables, entities with their REAL ids and \ +prices/values), compact JSON. This is what checks will be grounded in, so include real item ids. +- base_environment: {{summary, seed}} - what world must exist for it to run (mocked), with seed data \ +drawn from the real data +- real_use_cases: 8-15 one-line user-facing things people genuinely do with it, each naming the tool \ +and args it exercises where relevant +- signature_cases: 6-12 one-line cases its own engineer would insist on testing (constraint \ +enforcement, disambiguation, not-found, refusal, correction) - each grounded in a specific \ +constraint or data fact above +- grading_notes: 3-6 lines on how to check THIS agent (what state it changes, which tool arguments \ +carry the user's request, what "correct" means) +- anti_hallucination: interface-shaped names someone might plausibly invent for this agent that do \ +NOT exist (wrong tool names, wrong arg names, nonexistent menu/table entries)""" + + +def extract_contract(evidence_text: str, llm: LLMClient) -> AgentContract: + raw = llm.complete_json( + _EXTRACT_SYSTEM, + _EXTRACT_USER.format(evidence=evidence_text), + temperature=0.15, + max_tokens=10_000, + ) + if isinstance(raw, list): + raw = next((item for item in raw if isinstance(item, dict)), {}) + contract = AgentContract.model_validate(raw) + problems = validate_contract(contract) + if problems: + raise ValueError(f"extracted contract failed validation: {problems}") + return contract + + +def validate_contract(contract: AgentContract) -> list[str]: + problems: list[str] = [] + if not contract.agent.strip(): + problems.append("empty:agent") + if not contract.tools: + problems.append("no-tools") + for index, tool in enumerate(contract.tools): + if not tool.name.strip(): + problems.append(f"tool[{index}]:no-name") + if not contract.real_use_cases: + problems.append("no-use-cases") + seen = [name for name in contract.tool_names() if name] + if len(seen) != len(set(seen)): + problems.append("duplicate-tool-names") + return problems diff --git a/src/fi/alk/generation/emit.py b/src/fi/alk/generation/emit.py new file mode 100644 index 0000000..7a0b115 --- /dev/null +++ b/src/fi/alk/generation/emit.py @@ -0,0 +1,252 @@ +"""Emission: rich records to typed `fi.simulate` Scenarios, a runnable smoke manifest, a report. + +One generated record becomes one typed ``Scenario`` (kind ``task``) with a single-persona dataset: +``goal.states`` carries the sub-goal names, ``verification.checks`` carries one named check per +sub-goal (goal-machine vocabulary, with the full checkpoint definition preserved on the check dict), +and ``constraints.declared_tools`` bounds the action space. The persona keeps the legacy +``situation`` / ``outcome`` fields populated (the current simulator prompt drives off them) while the +typed ``knowledge`` facts carry disclosure rules for the instruction-following simulator. +""" + +from __future__ import annotations + +import json +import os +from typing import Any + +from fi.simulate.simulation.models import ( + CoverageDeclaration, + Persona, + PersonaFact, + Scenario, + ScenarioConstraints, + ScenarioGoal, + VerificationSpec, +) + +from .contract import AgentContract + +_KIND_TO_GOAL_MACHINE = { + "state": "world_success_condition", + "tool_call_args": "world_success_condition", + "conveyed": "world_success_condition", + "absent": "world_invariant", + "judge": "eval_template", +} + + +def _facts(record: dict) -> list[PersonaFact]: + facts: list[PersonaFact] = [] + for fact in record.get("facts") or []: + if isinstance(fact, dict) and fact.get("key"): + facts.append( + PersonaFact( + key=str(fact["key"]), + value=str(fact.get("value", "")), + disclosure=str(fact.get("disclosure", "on_request")), + ) + ) + return facts + + +def _referenced_tools(record: dict, contract: AgentContract) -> list[str]: + blob = json.dumps(record) + return sorted(name for name in contract.tool_names() if name in blob) + + +def to_alk_scenario(record: dict, contract: AgentContract) -> Scenario: + sub_goals = record.get("sub_goals") or [] + names = [str(sg.get("name")) for sg in sub_goals if isinstance(sg, dict) and sg.get("name")] + checks: list[dict[str, Any]] = [] + for sub_goal in sub_goals: + if not isinstance(sub_goal, dict) or not sub_goal.get("name"): + continue + checkpoint = sub_goal.get("checkpoint") or {} + kind = str(checkpoint.get("kind", "judge")) + checks.append( + { + "name": str(sub_goal["name"]), + "kind": _KIND_TO_GOAL_MACHINE.get(kind, "eval_template"), + "rung": "settle", + "checkpoint_kind": kind, + "deterministic": bool(checkpoint.get("deterministic")), + "detail": str(checkpoint.get("detail", "")), + "definition": checkpoint.get("definition") or {}, + "milestone": str(sub_goal.get("milestone", "")), + } + ) + + outcome = record.get("expected_outcome") or {} + persona_payload = dict(record.get("persona") or {}) + persona_payload.setdefault("name", "Caller") + + persona = Persona( + persona=persona_payload, + situation=str(record.get("agent_input", "")), + outcome=str(outcome.get("world_state") or record.get("goal", "")), + knowledge=_facts(record), + ) + referenced = _referenced_tools(record, contract) + return Scenario( + name=str(record.get("id") or record.get("use_case", "scenario")), + description=f"{record.get('use_case', '')} :: {record.get('situation', '')}".strip(" :"), + dataset=[persona], + kind="task", + goal=ScenarioGoal(states=names, success_state=names[-1] if names else None), + verification=VerificationSpec(checks=checks, threshold=1.0), + constraints=ScenarioConstraints( + declared_tools=referenced, + observable_state=dict((record.get("environment") or {}).get("seed") or {}), + max_user_knowledge=[fact.key for fact in _facts(record)], + ), + coverage=CoverageDeclaration( + intents=[str(record.get("use_case", ""))], + tool_obligations=[f"allow:{name}" for name in referenced], + ), + ) + + +def smoke_manifest(record: dict, contract: AgentContract) -> dict[str, Any]: + """A runnable chat-spine manifest for one record: mock tools + world-contract conditions. + + This is the offline proof that a generated scenario's deterministic state checks fire through + the real goal machine, in the exact shape the manifest loader accepts. + """ + environment = record.get("environment") or {} + conditions: list[dict[str, Any]] = [] + for sub_goal in record.get("sub_goals") or []: + checkpoint = (sub_goal or {}).get("checkpoint") or {} + definition = checkpoint.get("definition") or {} + if checkpoint.get("kind") == "state" and definition.get("must"): + conditions.append( + { + "name": str(sub_goal.get("name")), + "must": definition["must"], + **({"forbidden": definition["forbidden"]} if definition.get("forbidden") else {}), + } + ) + states = [c["name"] for c in conditions] + return { + "version": "agent-learning.run.v1", + "name": str(record.get("id", "generated")), + "agent": {"type": "scripted", "content": "done"}, + "evaluation": {"enabled": False}, + "scenario": { + "name": str(record.get("id", "generated")), + "dataset": [ + { + "persona": dict(record.get("persona") or {"name": "Caller"}), + "situation": str(record.get("agent_input", "")), + "outcome": str((record.get("expected_outcome") or {}).get("world_state", "")), + } + ], + "goal": {"states": states, "success_state": states[-1] if states else None}, + "verification": { + "checks": [ + {"name": name, "kind": "world_success_condition", "rung": "settle"} + for name in states + ] + }, + }, + "simulation": { + "engine": "local_text", + "max_turns": 2, + "min_turns": 1, + "environments": [ + { + "type": "tool_mock", + "tools": dict(environment.get("mock_responses") or {}), + "initial_state": dict(environment.get("seed") or {}), + }, + { + "type": "world_contract", + "name": "generated_world", + "initial_state": dict(environment.get("seed") or {}), + "success_conditions": conditions, + }, + ], + }, + } + + +def write_outputs( + out_dir: str, + *, + contract: AgentContract, + catalog: list[dict], + records: list[dict], + rejected: list[dict], + usage: dict[str, Any], +) -> None: + scenarios_dir = os.path.join(out_dir, "scenarios") + alk_dir = os.path.join(out_dir, "alk") + os.makedirs(scenarios_dir, exist_ok=True) + os.makedirs(alk_dir, exist_ok=True) + + def _dump(path: str, payload: Any) -> None: + with open(path, "w", encoding="utf-8") as fh: + json.dump(payload, fh, indent=2, ensure_ascii=False, default=str) + + _dump(os.path.join(out_dir, "contract.json"), contract.model_dump()) + _dump(os.path.join(out_dir, "subgoal_catalog.json"), catalog) + _dump(os.path.join(out_dir, "usage.json"), usage) + for record in records: + slug = str(record.get("id", "scenario")) + _dump(os.path.join(scenarios_dir, f"{slug}.json"), record) + alk = to_alk_scenario(record, contract) + _dump(os.path.join(alk_dir, f"{slug}.json"), alk.model_dump(exclude_none=True)) + if records: + _dump(os.path.join(out_dir, "smoke_manifest.json"), smoke_manifest(records[0], contract)) + with open(os.path.join(out_dir, "report.md"), "w", encoding="utf-8") as fh: + fh.write(render_report(contract, catalog, records, rejected, usage)) + + +def render_report( + contract: AgentContract, + catalog: list[dict], + records: list[dict], + rejected: list[dict], + usage: dict[str, Any], +) -> str: + catalog_names = {str(entry.get("name")) for entry in catalog} + reuse: dict[str, int] = {} + deterministic = 0 + total_checks = 0 + for record in records: + for sub_goal in record.get("sub_goals") or []: + name = str((sub_goal or {}).get("name", "")) + total_checks += 1 + if ((sub_goal or {}).get("checkpoint") or {}).get("deterministic"): + deterministic += 1 + if name in catalog_names: + reuse[name] = reuse.get(name, 0) + 1 + + lines = [ + f"# Generated scenarios: {contract.agent}", + "", + f"- scenarios accepted: **{len(records)}**, rejected by review: {len(rejected)}", + f"- checkpoints: {total_checks}, deterministic: {deterministic} " + f"({(100 * deterministic // max(total_checks, 1))}%)", + f"- shared sub-goals reused across scenarios: " + f"{sum(1 for count in reuse.values() if count >= 2)} of {len(catalog)} catalog entries", + f"- model usage: {usage}", + "", + "| # | Use case | Situation | Sub-goals | Det |", + "|---|---|---|---|---|", + ] + for index, record in enumerate(records, 1): + sub_goals = record.get("sub_goals") or [] + det = sum(1 for sg in sub_goals if ((sg or {}).get("checkpoint") or {}).get("deterministic")) + lines.append( + f"| {index} | {record.get('use_case', '')} | {record.get('situation', '')} " + f"| {len(sub_goals)} | {det}/{len(sub_goals)} |" + ) + if reuse: + lines += ["", "## Sub-goal roll-up (appearances across scenarios)", ""] + for name, count in sorted(reuse.items(), key=lambda item: -item[1]): + lines.append(f"- `{name}`: {count}") + if rejected: + lines += ["", "## Rejected in review", ""] + for record in rejected: + lines.append(f"- {record.get('id', '?')}: {record.get('_reject_reason', 'rejected')}") + return "\n".join(lines) + "\n" diff --git a/src/fi/alk/generation/explorer.py b/src/fi/alk/generation/explorer.py new file mode 100644 index 0000000..2f1b398 --- /dev/null +++ b/src/fi/alk/generation/explorer.py @@ -0,0 +1,301 @@ +"""Contract extraction as a bounded tool loop over the agent's repository. + +The model is given read-only tools (list a directory, read a file, search text) plus one submit tool, +and a turn budget. It decides what to open, exactly like a coding agent reading an unfamiliar repo. +The harness owns the loop: it executes tool calls inside a path-sandboxed root, feeds results back, +validates the submitted contract, and returns validator problems to the model for another attempt +instead of accepting a bad contract. If the turn budget runs out, the harness forces a submission. + +Every string the model sees is self-contained: the system prompt defines the task, the contract +schema, and the verification rules without assuming any outside context. +""" + +from __future__ import annotations + +import json +import logging +import os +import re +from typing import Any + +from .contract import AgentContract, validate_contract +from .llm import LLMClient + +logger = logging.getLogger(__name__) + +_MAX_TURNS = 20 +_MAX_READ_CHARS = 12_000 +_MAX_RESULT_CHARS = 14_000 + +_SYSTEM = """You are a senior software engineer. Your job: read the source code of an AI agent and +produce its testing CONTRACT, a JSON document that later test generation will treat as the complete +and only truth about this agent. Anything you put in the contract that is not verifiably in the code +will corrupt every test built on it, so you verify identifiers by reading the files where they are +defined, and you copy names character for character. + +You have tools to explore the repository: list directories, read files, and search for text. Explore +until you have verified, then call submit_contract exactly once with the finished contract. Work +efficiently: start from the README and the files that define the agent's tools, instructions, and +data; do not read files that cannot change the contract (build config, lockfiles, tests of the +framework itself). + +The contract fields, all required (use an empty list or empty string only when genuinely nothing +applies): +- agent: short name for the agent +- one_liner: what the agent does, one sentence +- modality: how a user reaches it: "voice" | "chat" | "browser" | "code" | "data_sql" | "research" | + "computer_use" | "other" +- conversational: true when a user talks with it across multiple turns (voice and chat agents), else + false +- system_prompt_excerpt: the 10-20 most behavior-defining lines of the agent's own instructions, + quoted verbatim from the code +- hard_constraints: rules the agent's instructions or code actually enforce (required elicitation, + refusals, limits, ordering rules), one line each, each traceable to a specific place in the code +- tools: the agent's real callable tools: [{"name": "", "args": [""], "arg_values": {"": []}, + "description": ""}]. Only tools that exist. Exact spelling. For enum-like parameters, + list the real valid values you found. +- data_schema: the real data the agent operates over (menu items, tables, records) with REAL ids, + names, prices or values, as compact JSON. Later tests take every concrete value from here, so + include the actual entries you found, not examples of their shape. +- base_environment: {"summary": "", "seed": {}} +- real_use_cases: 8-15 one-line things a user genuinely does with this agent, each naming the tool + and arguments it exercises where relevant +- signature_cases: 6-12 one-line test-worthy situations grounded in a specific constraint or data + fact you found (a constraint that forces a clarifying question, an id that does not exist, a + refusal the instructions demand, a correction mid-task) +- grading_notes: 3-6 lines on how to verify THIS agent behaved correctly: what state it changes, + which tool arguments carry the user's request, what "correct" means +- anti_hallucination: names someone might plausibly invent for this agent that do NOT exist (wrong + tool names, wrong argument names, ids that look valid but are not), so tests can be checked + against them + +If submit_contract returns validation problems, fix them and submit again.""" + +_TOOLS: list[dict[str, Any]] = [ + { + "type": "function", + "function": { + "name": "list_dir", + "description": "List entries of a directory inside the agent repository.", + "parameters": { + "type": "object", + "properties": {"path": {"type": "string", "description": "Relative path; '' for the root."}}, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "read_file", + "description": "Read a file inside the agent repository (truncated past 12000 chars; " + "pass offset to continue).", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "offset": {"type": "integer", "description": "Character offset to start from."}, + }, + "required": ["path"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "search_text", + "description": "Search all repository files for a plain substring; returns file:line hits.", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "submit_contract", + "description": "Submit the finished contract. Call exactly once, after verifying.", + "parameters": { + "type": "object", + "properties": {"contract": {"type": "object", "description": "The full contract JSON."}}, + "required": ["contract"], + }, + }, + }, +] + +_SKIP_DIRS = {".git", "node_modules", ".venv", "venv", "__pycache__", "dist", "build", ".omega"} + + +class _RepoTools: + """Path-sandboxed read tools over one repository root.""" + + def __init__(self, root: str) -> None: + self.root = os.path.abspath(root) + + def _resolve(self, path: str) -> str: + resolved = os.path.abspath(os.path.join(self.root, str(path or "").lstrip("/"))) + if resolved != self.root and not resolved.startswith(self.root + os.sep): + raise ValueError(f"path escapes the repository root: {path}") + return resolved + + def list_dir(self, path: str = "") -> str: + target = self._resolve(path) + if not os.path.isdir(target): + return f"not a directory: {path}" + entries = [] + for entry in sorted(os.listdir(target)): + if entry in _SKIP_DIRS: + continue + full = os.path.join(target, entry) + suffix = "/" if os.path.isdir(full) else f" ({os.path.getsize(full)} bytes)" + entries.append(f"{entry}{suffix}") + return "\n".join(entries) or "(empty)" + + def read_file(self, path: str, offset: int = 0) -> str: + target = self._resolve(path) + if not os.path.isfile(target): + return f"not a file: {path}" + try: + with open(target, encoding="utf-8", errors="ignore") as fh: + fh.seek(max(0, int(offset or 0))) + body = fh.read(_MAX_READ_CHARS) + except OSError as exc: + return f"read failed: {exc}" + marker = "" if len(body) < _MAX_READ_CHARS else f"\n... truncated; continue with offset={offset + _MAX_READ_CHARS}" + return body + marker + + def search_text(self, query: str) -> str: + query = str(query or "") + if not query.strip(): + return "empty query" + hits: list[str] = [] + for dirpath, dirnames, filenames in os.walk(self.root): + dirnames[:] = [d for d in dirnames if d not in _SKIP_DIRS] + for filename in filenames: + full = os.path.join(dirpath, filename) + try: + with open(full, encoding="utf-8", errors="ignore") as fh: + for line_number, line in enumerate(fh, 1): + if query in line: + rel = os.path.relpath(full, self.root) + hits.append(f"{rel}:{line_number}: {line.strip()[:160]}") + if len(hits) >= 60: + return "\n".join(hits) + "\n... (capped at 60 hits)" + except OSError: + continue + return "\n".join(hits) or "no hits" + + +def explore_contract(root: str, llm: LLMClient, *, max_turns: int = _MAX_TURNS) -> AgentContract: + """Run the exploration loop until a valid contract is submitted.""" + tools = _RepoTools(root) + messages: list[dict[str, Any]] = [ + {"role": "system", "content": _SYSTEM}, + { + "role": "user", + "content": ( + "The agent repository root is mounted for your tools. Explore it and submit the " + "contract.\n\nRoot listing:\n" + tools.list_dir("") + ), + }, + ] + submitted: AgentContract | None = None + for turn in range(max_turns): + forced = turn == max_turns - 1 + if forced: + messages.append( + { + "role": "user", + "content": "Turn budget exhausted. Call submit_contract NOW with your best " + "verified contract.", + } + ) + reply = llm.complete_turn(messages, tools=_TOOLS, temperature=0.15) + calls = reply.get("tool_calls") or [] + if not calls: + messages.append({"role": "assistant", "content": reply.get("content") or ""}) + messages.append( + {"role": "user", "content": "Use the tools. Explore the repository, then call submit_contract."} + ) + continue + messages.append(_assistant_message(reply)) + for call in calls: + name = call.get("name") + arguments = call.get("arguments") or {} + if name == "submit_contract": + result, submitted = _try_submit(arguments) + else: + result = _run_tool(tools, name, arguments) + messages.append( + { + "role": "tool", + "tool_call_id": call.get("id") or name, + "content": str(result)[:_MAX_RESULT_CHARS], + } + ) + if submitted is not None: + logger.info("contract submitted", extra={"turns": turn + 1}) + return submitted + raise RuntimeError(f"exploration ended after {max_turns} turns without a valid contract") + + +def _assistant_message(reply: dict[str, Any]) -> dict[str, Any]: + raw = reply.get("raw") + if raw is not None: + try: + return raw.model_dump() + except AttributeError: + pass + return { + "role": "assistant", + "content": reply.get("content") or "", + "tool_calls": [ + { + "id": call.get("id") or call.get("name"), + "type": "function", + "function": {"name": call.get("name"), "arguments": json.dumps(call.get("arguments") or {})}, + } + for call in reply.get("tool_calls") or [] + ], + } + + +def _run_tool(tools: _RepoTools, name: str, arguments: dict[str, Any]) -> str: + try: + if name == "list_dir": + return tools.list_dir(arguments.get("path", "")) + if name == "read_file": + return tools.read_file(arguments.get("path", ""), int(arguments.get("offset") or 0)) + if name == "search_text": + return tools.search_text(arguments.get("query", "")) + except ValueError as exc: + return str(exc) + return f"unknown tool: {name}" + + +def _try_submit(arguments: dict[str, Any]) -> tuple[str, AgentContract | None]: + payload = arguments.get("contract") + if isinstance(payload, str): + try: + payload = json.loads(payload) + except json.JSONDecodeError: + payload = None + if not isinstance(payload, dict): + return "submit_contract requires a JSON object under the 'contract' key; fix and resubmit", None + try: + contract = AgentContract.model_validate(payload) + except Exception as exc: # noqa: BLE001 - fed back to the model + return f"contract failed schema validation; fix and resubmit: {_short(exc)}", None + problems = validate_contract(contract) + if problems: + return f"contract failed checks; fix and resubmit: {problems}", None + return "accepted", contract + + +def _short(exc: Exception) -> str: + return re.sub(r"\s+", " ", str(exc))[:600] diff --git a/src/fi/alk/generation/llm.py b/src/fi/alk/generation/llm.py new file mode 100644 index 0000000..df64281 --- /dev/null +++ b/src/fi/alk/generation/llm.py @@ -0,0 +1,260 @@ +"""LLM client boundary: a two-method protocol, a litellm implementation, an offline fake. + +The pipeline only ever sees ``LLMClient``. The default implementation routes through litellm the +same way ``fi.simulate.suite`` does: ``vertex_ai/`` reaches Vertex AI with credentials from +``GOOGLE_APPLICATION_CREDENTIALS`` (or an explicit ``vertex_credentials`` path); any other +fully-qualified litellm model string works unchanged. Spend is metered per call against a hard USD +ceiling so an unattended run can never overshoot its budget. +""" + +from __future__ import annotations + +import json +import os +import re +import time +from dataclasses import dataclass, field +from typing import Any, Protocol + +DEFAULT_MODEL = os.environ.get("ALK_GENERATION_MODEL", "vertex_ai/gemini-2.5-flash") + +# USD per token, overridable per client. Defaults are Gemini 2.5 Flash list prices. +DEFAULT_INPUT_COST_PER_TOKEN = 0.30 / 1_000_000 +DEFAULT_OUTPUT_COST_PER_TOKEN = 2.50 / 1_000_000 + +_AUTH_MARKERS = ("401", "403", "unauthorized", "unauthenticated", "permission", "credential") + + +class BudgetExceeded(RuntimeError): + """Raised before a call that would push spend past the configured ceiling.""" + + +class AuthFailed(RuntimeError): + """Raised on provider auth errors; retrying these only burns time.""" + + +@dataclass +class Usage: + calls: int = 0 + prompt_tokens: int = 0 + completion_tokens: int = 0 + usd: float = 0.0 + + def as_dict(self) -> dict[str, Any]: + return { + "calls": self.calls, + "prompt_tokens": self.prompt_tokens, + "completion_tokens": self.completion_tokens, + "usd": round(self.usd, 4), + } + + +class LLMClient(Protocol): + """What the pipeline needs from a model. Implementations own transport and retries.""" + + def complete_json( + self, system: str, user: str, *, temperature: float = 0.3, max_tokens: int = 8000 + ) -> Any: ... + + def complete_turn( + self, + messages: list[dict[str, Any]], + *, + tools: list[dict[str, Any]] | None = None, + temperature: float = 0.2, + max_tokens: int = 8000, + ) -> dict[str, Any]: + """One chat turn. Returns ``{"content": str | None, "tool_calls": [{"id", "name", + "arguments": dict}, ...]}`` so a harness can run a bounded tool loop.""" + ... + + @property + def usage(self) -> Usage: ... + + +def _extract_json(text: str) -> Any: + """Parse the first JSON object or array in ``text``, tolerating code fences.""" + text = text.strip() + fenced = re.search(r"```(?:json)?\s*(.+?)```", text, re.S) + if fenced: + text = fenced.group(1).strip() + try: + return json.loads(text) + except json.JSONDecodeError: + pass + for opener, closer in (("{", "}"), ("[", "]")): + start = text.find(opener) + if start < 0: + continue + depth = 0 + for i in range(start, len(text)): + if text[i] == opener: + depth += 1 + elif text[i] == closer: + depth -= 1 + if depth == 0: + try: + return json.loads(text[start : i + 1]) + except json.JSONDecodeError: + break + raise ValueError(f"model returned no parseable JSON (first 200 chars: {text[:200]!r})") + + +@dataclass +class LiteLLMClient: + """litellm-backed client with per-call cost metering and a hard budget ceiling.""" + + model: str = DEFAULT_MODEL + budget_usd: float = 2.0 + vertex_location: str = os.environ.get("VERTEX_LOCATION", "global") + vertex_credentials: str | None = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + input_cost_per_token: float = DEFAULT_INPUT_COST_PER_TOKEN + output_cost_per_token: float = DEFAULT_OUTPUT_COST_PER_TOKEN + max_attempts: int = 4 + _usage: Usage = field(default_factory=Usage) + + @property + def usage(self) -> Usage: + return self._usage + + def complete_json( + self, system: str, user: str, *, temperature: float = 0.3, max_tokens: int = 8000 + ) -> Any: + self._check_budget() + text = self._chat(system, user, temperature=temperature, max_tokens=max_tokens) + return _extract_json(text) + + def complete_turn( + self, + messages: list[dict[str, Any]], + *, + tools: list[dict[str, Any]] | None = None, + temperature: float = 0.2, + max_tokens: int = 8000, + ) -> dict[str, Any]: + self._check_budget() + try: + import litellm + except Exception as exc: # pragma: no cover - import guard + raise RuntimeError("fi.alk.generation requires litellm; reinstall agent-learning-kit") from exc + litellm.drop_params = True + kwargs = self._provider_kwargs() + kwargs.update({"temperature": temperature, "max_tokens": max_tokens}) + if tools: + kwargs["tools"] = tools + last: Exception | None = None + for attempt in range(self.max_attempts): + try: + response = litellm.completion(model=self.model, messages=messages, **kwargs) + self._meter(response) + message = response.choices[0].message + calls = [] + for call in getattr(message, "tool_calls", None) or []: + function = getattr(call, "function", None) + try: + arguments = json.loads(getattr(function, "arguments", "") or "{}") + except json.JSONDecodeError: + arguments = {} + calls.append( + {"id": getattr(call, "id", ""), "name": getattr(function, "name", ""), + "arguments": arguments} + ) + return {"content": message.content, "tool_calls": calls, "raw": message} + except Exception as exc: # noqa: BLE001 - classified below + text = str(exc).lower() + if any(marker in text for marker in _AUTH_MARKERS): + raise AuthFailed(f"provider auth failed for {self.model}: {exc}") from exc + last = exc + time.sleep(min(2**attempt, 8)) + raise RuntimeError(f"model call failed after {self.max_attempts} attempts: {last}") + + def _check_budget(self) -> None: + if self._usage.usd >= self.budget_usd: + raise BudgetExceeded( + f"spend {self._usage.usd:.2f} USD reached the {self.budget_usd:.2f} USD ceiling" + ) + + def _provider_kwargs(self) -> dict[str, Any]: + kwargs: dict[str, Any] = {} + if self.model.startswith("vertex_ai/"): + kwargs["vertex_location"] = self.vertex_location + if self.vertex_credentials: + kwargs["vertex_credentials"] = self.vertex_credentials + try: + with open(self.vertex_credentials, encoding="utf-8") as fh: + kwargs["vertex_project"] = json.load(fh).get("project_id") + except OSError: + pass + return kwargs + + def _chat(self, system: str, user: str, *, temperature: float, max_tokens: int) -> str: + try: + import litellm + except Exception as exc: # pragma: no cover - import guard + raise RuntimeError("fi.alk.generation requires litellm; reinstall agent-learning-kit") from exc + + litellm.drop_params = True + kwargs = self._provider_kwargs() + kwargs.update({"temperature": temperature, "max_tokens": max_tokens}) + messages = [{"role": "system", "content": system}, {"role": "user", "content": user}] + last: Exception | None = None + for attempt in range(self.max_attempts): + try: + response = litellm.completion(model=self.model, messages=messages, **kwargs) + self._meter(response) + content = response.choices[0].message.content + if not content or not str(content).strip(): + raise ValueError("model returned empty content") + return str(content) + except Exception as exc: # noqa: BLE001 - classified below + message = str(exc).lower() + if any(marker in message for marker in _AUTH_MARKERS): + raise AuthFailed(f"provider auth failed for {self.model}: {exc}") from exc + last = exc + time.sleep(min(2**attempt, 8)) + raise RuntimeError(f"model call failed after {self.max_attempts} attempts: {last}") + + def _meter(self, response: Any) -> None: + self._usage.calls += 1 + usage = getattr(response, "usage", None) + prompt = int(getattr(usage, "prompt_tokens", 0) or 0) + completion = int(getattr(usage, "completion_tokens", 0) or 0) + self._usage.prompt_tokens += prompt + self._usage.completion_tokens += completion + self._usage.usd += prompt * self.input_cost_per_token + completion * self.output_cost_per_token + + +@dataclass +class FakeLLMClient: + """Deterministic offline client for tests: pops queued responses in order.""" + + responses: list[Any] = field(default_factory=list) + _usage: Usage = field(default_factory=Usage) + + @property + def usage(self) -> Usage: + return self._usage + + def complete_json( + self, system: str, user: str, *, temperature: float = 0.3, max_tokens: int = 8000 + ) -> Any: + if not self.responses: + raise AssertionError("FakeLLMClient exhausted; queue more responses") + self._usage.calls += 1 + return self.responses.pop(0) + + def complete_turn( + self, + messages: list[dict[str, Any]], + *, + tools: list[dict[str, Any]] | None = None, + temperature: float = 0.2, + max_tokens: int = 8000, + ) -> dict[str, Any]: + if not self.responses: + raise AssertionError("FakeLLMClient exhausted; queue more responses") + self._usage.calls += 1 + turn = self.responses.pop(0) + if isinstance(turn, dict) and ("tool_calls" in turn or "content" in turn): + return {"content": turn.get("content"), "tool_calls": turn.get("tool_calls", [])} + return {"content": json.dumps(turn), "tool_calls": []} diff --git a/src/fi/alk/generation/pipeline.py b/src/fi/alk/generation/pipeline.py new file mode 100644 index 0000000..06b6f7d --- /dev/null +++ b/src/fi/alk/generation/pipeline.py @@ -0,0 +1,293 @@ +"""The generation harness: bounded loops at every level, deterministic code between model calls. + +Three nested loops, each with a cap and a feedback path, in the generator-verifier shape: + +1. CONTRACT loop (explorer.py): the model reads the agent's repository through tools until a + contract survives validation; validator problems go back into the conversation. +2. SCENARIO loop (per planned scenario): materialise, run deterministic validators, run the reviewer + model, feed both back as fix instructions, repeat up to ``max_repairs``. +3. SUITE loop: after a batch is accepted, a coverage review names missing situations and + near-duplicates; gaps become new plans, duplicates are dropped, and the loop continues until the + target count or ``max_suite_rounds`` is reached. + +The model does semantics. Deterministic code does structure, dedup, grounding checks, budget, and +every loop's exit condition. +""" + +from __future__ import annotations + +import logging +import re +from dataclasses import dataclass, field +from typing import Any + +from . import prompts +from .contract import AgentContract, extract_contract +from .emit import write_outputs +from .explorer import explore_contract +from .llm import LLMClient +from .sources import AgentSource +from .validators import repair_hint, validate_scenario + +logger = logging.getLogger(__name__) + +_ACCEPT_FLOOR = 3 # every reviewer score must reach this, and the verdict must not be reject + + +@dataclass +class GenerationConfig: + n: int = 20 + max_row_rounds: int = 4 + max_repairs: int = 2 + max_suite_rounds: int = 2 + max_explore_turns: int = 20 + critic_enabled: bool = True + out_dir: str = "artifacts/generated-scenarios" + + +@dataclass +class GenerationResult: + contract: AgentContract + catalog: list[dict] = field(default_factory=list) + records: list[dict] = field(default_factory=list) + rejected: list[dict] = field(default_factory=list) + usage: dict[str, Any] = field(default_factory=dict) + + +def _slugify(value: str) -> str: + slug = re.sub(r"[^a-z0-9]+", "-", str(value).lower()).strip("-") + return slug[:60] or "scenario" + + +def build_contract(source: AgentSource, llm: LLMClient, config: GenerationConfig) -> AgentContract: + """Prefer the exploration loop when the source exposes a filesystem root.""" + evidence = source.describe() + root = (evidence.metadata or {}).get("root") + if root: + try: + return explore_contract(root, llm, max_turns=config.max_explore_turns) + except Exception as exc: # noqa: BLE001 - fall back to single-shot extraction + logger.warning("exploration failed, falling back to blob extraction: %s", exc) + return extract_contract(evidence.text, llm) + + +def derive_catalog(contract: AgentContract, llm: LLMClient) -> list[dict]: + raw = llm.complete_json( + prompts.SCENARIO_MODEL, + prompts.subgoal_catalog_prompt(contract.brief()), + temperature=0.3, + max_tokens=6000, + ) + catalog = raw.get("catalog", raw) if isinstance(raw, dict) else raw + entries: list[dict] = [] + seen: set[str] = set() + for entry in catalog if isinstance(catalog, list) else []: + if not isinstance(entry, dict): + continue + name = str(entry.get("name", "")).strip() + if not re.fullmatch(r"[a-z][a-z0-9_]*", name) or name in seen: + continue + seen.add(name) + entries.append(entry) + return entries + + +def derive_rows( + contract: AgentContract, + llm: LLMClient, + config: GenerationConfig, + *, + want: int, + existing: list[dict], + feedback: str = "", +) -> list[dict]: + brief = contract.brief() + rows: list[dict] = [] + seen = { + (str(r.get("use_case", "")).strip().lower(), str(r.get("situation", "")).strip().lower()) + for r in existing + } + for round_index in range(config.max_row_rounds): + remaining = want - len(rows) + if remaining <= 0: + break + raw = llm.complete_json( + prompts.SCENARIO_MODEL, + prompts.derive_rows_prompt( + brief, + want=remaining, + signature_cases=contract.signature_cases, + real_use_cases=contract.real_use_cases, + existing=[ + {"use_case": r.get("use_case"), "situation": r.get("situation")} + for r in existing + rows + ], + feedback=feedback, + first_round=round_index == 0 and not existing, + ), + temperature=0.4, + max_tokens=6000, + ) + for row in raw.get("rows", raw if isinstance(raw, list) else []): + if not isinstance(row, dict) or not row.get("situation"): + continue + key = ( + str(row.get("use_case", "")).strip().lower(), + str(row.get("situation", "")).strip().lower(), + ) + if key in seen: + continue + seen.add(key) + row["id"] = _slugify(row.get("id") or row.get("situation", "")) + rows.append(row) + return rows[:want] + + +def materialize_row( + contract: AgentContract, + row: dict, + catalog: list[dict], + llm: LLMClient, + config: GenerationConfig, +) -> tuple[dict | None, str]: + """One scenario through the generate-validate-review-repair loop.""" + brief = contract.brief() + hint = "" + best: dict | None = None + reason = "" + for _attempt in range(1 + config.max_repairs): + raw = llm.complete_json( + prompts.SCENARIO_MODEL, + prompts.materialize_prompt( + brief, + row=row, + base_environment=contract.base_environment, + catalog=catalog, + modality=contract.modality, + conversational=contract.conversational, + hint=hint, + ), + temperature=0.35, + max_tokens=9000, + ) + record = raw if isinstance(raw, dict) else next((x for x in raw if isinstance(x, dict)), {}) + for key in ("id", "use_case", "situation", "goal"): + record.setdefault(key, row.get(key)) + record["id"] = _slugify(record.get("id") or row.get("id", "")) + + problems = validate_scenario(record, contract) + if problems: + best, reason = record, f"validator: {problems[:6]}" + hint = repair_hint(problems) + continue + if not config.critic_enabled: + return record, "" + verdict = llm.complete_json( + prompts.CRITIC_SYSTEM, prompts.critic_prompt(brief, record), temperature=0.2, max_tokens=2500 + ) + if not isinstance(verdict, dict): + verdict = {} + record["_review"] = {k: verdict.get(k) for k in ("verdict", "scores", "problems")} + decision = str(verdict.get("verdict", "revise")).lower() + scores = verdict.get("scores") or {} + low = [k for k, v in scores.items() if isinstance(v, (int, float)) and v < _ACCEPT_FLOOR] + if decision == "accept" and not low: + return record, "" + if decision == "reject": + return None, f"reviewer reject: {verdict.get('problems', [])[:4]}" + best, reason = record, f"reviewer revise (low: {low}): {verdict.get('problems', [])[:4]}" + hint = str(verdict.get("fix_hints") or "") or repair_hint([]) + # Out of repair attempts: keep the best structurally-valid draft, flagged, rather than lose it. + if best is not None and not validate_scenario(best, contract): + best["_review_flag"] = reason + return best, "" + return None, reason + + +def suite_review( + contract: AgentContract, records: list[dict], llm: LLMClient +) -> tuple[list[dict], list[str], str]: + """Coverage pass over the accepted set: (gap rows feedback, duplicate ids to drop, feedback).""" + raw = llm.complete_json( + prompts.SUITE_REVIEW_SYSTEM, + prompts.suite_review_prompt(contract.brief(), records), + temperature=0.2, + max_tokens=2500, + ) + if not isinstance(raw, dict): + return [], [], "" + gaps = [g for g in raw.get("gaps") or [] if isinstance(g, dict) and g.get("situation")] + duplicate_ids: list[str] = [] + known = {str(r.get("id")) for r in records} + for pair in raw.get("near_duplicates") or []: + if isinstance(pair, list) and len(pair) == 2 and all(str(p) in known for p in pair): + duplicate_ids.append(str(pair[1])) + feedback = "; ".join( + f"missing: {g['situation']} ({g.get('why_it_matters', '')})" for g in gaps + ) + return gaps, duplicate_ids, feedback + + +def generate( + source: AgentSource, + llm: LLMClient, + config: GenerationConfig | None = None, +) -> GenerationResult: + config = config or GenerationConfig() + contract = build_contract(source, llm, config) + logger.info("contract ready", extra={"agent": contract.agent, "tools": len(contract.tools)}) + + catalog = derive_catalog(contract, llm) + records: list[dict] = [] + rejected: list[dict] = [] + feedback = "" + + def _flush() -> None: + write_outputs( + config.out_dir, + contract=contract, + catalog=catalog, + records=records, + rejected=rejected, + usage=llm.usage.as_dict(), + ) + + try: + for suite_round in range(1 + config.max_suite_rounds): + want = config.n - len(records) + if want <= 0: + break + rows = derive_rows( + contract, llm, config, want=want, existing=records + rejected, feedback=feedback + ) + if not rows: + break + for row in rows: + record, reason = materialize_row(contract, row, catalog, llm, config) + if record is not None: + records.append(record) + else: + rejected.append({**row, "_reject_reason": reason}) + if suite_round < config.max_suite_rounds and records: + gaps, duplicate_ids, feedback = suite_review(contract, records, llm) + if duplicate_ids: + dropped = [r for r in records if str(r.get("id")) in set(duplicate_ids)] + records = [r for r in records if str(r.get("id")) not in set(duplicate_ids)] + for record in dropped: + record["_reject_reason"] = "near-duplicate of an accepted scenario" + rejected.append(record) + if not gaps and len(records) >= config.n: + break + except Exception: + _flush() + raise + + result = GenerationResult( + contract=contract, + catalog=catalog, + records=records, + rejected=rejected, + usage=llm.usage.as_dict(), + ) + _flush() + return result diff --git a/src/fi/alk/generation/prompts.py b/src/fi/alk/generation/prompts.py new file mode 100644 index 0000000..40521b0 --- /dev/null +++ b/src/fi/alk/generation/prompts.py @@ -0,0 +1,287 @@ +"""Every prompt in the pipeline. Self-contained by rule: each prompt defines every term it uses. + +A fresh model with no context about this codebase or this team must be able to do the task from the +prompt alone. No internal shorthand, no references to meetings or documents, no undefined jargon. +All agent-specific grounding arrives through the contract brief injected at call time. +""" + +from __future__ import annotations + +import json + +# The scenario model, written as definitions a fresh model can act on. +SCENARIO_MODEL = """You help test an AI agent by designing test scenarios. Definitions used throughout: + +- AGENT UNDER TEST: the AI system being evaluated. Its real interface (tools, argument names, valid + values, data) is given to you as a CONTRACT. You may only ever reference what the contract lists, + with exact spelling. Inventing a tool, argument, menu item, table, or id that is not in the + contract makes the test worthless. +- USE CASE: one real job a user hires this agent for, stated from the user's side. Example for a + food-ordering agent: "Order a combo meal". Example for a database agent: "Ask for a sales total". +- SCENARIO: one concrete test. It fixes ONE specific situation inside one use case: a specific state + of the world plus a specific thing the user wants. Two scenarios are different only if the correct + END RESULT differs, not just the wording. "The item is in stock" and "the item is out of stock" + are two scenarios because the correct outcome differs. Never write two scenarios that are the same + situation reworded. +- SUB-GOAL: a milestone inside one scenario that must be true for the scenario to end correctly. + Example: "the drink was elicited", "the refund was recorded". 3 to 6 per scenario. A sub-goal is + something a product owner would recognise, not an internal implementation step like "the JSON + parsed" and not a micro-step like "the agent said hello". +- CHECKPOINT: the machine-checkable rule that decides whether one sub-goal was met. Checkpoints must + test the RIGHT VALUES, not just that something happened: if the user asked for 11 PM and the agent + booked 10 PM, a checkpoint that only verifies "a booking call happened" wrongly passes; the + checkpoint must assert the booked time equals 11 PM. +- ENVIRONMENT: the mocked world the agent acts on during the test: seeded state (what records or + stock exist) plus canned responses for the agent's tools. The agent's own reasoning is never + mocked; only the world it acts on is. +- SIMULATED USER: for conversational agents (voice or chat), a separate AI plays the user during the + test. It receives a situation instruction (who it is, what it wants, what it knows). It does NOT + see the checkpoints, the seeded environment, or the expected outcome. + +Three parts of every scenario stay strictly separate, because leaking one into another invalidates +the test: +(A) INPUT: what the agent or the simulated user is told. Never contains the answer, the checks, or + facts about the environment the user could not know. +(B) ENVIRONMENT: the seeded world state and mock tool responses. +(C) CHECKPOINTS: the hidden pass/fail rules, graded after the run. + +Quality bar for every scenario you write: +- A competent implementation of this agent could plausibly FAIL it. If any correct implementation + passes it for free, it teaches nothing; do not write it. +- A real user could plausibly bring this situation. No contrived or gimmicky setups. +- Concrete values everywhere, taken from the contract's real data. No placeholders, no variables, + no "example_id". +- User personality, accent, or language is NOT varied unless the scenario is specifically about it.""" + +AGENT_INPUT_BY_MODALITY = { + "voice": ( + "a situation instruction for the simulated caller, written in second person as lived " + "circumstance ('You are calling... You want...'). State their goal and what they know. Facts " + "the agent should have to ask for are listed separately (see `facts`), so do not volunteer " + "them here. Never write stage directions like 'tell the agent that X'; never script the " + "agent's side; no accent or voice notes" + ), + "chat": ( + "a situation instruction for the simulated user, second person, lived circumstance: their " + "goal and what they know. Facts the agent should elicit are listed separately in `facts`" + ), + "data_sql": "the plain-English question only: no SQL, no table or column names, no answer", + "code": ( + "the command the agent is invoked with (a real command from the contract) or the issue text " + "handed to it: never the fix, the patch, or the expected review" + ), + "browser": "the natural-language task plus only the starting URL: no selectors, no answer", + "research": "the research question or brief only: no expected findings", + "_default": "exactly what the agent receives at the start, in natural form: never the answer", +} + +CHECKPOINT_VOCABULARY = """CHECKPOINT kinds, strongest first. Use the strongest kind that applies; use +`judge` only when nothing inspectable exists. +- tool_call_args (deterministic): the agent must call a specific tool with specific argument values. + definition: {"tool": "", "args_equal": {"": + , ...}, "args_present": [""]}. + Put every argument whose value the user's request determines into args_equal. +- state (deterministic): the world must end in a specific state. definition: {"must": + {"": }, "forbidden": {"": }} evaluated against + the seeded environment state after the run. +- conveyed (deterministic): the agent must have told the user a specific grounded fact. definition: + {"must_include_any": ["", ""]} matched against the agent's + side of the transcript. Use real values from the contract data (a price, a total, an id). +- absent (deterministic): something must NOT happen. definition: {"no_tool_call": ""} or + {"no_tool_call_with": {"tool": "", "args_equal": {...}}}. +- judge (not deterministic, last resort): definition: {"rubric": ""}. +Each sub-goal is written as: {"name": "", "milestone": "", "checkpoint": {"kind": "", "detail": "", +"deterministic": true|false, "definition": {...}}}. +For conversational agents, checkpoints must be ORDER-INDEPENDENT: the agent may gather information +in any order, so assert final tool calls, final state, and captured facts, never a question order.""" + + +def subgoal_catalog_prompt(brief: str) -> str: + return f"""{brief} + +Task: derive this agent's SHARED SUB-GOAL CATALOG. + +A shared sub-goal is a milestone that will recur across MANY different test scenarios for this agent. +Naming these once, and reusing the same name everywhere, lets results aggregate: if 30 scenarios +include the sub-goal `payment_recorded` and it fails in 12, the team sees exactly where the agent +breaks. Scenario-specific values (which item, which amount) are filled in per scenario; the catalog +entry fixes the name, the meaning, and the shape of its checkpoint. + +{CHECKPOINT_VOCABULARY} + +Rules: +- 6 to 14 entries. Each must plausibly appear in several DIFFERENT scenarios for this agent. +- Names are snake_case, stable, and meaningful to a product owner. No internal plumbing, no + micro-steps. +- Each entry carries default_kind (the checkpoint kind it normally uses) and definition_template: + the definition shape with markers where a scenario supplies concrete values. +- Prefer deterministic kinds. If an entry must use `judge`, say in one line why nothing inspectable + exists for it. +Return JSON: {{"catalog": [{{"name": "...", "description": "...", "default_kind": "...", +"definition_template": {{...}}, "justification_if_judge": "..."}}]}}""" + + +def derive_rows_prompt(brief: str, *, want: int, signature_cases: list[str], + real_use_cases: list[str], existing: list[dict], feedback: str, + first_round: bool) -> str: + must = "" + if first_round and signature_cases: + must = ("Include one scenario for EACH of these required cases first (they come from the " + "agent's own constraints and data):\n - " + + "\n - ".join(str(s) for s in signature_cases) + "\n") + uses = "" + if real_use_cases: + uses = ("The agent's real use cases, to draw scenarios from:\n - " + + "\n - ".join(str(u) for u in real_use_cases) + "\n") + dedupe = "" + if existing: + dedupe = ("Scenarios already planned. Yours must test DIFFERENT situations with DIFFERENT " + f"correct outcomes; do not repeat or reword any of these:\n{json.dumps(existing)[:2200]}\n") + feedback_block = f"\nReviewer feedback on the previous round; act on all of it:\n{feedback}\n" if feedback else "" + return f"""{brief} + +Task: plan {want} distinct test scenarios for this agent. You are both the engineer who built it and +the product manager who answers for it in production; plan the tests those two people would insist +on before shipping. + +For each scenario return one line of planning, not the full test yet: +- id: a short slug +- use_case: the user-facing job it belongs to (sentence case; scenarios sharing a job repeat the + same use_case wording exactly) +- situation: ONE line naming the specific condition of the world or the user that this scenario + fixes, phrased from the user or world side. It must not mention the agent's tools, must not + prescribe what the agent should do, and must not contain the expected outcome. +- why_distinct: one line naming the distinct correct OUTCOME this situation produces +- goal: one line, the single end-objective of the test + +{uses}{must}Coverage rules: +- Different situations with the same correct outcome are ONE scenario; pick the strongest. +- Cover the failure-shaped situations a production owner worries about, where the contract makes + them real: the requested thing does not exist or is unavailable, the request is ambiguous and + needs a clarifying question, the user changes their mind or corrects an earlier statement mid-way, + the request violates one of the agent's hard constraints and must be declined, the user abandons. +- Also cover the core successful paths, including ones with several steps or several items. +- No scenarios about internal machinery (logging, config, retries): users never bring those. +{dedupe}{feedback_block}Return JSON: {{"rows": [{{"id": "...", "use_case": "...", "situation": "...", +"why_distinct": "...", "goal": "..."}}]}}""" + + +def materialize_prompt(brief: str, *, row: dict, base_environment: dict, catalog: list[dict], + modality: str, conversational: bool, hint: str = "") -> str: + input_spec = AGENT_INPUT_BY_MODALITY.get(modality, AGENT_INPUT_BY_MODALITY["_default"]) + conv = "" + if conversational: + conv = """- This agent is conversational: `agent_input` is the situation instruction handed to the + simulated user, and `facts` lists what that user knows. Every fact the agent is supposed to ask + for gets disclosure "on_request"; the simulated user volunteers only "volunteer" facts. +""" + catalog_block = json.dumps( + [{"name": c.get("name"), "description": c.get("description"), + "default_kind": c.get("default_kind"), "definition_template": c.get("definition_template")} + for c in catalog] + )[:3600] + fix = "" + if hint: + fix = f"\n\nA previous draft of this scenario failed review. Fix every one of these before returning:\n{hint}" + return f"""CONTRACT (the agent's real interface; use nothing outside it): +{brief} + +BASE ENVIRONMENT (exists before every test; each test declares only its changes): +{json.dumps(base_environment)[:1600]} + +SHARED SUB-GOAL CATALOG (when a milestone in your scenario matches an entry, use the entry's exact +name and fill its definition_template with this scenario's concrete values; invent a new sub-goal +name only when no entry fits): +{catalog_block} + +SCENARIO PLAN to expand into a full test: {json.dumps(row)} + +{CHECKPOINT_VOCABULARY} + +Write the complete test. Every value must be a real value from the contract's data. Keep the three +parts separate: the input never reveals the environment seeding, the checkpoints, or the outcome. + +Return JSON with ALL of these keys, none empty: +- id, use_case, situation, goal: carried from the plan (sharpen wording if needed, keep meaning) +- description: 2-3 sentences for a human reviewer: what is seeded, what the user wants, and what a + correct agent does. This is documentation, not part of the test input. +- agent_input: {input_spec} +- facts: [{{"key": "...", "value": "...", "disclosure": "volunteer" | "on_request" | "withhold"}}]. + The concrete information the simulated user holds. Empty list only for non-conversational agents. +- persona: {{"name": ""}} and nothing more, unless this scenario is + specifically about a user attribute +- environment: {{"seed": {{}}, + "mock_responses": {{"": {{"content": "", + "state_updates": {{}}}}}}}}. Mock only tools this scenario expects + the agent to call. +- sub_goals: 3 to 6, per the checkpoint vocabulary above, every definition fully concrete +- expected_outcome: {{"world_state": "", "must_convey": [""], "forbidden": [""]}} +- max_reasonable_turns: how many user turns a competent agent needs, as an integer{fix}""" + + +CRITIC_SYSTEM = SCENARIO_MODEL + """ + +Role: you are the reviewer who decides whether a proposed test scenario enters the team's test suite. +You did not write it, and your default answer is no. Approve only what you would defend to the +engineer who owns the agent. Review in this order: + +1. WORTH. Could a competent implementation of this agent plausibly fail this test? If every correct + implementation passes it for free, reject it however well it is written, and say what a good + agent could actually get wrong here if anything. +2. REAL. Would a real user plausibly bring this situation? +3. GROUNDED. Every tool, argument name, value, and id exists in the contract, spelled exactly. + Nothing contradicts the agent's hard constraints. Any invented interface or id is fatal. +4. CHECKABLE. Every deterministic checkpoint is computable from the seeded environment plus the + expected calls; expected values match what the input implies (an input asking for a large drink + must not be checked as medium); conversational checkpoints do not depend on question order. +5. SEPARATION. The input reveals nothing the user would not know: no seeded availability, no + internal ids, no expected outcome, no checkpoint contents. + +Return JSON: {"verdict": "accept" | "revise" | "reject", "scores": {"worth": 1-5, "real": 1-5, +"grounded": 1-5, "checkable": 1-5, "separation": 1-5}, "problems": [""], +"fix_hints": ""} +Reject means the situation itself is not worth testing; revise means the situation is good but the +execution has fixable problems.""" + + +def critic_prompt(brief: str, scenario: dict) -> str: + return f"""CONTRACT (the ground truth this test must respect): +{brief} + +PROPOSED TEST SCENARIO: +{json.dumps(scenario)[:7000]} + +Review it per your instructions and return the JSON verdict.""" + + +SUITE_REVIEW_SYSTEM = SCENARIO_MODEL + """ + +Role: you review a whole set of accepted test scenarios for COVERAGE, not for individual quality. +You answer one question: what is missing? Return specific, plannable gaps, each phrased as a +situation from the user or world side with its distinct correct outcome. Do not repeat situations +the set already covers. Return JSON: {"gaps": [{"situation": "", "why_it_matters": ""}], "near_duplicates": [["", ""]]} with at most 6 gaps, empty lists when the set is +genuinely complete.""" + + +def suite_review_prompt(brief: str, records: list[dict]) -> str: + summary = [ + { + "id": r.get("id"), + "use_case": r.get("use_case"), + "situation": r.get("situation"), + "outcome": (r.get("expected_outcome") or {}).get("world_state"), + } + for r in records + ] + return f"""CONTRACT: +{brief} + +ACCEPTED SCENARIOS so far: +{json.dumps(summary)[:6000]} + +What situations that matter in production are missing, and which pairs are near-duplicates?""" diff --git a/src/fi/alk/generation/sources.py b/src/fi/alk/generation/sources.py new file mode 100644 index 0000000..a1dc208 --- /dev/null +++ b/src/fi/alk/generation/sources.py @@ -0,0 +1,145 @@ +"""Agent connections: where the evidence about an agent comes from. + +An ``AgentSource`` turns "point me at an agent" into a bounded text blob the contract extractor can +ground in. The repo-folder source ships today; a Vapi or Retell config fetch, or a platform agent +definition, is a new class with three members registered under a new name. Registration reuses the +runtime's ``AdapterRegistry`` so third-party packages can plug in through the +``fi.alk.generation.sources`` entry-point group without editing this file. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from typing import Any, Protocol + +from fi.simulate.registry import AdapterRegistry + +SOURCE_ENTRY_POINT_GROUP = "fi.alk.generation.sources" + +source_registry: AdapterRegistry = AdapterRegistry("agent_source", SOURCE_ENTRY_POINT_GROUP) + + +def register_source(name: str, factory=None, *, override: bool = False): + return source_registry.register(name, factory, override=override) + + +@dataclass +class AgentEvidence: + """Bounded, grounded raw material about one agent.""" + + name: str + text: str + metadata: dict[str, Any] = field(default_factory=dict) + + +class AgentSource(Protocol): + """One way of reaching an agent's definition.""" + + name: str + + def describe(self) -> AgentEvidence: ... + + +# Path fragments that tend to hold the action surface: tools, prompts, commands, data. +_SURFACE_HINTS = ( + "controller", "tool", "tools", "action", "function", "command", "commands", "prompt", + "prompts", "skill", "skills", "registry", "capabilit", "agent", "database", "menu", + "order", "schema", "config", "assistant", "instruction", +) +_EXAMPLE_HINTS = ("example", "examples", "demo", "cookbook", "recipe") +_CODE_EXT = (".py", ".ts", ".js", ".yaml", ".yml", ".md", ".txt", ".toml", ".json") +_SKIP_DIRS = { + ".git", "node_modules", ".venv", "venv", "__pycache__", "dist", "build", ".next", + "frontend", "static", "assets", ".flox", "tests", "test", ".omega", "artifacts", +} +_MAX_FILE_CHARS = 9000 +_MAX_TOTAL_CHARS = 60_000 + + +def _read(path: str, limit: int = _MAX_FILE_CHARS) -> str: + try: + with open(path, encoding="utf-8", errors="ignore") as fh: + return fh.read(limit) + except OSError: + return "" + + +@register_source("repo") +@dataclass +class RepoFolderSource: + """Read an agent straight out of its repository folder. + + Collection is deterministic: README first, then files ranked by how strongly their path suggests + the action surface (tools, prompts, commands, data), then example task names. Nothing is + executed; nothing leaves the machine. + """ + + path: str + name: str = "repo" + + def describe(self) -> AgentEvidence: + root = os.path.abspath(self.path.rstrip("/")) + if not os.path.isdir(root): + raise FileNotFoundError(f"agent repo folder not found: {root}") + agent_name = os.path.basename(root) + parts: list[str] = [f"# AGENT REPO: {agent_name}\n"] + total = len(parts[0]) + + for candidate in ("README.md", "README.rst", "readme.md", "README"): + readme = os.path.join(root, candidate) + if os.path.isfile(readme): + body = _read(readme) + parts.append(f"\n## README\n{body}\n") + total += len(body) + break + + surface: list[tuple[int, str]] = [] + examples: list[str] = [] + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = [d for d in dirnames if d.lower() not in _SKIP_DIRS] + lowered_dir = dirpath.lower() + in_examples = any(hint in lowered_dir for hint in _EXAMPLE_HINTS) + for filename in filenames: + rel = os.path.relpath(os.path.join(dirpath, filename), root) + if in_examples: + if filename.endswith(_CODE_EXT): + examples.append(rel) + continue + if not filename.endswith(_CODE_EXT): + continue + lowered_file = filename.lower() + score = sum( + 2 * (hint in lowered_file) + (f"/{hint}" in lowered_dir) + for hint in _SURFACE_HINTS + ) + if score > 0: + surface.append((score, os.path.join(dirpath, filename))) + + surface.sort(key=lambda item: -item[0]) + if surface: + parts.append("\n## ACTION SURFACE (tool / prompt / command / data files)\n") + for _, filepath in surface: + if total > _MAX_TOTAL_CHARS: + break + body = _read(filepath) + if not body.strip(): + continue + chunk = f"\n### FILE: {os.path.relpath(filepath, root)}\n{body}\n" + parts.append(chunk) + total += len(chunk) + + if examples: + parts.append("\n## EXAMPLE TASKS (filenames reveal real use-cases)\n") + parts.append("\n".join(f"- {e}" for e in sorted(examples)[:120])) + + return AgentEvidence( + name=agent_name, + text="".join(parts)[: _MAX_TOTAL_CHARS + 12_000], + metadata={"source": self.name, "root": root, "surface_files": len(surface)}, + ) + + +def resolve_source(kind: str, **kwargs: Any) -> AgentSource: + """Build a registered source by name (``repo`` today; ``vapi``/``retell`` tomorrow).""" + return source_registry.create(kind, **kwargs) diff --git a/src/fi/alk/generation/validators.py b/src/fi/alk/generation/validators.py new file mode 100644 index 0000000..9a9cfb4 --- /dev/null +++ b/src/fi/alk/generation/validators.py @@ -0,0 +1,174 @@ +"""Deterministic structural validators. Free, run before any critic call. + +Structure and grounding are code's job, not the model's: completeness, the sub-goal/checkpoint join, +checkpoint definitions matching their declared kind, tool references existing in the contract, and +the hallucinated-interface guard (interface-shaped tokens that appear only in the contract's +anti_hallucination list). No domain vocabulary lives here. +""" + +from __future__ import annotations + +import json +import re + +from .contract import AgentContract + +_CHECK_KINDS = ("tool_call_args", "state", "conveyed", "absent", "judge") +_DISCLOSURES = ("volunteer", "on_request", "withhold") +_TOKEN = re.compile(r"/[a-z][a-z0-9_]{2,}|[A-Za-z_][A-Za-z0-9_]{2,}") + + +def _interface_shaped(token: str) -> bool: + return "_" in token or token.startswith("/") or bool(re.search(r"[a-z][A-Z]", token)) + + +def _legit_vocabulary(contract: AgentContract) -> set[str]: + payload = contract.model_dump(exclude={"anti_hallucination"}) + return {match.lower() for match in _TOKEN.findall(json.dumps(payload))} + + +def banned_tokens(contract: AgentContract) -> set[str]: + """Interface-shaped tokens appearing ONLY in anti_hallucination (the known fakes).""" + legit = _legit_vocabulary(contract) + banned: set[str] = set() + for entry in contract.anti_hallucination: + for match in _TOKEN.findall(str(entry)): + if _interface_shaped(match) and match.lower() not in legit: + banned.add(match) + return banned + + +def _validate_definition(kind: str, definition: dict, tool_names: set[str], where: str) -> list[str]: + problems: list[str] = [] + if kind == "tool_call_args": + tool = definition.get("tool") + if tool not in tool_names: + problems.append(f"{where}:unknown-tool:{tool}") + if not definition.get("args_equal") and not definition.get("args_present"): + problems.append(f"{where}:tool_call_args-without-args") + elif kind == "state": + if not definition.get("must") and not definition.get("forbidden"): + problems.append(f"{where}:state-without-must-or-forbidden") + elif kind == "conveyed": + variants = definition.get("must_include_any") + if not isinstance(variants, list) or not any(str(v).strip() for v in variants or []): + problems.append(f"{where}:conveyed-without-variants") + elif kind == "absent": + inner = definition.get("no_tool_call_with") or {} + tool = definition.get("no_tool_call") or inner.get("tool") + if not tool: + problems.append(f"{where}:absent-without-tool") + elif tool not in tool_names: + problems.append(f"{where}:unknown-tool:{tool}") + elif kind == "judge": + if not str(definition.get("rubric", "")).strip(): + problems.append(f"{where}:judge-without-rubric") + else: + problems.append(f"{where}:unknown-kind:{kind}") + return problems + + +def validate_scenario(scenario: dict, contract: AgentContract) -> list[str]: + """Return problems; empty means structurally complete and grounded enough for the critic.""" + problems: list[str] = [] + tool_names = contract.tool_names() + + for field in ("id", "use_case", "situation", "goal", "description", "agent_input", "expected_outcome"): + if scenario.get(field) in (None, "", [], {}): + problems.append(f"empty:{field}") + description = scenario.get("description") + if isinstance(description, str) and 0 < len(description) < 60: + problems.append("description-too-short") + + facts = scenario.get("facts") + if contract.conversational and not isinstance(facts, list): + problems.append("facts-not-a-list") + for index, fact in enumerate(facts or []): + if not isinstance(fact, dict) or not fact.get("key"): + problems.append(f"fact[{index}]:malformed") + elif fact.get("disclosure") not in _DISCLOSURES: + problems.append(f"fact[{index}]:bad-disclosure") + + sub_goals = scenario.get("sub_goals") + if not isinstance(sub_goals, list) or len(sub_goals) < 3: + problems.append("sub_goals<3") + else: + seen_names: set[str] = set() + deterministic_count = 0 + for index, sub_goal in enumerate(sub_goals): + where = f"sub_goal[{index}]" + if not isinstance(sub_goal, dict) or not sub_goal.get("name"): + problems.append(f"{where}:no-name") + continue + name = str(sub_goal["name"]) + if not re.fullmatch(r"[a-z][a-z0-9_]*", name): + problems.append(f"{where}:name-not-snake_case:{name}") + if name in seen_names: + problems.append(f"{where}:duplicate-name:{name}") + seen_names.add(name) + checkpoint = sub_goal.get("checkpoint") + if not isinstance(checkpoint, dict): + problems.append(f"{where}:no-checkpoint") + continue + kind = str(checkpoint.get("kind", "")) + definition = checkpoint.get("definition") + if not isinstance(definition, dict) or not definition: + problems.append(f"{where}:no-definition") + else: + problems += _validate_definition(kind, definition, tool_names, where) + deterministic = bool(checkpoint.get("deterministic")) + if deterministic and kind == "judge": + problems.append(f"{where}:judge-marked-deterministic") + if deterministic: + deterministic_count += 1 + if sub_goals and deterministic_count == 0: + problems.append("no-deterministic-checkpoint") + + outcome = scenario.get("expected_outcome") + if isinstance(outcome, dict): + if not str(outcome.get("world_state", "")).strip(): + problems.append("empty:expected_outcome.world_state") + + environment = scenario.get("environment") + if not isinstance(environment, dict): + problems.append("environment-not-a-dict") + else: + for tool in (environment.get("mock_responses") or {}): + if tool not in tool_names: + problems.append(f"mock_responses:unknown-tool:{tool}") + + blob = json.dumps(scenario) + if re.search(r"\{[a-z_]+\}", blob): + problems.append("template-placeholders-present") + banned = banned_tokens(contract) + hits = sorted({b for b in banned if re.search(r"(? str: + """Targeted, imperative fix instructions from validator problems.""" + lines: list[str] = [] + for problem in problems: + if problem.startswith("empty:"): + lines.append(f"- Field '{problem.split(':', 1)[1]}' was empty; fill it with real, complete content.") + elif problem == "description-too-short": + lines.append("- Write a proper 2-3 sentence description, not a stub.") + elif problem == "sub_goals<3": + lines.append("- Provide at least 3 branch-specific sub_goals, each with a concrete checkpoint, " + "ending with a final verification of the resulting state.") + elif ":unknown-tool:" in problem: + lines.append(f"- A checkpoint or mock references a tool that does not exist ({problem.split(':')[-1]}). " + "Use ONLY the contract's real tools with exact names.") + elif problem == "template-placeholders-present": + lines.append("- Remove every {placeholder}; write concrete values from the contract data.") + elif problem.startswith("banned-interface:"): + lines.append(f"- You referenced a non-existent interface ({problem.split(':', 1)[1]}). " + "Use only the contract's real tools, args and ids.") + elif problem == "no-deterministic-checkpoint": + lines.append("- Every checkpoint is a judge; make the tool-argument and end-state checks " + "deterministic per the vocabulary.") + elif ":" in problem: + lines.append(f"- Fix: {problem}") + return "\n".join(dict.fromkeys(lines)) diff --git a/tests/test_generation_pipeline.py b/tests/test_generation_pipeline.py new file mode 100644 index 0000000..8dbb42a --- /dev/null +++ b/tests/test_generation_pipeline.py @@ -0,0 +1,225 @@ +"""Offline pipeline test: fake LLM, real validators, real emission, real goal machine.""" + +from __future__ import annotations + +import json + +import pytest + +from fi.alk.generation import ( + AgentContract, + FakeLLMClient, + GenerationConfig, + RepoFolderSource, + ToolSpec, + generate, + smoke_manifest, + to_alk_scenario, + validate_scenario, +) + +CONTRACT = { + "agent": "cafe-order", + "one_liner": "Takes cafe orders over voice.", + "modality": "voice", + "conversational": True, + "hard_constraints": ["A combo requires a drink."], + "tools": [ + { + "name": "add_item", + "args": ["item_id", "size"], + "arg_values": {"item_id": ["latte", "mocha"], "size": ["M", "L"]}, + "description": "Add an item to the order.", + }, + {"name": "list_order", "args": [], "arg_values": {}, "description": "Read back the order."}, + ], + "data_schema": {"menu": {"latte": {"price": 4.5}, "mocha": {"price": 5.0}}}, + "base_environment": {"summary": "empty order", "seed": {"order": {"items": []}}}, + "real_use_cases": ["Order a latte -> add_item(item_id=latte)"], + "signature_cases": ["Unknown item is declined"], + "grading_notes": "The order state carries the truth.", + "anti_hallucination": ["remove_item (does not exist)", "add_to_cart (wrong name)"], +} + +CATALOG = { + "catalog": [ + { + "name": "item_added", + "description": "The requested item is in the order with the right attributes.", + "default_kind": "tool_call_args", + "definition_template": {"tool": "add_item", "args_equal": {"item_id": "", "size": ""}}, + }, + { + "name": "order_confirmed", + "description": "The final order was read back.", + "default_kind": "state", + "definition_template": {"must": {"order.confirmed": True}}, + }, + ] +} + +ROWS = { + "rows": [ + { + "id": "latte-medium", + "use_case": "Order a single item", + "situation": "The caller wants one medium latte and confirms", + "why_distinct": "Plain single-item success path", + "goal": "A medium latte is ordered and confirmed", + } + ] +} + +SCENARIO = { + "id": "latte-medium", + "use_case": "Order a single item", + "situation": "The caller wants one medium latte and confirms", + "goal": "A medium latte is ordered and confirmed", + "description": "A caller orders one medium latte, nothing else. The menu has lattes and mochas; " + "the order starts empty and the agent must add the right item at the right size.", + "agent_input": "You are calling a cafe. You want one latte, medium. If asked anything else, " + "decline politely.", + "facts": [{"key": "size", "value": "M", "disclosure": "on_request"}], + "persona": {"name": "Sam"}, + "environment": { + "seed": {"order": {"items": [], "confirmed": False}}, + "mock_responses": { + "add_item": { + "content": "added latte size M", + "state_updates": {"order": {"items": ["latte_M"], "confirmed": True}}, + } + }, + }, + "sub_goals": [ + { + "name": "item_added", + "milestone": "The latte is added at the requested size", + "checkpoint": { + "kind": "tool_call_args", + "detail": "add_item called with item_id=latte, size=M", + "deterministic": True, + "definition": {"tool": "add_item", "args_equal": {"item_id": "latte", "size": "M"}}, + }, + }, + { + "name": "order_confirmed", + "milestone": "The order ends confirmed", + "checkpoint": { + "kind": "state", + "detail": "order.confirmed is true", + "deterministic": True, + "definition": {"must": {"order.confirmed": True}}, + }, + }, + { + "name": "price_conveyed", + "milestone": "The caller hears the price", + "checkpoint": { + "kind": "conveyed", + "detail": "the agent states the latte price", + "deterministic": True, + "definition": {"must_include_any": ["4.5", "4.50"]}, + }, + }, + ], + "expected_outcome": { + "world_state": "The order holds one medium latte and is confirmed", + "must_convey": ["4.5"], + "forbidden": ["adding any second item"], + }, + "max_reasonable_turns": 6, +} + +VERDICT = { + "verdict": "accept", + "scores": {"worth": 4, "real": 5, "grounded": 5, "checkable": 4, "separation": 5}, + "problems": [], + "fix_hints": "", +} + + +@pytest.fixture() +def agent_repo(tmp_path): + repo = tmp_path / "cafe-agent" + repo.mkdir() + (repo / "README.md").write_text("# Cafe order agent\nTakes cafe orders.") + (repo / "tools.py").write_text("def add_item(item_id, size):\n ...\n") + return str(repo) + + +def test_full_pipeline_offline(agent_repo, tmp_path): + llm = FakeLLMClient( + responses=[ + {"tool_calls": [{"id": "c1", "name": "submit_contract", "arguments": {"contract": CONTRACT}}]}, + CATALOG, + ROWS, + SCENARIO, + VERDICT, + {"gaps": [], "near_duplicates": []}, + ] + ) + out = tmp_path / "out" + result = generate( + RepoFolderSource(path=agent_repo), + llm, + GenerationConfig(n=1, out_dir=str(out)), + ) + assert len(result.records) == 1 + assert not result.rejected + record = result.records[0] + assert record["_review"]["verdict"] == "accept" + + assert (out / "scenarios" / "latte-medium.json").is_file() + assert (out / "report.md").is_file() + alk = json.loads((out / "alk" / "latte-medium.json").read_text()) + assert alk["kind"] == "task" + assert alk["goal"]["states"] == ["item_added", "order_confirmed", "price_conveyed"] + check_names = {c["name"] for c in alk["verification"]["checks"]} + assert set(alk["goal"]["states"]) <= check_names + assert alk["dataset"][0]["knowledge"][0]["disclosure"] == "on_request" + + +def test_validators_catch_hallucinated_tool(): + contract = AgentContract( + agent="a", + tools=[ToolSpec(name="add_item", args=["item_id"])], + real_use_cases=["x"], + anti_hallucination=["remove_item (does not exist)"], + ) + bad = dict(SCENARIO) + bad = json.loads(json.dumps(bad).replace("add_item", "remove_item")) + problems = validate_scenario(bad, contract) + assert any("unknown-tool" in p or "banned-interface" in p for p in problems) + + +def test_smoke_manifest_state_checks_fire_through_goal_machine(): + contract = AgentContract.model_validate(CONTRACT) + manifest = smoke_manifest(SCENARIO, contract) + world = next(e for e in manifest["simulation"]["environments"] if e["type"] == "world_contract") + assert world["success_conditions"][0]["name"] == "order_confirmed" + + from fi.simulate.environment import WorldContractEnvironment + from fi.simulate.simulation import goal_machine + from fi.simulate.simulation.models import ScenarioGoal, VerificationSpec + + env = WorldContractEnvironment( + name="w", + initial_state={"order": {"items": ["latte_M"], "confirmed": True}}, + success_conditions=world["success_conditions"], + ) + snapshot = env.reset() + verdict = goal_machine.evaluate_settle( + ScenarioGoal(states=["order_confirmed"], success_state="order_confirmed"), + VerificationSpec(checks=manifest["scenario"]["verification"]["checks"]), + environment_state={"world_contract": snapshot.state.get("world_contract", env._summary())}, + ) + assert verdict["stop"] == "goal_success" + assert "order_confirmed" in verdict["states_reached"] + + +def test_alk_scenario_is_typed_and_content_addressed(): + contract = AgentContract.model_validate(CONTRACT) + scenario = to_alk_scenario(SCENARIO, contract) + assert scenario.kind == "task" + assert scenario.version and scenario.version.startswith("sha256:") + assert scenario.constraints.declared_tools == ["add_item"] From a78b93fe399e6f63f99a14073b41ac43108bcc76 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 13 Aug 2026 22:05:29 +0530 Subject: [PATCH 02/55] fix(generation): interpolate the conversational clause; format to house style --- src/fi/alk/generation/__init__.py | 9 ++- src/fi/alk/generation/cli.py | 24 ++++++-- src/fi/alk/generation/contract.py | 17 ++++-- src/fi/alk/generation/emit.py | 35 ++++++++--- src/fi/alk/generation/explorer.py | 80 +++++++++++++++++++----- src/fi/alk/generation/llm.py | 95 +++++++++++++++++++++++------ src/fi/alk/generation/pipeline.py | 76 ++++++++++++++++++----- src/fi/alk/generation/prompts.py | 81 ++++++++++++++++++------ src/fi/alk/generation/sources.py | 46 ++++++++++++-- src/fi/alk/generation/validators.py | 60 +++++++++++++----- tests/test_generation_pipeline.py | 37 +++++++++-- 11 files changed, 443 insertions(+), 117 deletions(-) diff --git a/src/fi/alk/generation/__init__.py b/src/fi/alk/generation/__init__.py index 244e278..19f6726 100644 --- a/src/fi/alk/generation/__init__.py +++ b/src/fi/alk/generation/__init__.py @@ -2,7 +2,14 @@ from .contract import AgentContract, ToolSpec, extract_contract, validate_contract from .emit import smoke_manifest, to_alk_scenario, write_outputs -from .llm import AuthFailed, BudgetExceeded, FakeLLMClient, LiteLLMClient, LLMClient, Usage +from .llm import ( + AuthFailed, + BudgetExceeded, + FakeLLMClient, + LiteLLMClient, + LLMClient, + Usage, +) from .pipeline import GenerationConfig, GenerationResult, generate from .sources import ( AgentEvidence, diff --git a/src/fi/alk/generation/cli.py b/src/fi/alk/generation/cli.py index 65f615d..a28f0d3 100644 --- a/src/fi/alk/generation/cli.py +++ b/src/fi/alk/generation/cli.py @@ -17,13 +17,23 @@ def build_parser() -> argparse.ArgumentParser: prog="python -m fi.alk.generation", description="Generate grounded, checkable test scenarios for an agent.", ) - parser.add_argument("--source", default="repo", help="agent connection kind (default: repo)") - parser.add_argument("--repo", help="path to the agent's repository folder (repo source)") + parser.add_argument( + "--source", default="repo", help="agent connection kind (default: repo)" + ) + parser.add_argument( + "--repo", help="path to the agent's repository folder (repo source)" + ) parser.add_argument("--n", type=int, default=20, help="target number of scenarios") parser.add_argument("--model", default=DEFAULT_MODEL, help="litellm model string") - parser.add_argument("--budget-usd", type=float, default=2.0, help="hard spend ceiling for this run") - parser.add_argument("--out", default="artifacts/generated-scenarios", help="output directory") - parser.add_argument("--no-critic", action="store_true", help="skip the QA review pass") + parser.add_argument( + "--budget-usd", type=float, default=2.0, help="hard spend ceiling for this run" + ) + parser.add_argument( + "--out", default="artifacts/generated-scenarios", help="output directory" + ) + parser.add_argument( + "--no-critic", action="store_true", help="skip the QA review pass" + ) parser.add_argument("--verbose", action="store_true") return parser @@ -42,7 +52,9 @@ def main(argv: list[str] | None = None) -> int: source_kwargs["path"] = args.repo source = resolve_source(args.source, **source_kwargs) llm = LiteLLMClient(model=args.model, budget_usd=args.budget_usd) - config = GenerationConfig(n=args.n, critic_enabled=not args.no_critic, out_dir=args.out) + config = GenerationConfig( + n=args.n, critic_enabled=not args.no_critic, out_dir=args.out + ) result = generate(source, llm, config) print( diff --git a/src/fi/alk/generation/contract.py b/src/fi/alk/generation/contract.py index e4a6bcb..12eef4f 100644 --- a/src/fi/alk/generation/contract.py +++ b/src/fi/alk/generation/contract.py @@ -46,12 +46,19 @@ def brief(self, *, full_schema: bool = True) -> str: """The grounding block handed to the model on every downstream call.""" lines: list[str] = [] for tool in self.tools: - values = f" [values: {json.dumps(tool.arg_values)[:300]}]" if tool.arg_values else "" - lines.append(f" - {tool.name}({', '.join(tool.args)}){values} : {tool.description[:140]}") + values = ( + f" [values: {json.dumps(tool.arg_values)[:300]}]" + if tool.arg_values + else "" + ) + lines.append( + f" - {tool.name}({', '.join(tool.args)}){values} : {tool.description[:140]}" + ) parts = [ f"AGENT: {self.agent} - {self.one_liner}", f"MODALITY: {self.modality}", - "REAL TOOLS (use ONLY these, with these exact arg names):\n" + ("\n".join(lines) or " (none)"), + "REAL TOOLS (use ONLY these, with these exact arg names):\n" + + ("\n".join(lines) or " (none)"), ] if self.hard_constraints: parts.append( @@ -64,7 +71,9 @@ def brief(self, *, full_schema: bool = True) -> str: + json.dumps(self.data_schema)[:2400] ) if self.grading_notes: - parts.append(f"GRADING NOTES (how checks must be written for THIS agent):\n{self.grading_notes[:900]}") + parts.append( + f"GRADING NOTES (how checks must be written for THIS agent):\n{self.grading_notes[:900]}" + ) if self.anti_hallucination: parts.append( "NEVER USE THESE (they do not exist / are wrong): " diff --git a/src/fi/alk/generation/emit.py b/src/fi/alk/generation/emit.py index 7a0b115..c1d4108 100644 --- a/src/fi/alk/generation/emit.py +++ b/src/fi/alk/generation/emit.py @@ -56,7 +56,11 @@ def _referenced_tools(record: dict, contract: AgentContract) -> list[str]: def to_alk_scenario(record: dict, contract: AgentContract) -> Scenario: sub_goals = record.get("sub_goals") or [] - names = [str(sg.get("name")) for sg in sub_goals if isinstance(sg, dict) and sg.get("name")] + names = [ + str(sg.get("name")) + for sg in sub_goals + if isinstance(sg, dict) and sg.get("name") + ] checks: list[dict[str, Any]] = [] for sub_goal in sub_goals: if not isinstance(sub_goal, dict) or not sub_goal.get("name"): @@ -89,7 +93,9 @@ def to_alk_scenario(record: dict, contract: AgentContract) -> Scenario: referenced = _referenced_tools(record, contract) return Scenario( name=str(record.get("id") or record.get("use_case", "scenario")), - description=f"{record.get('use_case', '')} :: {record.get('situation', '')}".strip(" :"), + description=f"{record.get('use_case', '')} :: {record.get('situation', '')}".strip( + " :" + ), dataset=[persona], kind="task", goal=ScenarioGoal(states=names, success_state=names[-1] if names else None), @@ -122,7 +128,11 @@ def smoke_manifest(record: dict, contract: AgentContract) -> dict[str, Any]: { "name": str(sub_goal.get("name")), "must": definition["must"], - **({"forbidden": definition["forbidden"]} if definition.get("forbidden") else {}), + **( + {"forbidden": definition["forbidden"]} + if definition.get("forbidden") + else {} + ), } ) states = [c["name"] for c in conditions] @@ -137,7 +147,9 @@ def smoke_manifest(record: dict, contract: AgentContract) -> dict[str, Any]: { "persona": dict(record.get("persona") or {"name": "Caller"}), "situation": str(record.get("agent_input", "")), - "outcome": str((record.get("expected_outcome") or {}).get("world_state", "")), + "outcome": str( + (record.get("expected_outcome") or {}).get("world_state", "") + ), } ], "goal": {"states": states, "success_state": states[-1] if states else None}, @@ -196,7 +208,10 @@ def _dump(path: str, payload: Any) -> None: alk = to_alk_scenario(record, contract) _dump(os.path.join(alk_dir, f"{slug}.json"), alk.model_dump(exclude_none=True)) if records: - _dump(os.path.join(out_dir, "smoke_manifest.json"), smoke_manifest(records[0], contract)) + _dump( + os.path.join(out_dir, "smoke_manifest.json"), + smoke_manifest(records[0], contract), + ) with open(os.path.join(out_dir, "report.md"), "w", encoding="utf-8") as fh: fh.write(render_report(contract, catalog, records, rejected, usage)) @@ -236,7 +251,11 @@ def render_report( ] for index, record in enumerate(records, 1): sub_goals = record.get("sub_goals") or [] - det = sum(1 for sg in sub_goals if ((sg or {}).get("checkpoint") or {}).get("deterministic")) + det = sum( + 1 + for sg in sub_goals + if ((sg or {}).get("checkpoint") or {}).get("deterministic") + ) lines.append( f"| {index} | {record.get('use_case', '')} | {record.get('situation', '')} " f"| {len(sub_goals)} | {det}/{len(sub_goals)} |" @@ -248,5 +267,7 @@ def render_report( if rejected: lines += ["", "## Rejected in review", ""] for record in rejected: - lines.append(f"- {record.get('id', '?')}: {record.get('_reject_reason', 'rejected')}") + lines.append( + f"- {record.get('id', '?')}: {record.get('_reject_reason', 'rejected')}" + ) return "\n".join(lines) + "\n" diff --git a/src/fi/alk/generation/explorer.py b/src/fi/alk/generation/explorer.py index 2f1b398..c5678ff 100644 --- a/src/fi/alk/generation/explorer.py +++ b/src/fi/alk/generation/explorer.py @@ -81,7 +81,12 @@ "description": "List entries of a directory inside the agent repository.", "parameters": { "type": "object", - "properties": {"path": {"type": "string", "description": "Relative path; '' for the root."}}, + "properties": { + "path": { + "type": "string", + "description": "Relative path; '' for the root.", + } + }, "required": [], }, }, @@ -96,7 +101,10 @@ "type": "object", "properties": { "path": {"type": "string"}, - "offset": {"type": "integer", "description": "Character offset to start from."}, + "offset": { + "type": "integer", + "description": "Character offset to start from.", + }, }, "required": ["path"], }, @@ -121,14 +129,28 @@ "description": "Submit the finished contract. Call exactly once, after verifying.", "parameters": { "type": "object", - "properties": {"contract": {"type": "object", "description": "The full contract JSON."}}, + "properties": { + "contract": { + "type": "object", + "description": "The full contract JSON.", + } + }, "required": ["contract"], }, }, }, ] -_SKIP_DIRS = {".git", "node_modules", ".venv", "venv", "__pycache__", "dist", "build", ".omega"} +_SKIP_DIRS = { + ".git", + "node_modules", + ".venv", + "venv", + "__pycache__", + "dist", + "build", + ".omega", +} class _RepoTools: @@ -152,7 +174,9 @@ def list_dir(self, path: str = "") -> str: if entry in _SKIP_DIRS: continue full = os.path.join(target, entry) - suffix = "/" if os.path.isdir(full) else f" ({os.path.getsize(full)} bytes)" + suffix = ( + "/" if os.path.isdir(full) else f" ({os.path.getsize(full)} bytes)" + ) entries.append(f"{entry}{suffix}") return "\n".join(entries) or "(empty)" @@ -166,7 +190,11 @@ def read_file(self, path: str, offset: int = 0) -> str: body = fh.read(_MAX_READ_CHARS) except OSError as exc: return f"read failed: {exc}" - marker = "" if len(body) < _MAX_READ_CHARS else f"\n... truncated; continue with offset={offset + _MAX_READ_CHARS}" + marker = ( + "" + if len(body) < _MAX_READ_CHARS + else f"\n... truncated; continue with offset={offset + _MAX_READ_CHARS}" + ) return body + marker def search_text(self, query: str) -> str: @@ -183,7 +211,9 @@ def search_text(self, query: str) -> str: for line_number, line in enumerate(fh, 1): if query in line: rel = os.path.relpath(full, self.root) - hits.append(f"{rel}:{line_number}: {line.strip()[:160]}") + hits.append( + f"{rel}:{line_number}: {line.strip()[:160]}" + ) if len(hits) >= 60: return "\n".join(hits) + "\n... (capped at 60 hits)" except OSError: @@ -191,7 +221,9 @@ def search_text(self, query: str) -> str: return "\n".join(hits) or "no hits" -def explore_contract(root: str, llm: LLMClient, *, max_turns: int = _MAX_TURNS) -> AgentContract: +def explore_contract( + root: str, llm: LLMClient, *, max_turns: int = _MAX_TURNS +) -> AgentContract: """Run the exploration loop until a valid contract is submitted.""" tools = _RepoTools(root) messages: list[dict[str, Any]] = [ @@ -218,9 +250,14 @@ def explore_contract(root: str, llm: LLMClient, *, max_turns: int = _MAX_TURNS) reply = llm.complete_turn(messages, tools=_TOOLS, temperature=0.15) calls = reply.get("tool_calls") or [] if not calls: - messages.append({"role": "assistant", "content": reply.get("content") or ""}) messages.append( - {"role": "user", "content": "Use the tools. Explore the repository, then call submit_contract."} + {"role": "assistant", "content": reply.get("content") or ""} + ) + messages.append( + { + "role": "user", + "content": "Use the tools. Explore the repository, then call submit_contract.", + } ) continue messages.append(_assistant_message(reply)) @@ -241,7 +278,9 @@ def explore_contract(root: str, llm: LLMClient, *, max_turns: int = _MAX_TURNS) if submitted is not None: logger.info("contract submitted", extra={"turns": turn + 1}) return submitted - raise RuntimeError(f"exploration ended after {max_turns} turns without a valid contract") + raise RuntimeError( + f"exploration ended after {max_turns} turns without a valid contract" + ) def _assistant_message(reply: dict[str, Any]) -> dict[str, Any]: @@ -258,7 +297,10 @@ def _assistant_message(reply: dict[str, Any]) -> dict[str, Any]: { "id": call.get("id") or call.get("name"), "type": "function", - "function": {"name": call.get("name"), "arguments": json.dumps(call.get("arguments") or {})}, + "function": { + "name": call.get("name"), + "arguments": json.dumps(call.get("arguments") or {}), + }, } for call in reply.get("tool_calls") or [] ], @@ -270,7 +312,9 @@ def _run_tool(tools: _RepoTools, name: str, arguments: dict[str, Any]) -> str: if name == "list_dir": return tools.list_dir(arguments.get("path", "")) if name == "read_file": - return tools.read_file(arguments.get("path", ""), int(arguments.get("offset") or 0)) + return tools.read_file( + arguments.get("path", ""), int(arguments.get("offset") or 0) + ) if name == "search_text": return tools.search_text(arguments.get("query", "")) except ValueError as exc: @@ -286,11 +330,17 @@ def _try_submit(arguments: dict[str, Any]) -> tuple[str, AgentContract | None]: except json.JSONDecodeError: payload = None if not isinstance(payload, dict): - return "submit_contract requires a JSON object under the 'contract' key; fix and resubmit", None + return ( + "submit_contract requires a JSON object under the 'contract' key; fix and resubmit", + None, + ) try: contract = AgentContract.model_validate(payload) except Exception as exc: # noqa: BLE001 - fed back to the model - return f"contract failed schema validation; fix and resubmit: {_short(exc)}", None + return ( + f"contract failed schema validation; fix and resubmit: {_short(exc)}", + None, + ) problems = validate_contract(contract) if problems: return f"contract failed checks; fix and resubmit: {problems}", None diff --git a/src/fi/alk/generation/llm.py b/src/fi/alk/generation/llm.py index df64281..d769b33 100644 --- a/src/fi/alk/generation/llm.py +++ b/src/fi/alk/generation/llm.py @@ -22,7 +22,14 @@ DEFAULT_INPUT_COST_PER_TOKEN = 0.30 / 1_000_000 DEFAULT_OUTPUT_COST_PER_TOKEN = 2.50 / 1_000_000 -_AUTH_MARKERS = ("401", "403", "unauthorized", "unauthenticated", "permission", "credential") +_AUTH_MARKERS = ( + "401", + "403", + "unauthorized", + "unauthenticated", + "permission", + "credential", +) class BudgetExceeded(RuntimeError): @@ -53,7 +60,12 @@ class LLMClient(Protocol): """What the pipeline needs from a model. Implementations own transport and retries.""" def complete_json( - self, system: str, user: str, *, temperature: float = 0.3, max_tokens: int = 8000 + self, + system: str, + user: str, + *, + temperature: float = 0.3, + max_tokens: int = 8000, ) -> Any: ... def complete_turn( @@ -97,7 +109,9 @@ def _extract_json(text: str) -> Any: return json.loads(text[start : i + 1]) except json.JSONDecodeError: break - raise ValueError(f"model returned no parseable JSON (first 200 chars: {text[:200]!r})") + raise ValueError( + f"model returned no parseable JSON (first 200 chars: {text[:200]!r})" + ) @dataclass @@ -118,7 +132,12 @@ def usage(self) -> Usage: return self._usage def complete_json( - self, system: str, user: str, *, temperature: float = 0.3, max_tokens: int = 8000 + self, + system: str, + user: str, + *, + temperature: float = 0.3, + max_tokens: int = 8000, ) -> Any: self._check_budget() text = self._chat(system, user, temperature=temperature, max_tokens=max_tokens) @@ -136,7 +155,9 @@ def complete_turn( try: import litellm except Exception as exc: # pragma: no cover - import guard - raise RuntimeError("fi.alk.generation requires litellm; reinstall agent-learning-kit") from exc + raise RuntimeError( + "fi.alk.generation requires litellm; reinstall agent-learning-kit" + ) from exc litellm.drop_params = True kwargs = self._provider_kwargs() kwargs.update({"temperature": temperature, "max_tokens": max_tokens}) @@ -145,28 +166,39 @@ def complete_turn( last: Exception | None = None for attempt in range(self.max_attempts): try: - response = litellm.completion(model=self.model, messages=messages, **kwargs) + response = litellm.completion( + model=self.model, messages=messages, **kwargs + ) self._meter(response) message = response.choices[0].message calls = [] for call in getattr(message, "tool_calls", None) or []: function = getattr(call, "function", None) try: - arguments = json.loads(getattr(function, "arguments", "") or "{}") + arguments = json.loads( + getattr(function, "arguments", "") or "{}" + ) except json.JSONDecodeError: arguments = {} calls.append( - {"id": getattr(call, "id", ""), "name": getattr(function, "name", ""), - "arguments": arguments} + { + "id": getattr(call, "id", ""), + "name": getattr(function, "name", ""), + "arguments": arguments, + } ) return {"content": message.content, "tool_calls": calls, "raw": message} except Exception as exc: # noqa: BLE001 - classified below text = str(exc).lower() if any(marker in text for marker in _AUTH_MARKERS): - raise AuthFailed(f"provider auth failed for {self.model}: {exc}") from exc + raise AuthFailed( + f"provider auth failed for {self.model}: {exc}" + ) from exc last = exc time.sleep(min(2**attempt, 8)) - raise RuntimeError(f"model call failed after {self.max_attempts} attempts: {last}") + raise RuntimeError( + f"model call failed after {self.max_attempts} attempts: {last}" + ) def _check_budget(self) -> None: if self._usage.usd >= self.budget_usd: @@ -187,20 +219,29 @@ def _provider_kwargs(self) -> dict[str, Any]: pass return kwargs - def _chat(self, system: str, user: str, *, temperature: float, max_tokens: int) -> str: + def _chat( + self, system: str, user: str, *, temperature: float, max_tokens: int + ) -> str: try: import litellm except Exception as exc: # pragma: no cover - import guard - raise RuntimeError("fi.alk.generation requires litellm; reinstall agent-learning-kit") from exc + raise RuntimeError( + "fi.alk.generation requires litellm; reinstall agent-learning-kit" + ) from exc litellm.drop_params = True kwargs = self._provider_kwargs() kwargs.update({"temperature": temperature, "max_tokens": max_tokens}) - messages = [{"role": "system", "content": system}, {"role": "user", "content": user}] + messages = [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ] last: Exception | None = None for attempt in range(self.max_attempts): try: - response = litellm.completion(model=self.model, messages=messages, **kwargs) + response = litellm.completion( + model=self.model, messages=messages, **kwargs + ) self._meter(response) content = response.choices[0].message.content if not content or not str(content).strip(): @@ -209,10 +250,14 @@ def _chat(self, system: str, user: str, *, temperature: float, max_tokens: int) except Exception as exc: # noqa: BLE001 - classified below message = str(exc).lower() if any(marker in message for marker in _AUTH_MARKERS): - raise AuthFailed(f"provider auth failed for {self.model}: {exc}") from exc + raise AuthFailed( + f"provider auth failed for {self.model}: {exc}" + ) from exc last = exc time.sleep(min(2**attempt, 8)) - raise RuntimeError(f"model call failed after {self.max_attempts} attempts: {last}") + raise RuntimeError( + f"model call failed after {self.max_attempts} attempts: {last}" + ) def _meter(self, response: Any) -> None: self._usage.calls += 1 @@ -221,7 +266,9 @@ def _meter(self, response: Any) -> None: completion = int(getattr(usage, "completion_tokens", 0) or 0) self._usage.prompt_tokens += prompt self._usage.completion_tokens += completion - self._usage.usd += prompt * self.input_cost_per_token + completion * self.output_cost_per_token + self._usage.usd += ( + prompt * self.input_cost_per_token + completion * self.output_cost_per_token + ) @dataclass @@ -236,7 +283,12 @@ def usage(self) -> Usage: return self._usage def complete_json( - self, system: str, user: str, *, temperature: float = 0.3, max_tokens: int = 8000 + self, + system: str, + user: str, + *, + temperature: float = 0.3, + max_tokens: int = 8000, ) -> Any: if not self.responses: raise AssertionError("FakeLLMClient exhausted; queue more responses") @@ -256,5 +308,8 @@ def complete_turn( self._usage.calls += 1 turn = self.responses.pop(0) if isinstance(turn, dict) and ("tool_calls" in turn or "content" in turn): - return {"content": turn.get("content"), "tool_calls": turn.get("tool_calls", [])} + return { + "content": turn.get("content"), + "tool_calls": turn.get("tool_calls", []), + } return {"content": json.dumps(turn), "tool_calls": []} diff --git a/src/fi/alk/generation/pipeline.py b/src/fi/alk/generation/pipeline.py index 06b6f7d..7d51fef 100644 --- a/src/fi/alk/generation/pipeline.py +++ b/src/fi/alk/generation/pipeline.py @@ -31,7 +31,9 @@ logger = logging.getLogger(__name__) -_ACCEPT_FLOOR = 3 # every reviewer score must reach this, and the verdict must not be reject +_ACCEPT_FLOOR = ( + 3 # every reviewer score must reach this, and the verdict must not be reject +) @dataclass @@ -59,7 +61,9 @@ def _slugify(value: str) -> str: return slug[:60] or "scenario" -def build_contract(source: AgentSource, llm: LLMClient, config: GenerationConfig) -> AgentContract: +def build_contract( + source: AgentSource, llm: LLMClient, config: GenerationConfig +) -> AgentContract: """Prefer the exploration loop when the source exposes a filesystem root.""" evidence = source.describe() root = (evidence.metadata or {}).get("root") @@ -67,7 +71,9 @@ def build_contract(source: AgentSource, llm: LLMClient, config: GenerationConfig try: return explore_contract(root, llm, max_turns=config.max_explore_turns) except Exception as exc: # noqa: BLE001 - fall back to single-shot extraction - logger.warning("exploration failed, falling back to blob extraction: %s", exc) + logger.warning( + "exploration failed, falling back to blob extraction: %s", exc + ) return extract_contract(evidence.text, llm) @@ -104,7 +110,10 @@ def derive_rows( brief = contract.brief() rows: list[dict] = [] seen = { - (str(r.get("use_case", "")).strip().lower(), str(r.get("situation", "")).strip().lower()) + ( + str(r.get("use_case", "")).strip().lower(), + str(r.get("situation", "")).strip().lower(), + ) for r in existing } for round_index in range(config.max_row_rounds): @@ -170,7 +179,11 @@ def materialize_row( temperature=0.35, max_tokens=9000, ) - record = raw if isinstance(raw, dict) else next((x for x in raw if isinstance(x, dict)), {}) + record = ( + raw + if isinstance(raw, dict) + else next((x for x in raw if isinstance(x, dict)), {}) + ) for key in ("id", "use_case", "situation", "goal"): record.setdefault(key, row.get(key)) record["id"] = _slugify(record.get("id") or row.get("id", "")) @@ -183,19 +196,31 @@ def materialize_row( if not config.critic_enabled: return record, "" verdict = llm.complete_json( - prompts.CRITIC_SYSTEM, prompts.critic_prompt(brief, record), temperature=0.2, max_tokens=2500 + prompts.CRITIC_SYSTEM, + prompts.critic_prompt(brief, record), + temperature=0.2, + max_tokens=2500, ) if not isinstance(verdict, dict): verdict = {} - record["_review"] = {k: verdict.get(k) for k in ("verdict", "scores", "problems")} + record["_review"] = { + k: verdict.get(k) for k in ("verdict", "scores", "problems") + } decision = str(verdict.get("verdict", "revise")).lower() scores = verdict.get("scores") or {} - low = [k for k, v in scores.items() if isinstance(v, (int, float)) and v < _ACCEPT_FLOOR] + low = [ + k + for k, v in scores.items() + if isinstance(v, (int, float)) and v < _ACCEPT_FLOOR + ] if decision == "accept" and not low: return record, "" if decision == "reject": return None, f"reviewer reject: {verdict.get('problems', [])[:4]}" - best, reason = record, f"reviewer revise (low: {low}): {verdict.get('problems', [])[:4]}" + best, reason = ( + record, + f"reviewer revise (low: {low}): {verdict.get('problems', [])[:4]}", + ) hint = str(verdict.get("fix_hints") or "") or repair_hint([]) # Out of repair attempts: keep the best structurally-valid draft, flagged, rather than lose it. if best is not None and not validate_scenario(best, contract): @@ -216,11 +241,17 @@ def suite_review( ) if not isinstance(raw, dict): return [], [], "" - gaps = [g for g in raw.get("gaps") or [] if isinstance(g, dict) and g.get("situation")] + gaps = [ + g for g in raw.get("gaps") or [] if isinstance(g, dict) and g.get("situation") + ] duplicate_ids: list[str] = [] known = {str(r.get("id")) for r in records} for pair in raw.get("near_duplicates") or []: - if isinstance(pair, list) and len(pair) == 2 and all(str(p) in known for p in pair): + if ( + isinstance(pair, list) + and len(pair) == 2 + and all(str(p) in known for p in pair) + ): duplicate_ids.append(str(pair[1])) feedback = "; ".join( f"missing: {g['situation']} ({g.get('why_it_matters', '')})" for g in gaps @@ -235,7 +266,9 @@ def generate( ) -> GenerationResult: config = config or GenerationConfig() contract = build_contract(source, llm, config) - logger.info("contract ready", extra={"agent": contract.agent, "tools": len(contract.tools)}) + logger.info( + "contract ready", extra={"agent": contract.agent, "tools": len(contract.tools)} + ) catalog = derive_catalog(contract, llm) records: list[dict] = [] @@ -258,7 +291,12 @@ def _flush() -> None: if want <= 0: break rows = derive_rows( - contract, llm, config, want=want, existing=records + rejected, feedback=feedback + contract, + llm, + config, + want=want, + existing=records + rejected, + feedback=feedback, ) if not rows: break @@ -271,10 +309,16 @@ def _flush() -> None: if suite_round < config.max_suite_rounds and records: gaps, duplicate_ids, feedback = suite_review(contract, records, llm) if duplicate_ids: - dropped = [r for r in records if str(r.get("id")) in set(duplicate_ids)] - records = [r for r in records if str(r.get("id")) not in set(duplicate_ids)] + dropped = [ + r for r in records if str(r.get("id")) in set(duplicate_ids) + ] + records = [ + r for r in records if str(r.get("id")) not in set(duplicate_ids) + ] for record in dropped: - record["_reject_reason"] = "near-duplicate of an accepted scenario" + record["_reject_reason"] = ( + "near-duplicate of an accepted scenario" + ) rejected.append(record) if not gaps and len(records) >= config.n: break diff --git a/src/fi/alk/generation/prompts.py b/src/fi/alk/generation/prompts.py index 40521b0..16674fe 100644 --- a/src/fi/alk/generation/prompts.py +++ b/src/fi/alk/generation/prompts.py @@ -123,23 +123,42 @@ def subgoal_catalog_prompt(brief: str) -> str: "definition_template": {{...}}, "justification_if_judge": "..."}}]}}""" -def derive_rows_prompt(brief: str, *, want: int, signature_cases: list[str], - real_use_cases: list[str], existing: list[dict], feedback: str, - first_round: bool) -> str: +def derive_rows_prompt( + brief: str, + *, + want: int, + signature_cases: list[str], + real_use_cases: list[str], + existing: list[dict], + feedback: str, + first_round: bool, +) -> str: must = "" if first_round and signature_cases: - must = ("Include one scenario for EACH of these required cases first (they come from the " - "agent's own constraints and data):\n - " - + "\n - ".join(str(s) for s in signature_cases) + "\n") + must = ( + "Include one scenario for EACH of these required cases first (they come from the " + "agent's own constraints and data):\n - " + + "\n - ".join(str(s) for s in signature_cases) + + "\n" + ) uses = "" if real_use_cases: - uses = ("The agent's real use cases, to draw scenarios from:\n - " - + "\n - ".join(str(u) for u in real_use_cases) + "\n") + uses = ( + "The agent's real use cases, to draw scenarios from:\n - " + + "\n - ".join(str(u) for u in real_use_cases) + + "\n" + ) dedupe = "" if existing: - dedupe = ("Scenarios already planned. Yours must test DIFFERENT situations with DIFFERENT " - f"correct outcomes; do not repeat or reword any of these:\n{json.dumps(existing)[:2200]}\n") - feedback_block = f"\nReviewer feedback on the previous round; act on all of it:\n{feedback}\n" if feedback else "" + dedupe = ( + "Scenarios already planned. Yours must test DIFFERENT situations with DIFFERENT " + f"correct outcomes; do not repeat or reword any of these:\n{json.dumps(existing)[:2200]}\n" + ) + feedback_block = ( + f"\nReviewer feedback on the previous round; act on all of it:\n{feedback}\n" + if feedback + else "" + ) return f"""{brief} Task: plan {want} distinct test scenarios for this agent. You are both the engineer who built it and @@ -168,9 +187,19 @@ def derive_rows_prompt(brief: str, *, want: int, signature_cases: list[str], "why_distinct": "...", "goal": "..."}}]}}""" -def materialize_prompt(brief: str, *, row: dict, base_environment: dict, catalog: list[dict], - modality: str, conversational: bool, hint: str = "") -> str: - input_spec = AGENT_INPUT_BY_MODALITY.get(modality, AGENT_INPUT_BY_MODALITY["_default"]) +def materialize_prompt( + brief: str, + *, + row: dict, + base_environment: dict, + catalog: list[dict], + modality: str, + conversational: bool, + hint: str = "", +) -> str: + input_spec = AGENT_INPUT_BY_MODALITY.get( + modality, AGENT_INPUT_BY_MODALITY["_default"] + ) conv = "" if conversational: conv = """- This agent is conversational: `agent_input` is the situation instruction handed to the @@ -178,9 +207,15 @@ def materialize_prompt(brief: str, *, row: dict, base_environment: dict, catalog for gets disclosure "on_request"; the simulated user volunteers only "volunteer" facts. """ catalog_block = json.dumps( - [{"name": c.get("name"), "description": c.get("description"), - "default_kind": c.get("default_kind"), "definition_template": c.get("definition_template")} - for c in catalog] + [ + { + "name": c.get("name"), + "description": c.get("description"), + "default_kind": c.get("default_kind"), + "definition_template": c.get("definition_template"), + } + for c in catalog + ] )[:3600] fix = "" if hint: @@ -202,7 +237,7 @@ def materialize_prompt(brief: str, *, row: dict, base_environment: dict, catalog Write the complete test. Every value must be a real value from the contract's data. Keep the three parts separate: the input never reveals the environment seeding, the checkpoints, or the outcome. - +{conv} Return JSON with ALL of these keys, none empty: - id, use_case, situation, goal: carried from the plan (sharpen wording if needed, keep meaning) - description: 2-3 sentences for a human reviewer: what is seeded, what the user wants, and what a @@ -223,7 +258,9 @@ def materialize_prompt(brief: str, *, row: dict, base_environment: dict, catalog - max_reasonable_turns: how many user turns a competent agent needs, as an integer{fix}""" -CRITIC_SYSTEM = SCENARIO_MODEL + """ +CRITIC_SYSTEM = ( + SCENARIO_MODEL + + """ Role: you are the reviewer who decides whether a proposed test scenario enters the team's test suite. You did not write it, and your default answer is no. Approve only what you would defend to the @@ -246,6 +283,7 @@ def materialize_prompt(brief: str, *, row: dict, base_environment: dict, catalog "fix_hints": ""} Reject means the situation itself is not worth testing; revise means the situation is good but the execution has fixable problems.""" +) def critic_prompt(brief: str, scenario: dict) -> str: @@ -258,7 +296,9 @@ def critic_prompt(brief: str, scenario: dict) -> str: Review it per your instructions and return the JSON verdict.""" -SUITE_REVIEW_SYSTEM = SCENARIO_MODEL + """ +SUITE_REVIEW_SYSTEM = ( + SCENARIO_MODEL + + """ Role: you review a whole set of accepted test scenarios for COVERAGE, not for individual quality. You answer one question: what is missing? Return specific, plannable gaps, each phrased as a @@ -266,6 +306,7 @@ def critic_prompt(brief: str, scenario: dict) -> str: the set already covers. Return JSON: {"gaps": [{"situation": "", "why_it_matters": ""}], "near_duplicates": [["", ""]]} with at most 6 gaps, empty lists when the set is genuinely complete.""" +) def suite_review_prompt(brief: str, records: list[dict]) -> str: diff --git a/src/fi/alk/generation/sources.py b/src/fi/alk/generation/sources.py index a1dc208..861b701 100644 --- a/src/fi/alk/generation/sources.py +++ b/src/fi/alk/generation/sources.py @@ -17,7 +17,9 @@ SOURCE_ENTRY_POINT_GROUP = "fi.alk.generation.sources" -source_registry: AdapterRegistry = AdapterRegistry("agent_source", SOURCE_ENTRY_POINT_GROUP) +source_registry: AdapterRegistry = AdapterRegistry( + "agent_source", SOURCE_ENTRY_POINT_GROUP +) def register_source(name: str, factory=None, *, override: bool = False): @@ -43,15 +45,47 @@ def describe(self) -> AgentEvidence: ... # Path fragments that tend to hold the action surface: tools, prompts, commands, data. _SURFACE_HINTS = ( - "controller", "tool", "tools", "action", "function", "command", "commands", "prompt", - "prompts", "skill", "skills", "registry", "capabilit", "agent", "database", "menu", - "order", "schema", "config", "assistant", "instruction", + "controller", + "tool", + "tools", + "action", + "function", + "command", + "commands", + "prompt", + "prompts", + "skill", + "skills", + "registry", + "capabilit", + "agent", + "database", + "menu", + "order", + "schema", + "config", + "assistant", + "instruction", ) _EXAMPLE_HINTS = ("example", "examples", "demo", "cookbook", "recipe") _CODE_EXT = (".py", ".ts", ".js", ".yaml", ".yml", ".md", ".txt", ".toml", ".json") _SKIP_DIRS = { - ".git", "node_modules", ".venv", "venv", "__pycache__", "dist", "build", ".next", - "frontend", "static", "assets", ".flox", "tests", "test", ".omega", "artifacts", + ".git", + "node_modules", + ".venv", + "venv", + "__pycache__", + "dist", + "build", + ".next", + "frontend", + "static", + "assets", + ".flox", + "tests", + "test", + ".omega", + "artifacts", } _MAX_FILE_CHARS = 9000 _MAX_TOTAL_CHARS = 60_000 diff --git a/src/fi/alk/generation/validators.py b/src/fi/alk/generation/validators.py index 9a9cfb4..5cc216f 100644 --- a/src/fi/alk/generation/validators.py +++ b/src/fi/alk/generation/validators.py @@ -19,7 +19,9 @@ def _interface_shaped(token: str) -> bool: - return "_" in token or token.startswith("/") or bool(re.search(r"[a-z][A-Z]", token)) + return ( + "_" in token or token.startswith("/") or bool(re.search(r"[a-z][A-Z]", token)) + ) def _legit_vocabulary(contract: AgentContract) -> set[str]: @@ -38,7 +40,9 @@ def banned_tokens(contract: AgentContract) -> set[str]: return banned -def _validate_definition(kind: str, definition: dict, tool_names: set[str], where: str) -> list[str]: +def _validate_definition( + kind: str, definition: dict, tool_names: set[str], where: str +) -> list[str]: problems: list[str] = [] if kind == "tool_call_args": tool = definition.get("tool") @@ -51,7 +55,9 @@ def _validate_definition(kind: str, definition: dict, tool_names: set[str], wher problems.append(f"{where}:state-without-must-or-forbidden") elif kind == "conveyed": variants = definition.get("must_include_any") - if not isinstance(variants, list) or not any(str(v).strip() for v in variants or []): + if not isinstance(variants, list) or not any( + str(v).strip() for v in variants or [] + ): problems.append(f"{where}:conveyed-without-variants") elif kind == "absent": inner = definition.get("no_tool_call_with") or {} @@ -73,7 +79,15 @@ def validate_scenario(scenario: dict, contract: AgentContract) -> list[str]: problems: list[str] = [] tool_names = contract.tool_names() - for field in ("id", "use_case", "situation", "goal", "description", "agent_input", "expected_outcome"): + for field in ( + "id", + "use_case", + "situation", + "goal", + "description", + "agent_input", + "expected_outcome", + ): if scenario.get(field) in (None, "", [], {}): problems.append(f"empty:{field}") description = scenario.get("description") @@ -133,7 +147,7 @@ def validate_scenario(scenario: dict, contract: AgentContract) -> list[str]: if not isinstance(environment, dict): problems.append("environment-not-a-dict") else: - for tool in (environment.get("mock_responses") or {}): + for tool in environment.get("mock_responses") or {}: if tool not in tool_names: problems.append(f"mock_responses:unknown-tool:{tool}") @@ -141,7 +155,9 @@ def validate_scenario(scenario: dict, contract: AgentContract) -> list[str]: if re.search(r"\{[a-z_]+\}", blob): problems.append("template-placeholders-present") banned = banned_tokens(contract) - hits = sorted({b for b in banned if re.search(r"(? str: lines: list[str] = [] for problem in problems: if problem.startswith("empty:"): - lines.append(f"- Field '{problem.split(':', 1)[1]}' was empty; fill it with real, complete content.") + lines.append( + f"- Field '{problem.split(':', 1)[1]}' was empty; fill it with real, complete content." + ) elif problem == "description-too-short": lines.append("- Write a proper 2-3 sentence description, not a stub.") elif problem == "sub_goals<3": - lines.append("- Provide at least 3 branch-specific sub_goals, each with a concrete checkpoint, " - "ending with a final verification of the resulting state.") + lines.append( + "- Provide at least 3 branch-specific sub_goals, each with a concrete checkpoint, " + "ending with a final verification of the resulting state." + ) elif ":unknown-tool:" in problem: - lines.append(f"- A checkpoint or mock references a tool that does not exist ({problem.split(':')[-1]}). " - "Use ONLY the contract's real tools with exact names.") + lines.append( + f"- A checkpoint or mock references a tool that does not exist ({problem.split(':')[-1]}). " + "Use ONLY the contract's real tools with exact names." + ) elif problem == "template-placeholders-present": - lines.append("- Remove every {placeholder}; write concrete values from the contract data.") + lines.append( + "- Remove every {placeholder}; write concrete values from the contract data." + ) elif problem.startswith("banned-interface:"): - lines.append(f"- You referenced a non-existent interface ({problem.split(':', 1)[1]}). " - "Use only the contract's real tools, args and ids.") + lines.append( + f"- You referenced a non-existent interface ({problem.split(':', 1)[1]}). " + "Use only the contract's real tools, args and ids." + ) elif problem == "no-deterministic-checkpoint": - lines.append("- Every checkpoint is a judge; make the tool-argument and end-state checks " - "deterministic per the vocabulary.") + lines.append( + "- Every checkpoint is a judge; make the tool-argument and end-state checks " + "deterministic per the vocabulary." + ) elif ":" in problem: lines.append(f"- Fix: {problem}") return "\n".join(dict.fromkeys(lines)) diff --git a/tests/test_generation_pipeline.py b/tests/test_generation_pipeline.py index 8dbb42a..542c7f4 100644 --- a/tests/test_generation_pipeline.py +++ b/tests/test_generation_pipeline.py @@ -31,7 +31,12 @@ "arg_values": {"item_id": ["latte", "mocha"], "size": ["M", "L"]}, "description": "Add an item to the order.", }, - {"name": "list_order", "args": [], "arg_values": {}, "description": "Read back the order."}, + { + "name": "list_order", + "args": [], + "arg_values": {}, + "description": "Read back the order.", + }, ], "data_schema": {"menu": {"latte": {"price": 4.5}, "mocha": {"price": 5.0}}}, "base_environment": {"summary": "empty order", "seed": {"order": {"items": []}}}, @@ -47,7 +52,10 @@ "name": "item_added", "description": "The requested item is in the order with the right attributes.", "default_kind": "tool_call_args", - "definition_template": {"tool": "add_item", "args_equal": {"item_id": "", "size": ""}}, + "definition_template": { + "tool": "add_item", + "args_equal": {"item_id": "", "size": ""}, + }, }, { "name": "order_confirmed", @@ -98,7 +106,10 @@ "kind": "tool_call_args", "detail": "add_item called with item_id=latte, size=M", "deterministic": True, - "definition": {"tool": "add_item", "args_equal": {"item_id": "latte", "size": "M"}}, + "definition": { + "tool": "add_item", + "args_equal": {"item_id": "latte", "size": "M"}, + }, }, }, { @@ -150,7 +161,15 @@ def agent_repo(tmp_path): def test_full_pipeline_offline(agent_repo, tmp_path): llm = FakeLLMClient( responses=[ - {"tool_calls": [{"id": "c1", "name": "submit_contract", "arguments": {"contract": CONTRACT}}]}, + { + "tool_calls": [ + { + "id": "c1", + "name": "submit_contract", + "arguments": {"contract": CONTRACT}, + } + ] + }, CATALOG, ROWS, SCENARIO, @@ -195,7 +214,11 @@ def test_validators_catch_hallucinated_tool(): def test_smoke_manifest_state_checks_fire_through_goal_machine(): contract = AgentContract.model_validate(CONTRACT) manifest = smoke_manifest(SCENARIO, contract) - world = next(e for e in manifest["simulation"]["environments"] if e["type"] == "world_contract") + world = next( + e + for e in manifest["simulation"]["environments"] + if e["type"] == "world_contract" + ) assert world["success_conditions"][0]["name"] == "order_confirmed" from fi.simulate.environment import WorldContractEnvironment @@ -211,7 +234,9 @@ def test_smoke_manifest_state_checks_fire_through_goal_machine(): verdict = goal_machine.evaluate_settle( ScenarioGoal(states=["order_confirmed"], success_state="order_confirmed"), VerificationSpec(checks=manifest["scenario"]["verification"]["checks"]), - environment_state={"world_contract": snapshot.state.get("world_contract", env._summary())}, + environment_state={ + "world_contract": snapshot.state.get("world_contract", env._summary()) + }, ) assert verdict["stop"] == "goal_success" assert "order_confirmed" in verdict["states_reached"] From b4053dc9edd892c931458de046634d45b39f1000 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 13 Aug 2026 22:10:17 +0530 Subject: [PATCH 03/55] feat(generation): benchmark-shaped planning (target_failure), operator guidance channel, critic catch-verification --- src/fi/alk/generation/cli.py | 4 ++ src/fi/alk/generation/pipeline.py | 11 +++- src/fi/alk/generation/prompts.py | 105 ++++++++++++++++++++++-------- 3 files changed, 91 insertions(+), 29 deletions(-) diff --git a/src/fi/alk/generation/cli.py b/src/fi/alk/generation/cli.py index a28f0d3..10f87c0 100644 --- a/src/fi/alk/generation/cli.py +++ b/src/fi/alk/generation/cli.py @@ -50,6 +50,10 @@ def main(argv: list[str] | None = None) -> int: print("--repo is required for the repo source", file=sys.stderr) return 2 source_kwargs["path"] = args.repo + guidance = args.guidance + if guidance.startswith("@"): + with open(guidance[1:], encoding="utf-8") as fh: + guidance = fh.read() source = resolve_source(args.source, **source_kwargs) llm = LiteLLMClient(model=args.model, budget_usd=args.budget_usd) config = GenerationConfig( diff --git a/src/fi/alk/generation/pipeline.py b/src/fi/alk/generation/pipeline.py index 7d51fef..9ea2c57 100644 --- a/src/fi/alk/generation/pipeline.py +++ b/src/fi/alk/generation/pipeline.py @@ -44,6 +44,7 @@ class GenerationConfig: max_suite_rounds: int = 2 max_explore_turns: int = 20 critic_enabled: bool = True + guidance: str = "" out_dir: str = "artifacts/generated-scenarios" @@ -133,6 +134,7 @@ def derive_rows( ], feedback=feedback, first_round=round_index == 0 and not existing, + guidance=config.guidance, ), temperature=0.4, max_tokens=6000, @@ -175,6 +177,7 @@ def materialize_row( modality=contract.modality, conversational=contract.conversational, hint=hint, + guidance=config.guidance, ), temperature=0.35, max_tokens=9000, @@ -230,12 +233,12 @@ def materialize_row( def suite_review( - contract: AgentContract, records: list[dict], llm: LLMClient + contract: AgentContract, records: list[dict], llm: LLMClient, *, guidance: str = "" ) -> tuple[list[dict], list[str], str]: """Coverage pass over the accepted set: (gap rows feedback, duplicate ids to drop, feedback).""" raw = llm.complete_json( prompts.SUITE_REVIEW_SYSTEM, - prompts.suite_review_prompt(contract.brief(), records), + prompts.suite_review_prompt(contract.brief(), records, guidance=guidance), temperature=0.2, max_tokens=2500, ) @@ -307,7 +310,9 @@ def _flush() -> None: else: rejected.append({**row, "_reject_reason": reason}) if suite_round < config.max_suite_rounds and records: - gaps, duplicate_ids, feedback = suite_review(contract, records, llm) + gaps, duplicate_ids, feedback = suite_review( + contract, records, llm, guidance=config.guidance + ) if duplicate_ids: dropped = [ r for r in records if str(r.get("id")) in set(duplicate_ids) diff --git a/src/fi/alk/generation/prompts.py b/src/fi/alk/generation/prompts.py index 16674fe..b220090 100644 --- a/src/fi/alk/generation/prompts.py +++ b/src/fi/alk/generation/prompts.py @@ -98,6 +98,21 @@ in any order, so assert final tool calls, final state, and captured facts, never a question order.""" +def guidance_block(guidance: str) -> str: + """Operator instructions, injected wherever they can steer the work. + + They choose WHAT to test (focus areas, situations to include or skip, emphasis); they never + override the contract's ground truth and never lower the quality bar. + """ + if not str(guidance or "").strip(): + return "" + return ( + "\nINSTRUCTIONS FROM THE TEST OWNER (follow them when choosing what to test; they never " + "permit inventing interfaces or weakening checkpoints):\n" + f"{str(guidance).strip()[:2000]}\n" + ) + + def subgoal_catalog_prompt(brief: str) -> str: return f"""{brief} @@ -132,6 +147,7 @@ def derive_rows_prompt( existing: list[dict], feedback: str, first_round: bool, + guidance: str = "", ) -> str: must = "" if first_round and signature_cases: @@ -161,30 +177,54 @@ def derive_rows_prompt( ) return f"""{brief} -Task: plan {want} distinct test scenarios for this agent. You are both the engineer who built it and -the product manager who answers for it in production; plan the tests those two people would insist -on before shipping. - -For each scenario return one line of planning, not the full test yet: +Task: plan {want} distinct test scenarios for this agent. + +How to author each scenario. Work through these steps in order, in your head, before writing its +plan line: +1. Pick the failure to catch. Name one specific wrong behavior a plausible implementation of THIS + agent could produce: it drops a detail the user stated, acts on an unstated assumption instead of + asking, mishandles a mid-conversation correction, ignores a rule it must enforce, proceeds when + the world cannot satisfy the request, loses track across several items. The scenario exists to + catch that failure; if you cannot name one, the scenario is not worth running. +2. Construct the request so exactly ONE final state is correct. The user's specific requirements are + what pin it down: each concrete detail they want (which item, which size, which time, what to + exclude) removes ambiguity about the correct end state, and each is something the agent can get + wrong. A request whose correct outcome is vague cannot be graded; sharpen it until one end state + is right and everything else is wrong. +3. Place the information. Decide what the user states up front, what they hold until asked, and what + only the environment knows (availability, stock, an existing record). The agent should have to + gather before it acts; a scenario where everything is handed over in the first sentence tests + only transcription. +4. Let steps interact when the use case allows it. Several requests where handling one affects + another (modify the earlier one, remove one of them, a running total) make an early mistake + visible in the final state. One isolated request hides errors; interacting ones expose them. +5. Define done. The final state that must hold, what the agent must have told the user, and what + must be left untouched. Everything is graded from that end state and transcript, never from + which path the agent took. + +For each scenario return one plan line, not the full test yet: - id: a short slug - use_case: the user-facing job it belongs to (sentence case; scenarios sharing a job repeat the same use_case wording exactly) -- situation: ONE line naming the specific condition of the world or the user that this scenario - fixes, phrased from the user or world side. It must not mention the agent's tools, must not - prescribe what the agent should do, and must not contain the expected outcome. -- why_distinct: one line naming the distinct correct OUTCOME this situation produces -- goal: one line, the single end-objective of the test - -{uses}{must}Coverage rules: -- Different situations with the same correct outcome are ONE scenario; pick the strongest. -- Cover the failure-shaped situations a production owner worries about, where the contract makes - them real: the requested thing does not exist or is unavailable, the request is ambiguous and - needs a clarifying question, the user changes their mind or corrects an earlier statement mid-way, - the request violates one of the agent's hard constraints and must be declined, the user abandons. -- Also cover the core successful paths, including ones with several steps or several items. +- situation: ONE line naming the specific condition of the world or the user this scenario fixes, + phrased from the user or world side. It must not mention the agent's tools, must not prescribe + what the agent should do, and must not contain the expected outcome. +- target_failure: the specific wrong behavior from step 1 that this scenario would catch +- unique_end_state: one line, the single correct final state from step 2 +- goal: one line, the end-objective of the test from the user's side + +{uses}{must}Coverage across the set: +- Different situations with the same correct end state are ONE scenario; keep the strongest. +- Spread the target failures: a set where ten scenarios catch the same failure type is worth two + scenarios, not ten. +- Include the situations the agent's own rules and data make real: a rule that forces a clarifying + question or a refusal, a requested thing that does not exist or is unavailable, a correction after + something was already handled, a request spanning several items or steps. +- Include the core successful paths too, at real complexity (several items, specific requirements), + not toy versions. - No scenarios about internal machinery (logging, config, retries): users never bring those. -{dedupe}{feedback_block}Return JSON: {{"rows": [{{"id": "...", "use_case": "...", "situation": "...", -"why_distinct": "...", "goal": "..."}}]}}""" +{dedupe}{feedback_block}{guidance_block(guidance)}Return JSON: {{"rows": [{{"id": "...", "use_case": "...", "situation": "...", +"target_failure": "...", "unique_end_state": "...", "goal": "..."}}]}}""" def materialize_prompt( @@ -196,6 +236,7 @@ def materialize_prompt( modality: str, conversational: bool, hint: str = "", + guidance: str = "", ) -> str: input_spec = AGENT_INPUT_BY_MODALITY.get( modality, AGENT_INPUT_BY_MODALITY["_default"] @@ -237,7 +278,17 @@ def materialize_prompt( Write the complete test. Every value must be a real value from the contract's data. Keep the three parts separate: the input never reveals the environment seeding, the checkpoints, or the outcome. -{conv} + +The plan names a target_failure: the wrong behavior this test exists to catch. Design the +checkpoints so that if the agent committed exactly that failure, at least one deterministic +checkpoint fails. Then cover the rest of "done": +- every specific requirement the user states becomes an asserted value somewhere (a tool argument, a + final-state field, or a conveyed fact); a requirement no checkpoint asserts is a requirement the + test silently allows the agent to drop; +- assert what must be left alone as well as what must change: a final checkpoint on the exact end + state (these items, nothing more) or an `absent` checkpoint catches collateral actions that + per-step checks miss. +{conv}{guidance_block(guidance)} Return JSON with ALL of these keys, none empty: - id, use_case, situation, goal: carried from the plan (sharpen wording if needed, keep meaning) - description: 2-3 sentences for a human reviewer: what is seeded, what the user wants, and what a @@ -266,9 +317,11 @@ def materialize_prompt( You did not write it, and your default answer is no. Approve only what you would defend to the engineer who owns the agent. Review in this order: -1. WORTH. Could a competent implementation of this agent plausibly fail this test? If every correct - implementation passes it for free, reject it however well it is written, and say what a good - agent could actually get wrong here if anything. +1. WORTH. The scenario declares a target_failure: the wrong behavior it exists to catch. Ask two + questions. Could a plausible implementation of this agent actually commit that failure? And if it + did, would at least one deterministic checkpoint fail? If the checkpoints would still pass while + the target failure happens, the test is broken; reject or demand the missing checkpoint. If no + plausible implementation could commit it, the test wastes a run; reject. 2. REAL. Would a real user plausibly bring this situation? 3. GROUNDED. Every tool, argument name, value, and id exists in the contract, spelled exactly. Nothing contradicts the agent's hard constraints. Any invented interface or id is fatal. @@ -309,7 +362,7 @@ def critic_prompt(brief: str, scenario: dict) -> str: ) -def suite_review_prompt(brief: str, records: list[dict]) -> str: +def suite_review_prompt(brief: str, records: list[dict], guidance: str = "") -> str: summary = [ { "id": r.get("id"), @@ -325,4 +378,4 @@ def suite_review_prompt(brief: str, records: list[dict]) -> str: ACCEPTED SCENARIOS so far: {json.dumps(summary)[:6000]} -What situations that matter in production are missing, and which pairs are near-duplicates?""" +{guidance_block(guidance)}What situations that matter in production are missing, and which pairs are near-duplicates?""" From ab890b48b0b6cca91da67e2fd524717ef794cd3d Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 13 Aug 2026 22:10:53 +0530 Subject: [PATCH 04/55] feat(generation): incremental artifact flush and progress line per scenario --- src/fi/alk/generation/pipeline.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/fi/alk/generation/pipeline.py b/src/fi/alk/generation/pipeline.py index 9ea2c57..e8db5d1 100644 --- a/src/fi/alk/generation/pipeline.py +++ b/src/fi/alk/generation/pipeline.py @@ -309,6 +309,12 @@ def _flush() -> None: records.append(record) else: rejected.append({**row, "_reject_reason": reason}) + _flush() # runs are long; keep every artifact inspectable while they go + print( + f"[generation] accepted={len(records)} rejected={len(rejected)} " + f"spent={llm.usage.as_dict().get('usd', 0)}", + flush=True, + ) if suite_round < config.max_suite_rounds and records: gaps, duplicate_ids, feedback = suite_review( contract, records, llm, guidance=config.guidance From 587bbd52ce9a2553637746c2cea3b149c1961d59 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 13 Aug 2026 22:11:54 +0530 Subject: [PATCH 05/55] feat(generation): deterministic run auditor against agent source --- scripts/audit_generated_scenarios.py | 97 ++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 scripts/audit_generated_scenarios.py diff --git a/scripts/audit_generated_scenarios.py b/scripts/audit_generated_scenarios.py new file mode 100644 index 0000000..a3c2e56 --- /dev/null +++ b/scripts/audit_generated_scenarios.py @@ -0,0 +1,97 @@ +"""Audit a generation run against the agent's real source. Deterministic, no model calls. + +Usage: python scripts/audit_generated_scenarios.py + +Checks, per scenario and in aggregate: +- every tool named in a checkpoint or mock exists in the agent source; +- every identifier-shaped argument value in a tool_call_args checkpoint appears in the agent source + (menu ids, enum values), so no checkpoint asserts an id that does not exist; +- checkpoint kind mix and deterministic share; +- sub-goal reuse across scenarios (the roll-up property); +- input/checkpoint separation smells: the agent_input leaking seeded ids that facts do not cover. +""" + +from __future__ import annotations + +import json +import os +import re +import sys +from collections import Counter + + +def _load(path: str) -> dict: + with open(path, encoding="utf-8") as fh: + return json.load(fh) + + +def _source_blob(agent_repo: str) -> str: + parts = [] + for dirpath, dirnames, filenames in os.walk(agent_repo): + dirnames[:] = [d for d in dirnames if d not in {".git", "__pycache__", ".venv"}] + for filename in filenames: + if filename.endswith((".py", ".md", ".json", ".yaml", ".yml", ".toml")): + try: + with open(os.path.join(dirpath, filename), encoding="utf-8", errors="ignore") as fh: + parts.append(fh.read()) + except OSError: + pass + return "\n".join(parts) + + +_IDENTIFIER = re.compile(r"^[a-z][a-z0-9_]{2,}$") + + +def audit(run_dir: str, agent_repo: str) -> int: + scenarios_dir = os.path.join(run_dir, "scenarios") + files = sorted(os.listdir(scenarios_dir)) if os.path.isdir(scenarios_dir) else [] + if not files: + print("no scenarios found") + return 1 + source = _source_blob(agent_repo) + + kind_mix: Counter = Counter() + subgoal_uses: Counter = Counter() + deterministic = total = 0 + failures: list[str] = [] + + for name in files: + record = _load(os.path.join(scenarios_dir, name)) + slug = record.get("id", name) + fact_values = {str(f.get("value", "")).lower() for f in record.get("facts") or []} + for sub_goal in record.get("sub_goals") or []: + checkpoint = (sub_goal or {}).get("checkpoint") or {} + definition = checkpoint.get("definition") or {} + kind = checkpoint.get("kind", "?") + kind_mix[kind] += 1 + subgoal_uses[str(sub_goal.get("name"))] += 1 + total += 1 + if checkpoint.get("deterministic"): + deterministic += 1 + tool = definition.get("tool") or definition.get("no_tool_call") or ( + definition.get("no_tool_call_with") or {} + ).get("tool") + if tool and f"{tool}" not in source: + failures.append(f"{slug}: tool `{tool}` not found in agent source") + for arg, value in (definition.get("args_equal") or {}).items(): + text = str(value) + if _IDENTIFIER.match(text) and text not in source: + failures.append(f"{slug}: args_equal {arg}={text} not found in agent source") + agent_input = str(record.get("agent_input", "")).lower() + for token in re.findall(r"[a-z][a-z0-9_]{4,}", agent_input): + if "_" in token and token in source and token not in fact_values: + failures.append(f"{slug}: agent_input leaks internal identifier `{token}`") + + reused = sum(1 for count in subgoal_uses.values() if count >= 2) + print(f"scenarios: {len(files)}") + print(f"checkpoints: {total}, deterministic: {deterministic} ({100 * deterministic // max(total, 1)}%)") + print(f"kind mix: {dict(kind_mix)}") + print(f"sub-goal names reused in >=2 scenarios: {reused} of {len(subgoal_uses)}") + print(f"grounding failures: {len(failures)}") + for failure in failures[:30]: + print(f" - {failure}") + return 0 if not failures else 2 + + +if __name__ == "__main__": + raise SystemExit(audit(sys.argv[1], sys.argv[2])) From b38ad4f7101fece251cb4299a0d5b9b55bebf173 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 13 Aug 2026 22:12:36 +0530 Subject: [PATCH 06/55] fix(generation): register the guidance flag the CLI reads --- src/fi/alk/generation/cli.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/fi/alk/generation/cli.py b/src/fi/alk/generation/cli.py index 10f87c0..f90183a 100644 --- a/src/fi/alk/generation/cli.py +++ b/src/fi/alk/generation/cli.py @@ -34,6 +34,11 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument( "--no-critic", action="store_true", help="skip the QA review pass" ) + parser.add_argument( + "--guidance", + default="", + help="operator instructions steering what to test (or @path/to/file to read them)", + ) parser.add_argument("--verbose", action="store_true") return parser @@ -57,7 +62,7 @@ def main(argv: list[str] | None = None) -> int: source = resolve_source(args.source, **source_kwargs) llm = LiteLLMClient(model=args.model, budget_usd=args.budget_usd) config = GenerationConfig( - n=args.n, critic_enabled=not args.no_critic, out_dir=args.out + n=args.n, critic_enabled=not args.no_critic, guidance=guidance, out_dir=args.out ) result = generate(source, llm, config) From 081bc0197afa7a7f1bda6aaec31539d5cf6b7330 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 13 Aug 2026 22:17:21 +0530 Subject: [PATCH 07/55] fix(generation): truncation-tolerant JSON parsing, parse retry, generous token budgets --- src/fi/alk/generation/contract.py | 2 +- src/fi/alk/generation/explorer.py | 4 +- src/fi/alk/generation/llm.py | 109 ++++++++++++++++++++++++++++-- src/fi/alk/generation/pipeline.py | 10 +-- 4 files changed, 111 insertions(+), 14 deletions(-) diff --git a/src/fi/alk/generation/contract.py b/src/fi/alk/generation/contract.py index 12eef4f..ce2ed6d 100644 --- a/src/fi/alk/generation/contract.py +++ b/src/fi/alk/generation/contract.py @@ -121,7 +121,7 @@ def extract_contract(evidence_text: str, llm: LLMClient) -> AgentContract: _EXTRACT_SYSTEM, _EXTRACT_USER.format(evidence=evidence_text), temperature=0.15, - max_tokens=10_000, + max_tokens=24_000, ) if isinstance(raw, list): raw = next((item for item in raw if isinstance(item, dict)), {}) diff --git a/src/fi/alk/generation/explorer.py b/src/fi/alk/generation/explorer.py index c5678ff..9e5ccfe 100644 --- a/src/fi/alk/generation/explorer.py +++ b/src/fi/alk/generation/explorer.py @@ -247,7 +247,9 @@ def explore_contract( "verified contract.", } ) - reply = llm.complete_turn(messages, tools=_TOOLS, temperature=0.15) + reply = llm.complete_turn( + messages, tools=_TOOLS, temperature=0.15, max_tokens=16_000 + ) calls = reply.get("tool_calls") or [] if not calls: messages.append( diff --git a/src/fi/alk/generation/llm.py b/src/fi/alk/generation/llm.py index d769b33..383149d 100644 --- a/src/fi/alk/generation/llm.py +++ b/src/fi/alk/generation/llm.py @@ -74,7 +74,7 @@ def complete_turn( *, tools: list[dict[str, Any]] | None = None, temperature: float = 0.2, - max_tokens: int = 8000, + max_tokens: int = 16_000, ) -> dict[str, Any]: """One chat turn. Returns ``{"content": str | None, "tool_calls": [{"id", "name", "arguments": dict}, ...]}`` so a harness can run a bounded tool loop.""" @@ -85,9 +85,15 @@ def usage(self) -> Usage: ... def _extract_json(text: str) -> Any: - """Parse the first JSON object or array in ``text``, tolerating code fences.""" + """Parse the first JSON object or array in ``text``. + + Tolerates code fences (including an unterminated fence when the output was cut off) and repairs + truncation by dropping the incomplete tail and closing the open brackets. Truncated model output + is a routine failure mode, not an exception, so parsing must degrade gracefully before the + caller decides to retry. + """ text = text.strip() - fenced = re.search(r"```(?:json)?\s*(.+?)```", text, re.S) + fenced = re.search(r"```(?:json)?\s*(.+?)(?:```|$)", text, re.S) if fenced: text = fenced.group(1).strip() try: @@ -109,11 +115,79 @@ def _extract_json(text: str) -> Any: return json.loads(text[start : i + 1]) except json.JSONDecodeError: break + repaired = _repair_truncated(text) + if repaired is not None: + return repaired raise ValueError( f"model returned no parseable JSON (first 200 chars: {text[:200]!r})" ) +def _repair_truncated(text: str) -> Any | None: + """Best-effort parse of JSON that was cut off mid-stream. + + Walks the text tracking string and bracket state, discards the incomplete trailing element at + each failure, and closes whatever remains open. Returns None when nothing parseable survives. + """ + start = min((i for i in (text.find("{"), text.find("[")) if i >= 0), default=-1) + if start < 0: + return None + stack: list[str] = [] + in_string = False + escaped = False + last_complete = start + for index in range(start, len(text)): + char = text[index] + if in_string: + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif char == '"': + in_string = False + continue + if char == '"': + in_string = True + elif char in "{[": + stack.append("}" if char == "{" else "]") + elif char in "}]": + if stack: + stack.pop() + last_complete = index + elif char == ",": + last_complete = index + # Retry from progressively earlier cut points: full text, then the last complete element. + for cut in (len(text), last_complete): + candidate = text[start:cut].rstrip().rstrip(",") + # Recompute the open stack for this candidate. + open_stack: list[str] = [] + in_str = False + esc = False + for char in candidate: + if in_str: + if esc: + esc = False + elif char == "\\": + esc = True + elif char == '"': + in_str = False + continue + if char == '"': + in_str = True + elif char in "{[": + open_stack.append("}" if char == "{" else "]") + elif char in "}]" and open_stack: + open_stack.pop() + if in_str: + candidate += '"' + candidate += "".join(reversed(open_stack)) + try: + return json.loads(candidate) + except json.JSONDecodeError: + continue + return None + + @dataclass class LiteLLMClient: """litellm-backed client with per-call cost metering and a hard budget ceiling.""" @@ -137,11 +211,32 @@ def complete_json( user: str, *, temperature: float = 0.3, - max_tokens: int = 8000, + max_tokens: int = 20_000, ) -> Any: + """Chat completion parsed as JSON, retrying once on a cut-off or malformed reply. + + Gemini-family models spend part of ``max_tokens`` on internal reasoning, so a reply can + arrive truncated even when the visible JSON would have fit. The retry names the problem to + the model and raises the output budget. + """ self._check_budget() text = self._chat(system, user, temperature=temperature, max_tokens=max_tokens) - return _extract_json(text) + try: + return _extract_json(text) + except ValueError: + self._check_budget() + retry_user = ( + user + + "\n\nYour previous reply was cut off or was not valid JSON. Return ONLY the " + "complete JSON, with no code fences and no prose." + ) + text = self._chat( + system, + retry_user, + temperature=temperature, + max_tokens=min(max_tokens * 2, 50_000), + ) + return _extract_json(text) def complete_turn( self, @@ -149,7 +244,7 @@ def complete_turn( *, tools: list[dict[str, Any]] | None = None, temperature: float = 0.2, - max_tokens: int = 8000, + max_tokens: int = 16_000, ) -> dict[str, Any]: self._check_budget() try: @@ -301,7 +396,7 @@ def complete_turn( *, tools: list[dict[str, Any]] | None = None, temperature: float = 0.2, - max_tokens: int = 8000, + max_tokens: int = 16_000, ) -> dict[str, Any]: if not self.responses: raise AssertionError("FakeLLMClient exhausted; queue more responses") diff --git a/src/fi/alk/generation/pipeline.py b/src/fi/alk/generation/pipeline.py index e8db5d1..e3151fa 100644 --- a/src/fi/alk/generation/pipeline.py +++ b/src/fi/alk/generation/pipeline.py @@ -83,7 +83,7 @@ def derive_catalog(contract: AgentContract, llm: LLMClient) -> list[dict]: prompts.SCENARIO_MODEL, prompts.subgoal_catalog_prompt(contract.brief()), temperature=0.3, - max_tokens=6000, + max_tokens=16_000, ) catalog = raw.get("catalog", raw) if isinstance(raw, dict) else raw entries: list[dict] = [] @@ -137,7 +137,7 @@ def derive_rows( guidance=config.guidance, ), temperature=0.4, - max_tokens=6000, + max_tokens=20_000, ) for row in raw.get("rows", raw if isinstance(raw, list) else []): if not isinstance(row, dict) or not row.get("situation"): @@ -180,7 +180,7 @@ def materialize_row( guidance=config.guidance, ), temperature=0.35, - max_tokens=9000, + max_tokens=20_000, ) record = ( raw @@ -202,7 +202,7 @@ def materialize_row( prompts.CRITIC_SYSTEM, prompts.critic_prompt(brief, record), temperature=0.2, - max_tokens=2500, + max_tokens=12_000, ) if not isinstance(verdict, dict): verdict = {} @@ -240,7 +240,7 @@ def suite_review( prompts.SUITE_REVIEW_SYSTEM, prompts.suite_review_prompt(contract.brief(), records, guidance=guidance), temperature=0.2, - max_tokens=2500, + max_tokens=12_000, ) if not isinstance(raw, dict): return [], [], "" From e38f86485923ed8f5e12ba93780840e8e7e7bf38 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 13 Aug 2026 22:22:23 +0530 Subject: [PATCH 08/55] fix(generation): conveyed checks match values not phrasings; judge rubrics yes-means-met --- src/fi/alk/generation/prompts.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/fi/alk/generation/prompts.py b/src/fi/alk/generation/prompts.py index b220090..b7cbb8e 100644 --- a/src/fi/alk/generation/prompts.py +++ b/src/fi/alk/generation/prompts.py @@ -84,13 +84,19 @@ - state (deterministic): the world must end in a specific state. definition: {"must": {"": }, "forbidden": {"": }} evaluated against the seeded environment state after the run. -- conveyed (deterministic): the agent must have told the user a specific grounded fact. definition: - {"must_include_any": ["", ""]} matched against the agent's - side of the transcript. Use real values from the contract data (a price, a total, an id). +- conveyed (deterministic): the agent must have told the user a specific VALUE: a price, a total, a + time, a name from the data. definition: {"must_include_any": ["", ""]} matched against the agent's transcript turns. Only values work here; NEVER match + question phrasings or sentence wordings, because a correct agent can phrase anything a hundred + ways. To verify the agent gathered a piece of information from the user, do not check its + question: the proof is the gathered value appearing in the final tool call (tool_call_args), since + the user only reveals on_request facts when asked. - absent (deterministic): something must NOT happen. definition: {"no_tool_call": ""} or {"no_tool_call_with": {"tool": "", "args_equal": {...}}}. - judge (not deterministic, last resort): definition: {"rubric": ""}. + grader answers from the transcript>"}. Phrase the rubric so that YES means the sub-goal was MET: + for a sub-goal that something must not happen, ask "Did the agent refrain from ...?", never "Did + the agent do ...?". Each sub-goal is written as: {"name": "", "milestone": "", "checkpoint": {"kind": "", "detail": "", "deterministic": true|false, "definition": {...}}}. From ccba2c2ab32abbe7e089779eac5440ff7e67c488 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 13 Aug 2026 22:25:53 +0530 Subject: [PATCH 09/55] refactor(generation): principle-driven prompt language; runnability gates; final-action closure rule --- src/fi/alk/generation/prompts.py | 91 ++++++++++++++++++-------------- 1 file changed, 50 insertions(+), 41 deletions(-) diff --git a/src/fi/alk/generation/prompts.py b/src/fi/alk/generation/prompts.py index b7cbb8e..fd9cd5e 100644 --- a/src/fi/alk/generation/prompts.py +++ b/src/fi/alk/generation/prompts.py @@ -16,21 +16,19 @@ values, data) is given to you as a CONTRACT. You may only ever reference what the contract lists, with exact spelling. Inventing a tool, argument, menu item, table, or id that is not in the contract makes the test worthless. -- USE CASE: one real job a user hires this agent for, stated from the user's side. Example for a - food-ordering agent: "Order a combo meal". Example for a database agent: "Ask for a sales total". +- USE CASE: one real job a user hires this agent for, named from the user's side in the user's own + words. - SCENARIO: one concrete test. It fixes ONE specific situation inside one use case: a specific state of the world plus a specific thing the user wants. Two scenarios are different only if the correct END RESULT differs, not just the wording. "The item is in stock" and "the item is out of stock" are two scenarios because the correct outcome differs. Never write two scenarios that are the same situation reworded. -- SUB-GOAL: a milestone inside one scenario that must be true for the scenario to end correctly. - Example: "the drink was elicited", "the refund was recorded". 3 to 6 per scenario. A sub-goal is - something a product owner would recognise, not an internal implementation step like "the JSON - parsed" and not a micro-step like "the agent said hello". -- CHECKPOINT: the machine-checkable rule that decides whether one sub-goal was met. Checkpoints must - test the RIGHT VALUES, not just that something happened: if the user asked for 11 PM and the agent - booked 10 PM, a checkpoint that only verifies "a booking call happened" wrongly passes; the - checkpoint must assert the booked time equals 11 PM. +- SUB-GOAL: a milestone inside one scenario that must be true for the scenario to end correctly, + 3 to 6 per scenario. A sub-goal is an outcome a product owner would recognise and care about; + internal implementation steps and conversational pleasantries are not sub-goals. +- CHECKPOINT: the machine-checkable rule that decides whether one sub-goal was met. A checkpoint + witnesses the VALUES the user's request determined, because a check that only confirms an action + occurred cannot tell acting correctly apart from acting wrongly. - ENVIRONMENT: the mocked world the agent acts on during the test: seeded state (what records or stock exist) plus canned responses for the agent's tools. The agent's own reasoning is never mocked; only the world it acts on is. @@ -49,17 +47,17 @@ - A competent implementation of this agent could plausibly FAIL it. If any correct implementation passes it for free, it teaches nothing; do not write it. - A real user could plausibly bring this situation. No contrived or gimmicky setups. -- Concrete values everywhere, taken from the contract's real data. No placeholders, no variables, - no "example_id". +- Every concrete value is a real entry from the contract's data; a value that cannot be found in + the contract does not belong in a test. - User personality, accent, or language is NOT varied unless the scenario is specifically about it.""" AGENT_INPUT_BY_MODALITY = { "voice": ( - "a situation instruction for the simulated caller, written in second person as lived " - "circumstance ('You are calling... You want...'). State their goal and what they know. Facts " - "the agent should have to ask for are listed separately (see `facts`), so do not volunteer " - "them here. Never write stage directions like 'tell the agent that X'; never script the " - "agent's side; no accent or voice notes" + "a situation instruction for the simulated caller, written in second person as the caller's " + "own lived circumstance: who they are, what is happening, and what they want. It describes " + "their experience and goal, never instructions about what to say, and never the other " + "side's turns. Facts the agent is expected to ask for live in `facts`, not here. No accent " + "or voice notes" ), "chat": ( "a situation instruction for the simulated user, second person, lived circumstance: their " @@ -75,33 +73,34 @@ "_default": "exactly what the agent receives at the start, in natural form: never the answer", } -CHECKPOINT_VOCABULARY = """CHECKPOINT kinds, strongest first. Use the strongest kind that applies; use -`judge` only when nothing inspectable exists. -- tool_call_args (deterministic): the agent must call a specific tool with specific argument values. - definition: {"tool": "", "args_equal": {"": - , ...}, "args_present": [""]}. - Put every argument whose value the user's request determines into args_equal. -- state (deterministic): the world must end in a specific state. definition: {"must": - {"": }, "forbidden": {"": }} evaluated against - the seeded environment state after the run. -- conveyed (deterministic): the agent must have told the user a specific VALUE: a price, a total, a - time, a name from the data. definition: {"must_include_any": ["", ""]} matched against the agent's transcript turns. Only values work here; NEVER match - question phrasings or sentence wordings, because a correct agent can phrase anything a hundred - ways. To verify the agent gathered a piece of information from the user, do not check its - question: the proof is the gathered value appearing in the final tool call (tool_call_args), since - the user only reveals on_request facts when asked. -- absent (deterministic): something must NOT happen. definition: {"no_tool_call": ""} or - {"no_tool_call_with": {"tool": "", "args_equal": {...}}}. -- judge (not deterministic, last resort): definition: {"rubric": ""}. Phrase the rubric so that YES means the sub-goal was MET: - for a sub-goal that something must not happen, ask "Did the agent refrain from ...?", never "Did - the agent do ...?". +CHECKPOINT_VOCABULARY = """CHECKPOINT kinds, strongest first. Choose the strongest kind the sub-goal +allows; `judge` exists only for sub-goals no state, call, or data value can witness. +- tool_call_args (deterministic): passes when the agent called the named tool and every argument + listed carried the expected value. definition: {"tool": "", "args_equal": + {"": , ...}, "args_present": [""]}. args_equal holds each argument whose correct value the user's request determines; an + argument left out of args_equal is a requirement the test does not protect. +- state (deterministic): passes when the world's final state carries the expected values. + definition: {"must": {"": }, "forbidden": {"": }}, + evaluated against the seeded environment state after the run. +- conveyed (deterministic): passes when a specific value from the environment's data (a price, a + total, a time, a name) appears in the agent's transcript turns. definition: {"must_include_any": + ["", ""]}. The agent's wording is its own; + only data values are matchable, because correct phrasing is unbounded. +- absent (deterministic): passes when a named action never occurred. definition: {"no_tool_call": + ""} or {"no_tool_call_with": {"tool": "", "args_equal": {...}}}. +- judge (not deterministic): definition: {"rubric": ""}. Each sub-goal is written as: {"name": "", "milestone": "", "checkpoint": {"kind": "", "detail": "", "deterministic": true|false, "definition": {...}}}. -For conversational agents, checkpoints must be ORDER-INDEPENDENT: the agent may gather information -in any order, so assert final tool calls, final state, and captured facts, never a question order.""" +Properties every scenario's checkpoints hold together: +- Information the user reveals only when asked is witnessed by its value arriving in a tool call or + the final state; conversation wording cannot witness it. +- A scenario whose goal changes the world closes with a checkpoint asserting the complete final + action and its argument values; one whose goal is that the world stays unchanged closes with the + checkpoint asserting that absence. +- For conversational agents, every checkpoint holds under any order of conversation.""" def guidance_block(guidance: str) -> str: @@ -285,6 +284,13 @@ def materialize_prompt( Write the complete test. Every value must be a real value from the contract's data. Keep the three parts separate: the input never reveals the environment seeding, the checkpoints, or the outcome. +The test must be runnable against the real agent exactly as it ships: +- The simulated user must be able to carry the whole conversation from agent_input plus facts alone: + every question the agent will predictably ask in this scenario has its answer among the facts. +- The environment seed may only change what a test setup can actually control. When the agent ships + with fixed data, the scenario draws its conditions from that data as it is; a condition that would + require altering data the agent's repository fixes makes the test unrunnable. + The plan names a target_failure: the wrong behavior this test exists to catch. Design the checkpoints so that if the agent committed exactly that failure, at least one deterministic checkpoint fails. Then cover the rest of "done": @@ -336,6 +342,9 @@ def materialize_prompt( must not be checked as medium); conversational checkpoints do not depend on question order. 5. SEPARATION. The input reveals nothing the user would not know: no seeded availability, no internal ids, no expected outcome, no checkpoint contents. +6. RUNNABLE. The simulated user can finish the conversation from agent_input plus facts alone, and + the environment requires nothing a test setup cannot control: a condition that depends on + altering data the agent's repository fixes makes the test unrunnable as shipped. Return JSON: {"verdict": "accept" | "revise" | "reject", "scores": {"worth": 1-5, "real": 1-5, "grounded": 1-5, "checkable": 1-5, "separation": 1-5}, "problems": [""], From 1eda880800e012689058613f4ab3fb00f3e6fda2 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 13 Aug 2026 22:27:29 +0530 Subject: [PATCH 10/55] feat(generation): pure-python checkpoint evaluator; prove record-to-runtime-mock-to-verdict path --- src/fi/alk/generation/__init__.py | 4 + src/fi/alk/generation/checks.py | 157 ++++++++++++++++++++++++++++++ tests/test_generation_pipeline.py | 48 +++++++++ 3 files changed, 209 insertions(+) create mode 100644 src/fi/alk/generation/checks.py diff --git a/src/fi/alk/generation/__init__.py b/src/fi/alk/generation/__init__.py index 19f6726..b488f37 100644 --- a/src/fi/alk/generation/__init__.py +++ b/src/fi/alk/generation/__init__.py @@ -1,5 +1,6 @@ """Local-first scenario generation: point at an agent, get reviewed, checkable test scenarios.""" +from .checks import CheckResult, evaluate_checkpoint, evaluate_scenario from .contract import AgentContract, ToolSpec, extract_contract, validate_contract from .emit import smoke_manifest, to_alk_scenario, write_outputs from .llm import ( @@ -23,6 +24,9 @@ __all__ = [ "AgentContract", + "CheckResult", + "evaluate_checkpoint", + "evaluate_scenario", "AgentEvidence", "AgentSource", "AuthFailed", diff --git a/src/fi/alk/generation/checks.py b/src/fi/alk/generation/checks.py new file mode 100644 index 0000000..32a0061 --- /dev/null +++ b/src/fi/alk/generation/checks.py @@ -0,0 +1,157 @@ +"""Pure-Python evaluation of generated checkpoints. No model calls, ever. + +This is the downstream consumer contract: after a simulation run, feed the recorded tool calls, the +agent's transcript turns, and the final environment state to ``evaluate_scenario`` and get a +pass/fail verdict per sub-goal. Checkpoints of kind ``judge`` are returned as ``skipped`` (they are +the one non-deterministic kind, flagged as such at generation time); everything else is plain +comparisons. + +Expected inputs: +- tool_calls: [{"name": str, "arguments": {...}}, ...] in call order +- transcript_turns: the agent-side utterances as strings +- final_state: nested dict of the world state after the run (dotted paths resolve into it) +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Mapping, Sequence + + +@dataclass(frozen=True) +class CheckResult: + name: str + kind: str + passed: bool | None # None = not evaluated here (judge) + reason: str + + +def _resolve_path(state: Mapping[str, Any], path: str) -> tuple[bool, Any]: + cursor: Any = state + for part in str(path).split("."): + if isinstance(cursor, Mapping) and part in cursor: + cursor = cursor[part] + else: + return False, None + return True, cursor + + +def _call_matches( + call: Mapping[str, Any], tool: str, args_equal: Mapping[str, Any] +) -> bool: + if str(call.get("name") or call.get("tool") or "") != tool: + return False + arguments = call.get("arguments") or call.get("args") or {} + if not isinstance(arguments, Mapping): + return False + return all(arguments.get(key) == value for key, value in args_equal.items()) + + +def _eval_tool_call_args( + definition: Mapping[str, Any], tool_calls: Sequence[Mapping[str, Any]] +) -> tuple[bool, str]: + tool = str(definition.get("tool", "")) + args_equal = definition.get("args_equal") or {} + args_present = definition.get("args_present") or [] + for call in tool_calls: + if not _call_matches(call, tool, args_equal): + continue + arguments = call.get("arguments") or call.get("args") or {} + missing = [arg for arg in args_present if arg not in arguments] + if missing: + continue + return True, f"call to {tool} matched" + return False, f"no call to {tool} carried the expected arguments" + + +def _eval_state( + definition: Mapping[str, Any], final_state: Mapping[str, Any] +) -> tuple[bool, str]: + for path, expected in (definition.get("must") or {}).items(): + found, actual = _resolve_path(final_state, path) + if not found or actual != expected: + return False, f"state {path} = {actual!r}, expected {expected!r}" + for path, forbidden in (definition.get("forbidden") or {}).items(): + found, actual = _resolve_path(final_state, path) + if found and actual == forbidden: + return False, f"state {path} carries the forbidden value {forbidden!r}" + return True, "state matched" + + +def _eval_conveyed( + definition: Mapping[str, Any], transcript_turns: Sequence[str] +) -> tuple[bool, str]: + variants = [str(v) for v in definition.get("must_include_any") or []] + joined = "\n".join(str(turn) for turn in transcript_turns) + for variant in variants: + if variant and variant.lower() in joined.lower(): + return True, f"value {variant!r} conveyed" + return False, f"none of {variants!r} appeared in the agent's turns" + + +def _eval_absent( + definition: Mapping[str, Any], tool_calls: Sequence[Mapping[str, Any]] +) -> tuple[bool, str]: + tool = definition.get("no_tool_call") + if tool: + hit = any( + str(c.get("name") or c.get("tool") or "") == str(tool) for c in tool_calls + ) + return (not hit, f"call to {tool} {'occurred' if hit else 'never occurred'}") + inner = definition.get("no_tool_call_with") or {} + tool = str(inner.get("tool", "")) + args_equal = inner.get("args_equal") or {} + hit = any(_call_matches(call, tool, args_equal) for call in tool_calls) + return ( + not hit, + f"matching call to {tool} {'occurred' if hit else 'never occurred'}", + ) + + +def evaluate_checkpoint( + kind: str, + definition: Mapping[str, Any], + *, + tool_calls: Sequence[Mapping[str, Any]] = (), + transcript_turns: Sequence[str] = (), + final_state: Mapping[str, Any] | None = None, +) -> tuple[bool | None, str]: + """Evaluate one checkpoint definition. Returns (passed, reason); passed None for judge.""" + if kind == "tool_call_args": + return _eval_tool_call_args(definition, tool_calls) + if kind == "state": + return _eval_state(definition, final_state or {}) + if kind == "conveyed": + return _eval_conveyed(definition, transcript_turns) + if kind == "absent": + return _eval_absent(definition, tool_calls) + if kind == "judge": + return None, "judge checkpoints are not evaluated deterministically" + return False, f"unknown checkpoint kind: {kind}" + + +def evaluate_scenario( + record: Mapping[str, Any], + *, + tool_calls: Sequence[Mapping[str, Any]] = (), + transcript_turns: Sequence[str] = (), + final_state: Mapping[str, Any] | None = None, +) -> list[CheckResult]: + """Evaluate every sub-goal of one generated scenario record against run evidence.""" + results: list[CheckResult] = [] + for sub_goal in record.get("sub_goals") or []: + checkpoint = (sub_goal or {}).get("checkpoint") or {} + kind = str(checkpoint.get("kind", "")) + passed, reason = evaluate_checkpoint( + kind, + checkpoint.get("definition") or {}, + tool_calls=tool_calls, + transcript_turns=transcript_turns, + final_state=final_state, + ) + results.append( + CheckResult( + name=str(sub_goal.get("name")), kind=kind, passed=passed, reason=reason + ) + ) + return results diff --git a/tests/test_generation_pipeline.py b/tests/test_generation_pipeline.py index 542c7f4..2cab3cb 100644 --- a/tests/test_generation_pipeline.py +++ b/tests/test_generation_pipeline.py @@ -248,3 +248,51 @@ def test_alk_scenario_is_typed_and_content_addressed(): assert scenario.kind == "task" assert scenario.version and scenario.version.startswith("sha256:") assert scenario.constraints.declared_tools == ["add_item"] + + +def test_record_drives_runtime_mock_and_python_checks_directly(): + """The golden-artifact property: a generated record feeds the real runtime mock builder and the + pure-Python checker with no translation step in between.""" + from fi.simulate.environments.chat import _mock_world_from_config + + from fi.alk.generation.checks import evaluate_scenario + + # 1. The record's environment block IS the runtime mock config, verbatim. + world = _mock_world_from_config( + { + "mock_tools": SCENARIO["environment"]["mock_responses"], + "tool_initial_state": SCENARIO["environment"]["seed"], + } + ) + assert world is not None + world.reset() + + # 2. The agent under test calls a tool; the mock answers and mutates world state. + result = world.handle_tool_call( + {"id": "c1", "name": "add_item", "arguments": {"item_id": "latte", "size": "M"}} + ) + assert result is not None and result.success + + # 3. Run evidence (tool-call log, transcript, final state) feeds plain-Python checks. + tool_calls = [{"name": "add_item", "arguments": {"item_id": "latte", "size": "M"}}] + verdicts = evaluate_scenario( + SCENARIO, + tool_calls=tool_calls, + transcript_turns=["That is one medium latte, 4.5 total. Anything else?"], + final_state=world.state, + ) + by_name = {v.name: v for v in verdicts} + assert by_name["item_added"].passed is True + assert by_name["order_confirmed"].passed is True + assert by_name["price_conveyed"].passed is True + + # 4. Wrong arguments fail the argument checkpoint: the check tests values, not activity. + wrong = evaluate_scenario( + SCENARIO, + tool_calls=[ + {"name": "add_item", "arguments": {"item_id": "mocha", "size": "L"}} + ], + transcript_turns=[], + final_state=world.state, + ) + assert {v.name: v for v in wrong}["item_added"].passed is False From edd12e0d382831acc5191024618c9c94c3f65e9a Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 13 Aug 2026 22:32:02 +0530 Subject: [PATCH 11/55] feat(generation): identifier grounding validator catches transposed ids in checkpoints --- src/fi/alk/generation/validators.py | 41 +++++++++++++++++++++++++++-- tests/test_generation_pipeline.py | 10 +++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/src/fi/alk/generation/validators.py b/src/fi/alk/generation/validators.py index 5cc216f..5df6e38 100644 --- a/src/fi/alk/generation/validators.py +++ b/src/fi/alk/generation/validators.py @@ -40,10 +40,38 @@ def banned_tokens(contract: AgentContract) -> set[str]: return banned +def _identifier_values(payload) -> set[str]: + """Underscore-shaped string values anywhere in a definition (the id-like ones).""" + values: set[str] = set() + if isinstance(payload, str): + if re.fullmatch(r"[a-z][a-z0-9]*(_[a-z0-9]+)+", payload): + values.add(payload) + elif isinstance(payload, dict): + for value in payload.values(): + values |= _identifier_values(value) + elif isinstance(payload, list): + for value in payload: + values |= _identifier_values(value) + return values + + def _validate_definition( - kind: str, definition: dict, tool_names: set[str], where: str + kind: str, + definition: dict, + tool_names: set[str], + where: str, + legit_vocabulary: set[str], ) -> list[str]: problems: list[str] = [] + unknown_ids = sorted( + value + for value in _identifier_values( + {k: v for k, v in definition.items() if k != "tool"} + ) + if value.lower() not in legit_vocabulary + ) + if unknown_ids: + problems.append(f"{where}:unknown-id:{','.join(unknown_ids)[:100]}") if kind == "tool_call_args": tool = definition.get("tool") if tool not in tool_names: @@ -78,6 +106,7 @@ def validate_scenario(scenario: dict, contract: AgentContract) -> list[str]: """Return problems; empty means structurally complete and grounded enough for the critic.""" problems: list[str] = [] tool_names = contract.tool_names() + legit_vocabulary = _legit_vocabulary(contract) for field in ( "id", @@ -129,7 +158,9 @@ def validate_scenario(scenario: dict, contract: AgentContract) -> list[str]: if not isinstance(definition, dict) or not definition: problems.append(f"{where}:no-definition") else: - problems += _validate_definition(kind, definition, tool_names, where) + problems += _validate_definition( + kind, definition, tool_names, where, legit_vocabulary + ) deterministic = bool(checkpoint.get("deterministic")) if deterministic and kind == "judge": problems.append(f"{where}:judge-marked-deterministic") @@ -178,6 +209,12 @@ def repair_hint(problems: list[str]) -> str: "- Provide at least 3 branch-specific sub_goals, each with a concrete checkpoint, " "ending with a final verification of the resulting state." ) + elif ":unknown-id:" in problem: + lines.append( + f"- A checkpoint uses an identifier that does not exist in the contract " + f"({problem.split(':')[-1]}). Copy ids character for character from the contract's " + "data and arg values; do not reorder or rename their parts." + ) elif ":unknown-tool:" in problem: lines.append( f"- A checkpoint or mock references a tool that does not exist ({problem.split(':')[-1]}). " diff --git a/tests/test_generation_pipeline.py b/tests/test_generation_pipeline.py index 2cab3cb..783d4ea 100644 --- a/tests/test_generation_pipeline.py +++ b/tests/test_generation_pipeline.py @@ -296,3 +296,13 @@ def test_record_drives_runtime_mock_and_python_checks_directly(): final_state=world.state, ) assert {v.name: v for v in wrong}["item_added"].passed is False + + +def test_validator_rejects_transposed_identifier(): + contract = AgentContract.model_validate(CONTRACT) + bad = json.loads(json.dumps(SCENARIO)) + bad["sub_goals"][0]["checkpoint"]["definition"]["args_equal"]["item_id"] = ( + "item_latte_big" + ) + problems = validate_scenario(bad, contract) + assert any("unknown-id" in p for p in problems) From 9e8c904058484183a495d9b43c2a69afc53ae0ed Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 13 Aug 2026 22:36:49 +0530 Subject: [PATCH 12/55] fix(generation): actionable repair hint for valueless conveyed checkpoints --- src/fi/alk/generation/validators.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/fi/alk/generation/validators.py b/src/fi/alk/generation/validators.py index 5df6e38..c56c99c 100644 --- a/src/fi/alk/generation/validators.py +++ b/src/fi/alk/generation/validators.py @@ -234,6 +234,13 @@ def repair_hint(problems: list[str]) -> str: "- Every checkpoint is a judge; make the tool-argument and end-state checks " "deterministic per the vocabulary." ) + elif ":conveyed-without-variants" in problem: + lines.append( + "- A conveyed checkpoint listed no values. must_include_any needs at least one real " + "value from the contract data; when no data value can witness this sub-goal, change " + "the checkpoint to the kind that can (absent for something that must not happen, " + "judge as the last resort)." + ) elif ":" in problem: lines.append(f"- Fix: {problem}") return "\n".join(dict.fromkeys(lines)) From 2f9912b0e385753bb6620ac905099bc4867a6019 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 13 Aug 2026 22:46:47 +0530 Subject: [PATCH 13/55] fix(generation): reviewer sees checkpoint vocabulary; target_failure carried and required --- src/fi/alk/generation/pipeline.py | 9 ++++++++- src/fi/alk/generation/prompts.py | 2 ++ src/fi/alk/generation/validators.py | 1 + tests/test_generation_pipeline.py | 1 + 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/fi/alk/generation/pipeline.py b/src/fi/alk/generation/pipeline.py index e3151fa..1ef33a6 100644 --- a/src/fi/alk/generation/pipeline.py +++ b/src/fi/alk/generation/pipeline.py @@ -187,7 +187,14 @@ def materialize_row( if isinstance(raw, dict) else next((x for x in raw if isinstance(x, dict)), {}) ) - for key in ("id", "use_case", "situation", "goal"): + for key in ( + "id", + "use_case", + "situation", + "goal", + "target_failure", + "unique_end_state", + ): record.setdefault(key, row.get(key)) record["id"] = _slugify(record.get("id") or row.get("id", "")) diff --git a/src/fi/alk/generation/prompts.py b/src/fi/alk/generation/prompts.py index fd9cd5e..17c2309 100644 --- a/src/fi/alk/generation/prompts.py +++ b/src/fi/alk/generation/prompts.py @@ -358,6 +358,8 @@ def critic_prompt(brief: str, scenario: dict) -> str: return f"""CONTRACT (the ground truth this test must respect): {brief} +{CHECKPOINT_VOCABULARY} + PROPOSED TEST SCENARIO: {json.dumps(scenario)[:7000]} diff --git a/src/fi/alk/generation/validators.py b/src/fi/alk/generation/validators.py index c56c99c..70829f9 100644 --- a/src/fi/alk/generation/validators.py +++ b/src/fi/alk/generation/validators.py @@ -116,6 +116,7 @@ def validate_scenario(scenario: dict, contract: AgentContract) -> list[str]: "description", "agent_input", "expected_outcome", + "target_failure", ): if scenario.get(field) in (None, "", [], {}): problems.append(f"empty:{field}") diff --git a/tests/test_generation_pipeline.py b/tests/test_generation_pipeline.py index 783d4ea..c883590 100644 --- a/tests/test_generation_pipeline.py +++ b/tests/test_generation_pipeline.py @@ -82,6 +82,7 @@ "id": "latte-medium", "use_case": "Order a single item", "situation": "The caller wants one medium latte and confirms", + "target_failure": "The agent adds the wrong item or size, or never confirms the order", "goal": "A medium latte is ordered and confirmed", "description": "A caller orders one medium latte, nothing else. The menu has lattes and mochas; " "the order starts empty and the agent must add the right item at the right size.", From fc55a60a5170ce9b55c361f3ec4c98d799dfccef Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 13 Aug 2026 22:49:14 +0530 Subject: [PATCH 14/55] fix(generation): validators tolerate malformed definition shapes instead of raising --- src/fi/alk/generation/validators.py | 12 ++++++++++-- tests/test_generation_pipeline.py | 20 ++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/fi/alk/generation/validators.py b/src/fi/alk/generation/validators.py index 70829f9..707074d 100644 --- a/src/fi/alk/generation/validators.py +++ b/src/fi/alk/generation/validators.py @@ -74,7 +74,7 @@ def _validate_definition( problems.append(f"{where}:unknown-id:{','.join(unknown_ids)[:100]}") if kind == "tool_call_args": tool = definition.get("tool") - if tool not in tool_names: + if not isinstance(tool, str) or tool not in tool_names: problems.append(f"{where}:unknown-tool:{tool}") if not definition.get("args_equal") and not definition.get("args_present"): problems.append(f"{where}:tool_call_args-without-args") @@ -88,10 +88,13 @@ def _validate_definition( ): problems.append(f"{where}:conveyed-without-variants") elif kind == "absent": - inner = definition.get("no_tool_call_with") or {} + inner = definition.get("no_tool_call_with") + inner = inner if isinstance(inner, dict) else {} tool = definition.get("no_tool_call") or inner.get("tool") if not tool: problems.append(f"{where}:absent-without-tool") + elif not isinstance(tool, str): + problems.append(f"{where}:absent-tool-not-a-single-name:{str(tool)[:60]}") elif tool not in tool_names: problems.append(f"{where}:unknown-tool:{tool}") elif kind == "judge": @@ -235,6 +238,11 @@ def repair_hint(problems: list[str]) -> str: "- Every checkpoint is a judge; make the tool-argument and end-state checks " "deterministic per the vocabulary." ) + elif ":absent-tool-not-a-single-name" in problem: + lines.append( + "- An absent checkpoint names several tools at once; write one absent checkpoint " + "per tool, each with a single tool name." + ) elif ":conveyed-without-variants" in problem: lines.append( "- A conveyed checkpoint listed no values. must_include_any needs at least one real " diff --git a/tests/test_generation_pipeline.py b/tests/test_generation_pipeline.py index c883590..bb899b9 100644 --- a/tests/test_generation_pipeline.py +++ b/tests/test_generation_pipeline.py @@ -307,3 +307,23 @@ def test_validator_rejects_transposed_identifier(): ) problems = validate_scenario(bad, contract) assert any("unknown-id" in p for p in problems) + + +def test_validator_survives_malformed_definition_shapes(): + """Model JSON is arbitrary; the validator reports problems, never raises.""" + contract = AgentContract.model_validate(CONTRACT) + bad = json.loads(json.dumps(SCENARIO)) + bad["sub_goals"].append( + { + "name": "no_extra_items", + "milestone": "nothing else ordered", + "checkpoint": { + "kind": "absent", + "detail": "no other tool fires", + "deterministic": True, + "definition": {"no_tool_call": ["add_item", "list_order"]}, + }, + } + ) + problems = validate_scenario(bad, contract) + assert any("absent-tool-not-a-single-name" in p for p in problems) From 07a548a7d2c603b8d7ef94db3f8b1e521f07125b Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 13 Aug 2026 22:58:17 +0530 Subject: [PATCH 15/55] fix(generation): runtime-generated values rule, duplicate-name hint, extra repair attempt --- src/fi/alk/generation/pipeline.py | 2 +- src/fi/alk/generation/prompts.py | 4 +++- src/fi/alk/generation/validators.py | 5 +++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/fi/alk/generation/pipeline.py b/src/fi/alk/generation/pipeline.py index 1ef33a6..4789f51 100644 --- a/src/fi/alk/generation/pipeline.py +++ b/src/fi/alk/generation/pipeline.py @@ -40,7 +40,7 @@ class GenerationConfig: n: int = 20 max_row_rounds: int = 4 - max_repairs: int = 2 + max_repairs: int = 3 max_suite_rounds: int = 2 max_explore_turns: int = 20 critic_enabled: bool = True diff --git a/src/fi/alk/generation/prompts.py b/src/fi/alk/generation/prompts.py index 17c2309..ad4f36d 100644 --- a/src/fi/alk/generation/prompts.py +++ b/src/fi/alk/generation/prompts.py @@ -79,7 +79,9 @@ listed carried the expected value. definition: {"tool": "", "args_equal": {"": , ...}, "args_present": [""]}. args_equal holds each argument whose correct value the user's request determines; an - argument left out of args_equal is a requirement the test does not protect. + argument left out of args_equal is a requirement the test does not protect. A value that only + comes into existence during the run (a generated id, a session handle) cannot be known in advance + and belongs in args_present, never in args_equal. - state (deterministic): passes when the world's final state carries the expected values. definition: {"must": {"": }, "forbidden": {"": }}, evaluated against the seeded environment state after the run. diff --git a/src/fi/alk/generation/validators.py b/src/fi/alk/generation/validators.py index 707074d..622a3d7 100644 --- a/src/fi/alk/generation/validators.py +++ b/src/fi/alk/generation/validators.py @@ -238,6 +238,11 @@ def repair_hint(problems: list[str]) -> str: "- Every checkpoint is a judge; make the tool-argument and end-state checks " "deterministic per the vocabulary." ) + elif ":duplicate-name:" in problem: + lines.append( + f"- Two sub_goals share the name {problem.split(':')[-1]!r}; every sub-goal needs " + "its own distinct snake_case name." + ) elif ":absent-tool-not-a-single-name" in problem: lines.append( "- An absent checkpoint names several tools at once; write one absent checkpoint " From a43b948a2f83878b627349c6f510ab0d97136e45 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 13 Aug 2026 23:09:07 +0530 Subject: [PATCH 16/55] feat(generation): chunked planning and scaled visibility caps for large scenario counts --- src/fi/alk/generation/pipeline.py | 5 +++-- src/fi/alk/generation/prompts.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/fi/alk/generation/pipeline.py b/src/fi/alk/generation/pipeline.py index 4789f51..fd3c8ee 100644 --- a/src/fi/alk/generation/pipeline.py +++ b/src/fi/alk/generation/pipeline.py @@ -117,7 +117,8 @@ def derive_rows( ) for r in existing } - for round_index in range(config.max_row_rounds): + rounds = max(config.max_row_rounds, -(-want // 25) + 1) + for round_index in range(rounds): remaining = want - len(rows) if remaining <= 0: break @@ -125,7 +126,7 @@ def derive_rows( prompts.SCENARIO_MODEL, prompts.derive_rows_prompt( brief, - want=remaining, + want=min(remaining, 25), signature_cases=contract.signature_cases, real_use_cases=contract.real_use_cases, existing=[ diff --git a/src/fi/alk/generation/prompts.py b/src/fi/alk/generation/prompts.py index ad4f36d..ed117c1 100644 --- a/src/fi/alk/generation/prompts.py +++ b/src/fi/alk/generation/prompts.py @@ -175,7 +175,7 @@ def derive_rows_prompt( if existing: dedupe = ( "Scenarios already planned. Yours must test DIFFERENT situations with DIFFERENT " - f"correct outcomes; do not repeat or reword any of these:\n{json.dumps(existing)[:2200]}\n" + f"correct outcomes; do not repeat or reword any of these:\n{json.dumps(existing)[:12000]}\n" ) feedback_block = ( f"\nReviewer feedback on the previous round; act on all of it:\n{feedback}\n" @@ -395,6 +395,6 @@ def suite_review_prompt(brief: str, records: list[dict], guidance: str = "") -> {brief} ACCEPTED SCENARIOS so far: -{json.dumps(summary)[:6000]} +{json.dumps(summary)[:20000]} {guidance_block(guidance)}What situations that matter in production are missing, and which pairs are near-duplicates?""" From 6e8dd9170b46845c69e90ce29b09c50264346b09 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 13 Aug 2026 23:15:19 +0530 Subject: [PATCH 17/55] fix(generation): unknown-id hint distinguishes runtime values from transposed ids --- src/fi/alk/generation/validators.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/fi/alk/generation/validators.py b/src/fi/alk/generation/validators.py index 622a3d7..688ed90 100644 --- a/src/fi/alk/generation/validators.py +++ b/src/fi/alk/generation/validators.py @@ -216,8 +216,10 @@ def repair_hint(problems: list[str]) -> str: elif ":unknown-id:" in problem: lines.append( f"- A checkpoint uses an identifier that does not exist in the contract " - f"({problem.split(':')[-1]}). Copy ids character for character from the contract's " - "data and arg values; do not reorder or rename their parts." + f"({problem.split(':')[-1]}). If it names a real entity, copy its id character for " + "character from the contract's data. If its value only comes into existence during " + "the run (an order id, a generated handle), it cannot be pinned: move that argument " + "to args_present and pin the arguments whose values the user's request determines." ) elif ":unknown-tool:" in problem: lines.append( From 75af66a1db86e06811c98b2cdf2890b85783e7db Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 13 Aug 2026 23:26:56 +0530 Subject: [PATCH 18/55] feat(generation): coverage-tree planning with per-node context and deterministic near-dup filter --- src/fi/alk/generation/__init__.py | 3 + src/fi/alk/generation/dedup.py | 51 ++++++++++++ src/fi/alk/generation/pipeline.py | 126 ++++++++++++++++++++++-------- src/fi/alk/generation/prompts.py | 46 ++++++++++- tests/test_generation_pipeline.py | 10 +++ 5 files changed, 204 insertions(+), 32 deletions(-) create mode 100644 src/fi/alk/generation/dedup.py diff --git a/src/fi/alk/generation/__init__.py b/src/fi/alk/generation/__init__.py index b488f37..8000aaf 100644 --- a/src/fi/alk/generation/__init__.py +++ b/src/fi/alk/generation/__init__.py @@ -1,6 +1,7 @@ """Local-first scenario generation: point at an agent, get reviewed, checkable test scenarios.""" from .checks import CheckResult, evaluate_checkpoint, evaluate_scenario +from .dedup import near_duplicate, similarity from .contract import AgentContract, ToolSpec, extract_contract, validate_contract from .emit import smoke_manifest, to_alk_scenario, write_outputs from .llm import ( @@ -40,6 +41,8 @@ "ToolSpec", "Usage", "banned_tokens", + "near_duplicate", + "similarity", "extract_contract", "generate", "register_source", diff --git a/src/fi/alk/generation/dedup.py b/src/fi/alk/generation/dedup.py new file mode 100644 index 0000000..90c9695 --- /dev/null +++ b/src/fi/alk/generation/dedup.py @@ -0,0 +1,51 @@ +"""Deterministic near-duplicate detection. No model calls, no context limits. + +Two planned scenarios are near-duplicates when the words describing their situation and outcome +mostly overlap. Token-set similarity is crude next to embeddings, but it is free, deterministic, +dependency-free, and catches the rewording-of-the-same-situation failure that matters at scale; +an embedding backend can replace ``similarity`` later without touching callers. +""" + +from __future__ import annotations + +import re +from typing import Iterable, Mapping + +_WORD = re.compile(r"[a-z0-9]+") +_STOPWORDS = frozenset( + "a an and are as at be but by for if in into is it no not of on or such that the " + "their then there these they this to was will with without user agent".split() +) + + +def _tokens(text: str) -> frozenset[str]: + return frozenset( + token for token in _WORD.findall(str(text).lower()) if token not in _STOPWORDS + ) + + +def _signature(row: Mapping) -> frozenset[str]: + return _tokens( + " ".join( + str(row.get(key, "")) + for key in ("situation", "unique_end_state", "target_failure") + ) + ) + + +def similarity(a: Mapping, b: Mapping) -> float: + """Jaccard similarity of the rows' descriptive token sets, 0..1.""" + ta, tb = _signature(a), _signature(b) + if not ta or not tb: + return 0.0 + return len(ta & tb) / len(ta | tb) + + +def near_duplicate( + row: Mapping, existing: Iterable[Mapping], *, threshold: float = 0.6 +) -> Mapping | None: + """Return the first existing row this one nearly duplicates, or None.""" + for other in existing: + if similarity(row, other) >= threshold: + return other + return None diff --git a/src/fi/alk/generation/pipeline.py b/src/fi/alk/generation/pipeline.py index fd3c8ee..3898a33 100644 --- a/src/fi/alk/generation/pipeline.py +++ b/src/fi/alk/generation/pipeline.py @@ -23,6 +23,7 @@ from . import prompts from .contract import AgentContract, extract_contract +from .dedup import near_duplicate from .emit import write_outputs from .explorer import explore_contract from .llm import LLMClient @@ -99,6 +100,39 @@ def derive_catalog(contract: AgentContract, llm: LLMClient) -> list[dict]: return entries +def derive_coverage_plan( + contract: AgentContract, llm: LLMClient, config: GenerationConfig +) -> list[dict]: + """Partition the target count across use-case nodes. The plan is O(use cases), never O(n), + so planning context stays bounded at any scenario count.""" + raw = llm.complete_json( + prompts.COVERAGE_PLAN_SYSTEM, + prompts.coverage_plan_prompt( + contract.brief(), total=config.n, guidance=config.guidance + ), + temperature=0.3, + max_tokens=16_000, + ) + nodes = raw.get("nodes", raw) if isinstance(raw, dict) else raw + plan: list[dict] = [] + for node in nodes if isinstance(nodes, list) else []: + if not isinstance(node, dict) or not node.get("use_case"): + continue + count = node.get("count") + node["count"] = max(1, int(count)) if isinstance(count, (int, float)) else 1 + plan.append(node) + total = sum(node["count"] for node in plan) + if plan and total != config.n: # renormalise counts to the requested total + scaled = [max(1, round(node["count"] * config.n / total)) for node in plan] + while sum(scaled) > config.n: + scaled[scaled.index(max(scaled))] -= 1 + while sum(scaled) < config.n: + scaled[scaled.index(min(scaled))] += 1 + for node, count in zip(plan, scaled): + node["count"] = count + return plan + + def derive_rows( contract: AgentContract, llm: LLMClient, @@ -107,6 +141,7 @@ def derive_rows( want: int, existing: list[dict], feedback: str = "", + node: dict | None = None, ) -> list[dict]: brief = contract.brief() rows: list[dict] = [] @@ -136,6 +171,7 @@ def derive_rows( feedback=feedback, first_round=round_index == 0 and not existing, guidance=config.guidance, + node=node, ), temperature=0.4, max_tokens=20_000, @@ -149,6 +185,8 @@ def derive_rows( ) if key in seen: continue + if near_duplicate(row, existing) or near_duplicate(row, rows): + continue seen.add(key) row["id"] = _slugify(row.get("id") or row.get("situation", "")) rows.append(row) @@ -296,8 +334,63 @@ def _flush() -> None: usage=llm.usage.as_dict(), ) + def _materialize_batch(rows: list[dict]) -> None: + for row in rows: + record, reason = materialize_row(contract, row, catalog, llm, config) + if record is not None: + records.append(record) + else: + rejected.append({**row, "_reject_reason": reason}) + _flush() # runs are long; keep every artifact inspectable while they go + print( + f"[generation] accepted={len(records)} rejected={len(rejected)} " + f"spent={llm.usage.as_dict().get('usd', 0)}", + flush=True, + ) + + def _node_rows(node: dict) -> list[dict]: + """Existing rows belonging to one coverage node (its local dedup context).""" + label = str(node.get("use_case", "")).strip().lower() + return [ + r + for r in records + rejected + if str(r.get("use_case", "")).strip().lower() == label + ] + try: - for suite_round in range(1 + config.max_suite_rounds): + # Coverage-tree planning: partition n across use-case nodes, then plan each node + # separately. Planning context is bounded by the node, never by the whole suite; + # cross-node overlap is prevented structurally and by the deterministic dedup filter. + plan = derive_coverage_plan(contract, llm, config) + logger.info("coverage plan", extra={"nodes": len(plan)}) + for node in plan: + rows = derive_rows( + contract, + llm, + config, + want=int(node["count"]), + existing=_node_rows(node), + node=node, + ) + _materialize_batch(rows) + + # Replenishment: coverage review names gaps and near-duplicates, then plans the + # shortfall suite-wide until the target count or the round cap is reached. + for suite_round in range(config.max_suite_rounds): + want = config.n - len(records) + if want <= 0 and suite_round > 0: + break + gaps, duplicate_ids, feedback = suite_review( + contract, records, llm, guidance=config.guidance + ) + if duplicate_ids: + dropped = [r for r in records if str(r.get("id")) in set(duplicate_ids)] + records[:] = [ + r for r in records if str(r.get("id")) not in set(duplicate_ids) + ] + for record in dropped: + record["_reject_reason"] = "near-duplicate of an accepted scenario" + rejected.append(record) want = config.n - len(records) if want <= 0: break @@ -311,36 +404,7 @@ def _flush() -> None: ) if not rows: break - for row in rows: - record, reason = materialize_row(contract, row, catalog, llm, config) - if record is not None: - records.append(record) - else: - rejected.append({**row, "_reject_reason": reason}) - _flush() # runs are long; keep every artifact inspectable while they go - print( - f"[generation] accepted={len(records)} rejected={len(rejected)} " - f"spent={llm.usage.as_dict().get('usd', 0)}", - flush=True, - ) - if suite_round < config.max_suite_rounds and records: - gaps, duplicate_ids, feedback = suite_review( - contract, records, llm, guidance=config.guidance - ) - if duplicate_ids: - dropped = [ - r for r in records if str(r.get("id")) in set(duplicate_ids) - ] - records = [ - r for r in records if str(r.get("id")) not in set(duplicate_ids) - ] - for record in dropped: - record["_reject_reason"] = ( - "near-duplicate of an accepted scenario" - ) - rejected.append(record) - if not gaps and len(records) >= config.n: - break + _materialize_batch(rows) except Exception: _flush() raise diff --git a/src/fi/alk/generation/prompts.py b/src/fi/alk/generation/prompts.py index ed117c1..53eb5e2 100644 --- a/src/fi/alk/generation/prompts.py +++ b/src/fi/alk/generation/prompts.py @@ -145,6 +145,40 @@ def subgoal_catalog_prompt(brief: str) -> str: "definition_template": {{...}}, "justification_if_judge": "..."}}]}}""" +COVERAGE_PLAN_SYSTEM = ( + SCENARIO_MODEL + + """ + +Role: before individual tests are written, you partition the whole testing effort. The partition is +what keeps a large test suite diverse: each part is planned separately, so parts must not overlap, +and together they must cover everything worth testing about this agent.""" +) + + +def coverage_plan_prompt(brief: str, *, total: int, guidance: str = "") -> str: + return f"""{brief} + +Task: partition {total} test scenarios across this agent's use cases. + +Return the partition as nodes. Each node is one use case (one real job users hire this agent for) +with a share of the {total} scenarios proportional to how much can genuinely go wrong in it: use +cases with rules to enforce, information to gather, or state to modify earn larger shares; a use +case where little can fail earns a small one. Every node also lists the distinct ANGLES worth +testing inside it: an angle is a one-line direction (a condition of the world or the user) that +would make scenarios within the node differ in their correct outcome. + +Rules: +- Nodes are mutually exclusive and jointly cover the agent's real jobs. No node for internal + machinery. +- Counts sum to {total}. A node's angle list should be at least as long as its count would need; + when a use case cannot support its share with genuinely distinct angles, give the surplus to one + that can. +- Angles within a node must each produce a DIFFERENT correct outcome, not the same outcome under + different wording. +{guidance_block(guidance)}Return JSON: {{"nodes": [{{"use_case": "...", "description": "", +"count": , "angles": ["", ...]}}]}}""" + + def derive_rows_prompt( brief: str, *, @@ -155,6 +189,7 @@ def derive_rows_prompt( feedback: str, first_round: bool, guidance: str = "", + node: dict | None = None, ) -> str: must = "" if first_round and signature_cases: @@ -182,8 +217,17 @@ def derive_rows_prompt( if feedback else "" ) + node_block = "" + if node: + angles = "".join(f"\n - {a}" for a in node.get("angles") or []) + node_block = ( + f"\nThis planning call covers ONE use case only. Every scenario you return belongs to it, " + f"and its use_case field repeats this wording exactly.\nUSE CASE: {node.get('use_case')}" + f"\n{node.get('description', '')}\nAngles worth testing here (each produces a different " + f"correct outcome; draw on them and add better ones if you see them):{angles}\n" + ) return f"""{brief} - +{node_block} Task: plan {want} distinct test scenarios for this agent. How to author each scenario. Work through these steps in order, in your head, before writing its diff --git a/tests/test_generation_pipeline.py b/tests/test_generation_pipeline.py index bb899b9..7b04b5a 100644 --- a/tests/test_generation_pipeline.py +++ b/tests/test_generation_pipeline.py @@ -172,6 +172,16 @@ def test_full_pipeline_offline(agent_repo, tmp_path): ] }, CATALOG, + { + "nodes": [ + { + "use_case": "Order a single item", + "description": "One item ordered and confirmed", + "count": 1, + "angles": ["plain single-item success"], + } + ] + }, ROWS, SCENARIO, VERDICT, From 51e2c2db477ff892a2e3d922b9eb6175c82ebc9c Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 13 Aug 2026 23:37:03 +0530 Subject: [PATCH 19/55] fix(generation): remove artifacts of scenarios later dropped as duplicates --- src/fi/alk/generation/emit.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/fi/alk/generation/emit.py b/src/fi/alk/generation/emit.py index c1d4108..8887429 100644 --- a/src/fi/alk/generation/emit.py +++ b/src/fi/alk/generation/emit.py @@ -194,6 +194,10 @@ def write_outputs( alk_dir = os.path.join(out_dir, "alk") os.makedirs(scenarios_dir, exist_ok=True) os.makedirs(alk_dir, exist_ok=True) + current = {f"{record.get('id', 'scenario')}.json" for record in records} + for directory in (scenarios_dir, alk_dir): + for stale in set(os.listdir(directory)) - current: + os.remove(os.path.join(directory, stale)) def _dump(path: str, payload: Any) -> None: with open(path, "w", encoding="utf-8") as fh: From 625932736856317030390f7c02ea63d7559f6116 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 13 Aug 2026 23:44:37 +0530 Subject: [PATCH 20/55] feat(generation): blueprint review gate, oracle self-consistency execution check, purpose register --- src/fi/alk/generation/emit.py | 6 ++ src/fi/alk/generation/oracle.py | 90 +++++++++++++++++++++++++++++ src/fi/alk/generation/pipeline.py | 50 +++++++++++++++- src/fi/alk/generation/prompts.py | 33 ++++++++++- src/fi/alk/generation/validators.py | 1 + tests/test_generation_pipeline.py | 36 +++++++++++- 6 files changed, 213 insertions(+), 3 deletions(-) create mode 100644 src/fi/alk/generation/oracle.py diff --git a/src/fi/alk/generation/emit.py b/src/fi/alk/generation/emit.py index 8887429..d6f5edd 100644 --- a/src/fi/alk/generation/emit.py +++ b/src/fi/alk/generation/emit.py @@ -264,6 +264,12 @@ def render_report( f"| {index} | {record.get('use_case', '')} | {record.get('situation', '')} " f"| {len(sub_goals)} | {det}/{len(sub_goals)} |" ) + lines += ["", "## Purpose per scenario (the definitive-yes register)", ""] + for record in records: + lines.append( + f"- **{record.get('id')}**: catches `{record.get('target_failure', '')}`. " + f"Matters because: {record.get('why_it_matters', '')}" + ) if reuse: lines += ["", "## Sub-goal roll-up (appearances across scenarios)", ""] for name, count in sorted(reuse.items(), key=lambda item: -item[1]): diff --git a/src/fi/alk/generation/oracle.py b/src/fi/alk/generation/oracle.py new file mode 100644 index 0000000..4a32621 --- /dev/null +++ b/src/fi/alk/generation/oracle.py @@ -0,0 +1,90 @@ +"""Oracle self-consistency: a scenario must pass the run it itself predicts. Pure code. + +A well-formed scenario fully determines one predicted run: the tool calls its checkpoints pin, the +final state its seed plus declared mock state updates produce, and the values it says the agent must +convey. Evaluating the scenario's own deterministic checkpoints against that predicted evidence is +an execution check with no model involved: a checkpoint that fails on the run its own scenario +predicts can never pass a real run, so the scenario is internally contradictory and must be +repaired before it costs anything downstream. +""" + +from __future__ import annotations + +import copy +from typing import Any, Mapping + +from .checks import evaluate_checkpoint + + +def _deep_merge(target: dict, updates: Mapping[str, Any]) -> None: + for key, value in updates.items(): + if isinstance(value, Mapping) and isinstance(target.get(key), dict): + _deep_merge(target[key], value) + else: + target[key] = copy.deepcopy(value) + + +def predicted_evidence(record: Mapping[str, Any]) -> dict[str, Any]: + """The run this scenario predicts, derived from its own definitions only.""" + tool_calls: list[dict[str, Any]] = [] + for sub_goal in record.get("sub_goals") or []: + checkpoint = (sub_goal or {}).get("checkpoint") or {} + definition = checkpoint.get("definition") or {} + if checkpoint.get("kind") == "tool_call_args": + arguments = dict(definition.get("args_equal") or {}) + for arg in definition.get("args_present") or []: + arguments.setdefault(str(arg), "") + tool_calls.append({"name": definition.get("tool"), "arguments": arguments}) + + # The transcript is predicted ONLY from what the scenario says the agent must communicate. + # Seeding it from the conveyed definitions themselves would make those checks self-satisfying; + # sourcing it from must_convey makes the oracle verify that every conveyed checkpoint asserts + # a value the scenario actually commits the agent to saying. + transcript: list[str] = [] + + environment = record.get("environment") or {} + final_state: dict[str, Any] = copy.deepcopy(dict(environment.get("seed") or {})) + for mock in (environment.get("mock_responses") or {}).values(): + if isinstance(mock, Mapping) and isinstance(mock.get("state_updates"), Mapping): + _deep_merge(final_state, mock["state_updates"]) + + outcome = record.get("expected_outcome") or {} + for value in outcome.get("must_convey") or []: + transcript.append(str(value)) + return { + "tool_calls": tool_calls, + "transcript_turns": transcript, + "final_state": final_state, + } + + +def oracle_problems(record: Mapping[str, Any]) -> list[str]: + """Deterministic checkpoints that fail the scenario's own predicted run.""" + evidence = predicted_evidence(record) + problems: list[str] = [] + for sub_goal in record.get("sub_goals") or []: + checkpoint = (sub_goal or {}).get("checkpoint") or {} + if not checkpoint.get("deterministic"): + continue + passed, reason = evaluate_checkpoint( + str(checkpoint.get("kind", "")), + checkpoint.get("definition") or {}, + tool_calls=evidence["tool_calls"], + transcript_turns=evidence["transcript_turns"], + final_state=evidence["final_state"], + ) + if passed is False: + problems.append(f"oracle:{sub_goal.get('name')}:{reason}") + return problems + + +def oracle_hint(problems: list[str]) -> str: + lines = [ + "- These checkpoints fail even on the run this scenario itself predicts, so they can never " + "pass a real run. Make the scenario self-consistent: the environment seed plus the declared " + "mock state_updates must produce the state the `state` checkpoints assert, expected values " + "asserted by `conveyed` checkpoints must appear in must_convey, and `absent` checkpoints " + "must not name calls the other checkpoints require:" + ] + lines += [f" - {p}" for p in problems[:8]] + return "\n".join(lines) diff --git a/src/fi/alk/generation/pipeline.py b/src/fi/alk/generation/pipeline.py index 3898a33..d5a1ff3 100644 --- a/src/fi/alk/generation/pipeline.py +++ b/src/fi/alk/generation/pipeline.py @@ -25,6 +25,7 @@ from .contract import AgentContract, extract_contract from .dedup import near_duplicate from .emit import write_outputs +from .oracle import oracle_hint, oracle_problems from .explorer import explore_contract from .llm import LLMClient from .sources import AgentSource @@ -193,6 +194,40 @@ def derive_rows( return rows[:want] +def review_plan( + contract: AgentContract, rows: list[dict], llm: LLMClient +) -> list[dict]: + """Blueprint gate: cheap review of plans before the expensive materialize+review spend. + + Fail-open: a malformed reviewer reply keeps the original rows, because this stage exists to + save cost and lift quality, never to lose work. + """ + if not rows: + return rows + try: + raw = llm.complete_json( + prompts.PLAN_REVIEW_SYSTEM, + prompts.plan_review_prompt(contract.brief(), rows), + temperature=0.2, + max_tokens=16_000, + ) + except Exception as exc: # noqa: BLE001 - reviewer trouble must not lose plans + logger.warning("plan review failed open: %s", exc) + return rows + reviewed = raw.get("rows", raw) if isinstance(raw, dict) else raw + survivors = [ + row + for row in (reviewed if isinstance(reviewed, list) else []) + if isinstance(row, dict) and row.get("situation") and row.get("target_failure") + ] + if not survivors or len(survivors) > len(rows): + return rows + for row in survivors: + row["id"] = _slugify(row.get("id") or row.get("situation", "")) + logger.info("plan review", extra={"in": len(rows), "kept": len(survivors)}) + return survivors + + def materialize_row( contract: AgentContract, row: dict, @@ -242,6 +277,11 @@ def materialize_row( best, reason = record, f"validator: {problems[:6]}" hint = repair_hint(problems) continue + inconsistencies = oracle_problems(record) + if inconsistencies: + best, reason = record, f"oracle: {inconsistencies[:4]}" + hint = oracle_hint(inconsistencies) + continue if not config.critic_enabled: return record, "" verdict = llm.complete_json( @@ -272,7 +312,11 @@ def materialize_row( ) hint = str(verdict.get("fix_hints") or "") or repair_hint([]) # Out of repair attempts: keep the best structurally-valid draft, flagged, rather than lose it. - if best is not None and not validate_scenario(best, contract): + if ( + best is not None + and not validate_scenario(best, contract) + and not oracle_problems(best) + ): best["_review_flag"] = reason return best, "" return None, reason @@ -372,6 +416,8 @@ def _node_rows(node: dict) -> list[dict]: existing=_node_rows(node), node=node, ) + if config.critic_enabled: + rows = review_plan(contract, rows, llm) _materialize_batch(rows) # Replenishment: coverage review names gaps and near-duplicates, then plans the @@ -404,6 +450,8 @@ def _node_rows(node: dict) -> list[dict]: ) if not rows: break + if config.critic_enabled: + rows = review_plan(contract, rows, llm) _materialize_batch(rows) except Exception: _flush() diff --git a/src/fi/alk/generation/prompts.py b/src/fi/alk/generation/prompts.py index 53eb5e2..46a4684 100644 --- a/src/fi/alk/generation/prompts.py +++ b/src/fi/alk/generation/prompts.py @@ -261,6 +261,8 @@ def derive_rows_prompt( phrased from the user or world side. It must not mention the agent's tools, must not prescribe what the agent should do, and must not contain the expected outcome. - target_failure: the specific wrong behavior from step 1 that this scenario would catch +- why_it_matters: one line naming the production consequence if that failure shipped (what a real + user or the business loses). A scenario whose consequence you cannot name is not worth running. - unique_end_state: one line, the single correct final state from step 2 - goal: one line, the end-objective of the test from the user's side @@ -275,7 +277,36 @@ def derive_rows_prompt( not toy versions. - No scenarios about internal machinery (logging, config, retries): users never bring those. {dedupe}{feedback_block}{guidance_block(guidance)}Return JSON: {{"rows": [{{"id": "...", "use_case": "...", "situation": "...", -"target_failure": "...", "unique_end_state": "...", "goal": "..."}}]}}""" +"target_failure": "...", "why_it_matters": "...", "unique_end_state": "...", "goal": "..."}}]}}""" + + +PLAN_REVIEW_SYSTEM = ( + SCENARIO_MODEL + + """ + +Role: you review PLANNED scenarios before any of them is written in full. Full tests are expensive; +your job is to make sure only plans that deserve the spend go forward. For each plan you return one +of three outcomes: keep it as is, fix it in place (rewrite its weak fields, keep its id), or drop it. +Judge each plan on: +1. PURPOSE. target_failure names a wrong behavior a plausible implementation could actually commit, + and why_it_matters names a real consequence. Plans with generic failures (the agent errs) or no + nameable consequence are dropped. +2. FEASIBLE. The situation can be set up with the agent's real data as it ships, and a simulated + user could genuinely play it. +3. DETERMINATE. unique_end_state pins exactly one correct final state under the agent's rules. +4. DISTINCT. No two surviving plans share the same correct end state. +Return JSON: {"rows": []}.""" +) + + +def plan_review_prompt(brief: str, rows: list[dict]) -> str: + return f"""{brief} + +PLANNED SCENARIOS to review: +{json.dumps(rows)[:14000]} + +Review per your instructions. Return only the surviving plans, fixed in place where fixing was +cheaper than dropping.""" def materialize_prompt( diff --git a/src/fi/alk/generation/validators.py b/src/fi/alk/generation/validators.py index 688ed90..43c8a4c 100644 --- a/src/fi/alk/generation/validators.py +++ b/src/fi/alk/generation/validators.py @@ -120,6 +120,7 @@ def validate_scenario(scenario: dict, contract: AgentContract) -> list[str]: "agent_input", "expected_outcome", "target_failure", + "why_it_matters", ): if scenario.get(field) in (None, "", [], {}): problems.append(f"empty:{field}") diff --git a/tests/test_generation_pipeline.py b/tests/test_generation_pipeline.py index 7b04b5a..ffbc1a7 100644 --- a/tests/test_generation_pipeline.py +++ b/tests/test_generation_pipeline.py @@ -72,7 +72,8 @@ "id": "latte-medium", "use_case": "Order a single item", "situation": "The caller wants one medium latte and confirms", - "why_distinct": "Plain single-item success path", + "target_failure": "The agent adds the wrong item or size", + "why_it_matters": "A wrong order reaches a paying customer", "goal": "A medium latte is ordered and confirmed", } ] @@ -83,6 +84,7 @@ "use_case": "Order a single item", "situation": "The caller wants one medium latte and confirms", "target_failure": "The agent adds the wrong item or size, or never confirms the order", + "why_it_matters": "A wrong order reaches a paying customer", "goal": "A medium latte is ordered and confirmed", "description": "A caller orders one medium latte, nothing else. The menu has lattes and mochas; " "the order starts empty and the agent must add the right item at the right size.", @@ -183,6 +185,7 @@ def test_full_pipeline_offline(agent_repo, tmp_path): ] }, ROWS, + ROWS, # plan review returns the same surviving plans SCENARIO, VERDICT, {"gaps": [], "near_duplicates": []}, @@ -337,3 +340,34 @@ def test_validator_survives_malformed_definition_shapes(): ) problems = validate_scenario(bad, contract) assert any("absent-tool-not-a-single-name" in p for p in problems) + + +def test_oracle_rejects_internally_contradictory_scenario(): + from fi.alk.generation.oracle import oracle_problems + + assert oracle_problems(SCENARIO) == [] # the fixture predicts a run it passes + + broken = json.loads(json.dumps(SCENARIO)) + # State checkpoint asserts a state the seed plus declared updates never produce. + broken["environment"]["mock_responses"]["add_item"]["state_updates"] = { + "order": {"items": ["latte_M"], "confirmed": False} + } + problems = oracle_problems(broken) + assert any("order_confirmed" in p for p in problems) + + contradiction = json.loads(json.dumps(SCENARIO)) + # An absent checkpoint forbids the very call another checkpoint requires. + contradiction["sub_goals"].append( + { + "name": "no_add_item_call", + "milestone": "contradicts the required call", + "checkpoint": { + "kind": "absent", + "detail": "add_item never fires", + "deterministic": True, + "definition": {"no_tool_call": "add_item"}, + }, + } + ) + problems = oracle_problems(contradiction) + assert any("no_add_item_call" in p for p in problems) From 5b34c8927970e1623d49cc0514066aba23ef9eed Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Thu, 13 Aug 2026 23:57:53 +0530 Subject: [PATCH 21/55] fix(generation): contract schema normalizes benign model-JSON shape variance --- src/fi/alk/generation/contract.py | 44 ++++++++++++++++++++++++++++++- tests/test_generation_pipeline.py | 11 ++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/src/fi/alk/generation/contract.py b/src/fi/alk/generation/contract.py index ce2ed6d..2da75bb 100644 --- a/src/fi/alk/generation/contract.py +++ b/src/fi/alk/generation/contract.py @@ -10,10 +10,25 @@ import json from typing import Any -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, model_validator from .llm import LLMClient +_STRING_FIELDS = ( + "agent", + "one_liner", + "modality", + "system_prompt_excerpt", + "grading_notes", +) +_LIST_FIELDS = ( + "hard_constraints", + "real_use_cases", + "signature_cases", + "anti_hallucination", +) +_DICT_FIELDS = ("data_schema", "base_environment") + class ToolSpec(BaseModel): name: str @@ -25,6 +40,33 @@ class ToolSpec(BaseModel): class AgentContract(BaseModel): """What the agent verifiably is. Nothing downstream may contradict this.""" + @model_validator(mode="before") + @classmethod + def _normalize_shapes(cls, payload: Any) -> Any: + """Model JSON varies in benign ways (a list where prose was asked, a bare string where a + list was). Normalize instead of rejecting: shape variance is not a grounding error.""" + if not isinstance(payload, dict): + return payload + for key in _STRING_FIELDS: + value = payload.get(key) + if isinstance(value, list): + payload[key] = "\n".join(str(item) for item in value) + elif value is not None and not isinstance(value, str): + payload[key] = str(value) + for key in _LIST_FIELDS: + value = payload.get(key) + if isinstance(value, str): + payload[key] = [value] + elif isinstance(value, list): + payload[key] = [ + str(item) if not isinstance(item, str) else item for item in value + ] + for key in _DICT_FIELDS: + value = payload.get(key) + if value is not None and not isinstance(value, dict): + payload[key] = {"value": value} + return payload + agent: str one_liner: str = "" modality: str = "chat" diff --git a/tests/test_generation_pipeline.py b/tests/test_generation_pipeline.py index ffbc1a7..134da9f 100644 --- a/tests/test_generation_pipeline.py +++ b/tests/test_generation_pipeline.py @@ -371,3 +371,14 @@ def test_oracle_rejects_internally_contradictory_scenario(): ) problems = oracle_problems(contradiction) assert any("no_add_item_call" in p for p in problems) + + +def test_contract_normalizes_benign_shape_variance(): + payload = json.loads(json.dumps(CONTRACT)) + payload["grading_notes"] = ["line one", "line two"] + payload["hard_constraints"] = "a single rule as a bare string" + payload["one_liner"] = ["joined", "sentence"] + contract = AgentContract.model_validate(payload) + assert contract.grading_notes == "line one\nline two" + assert contract.hard_constraints == ["a single rule as a bare string"] + assert "joined" in contract.one_liner From c6ed3ac1defb3f6c2f47d2b63abb2da4d5488af9 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 14 Aug 2026 00:23:14 +0530 Subject: [PATCH 22/55] feat(generation): parallel planning and materialization, contract caching, thread-safe metering --- src/fi/alk/generation/cli.py | 18 ++++++++++- src/fi/alk/generation/llm.py | 2 ++ src/fi/alk/generation/pipeline.py | 53 +++++++++++++++++++++++++++---- 3 files changed, 66 insertions(+), 7 deletions(-) diff --git a/src/fi/alk/generation/cli.py b/src/fi/alk/generation/cli.py index f90183a..b48d8e7 100644 --- a/src/fi/alk/generation/cli.py +++ b/src/fi/alk/generation/cli.py @@ -39,6 +39,17 @@ def build_parser() -> argparse.ArgumentParser: default="", help="operator instructions steering what to test (or @path/to/file to read them)", ) + parser.add_argument( + "--workers", + type=int, + default=8, + help="parallel scenario workers (1 = sequential)", + ) + parser.add_argument( + "--contract", + default="", + help="reuse a previously extracted contract.json (skips exploration)", + ) parser.add_argument("--verbose", action="store_true") return parser @@ -62,7 +73,12 @@ def main(argv: list[str] | None = None) -> int: source = resolve_source(args.source, **source_kwargs) llm = LiteLLMClient(model=args.model, budget_usd=args.budget_usd) config = GenerationConfig( - n=args.n, critic_enabled=not args.no_critic, guidance=guidance, out_dir=args.out + n=args.n, + critic_enabled=not args.no_critic, + guidance=guidance, + max_workers=args.workers, + contract_path=args.contract, + out_dir=args.out, ) result = generate(source, llm, config) diff --git a/src/fi/alk/generation/llm.py b/src/fi/alk/generation/llm.py index 383149d..63413d2 100644 --- a/src/fi/alk/generation/llm.py +++ b/src/fi/alk/generation/llm.py @@ -13,6 +13,7 @@ import os import re import time +import threading from dataclasses import dataclass, field from typing import Any, Protocol @@ -200,6 +201,7 @@ class LiteLLMClient: output_cost_per_token: float = DEFAULT_OUTPUT_COST_PER_TOKEN max_attempts: int = 4 _usage: Usage = field(default_factory=Usage) + _lock: threading.Lock = field(default_factory=threading.Lock, repr=False) @property def usage(self) -> Usage: diff --git a/src/fi/alk/generation/pipeline.py b/src/fi/alk/generation/pipeline.py index d5a1ff3..2e1920b 100644 --- a/src/fi/alk/generation/pipeline.py +++ b/src/fi/alk/generation/pipeline.py @@ -18,6 +18,8 @@ import logging import re +import threading +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from typing import Any @@ -47,6 +49,8 @@ class GenerationConfig: max_explore_turns: int = 20 critic_enabled: bool = True guidance: str = "" + max_workers: int = 8 + contract_path: str = "" out_dir: str = "artifacts/generated-scenarios" @@ -67,7 +71,16 @@ def _slugify(value: str) -> str: def build_contract( source: AgentSource, llm: LLMClient, config: GenerationConfig ) -> AgentContract: - """Prefer the exploration loop when the source exposes a filesystem root.""" + """Prefer a cached contract, then the exploration loop, then blob extraction. + + Contract extraction is once-per-agent work; caching it turns every regeneration run into + planning plus materialization only, which is where the wanted scenarios actually come from. + """ + if config.contract_path: + import json as _json + + with open(config.contract_path, encoding="utf-8") as fh: + return AgentContract.model_validate(_json.load(fh)) evidence = source.describe() root = (evidence.metadata or {}).get("root") if root: @@ -378,9 +391,11 @@ def _flush() -> None: usage=llm.usage.as_dict(), ) - def _materialize_batch(rows: list[dict]) -> None: - for row in rows: - record, reason = materialize_row(contract, row, catalog, llm, config) + lock = threading.Lock() + + def _materialize_one(row: dict) -> None: + record, reason = materialize_row(contract, row, catalog, llm, config) + with lock: if record is not None: records.append(record) else: @@ -392,6 +407,16 @@ def _materialize_batch(rows: list[dict]) -> None: flush=True, ) + def _materialize_batch(rows: list[dict]) -> None: + if not rows: + return + if config.max_workers <= 1 or len(rows) == 1: + for row in rows: + _materialize_one(row) + return + with ThreadPoolExecutor(max_workers=min(config.max_workers, len(rows))) as pool: + list(pool.map(_materialize_one, rows)) + def _node_rows(node: dict) -> list[dict]: """Existing rows belonging to one coverage node (its local dedup context).""" label = str(node.get("use_case", "")).strip().lower() @@ -407,7 +432,8 @@ def _node_rows(node: dict) -> list[dict]: # cross-node overlap is prevented structurally and by the deterministic dedup filter. plan = derive_coverage_plan(contract, llm, config) logger.info("coverage plan", extra={"nodes": len(plan)}) - for node in plan: + + def _plan_node(node: dict) -> list[dict]: rows = derive_rows( contract, llm, @@ -418,7 +444,22 @@ def _node_rows(node: dict) -> list[dict]: ) if config.critic_enabled: rows = review_plan(contract, rows, llm) - _materialize_batch(rows) + return rows + + if config.max_workers > 1 and len(plan) > 1: + with ThreadPoolExecutor( + max_workers=min(config.max_workers, len(plan)) + ) as pool: + node_rows = list(pool.map(_plan_node, plan)) + else: + node_rows = [_plan_node(node) for node in plan] + # Cross-node near-dup guard after parallel planning (nodes could not see each other). + vetted: list[dict] = [] + for rows in node_rows: + for row in rows: + if not near_duplicate(row, vetted): + vetted.append(row) + _materialize_batch(vetted) # Replenishment: coverage review names gaps and near-duplicates, then plans the # shortfall suite-wide until the target count or the round cap is reached. From c9b244e6f485560745ba53e0ca414fdf7276d568 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 14 Aug 2026 00:35:36 +0530 Subject: [PATCH 23/55] feat(generation): checkpoint argument values validated against the contract's per-arg valid sets --- src/fi/alk/generation/validators.py | 17 ++++++++++++++++- tests/test_generation_pipeline.py | 8 ++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/fi/alk/generation/validators.py b/src/fi/alk/generation/validators.py index 43c8a4c..4a9de16 100644 --- a/src/fi/alk/generation/validators.py +++ b/src/fi/alk/generation/validators.py @@ -61,6 +61,7 @@ def _validate_definition( tool_names: set[str], where: str, legit_vocabulary: set[str], + arg_values: dict[str, dict], ) -> list[str]: problems: list[str] = [] unknown_ids = sorted( @@ -78,6 +79,13 @@ def _validate_definition( problems.append(f"{where}:unknown-tool:{tool}") if not definition.get("args_equal") and not definition.get("args_present"): problems.append(f"{where}:tool_call_args-without-args") + for arg, value in (definition.get("args_equal") or {}).items(): + allowed = arg_values.get(str(tool), {}).get(str(arg)) + if not isinstance(allowed, list) or not allowed: + continue + candidates = {str(item).lower() for item in allowed} | {"null", "none"} + if value is not None and str(value).lower() not in candidates: + problems.append(f"{where}:arg-value-not-allowed:{arg}={value}") elif kind == "state": if not definition.get("must") and not definition.get("forbidden"): problems.append(f"{where}:state-without-must-or-forbidden") @@ -110,6 +118,7 @@ def validate_scenario(scenario: dict, contract: AgentContract) -> list[str]: problems: list[str] = [] tool_names = contract.tool_names() legit_vocabulary = _legit_vocabulary(contract) + arg_values = {tool.name: dict(tool.arg_values or {}) for tool in contract.tools} for field in ( "id", @@ -164,7 +173,7 @@ def validate_scenario(scenario: dict, contract: AgentContract) -> list[str]: problems.append(f"{where}:no-definition") else: problems += _validate_definition( - kind, definition, tool_names, where, legit_vocabulary + kind, definition, tool_names, where, legit_vocabulary, arg_values ) deterministic = bool(checkpoint.get("deterministic")) if deterministic and kind == "judge": @@ -241,6 +250,12 @@ def repair_hint(problems: list[str]) -> str: "- Every checkpoint is a judge; make the tool-argument and end-state checks " "deterministic per the vocabulary." ) + elif ":arg-value-not-allowed:" in problem: + lines.append( + f"- A checkpoint pins an argument to a value the contract does not list as valid " + f"({problem.split(':')[-1]}). Choose the value from that argument's listed valid " + "values in the contract." + ) elif ":duplicate-name:" in problem: lines.append( f"- Two sub_goals share the name {problem.split(':')[-1]!r}; every sub-goal needs " diff --git a/tests/test_generation_pipeline.py b/tests/test_generation_pipeline.py index 134da9f..5ae5b20 100644 --- a/tests/test_generation_pipeline.py +++ b/tests/test_generation_pipeline.py @@ -382,3 +382,11 @@ def test_contract_normalizes_benign_shape_variance(): assert contract.grading_notes == "line one\nline two" assert contract.hard_constraints == ["a single rule as a bare string"] assert "joined" in contract.one_liner + + +def test_validator_rejects_disallowed_arg_value(): + contract = AgentContract.model_validate(CONTRACT) + bad = json.loads(json.dumps(SCENARIO)) + bad["sub_goals"][0]["checkpoint"]["definition"]["args_equal"]["size"] = "XL" + problems = validate_scenario(bad, contract) + assert any("arg-value-not-allowed:size=XL" in p for p in problems) From c9448b3b326280a0f31b8544a4ab68f0b938d27f Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 14 Aug 2026 00:41:08 +0530 Subject: [PATCH 24/55] fix(generation): pinned argument values must exist in the contract vocabulary --- src/fi/alk/generation/validators.py | 25 +++++++++++++++++++++---- tests/test_generation_pipeline.py | 11 +++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/src/fi/alk/generation/validators.py b/src/fi/alk/generation/validators.py index 4a9de16..be1b667 100644 --- a/src/fi/alk/generation/validators.py +++ b/src/fi/alk/generation/validators.py @@ -81,11 +81,22 @@ def _validate_definition( problems.append(f"{where}:tool_call_args-without-args") for arg, value in (definition.get("args_equal") or {}).items(): allowed = arg_values.get(str(tool), {}).get(str(arg)) - if not isinstance(allowed, list) or not allowed: + if isinstance(allowed, list) and allowed: + candidates = {str(item).lower() for item in allowed} | {"null", "none"} + if value is not None and str(value).lower() not in candidates: + problems.append(f"{where}:arg-value-not-allowed:{arg}={value}") continue - candidates = {str(item).lower() for item in allowed} | {"null", "none"} - if value is not None and str(value).lower() not in candidates: - problems.append(f"{where}:arg-value-not-allowed:{arg}={value}") + # No listed valid values for this argument: a pinned string must still come from + # the contract's own vocabulary. A value found nowhere in the contract is either + # invented or runtime-generated; neither can be pinned in advance. + if ( + isinstance(value, str) + and len(value) >= 3 + and not value.replace(".", "").replace("-", "").isdigit() + and value.lower() not in ("null", "none") + and value.lower() not in legit_vocabulary + ): + problems.append(f"{where}:pinned-value-not-in-contract:{arg}={value}") elif kind == "state": if not definition.get("must") and not definition.get("forbidden"): problems.append(f"{where}:state-without-must-or-forbidden") @@ -250,6 +261,12 @@ def repair_hint(problems: list[str]) -> str: "- Every checkpoint is a judge; make the tool-argument and end-state checks " "deterministic per the vocabulary." ) + elif ":pinned-value-not-in-contract:" in problem: + lines.append( + f"- A checkpoint pins an argument to a value found nowhere in the contract " + f"({problem.split(':')[-1]}). If the value is real, copy it from the contract's " + "data; if it only exists at run time, move the argument to args_present." + ) elif ":arg-value-not-allowed:" in problem: lines.append( f"- A checkpoint pins an argument to a value the contract does not list as valid " diff --git a/tests/test_generation_pipeline.py b/tests/test_generation_pipeline.py index 5ae5b20..61471c2 100644 --- a/tests/test_generation_pipeline.py +++ b/tests/test_generation_pipeline.py @@ -390,3 +390,14 @@ def test_validator_rejects_disallowed_arg_value(): bad["sub_goals"][0]["checkpoint"]["definition"]["args_equal"]["size"] = "XL" problems = validate_scenario(bad, contract) assert any("arg-value-not-allowed:size=XL" in p for p in problems) + + +def test_validator_rejects_pinned_value_absent_from_contract(): + contract = AgentContract.model_validate(CONTRACT) + bad = json.loads(json.dumps(SCENARIO)) + bad["sub_goals"][0]["checkpoint"]["definition"] = { + "tool": "list_order", + "args_equal": {"order_ref": "def2x"}, + } + problems = validate_scenario(bad, contract) + assert any("pinned-value-not-in-contract" in p for p in problems) From f5d8557b3f9626c8b0e247d5505af34a913427a7 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 14 Aug 2026 00:51:17 +0530 Subject: [PATCH 25/55] feat(generation): min_count semantics for repeated-call checkpoints (call_nth honored) --- src/fi/alk/generation/checks.py | 22 ++++++++++++++++++---- src/fi/alk/generation/prompts.py | 4 +++- tests/test_generation_pipeline.py | 19 +++++++++++++++++++ 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/src/fi/alk/generation/checks.py b/src/fi/alk/generation/checks.py index 32a0061..c29d750 100644 --- a/src/fi/alk/generation/checks.py +++ b/src/fi/alk/generation/checks.py @@ -53,15 +53,29 @@ def _eval_tool_call_args( tool = str(definition.get("tool", "")) args_equal = definition.get("args_equal") or {} args_present = definition.get("args_present") or [] + # min_count: how many matching calls the run must contain (quantity semantics). + # call_nth is a synonym models produce naturally; nth-call-exists == at least n matches. + raw_count = definition.get("min_count", definition.get("call_nth", 1)) + try: + required = max(1, int(raw_count)) + except (TypeError, ValueError): + required = 1 + matched = 0 for call in tool_calls: if not _call_matches(call, tool, args_equal): continue arguments = call.get("arguments") or call.get("args") or {} - missing = [arg for arg in args_present if arg not in arguments] - if missing: + if any(arg not in arguments for arg in args_present): continue - return True, f"call to {tool} matched" - return False, f"no call to {tool} carried the expected arguments" + matched += 1 + if matched >= required: + suffix = f" x{matched}" if required > 1 else "" + return True, f"call to {tool} matched{suffix}" + return False, ( + f"only {matched} of {required} required matching calls to {tool}" + if required > 1 + else f"no call to {tool} carried the expected arguments" + )" def _eval_state( diff --git a/src/fi/alk/generation/prompts.py b/src/fi/alk/generation/prompts.py index 46a4684..2c4fa44 100644 --- a/src/fi/alk/generation/prompts.py +++ b/src/fi/alk/generation/prompts.py @@ -81,7 +81,9 @@ value>"]}. args_equal holds each argument whose correct value the user's request determines; an argument left out of args_equal is a requirement the test does not protect. A value that only comes into existence during the run (a generated id, a session handle) cannot be known in advance - and belongs in args_present, never in args_equal. + and belongs in args_present, never in args_equal. When the same call must happen several times + (a quantity of identical items), one checkpoint with "min_count": asserts it; separate + identical checkpoints do not. - state (deterministic): passes when the world's final state carries the expected values. definition: {"must": {"": }, "forbidden": {"": }}, evaluated against the seeded environment state after the run. diff --git a/tests/test_generation_pipeline.py b/tests/test_generation_pipeline.py index 61471c2..6b4d668 100644 --- a/tests/test_generation_pipeline.py +++ b/tests/test_generation_pipeline.py @@ -401,3 +401,22 @@ def test_validator_rejects_pinned_value_absent_from_contract(): } problems = validate_scenario(bad, contract) assert any("pinned-value-not-in-contract" in p for p in problems) + + +def test_tool_call_args_min_count_requires_multiple_calls(): + from fi.alk.generation.checks import evaluate_checkpoint + + definition = { + "tool": "add_item", + "args_equal": {"item_id": "latte"}, + "min_count": 2, + } + one = [{"name": "add_item", "arguments": {"item_id": "latte"}}] + passed, reason = evaluate_checkpoint("tool_call_args", definition, tool_calls=one) + assert passed is False and "1 of 2" in reason + passed, _ = evaluate_checkpoint("tool_call_args", definition, tool_calls=one * 2) + assert passed is True + # call_nth (the shape models produce unprompted) behaves as min_count + legacy = {"tool": "add_item", "args_equal": {"item_id": "latte"}, "call_nth": 2} + passed, _ = evaluate_checkpoint("tool_call_args", legacy, tool_calls=one) + assert passed is False From b9afe3ef781be6dcf74f27f655a60982d16f5f4d Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 14 Aug 2026 00:51:40 +0530 Subject: [PATCH 26/55] fix(generation): stray quote from patch script --- src/fi/alk/generation/checks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fi/alk/generation/checks.py b/src/fi/alk/generation/checks.py index c29d750..4d67433 100644 --- a/src/fi/alk/generation/checks.py +++ b/src/fi/alk/generation/checks.py @@ -75,7 +75,7 @@ def _eval_tool_call_args( f"only {matched} of {required} required matching calls to {tool}" if required > 1 else f"no call to {tool} carried the expected arguments" - )" + ) def _eval_state( From a616c7ea43ab6f8eeaddeb5f424e0b8fefa26dab Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 14 Aug 2026 00:57:20 +0530 Subject: [PATCH 27/55] fix(generation): reject-mining fixes - targeted hints, plan-review field merge, 2-subgoal floor, rubric specificity --- src/fi/alk/generation/pipeline.py | 24 ++++++++++++++++------- src/fi/alk/generation/prompts.py | 9 ++++++--- src/fi/alk/generation/validators.py | 23 +++++++++++++++++----- tests/test_generation_pipeline.py | 30 +++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 15 deletions(-) diff --git a/src/fi/alk/generation/pipeline.py b/src/fi/alk/generation/pipeline.py index 2e1920b..70f7dcd 100644 --- a/src/fi/alk/generation/pipeline.py +++ b/src/fi/alk/generation/pipeline.py @@ -228,15 +228,25 @@ def review_plan( logger.warning("plan review failed open: %s", exc) return rows reviewed = raw.get("rows", raw) if isinstance(raw, dict) else raw - survivors = [ - row - for row in (reviewed if isinstance(reviewed, list) else []) - if isinstance(row, dict) and row.get("situation") and row.get("target_failure") - ] + originals = {str(row.get("id")): row for row in rows} + survivors: list[dict] = [] + for row in reviewed if isinstance(reviewed, list) else []: + if ( + not isinstance(row, dict) + or not row.get("situation") + or not row.get("target_failure") + ): + continue + slug = _slugify(row.get("id") or row.get("situation", "")) + # The reviewer may override fields but never erase them: merge over the original plan. + merged = { + **originals.get(slug, {}), + **{k: v for k, v in row.items() if v not in (None, "")}, + } + merged["id"] = slug + survivors.append(merged) if not survivors or len(survivors) > len(rows): return rows - for row in survivors: - row["id"] = _slugify(row.get("id") or row.get("situation", "")) logger.info("plan review", extra={"in": len(rows), "kept": len(survivors)}) return survivors diff --git a/src/fi/alk/generation/prompts.py b/src/fi/alk/generation/prompts.py index 2c4fa44..bf18760 100644 --- a/src/fi/alk/generation/prompts.py +++ b/src/fi/alk/generation/prompts.py @@ -24,7 +24,7 @@ are two scenarios because the correct outcome differs. Never write two scenarios that are the same situation reworded. - SUB-GOAL: a milestone inside one scenario that must be true for the scenario to end correctly, - 3 to 6 per scenario. A sub-goal is an outcome a product owner would recognise and care about; + 2 to 6 per scenario, as many as the scenario genuinely needs and no more. A sub-goal is an outcome a product owner would recognise and care about; internal implementation steps and conversational pleasantries are not sub-goals. - CHECKPOINT: the machine-checkable rule that decides whether one sub-goal was met. A checkpoint witnesses the VALUES the user's request determined, because a check that only confirms an action @@ -352,8 +352,9 @@ def materialize_prompt( {json.dumps(base_environment)[:1600]} SHARED SUB-GOAL CATALOG (when a milestone in your scenario matches an entry, use the entry's exact -name and fill its definition_template with this scenario's concrete values; invent a new sub-goal -name only when no entry fits): +name and fill its definition_template with this scenario's concrete values; a template is filled +only when every generality in it has been replaced by this scenario's specifics, so a judge rubric +names the one thing this scenario checks; invent a new sub-goal name only when no entry fits): {catalog_block} SCENARIO PLAN to expand into a full test: {json.dumps(row)} @@ -419,6 +420,8 @@ def materialize_prompt( 4. CHECKABLE. Every deterministic checkpoint is computable from the seeded environment plus the expected calls; expected values match what the input implies (an input asking for a large drink must not be checked as medium); conversational checkpoints do not depend on question order. + When more than half the checkpoints are judges, demand deterministic replacements for every one + the vocabulary can express deterministically before accepting. 5. SEPARATION. The input reveals nothing the user would not know: no seeded availability, no internal ids, no expected outcome, no checkpoint contents. 6. RUNNABLE. The simulated user can finish the conversation from agent_input plus facts alone, and diff --git a/src/fi/alk/generation/validators.py b/src/fi/alk/generation/validators.py index be1b667..030c958 100644 --- a/src/fi/alk/generation/validators.py +++ b/src/fi/alk/generation/validators.py @@ -158,8 +158,8 @@ def validate_scenario(scenario: dict, contract: AgentContract) -> list[str]: problems.append(f"fact[{index}]:bad-disclosure") sub_goals = scenario.get("sub_goals") - if not isinstance(sub_goals, list) or len(sub_goals) < 3: - problems.append("sub_goals<3") + if not isinstance(sub_goals, list) or len(sub_goals) < 2: + problems.append("sub_goals<2") else: seen_names: set[str] = set() deterministic_count = 0 @@ -229,10 +229,10 @@ def repair_hint(problems: list[str]) -> str: ) elif problem == "description-too-short": lines.append("- Write a proper 2-3 sentence description, not a stub.") - elif problem == "sub_goals<3": + elif problem == "sub_goals<2": lines.append( - "- Provide at least 3 branch-specific sub_goals, each with a concrete checkpoint, " - "ending with a final verification of the resulting state." + "- Provide at least 2 sub_goals: the decisive check on the correct end result, plus " + "the behavior that leads there. Do not pad with filler; do not stop at one." ) elif ":unknown-id:" in problem: lines.append( @@ -261,6 +261,19 @@ def repair_hint(problems: list[str]) -> str: "- Every checkpoint is a judge; make the tool-argument and end-state checks " "deterministic per the vocabulary." ) + elif ":no-definition" in problem: + lines.append( + "- A checkpoint has prose but no definition object. Every checkpoint carries the " + "machine-readable definition for its kind exactly as the vocabulary specifies; the " + "detail sentence never replaces it." + ) + elif ":tool_call_args-without-args" in problem: + lines.append( + "- A tool_call_args checkpoint lists no arguments. Pin every argument the user's " + "request determines in args_equal; when every argument of the tool only exists at " + "run time, list those argument names in args_present instead, and never leave both " + "empty." + ) elif ":pinned-value-not-in-contract:" in problem: lines.append( f"- A checkpoint pins an argument to a value found nowhere in the contract " diff --git a/tests/test_generation_pipeline.py b/tests/test_generation_pipeline.py index 6b4d668..23e1426 100644 --- a/tests/test_generation_pipeline.py +++ b/tests/test_generation_pipeline.py @@ -420,3 +420,33 @@ def test_tool_call_args_min_count_requires_multiple_calls(): legacy = {"tool": "add_item", "args_equal": {"item_id": "latte"}, "call_nth": 2} passed, _ = evaluate_checkpoint("tool_call_args", legacy, tool_calls=one) assert passed is False + + +def test_two_subgoal_refusal_scenario_is_valid(): + contract = AgentContract.model_validate(CONTRACT) + lean = json.loads(json.dumps(SCENARIO)) + lean["sub_goals"] = [ + { + "name": "no_item_ordered", + "milestone": "nothing is added", + "checkpoint": { + "kind": "absent", + "detail": "no add_item call", + "deterministic": True, + "definition": {"no_tool_call": "add_item"}, + }, + }, + { + "name": "unavailability_declared", + "milestone": "the caller is told", + "checkpoint": { + "kind": "judge", + "detail": "agent states the item is unavailable", + "deterministic": False, + "definition": { + "rubric": "Did the agent state the requested item is unavailable?" + }, + }, + }, + ] + assert validate_scenario(lean, contract) == [] From 2018db4e433cf0ab1eb4e4ebd1cf3771f65e7ce9 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 14 Aug 2026 01:25:16 +0530 Subject: [PATCH 28/55] feat(generation): exact-N contract - stop at target, progress-based replenishment, explicit exhaustion verdict --- src/fi/alk/generation/pipeline.py | 32 ++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/src/fi/alk/generation/pipeline.py b/src/fi/alk/generation/pipeline.py index 70f7dcd..e6944d0 100644 --- a/src/fi/alk/generation/pipeline.py +++ b/src/fi/alk/generation/pipeline.py @@ -404,8 +404,16 @@ def _flush() -> None: lock = threading.Lock() def _materialize_one(row: dict) -> None: + with lock: + if len(records) >= config.n: + return # target reached; spend nothing further on this batch record, reason = materialize_row(contract, row, catalog, llm, config) with lock: + if record is not None and len(records) >= config.n: + rejected.append( + {**row, "_reject_reason": "surplus: target already reached"} + ) + return if record is not None: records.append(record) else: @@ -472,11 +480,17 @@ def _plan_node(node: dict) -> list[dict]: _materialize_batch(vetted) # Replenishment: coverage review names gaps and near-duplicates, then plans the - # shortfall suite-wide until the target count or the round cap is reached. - for suite_round in range(config.max_suite_rounds): + # shortfall suite-wide. Termination is by PROGRESS, not a fixed round count: the loop + # continues while rounds still produce accepted scenarios and ends the first time a + # round yields none, which is the empirical signal that the agent's genuinely distinct + # scenario space is exhausted below the requested target. + suite_round = 0 + while suite_round < max(config.max_suite_rounds, 8): + suite_round += 1 want = config.n - len(records) - if want <= 0 and suite_round > 0: + if want <= 0 and suite_round > 1: break + accepted_before = len(records) gaps, duplicate_ids, feedback = suite_review( contract, records, llm, guidance=config.guidance ) @@ -504,6 +518,18 @@ def _plan_node(node: dict) -> list[dict]: if config.critic_enabled: rows = review_plan(contract, rows, llm) _materialize_batch(rows) + if len(records) == accepted_before: + logger.warning( + "scenario space exhausted at %d of %d requested; a round produced no accepts", + len(records), + config.n, + ) + print( + f"[generation] EXHAUSTED: {len(records)} of {config.n} requested; " + "the last replenishment round produced no new accepted scenario", + flush=True, + ) + break except Exception: _flush() raise From 590ef158ee1079cea25d8ec266700188ca0bb078 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 14 Aug 2026 01:26:05 +0530 Subject: [PATCH 29/55] test(generation): exact-N delivery proven; surplus plans accounted not silently dropped --- src/fi/alk/generation/pipeline.py | 6 +++- tests/test_generation_pipeline.py | 47 +++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/fi/alk/generation/pipeline.py b/src/fi/alk/generation/pipeline.py index e6944d0..74ea374 100644 --- a/src/fi/alk/generation/pipeline.py +++ b/src/fi/alk/generation/pipeline.py @@ -406,7 +406,11 @@ def _flush() -> None: def _materialize_one(row: dict) -> None: with lock: if len(records) >= config.n: - return # target reached; spend nothing further on this batch + # Target reached: spend nothing, but account for the skipped plan. + rejected.append( + {**row, "_reject_reason": "surplus: target already reached"} + ) + return record, reason = materialize_row(contract, row, catalog, llm, config) with lock: if record is not None and len(records) >= config.n: diff --git a/tests/test_generation_pipeline.py b/tests/test_generation_pipeline.py index 23e1426..4c78ea5 100644 --- a/tests/test_generation_pipeline.py +++ b/tests/test_generation_pipeline.py @@ -450,3 +450,50 @@ def test_two_subgoal_refusal_scenario_is_valid(): }, ] assert validate_scenario(lean, contract) == [] + + +def test_exactly_n_scenarios_never_more(agent_repo, tmp_path): + """Two viable plans, n=1: exactly one accepted, the surplus accounted, no extra spend.""" + second_row = dict( + ROWS["rows"][0], id="latte-large", situation="The caller wants one large latte" + ) + two_rows = {"rows": [ROWS["rows"][0], second_row]} + llm = FakeLLMClient( + responses=[ + { + "tool_calls": [ + { + "id": "c1", + "name": "submit_contract", + "arguments": {"contract": CONTRACT}, + } + ] + }, + CATALOG, + { + "nodes": [ + { + "use_case": "Order a single item", + "description": "d", + "count": 1, + "angles": ["single item"], + } + ] + }, + two_rows, + two_rows, # plan review echoes both survivors + SCENARIO, + VERDICT, + {"gaps": [], "near_duplicates": []}, + ] + ) + config = GenerationConfig(n=1, out_dir=str(tmp_path / "out")) + config.max_workers = 1 + result = generate(RepoFolderSource(path=agent_repo), llm, config) + assert len(result.records) == 1 + assert any( + str(r.get("_reject_reason", "")).startswith("surplus") for r in result.rejected + ) + assert ( + not llm.responses + ) # every queued response consumed, none needed beyond the plan From 5ddf4a2ffa9b71cd7041a191ee63acd9ccbdaefc Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 14 Aug 2026 01:27:07 +0530 Subject: [PATCH 30/55] test(generation): exact-N assertion matches the structural guarantee --- tests/test_generation_pipeline.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_generation_pipeline.py b/tests/test_generation_pipeline.py index 4c78ea5..1f68634 100644 --- a/tests/test_generation_pipeline.py +++ b/tests/test_generation_pipeline.py @@ -453,7 +453,8 @@ def test_two_subgoal_refusal_scenario_is_valid(): def test_exactly_n_scenarios_never_more(agent_repo, tmp_path): - """Two viable plans, n=1: exactly one accepted, the surplus accounted, no extra spend.""" + """Two viable candidates, n=1: planning renormalizes to the target, so exactly one + scenario is planned, materialized, and delivered — never more.""" second_row = dict( ROWS["rows"][0], id="latte-large", situation="The caller wants one large latte" ) @@ -491,9 +492,6 @@ def test_exactly_n_scenarios_never_more(agent_repo, tmp_path): config.max_workers = 1 result = generate(RepoFolderSource(path=agent_repo), llm, config) assert len(result.records) == 1 - assert any( - str(r.get("_reject_reason", "")).startswith("surplus") for r in result.rejected - ) assert ( not llm.responses ) # every queued response consumed, none needed beyond the plan From 83677a045b70e9382e3132b695455af1acc6ef37 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 14 Aug 2026 01:27:51 +0530 Subject: [PATCH 31/55] feat(generation): stop planning a node the first round it yields nothing new --- src/fi/alk/generation/pipeline.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/fi/alk/generation/pipeline.py b/src/fi/alk/generation/pipeline.py index 74ea374..09fcbbe 100644 --- a/src/fi/alk/generation/pipeline.py +++ b/src/fi/alk/generation/pipeline.py @@ -171,6 +171,7 @@ def derive_rows( remaining = want - len(rows) if remaining <= 0: break + added_before = len(rows) raw = llm.complete_json( prompts.SCENARIO_MODEL, prompts.derive_rows_prompt( @@ -204,6 +205,10 @@ def derive_rows( seen.add(key) row["id"] = _slugify(row.get("id") or row.get("situation", "")) rows.append(row) + if len(rows) == added_before: + # A whole round survived neither exact nor near-dup filtering: this + # planning space is dry. Stop paying for rewordings of it. + break return rows[:want] From 278e9cc78f65de2a972e0deffa4e36004a5f1393 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 14 Aug 2026 01:41:23 +0530 Subject: [PATCH 32/55] feat(generation): contributor-stance rotation per planning round; dryness needs two empty stances --- src/fi/alk/generation/pipeline.py | 14 +++++++++++--- src/fi/alk/generation/prompts.py | 25 +++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/src/fi/alk/generation/pipeline.py b/src/fi/alk/generation/pipeline.py index 09fcbbe..1c7937f 100644 --- a/src/fi/alk/generation/pipeline.py +++ b/src/fi/alk/generation/pipeline.py @@ -167,11 +167,13 @@ def derive_rows( for r in existing } rounds = max(config.max_row_rounds, -(-want // 25) + 1) + consecutive_empty = 0 for round_index in range(rounds): remaining = want - len(rows) if remaining <= 0: break added_before = len(rows) + # rounds differ by contributor stance, so one empty round does not prove dryness raw = llm.complete_json( prompts.SCENARIO_MODEL, prompts.derive_rows_prompt( @@ -187,6 +189,9 @@ def derive_rows( first_round=round_index == 0 and not existing, guidance=config.guidance, node=node, + stance=prompts.CONTRIBUTOR_STANCES[ + round_index % len(prompts.CONTRIBUTOR_STANCES) + ], ), temperature=0.4, max_tokens=20_000, @@ -206,9 +211,12 @@ def derive_rows( row["id"] = _slugify(row.get("id") or row.get("situation", "")) rows.append(row) if len(rows) == added_before: - # A whole round survived neither exact nor near-dup filtering: this - # planning space is dry. Stop paying for rewordings of it. - break + consecutive_empty += 1 + if consecutive_empty >= 2: + # Two stances in a row produced nothing new: the space is dry. + break + else: + consecutive_empty = 0 return rows[:want] diff --git a/src/fi/alk/generation/prompts.py b/src/fi/alk/generation/prompts.py index bf18760..b3ca42e 100644 --- a/src/fi/alk/generation/prompts.py +++ b/src/fi/alk/generation/prompts.py @@ -181,6 +181,18 @@ def coverage_plan_prompt(brief: str, *, total: int, guidance: str = "") -> str: "count": , "angles": ["", ...]}}]}}""" +# Contributor stances: benchmark suites get their diversity from many independent contributors +# with different priors. Each planning round adopts a different contributor, so successive rounds +# over one node search the space from genuinely different angles. +CONTRIBUTOR_STANCES = ( + "the engineer who built this agent, testing what they know is fragile in their own code", + "an adversarial tester hunting the requests that sit right on the agent's rules and limits", + "a first-time user who does not know the agent's vocabulary and asks in their own words", + "an operations owner recreating the kinds of incidents real production traffic produces", + "a product manager testing the promises made about this agent, one promise at a time", +) + + def derive_rows_prompt( brief: str, *, @@ -192,6 +204,7 @@ def derive_rows_prompt( first_round: bool, guidance: str = "", node: dict | None = None, + stance: str = "", ) -> str: must = "" if first_round and signature_cases: @@ -230,7 +243,13 @@ def derive_rows_prompt( ) return f"""{brief} {node_block} -Task: plan {want} distinct test scenarios for this agent. +Task: plan {want} distinct test scenarios for this agent.{ + f''' +Adopt this contributor's viewpoint while planning: you are {stance}. Plan the scenarios THAT person +would insist on, in their voice of concern; every other rule below still applies.''' + if stance + else "" + } How to author each scenario. Work through these steps in order, in your head, before writing its plan line: @@ -278,7 +297,9 @@ def derive_rows_prompt( - Include the core successful paths too, at real complexity (several items, specific requirements), not toy versions. - No scenarios about internal machinery (logging, config, retries): users never bring those. -{dedupe}{feedback_block}{guidance_block(guidance)}Return JSON: {{"rows": [{{"id": "...", "use_case": "...", "situation": "...", +{dedupe}{feedback_block}{ + guidance_block(guidance) + }Return JSON: {{"rows": [{{"id": "...", "use_case": "...", "situation": "...", "target_failure": "...", "why_it_matters": "...", "unique_end_state": "...", "goal": "..."}}]}}""" From 14937ddf3738a93d020aaa34dfdfed33bf318672 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 14 Aug 2026 01:46:47 +0530 Subject: [PATCH 33/55] feat(generation): production-trace grounding - mined scenario plans with provenance, traces first then coverage --- src/fi/alk/generation/cli.py | 6 ++ src/fi/alk/generation/pipeline.py | 45 ++++++++++-- src/fi/alk/generation/traces.py | 117 ++++++++++++++++++++++++++++++ tests/test_generation_pipeline.py | 37 ++++++++++ 4 files changed, 197 insertions(+), 8 deletions(-) create mode 100644 src/fi/alk/generation/traces.py diff --git a/src/fi/alk/generation/cli.py b/src/fi/alk/generation/cli.py index b48d8e7..2d7626e 100644 --- a/src/fi/alk/generation/cli.py +++ b/src/fi/alk/generation/cli.py @@ -45,6 +45,11 @@ def build_parser() -> argparse.ArgumentParser: default=8, help="parallel scenario workers (1 = sequential)", ) + parser.add_argument( + "--traces", + default="", + help="file or folder of production transcripts; scenarios recreating them are generated first", + ) parser.add_argument( "--contract", default="", @@ -78,6 +83,7 @@ def main(argv: list[str] | None = None) -> int: guidance=guidance, max_workers=args.workers, contract_path=args.contract, + traces_path=args.traces, out_dir=args.out, ) diff --git a/src/fi/alk/generation/pipeline.py b/src/fi/alk/generation/pipeline.py index 1c7937f..bdb980d 100644 --- a/src/fi/alk/generation/pipeline.py +++ b/src/fi/alk/generation/pipeline.py @@ -31,6 +31,7 @@ from .explorer import explore_contract from .llm import LLMClient from .sources import AgentSource +from .traces import load_traces, mine_traces from .validators import repair_hint, validate_scenario logger = logging.getLogger(__name__) @@ -51,6 +52,7 @@ class GenerationConfig: guidance: str = "" max_workers: int = 8 contract_path: str = "" + traces_path: str = "" out_dir: str = "artifacts/generated-scenarios" @@ -115,14 +117,19 @@ def derive_catalog(contract: AgentContract, llm: LLMClient) -> list[dict]: def derive_coverage_plan( - contract: AgentContract, llm: LLMClient, config: GenerationConfig + contract: AgentContract, + llm: LLMClient, + config: GenerationConfig, + *, + total: int | None = None, ) -> list[dict]: """Partition the target count across use-case nodes. The plan is O(use cases), never O(n), so planning context stays bounded at any scenario count.""" + target = config.n if total is None else total raw = llm.complete_json( prompts.COVERAGE_PLAN_SYSTEM, prompts.coverage_plan_prompt( - contract.brief(), total=config.n, guidance=config.guidance + contract.brief(), total=target, guidance=config.guidance ), temperature=0.3, max_tokens=16_000, @@ -135,12 +142,12 @@ def derive_coverage_plan( count = node.get("count") node["count"] = max(1, int(count)) if isinstance(count, (int, float)) else 1 plan.append(node) - total = sum(node["count"] for node in plan) - if plan and total != config.n: # renormalise counts to the requested total - scaled = [max(1, round(node["count"] * config.n / total)) for node in plan] - while sum(scaled) > config.n: + planned = sum(node["count"] for node in plan) + if plan and planned != target: # renormalise counts to the requested total + scaled = [max(1, round(node["count"] * target / planned)) for node in plan] + while sum(scaled) > target and max(scaled) > 1: scaled[scaled.index(max(scaled))] -= 1 - while sum(scaled) < config.n: + while sum(scaled) < target: scaled[scaled.index(min(scaled))] += 1 for node, count in zip(plan, scaled): node["count"] = count @@ -462,10 +469,32 @@ def _node_rows(node: dict) -> list[dict]: ] try: + # Production traces first: a scenario that recreates a real interaction outranks an + # invented one, so mined plans take their share of N before coverage planning fills + # the remainder. Mined plans pass the same gates as everything else. + if config.traces_path: + raw_traces = load_traces(config.traces_path) + if raw_traces: + mined = mine_traces(contract, raw_traces, llm, guidance=config.guidance) + for row in mined: + row["id"] = _slugify(row.get("id") or row.get("situation", "")) + if config.critic_enabled: + mined = review_plan(contract, mined, llm) + logger.info( + "trace mining", + extra={"traces": len(raw_traces), "plans": len(mined)}, + ) + _materialize_batch(mined[: config.n]) + # Coverage-tree planning: partition n across use-case nodes, then plan each node # separately. Planning context is bounded by the node, never by the whole suite; # cross-node overlap is prevented structurally and by the deterministic dedup filter. - plan = derive_coverage_plan(contract, llm, config) + remaining_target = config.n - len(records) + plan = ( + derive_coverage_plan(contract, llm, config, total=remaining_target) + if remaining_target > 0 + else [] + ) logger.info("coverage plan", extra={"nodes": len(plan)}) def _plan_node(node: dict) -> list[dict]: diff --git a/src/fi/alk/generation/traces.py b/src/fi/alk/generation/traces.py new file mode 100644 index 0000000..805389d --- /dev/null +++ b/src/fi/alk/generation/traces.py @@ -0,0 +1,117 @@ +"""Production-trace grounding: turn real interactions into test scenarios. + +The strongest grounding a test can have is that it already happened. Given transcripts of real +calls (or chat logs) alongside the agent's contract, mining distills each interaction into a +scenario plan in the standard schema, with provenance pinned to the source trace. Mined plans then +pass the same gates as invented ones: reality supplies the situation, the contract still supplies +every id and value, and the validators still refuse anything ungrounded. +""" + +from __future__ import annotations + +import json +import os +from typing import Any + +from .contract import AgentContract +from .llm import LLMClient +from .prompts import SCENARIO_MODEL, guidance_block + +_TRACE_EXTENSIONS = (".json", ".jsonl", ".txt", ".md", ".csv") +_MAX_TRACE_CHARS = 7000 +_MAX_TRACES_PER_CALL = 4 + + +def load_traces(path: str) -> list[dict[str, str]]: + """Load raw traces from a file or folder: [{"ref": , "text": }].""" + paths: list[str] = [] + if os.path.isfile(path): + paths = [path] + elif os.path.isdir(path): + for name in sorted(os.listdir(path)): + if name.endswith(_TRACE_EXTENSIONS) and not name.startswith("."): + paths.append(os.path.join(path, name)) + traces: list[dict[str, str]] = [] + for file_path in paths: + try: + with open(file_path, encoding="utf-8", errors="ignore") as fh: + text = fh.read(_MAX_TRACE_CHARS) + except OSError: + continue + if text.strip(): + traces.append({"ref": os.path.basename(file_path), "text": text}) + return traces + + +def _mining_prompt(brief: str, batch: list[dict[str, str]], guidance: str) -> str: + blob = "\n\n".join(f"--- TRACE {t['ref']} ---\n{t['text']}" for t in batch) + return f"""{brief} + +Below are transcripts of REAL interactions this agent (or its production predecessor) had with real +users. Turn each into a test scenario plan that RECREATES the interaction, so the current agent can +be tested against situations that verifiably occur in production. + +{blob} + +For each trace, extract: +- what the user actually wanted, and which facts they stated up front versus only when asked; +- the condition of the world the interaction reveals (what existed, what was unavailable); +- how it ended, and whether the agent handled it correctly. + +Then write one plan per trace, in this exact schema: +- id: a short slug +- trace_ref: the trace name it recreates, exactly as given above +- use_case: the user-facing job, in the user's words +- situation: ONE line naming the condition this interaction fixes, from the user or world side +- target_failure: if the traced agent failed, the failure it committed; if it succeeded, the + regression that would break this real interaction +- why_it_matters: this happened with a real user; say what was or would be lost +- unique_end_state: the single correct final state for this interaction +- goal: one line, the end-objective from the user's side + +Rules: +- Ground every reference in the contract above: where the trace mentions an item or value, map it to + the contract's real id; where it mentions something outside the contract, the plan tests how the + agent handles exactly that request, never an invented interface. +- One plan per trace. Traces showing the same situation with the same correct outcome produce ONE + plan citing both refs. +{guidance_block(guidance)}Return JSON: {{"rows": [{{"id": "...", "trace_ref": "...", "use_case": "...", +"situation": "...", "target_failure": "...", "why_it_matters": "...", "unique_end_state": "...", +"goal": "..."}}]}}""" + + +def mine_traces( + contract: AgentContract, + traces: list[dict[str, str]], + llm: LLMClient, + *, + guidance: str = "", +) -> list[dict[str, Any]]: + """Distill raw traces into scenario plans carrying trace provenance.""" + brief = contract.brief() + plans: list[dict[str, Any]] = [] + for start in range(0, len(traces), _MAX_TRACES_PER_CALL): + batch = traces[start : start + _MAX_TRACES_PER_CALL] + raw = llm.complete_json( + SCENARIO_MODEL, + _mining_prompt(brief, batch, guidance), + temperature=0.2, + max_tokens=16_000, + ) + rows = raw.get("rows", raw) if isinstance(raw, dict) else raw + for row in rows if isinstance(rows, list) else []: + if ( + isinstance(row, dict) + and row.get("situation") + and row.get("target_failure") + ): + row["provenance"] = { + "kind": "production_trace", + "trace_ref": str(row.get("trace_ref", "")), + } + plans.append(row) + return plans + + +def _self_test() -> str: # pragma: no cover - imported for existence checks only + return json.dumps({"module": "traces"}) diff --git a/tests/test_generation_pipeline.py b/tests/test_generation_pipeline.py index 1f68634..a987b95 100644 --- a/tests/test_generation_pipeline.py +++ b/tests/test_generation_pipeline.py @@ -495,3 +495,40 @@ def test_exactly_n_scenarios_never_more(agent_repo, tmp_path): assert ( not llm.responses ) # every queued response consumed, none needed beyond the plan + + +def test_trace_mining_produces_provenance_pinned_plans(tmp_path): + from fi.alk.generation.traces import load_traces, mine_traces + + trace = tmp_path / "call_001.txt" + trace.write_text( + "USER: one medium latte please\nAGENT: sure, that is 4.5\nUSER: perfect" + ) + traces = load_traces(str(tmp_path)) + assert traces and traces[0]["ref"] == "call_001.txt" + + contract = AgentContract.model_validate(CONTRACT) + llm = FakeLLMClient( + responses=[ + { + "rows": [ + { + "id": "recreate-call-001", + "trace_ref": "call_001.txt", + "use_case": "Order a single item", + "situation": "A caller orders one medium latte and confirms the price", + "target_failure": "The agent misprices or mis-sizes the real order", + "why_it_matters": "This exact interaction happened with a real customer", + "unique_end_state": "One medium latte ordered at 4.5", + "goal": "Recreate the real call correctly", + } + ] + } + ] + ) + plans = mine_traces(contract, traces, llm) + assert len(plans) == 1 + assert plans[0]["provenance"] == { + "kind": "production_trace", + "trace_ref": "call_001.txt", + } From b226182775a9762e93d4361e214b4a6cb0a25327 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 14 Aug 2026 01:55:31 +0530 Subject: [PATCH 34/55] feat(generation): operator-requested scenarios planned first with provenance, baseline coverage fills the rest --- src/fi/alk/generation/pipeline.py | 31 ++++++++++++++++++++++++++++ src/fi/alk/generation/prompts.py | 17 ++++++++++++++++ tests/test_generation_pipeline.py | 34 +++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+) diff --git a/src/fi/alk/generation/pipeline.py b/src/fi/alk/generation/pipeline.py index bdb980d..98d71ba 100644 --- a/src/fi/alk/generation/pipeline.py +++ b/src/fi/alk/generation/pipeline.py @@ -486,6 +486,37 @@ def _node_rows(node: dict) -> list[dict]: ) _materialize_batch(mined[: config.n]) + # Operator request next: scenarios answering what the requester explicitly asked to + # test claim their share of N before baseline coverage. Same schema, same gates, + # provenance-marked so the report separates "what you asked for" from "what a full + # suite must contain anyway". + if config.guidance and len(records) < config.n: + raw = llm.complete_json( + prompts.SCENARIO_MODEL, + prompts.request_plan_prompt( + contract.brief(), + request=config.guidance, + want=config.n - len(records), + ), + temperature=0.3, + max_tokens=16_000, + ) + requested = raw.get("rows", raw) if isinstance(raw, dict) else raw + requested = [ + row + for row in (requested if isinstance(requested, list) else []) + if isinstance(row, dict) + and row.get("situation") + and row.get("target_failure") + ] + for row in requested: + row["id"] = _slugify(row.get("id") or row.get("situation", "")) + row["provenance"] = {"kind": "operator_request"} + if config.critic_enabled: + requested = review_plan(contract, requested, llm) + logger.info("request planning", extra={"plans": len(requested)}) + _materialize_batch(requested[: config.n - len(records)]) + # Coverage-tree planning: partition n across use-case nodes, then plan each node # separately. Planning context is bounded by the node, never by the whole suite; # cross-node overlap is prevented structurally and by the deterministic dedup filter. diff --git a/src/fi/alk/generation/prompts.py b/src/fi/alk/generation/prompts.py index b3ca42e..e7a748b 100644 --- a/src/fi/alk/generation/prompts.py +++ b/src/fi/alk/generation/prompts.py @@ -193,6 +193,23 @@ def coverage_plan_prompt(brief: str, *, total: int, guidance: str = "") -> str: ) +def request_plan_prompt(brief: str, *, request: str, want: int) -> str: + return f"""{brief} + +The person responsible for testing this agent has asked for scenarios in their own words: + +REQUEST: {str(request).strip()[:2000]} + +Task: plan the scenarios that test exactly what they asked for, up to {want} of them. Return only as +many as the request genuinely supports with DIFFERENT correct outcomes; when the request is narrow, +a few precise scenarios serve it better than padding. Every plan follows the standard rules: the +situation is real, the correct end state is unique, every reference exists in the contract, and each +plan names target_failure and why_it_matters (here: why it matters to what the requester is testing). + +Return JSON: {{"rows": [{{"id": "...", "use_case": "...", "situation": "...", +"target_failure": "...", "why_it_matters": "...", "unique_end_state": "...", "goal": "..."}}]}}""" + + def derive_rows_prompt( brief: str, *, diff --git a/tests/test_generation_pipeline.py b/tests/test_generation_pipeline.py index a987b95..a622b9b 100644 --- a/tests/test_generation_pipeline.py +++ b/tests/test_generation_pipeline.py @@ -532,3 +532,37 @@ def test_trace_mining_produces_provenance_pinned_plans(tmp_path): "kind": "production_trace", "trace_ref": "call_001.txt", } + + +def test_operator_request_scenarios_come_first_with_provenance(agent_repo, tmp_path): + requested_scenario = json.loads(json.dumps(SCENARIO)) + requested_scenario["provenance"] = {"kind": "operator_request"} + llm = FakeLLMClient( + responses=[ + { + "tool_calls": [ + { + "id": "c1", + "name": "submit_contract", + "arguments": {"contract": CONTRACT}, + } + ] + }, + CATALOG, + ROWS, # the dedicated request-planning reply + ROWS, # blueprint review echoes the survivor + requested_scenario, + VERDICT, + {"gaps": [], "near_duplicates": []}, + ] + ) + config = GenerationConfig( + n=1, + guidance="test single-item ordering accuracy", + out_dir=str(tmp_path / "out"), + ) + config.max_workers = 1 + result = generate(RepoFolderSource(path=agent_repo), llm, config) + assert len(result.records) == 1 + assert result.records[0]["provenance"]["kind"] == "operator_request" + assert not llm.responses # request filled N; coverage planning never ran From 2846fa1dffe516fbb425e72a1f574091b7e4a365 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 14 Aug 2026 02:03:51 +0530 Subject: [PATCH 35/55] feat(generation): trace head+tail windowing and dedup sampling; explorer completeness challenge restored --- src/fi/alk/generation/explorer.py | 14 ++++++++++++++ src/fi/alk/generation/traces.py | 29 ++++++++++++++++++++++++++--- tests/test_generation_pipeline.py | 27 +++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 3 deletions(-) diff --git a/src/fi/alk/generation/explorer.py b/src/fi/alk/generation/explorer.py index 9e5ccfe..df5a4ed 100644 --- a/src/fi/alk/generation/explorer.py +++ b/src/fi/alk/generation/explorer.py @@ -237,6 +237,7 @@ def explore_contract( }, ] submitted: AgentContract | None = None + challenged = False for turn in range(max_turns): forced = turn == max_turns - 1 if forced: @@ -268,6 +269,19 @@ def explore_contract( arguments = call.get("arguments") or {} if name == "submit_contract": result, submitted = _try_submit(arguments) + if submitted is not None and not challenged and turn < max_turns - 1: + # Completeness challenge: a structurally valid contract can still be + # missing tools, and everything downstream inherits that hole. The first + # valid submission is challenged once; only the resubmission is accepted. + challenged = True + submitted = None + result = ( + "Before this is accepted, verify completeness: search the repository " + "for tool or function registrations you have not listed (decorators, " + "registries, dispatch tables, configuration files that add tools). " + "Also verify the data section holds the real entries, not examples. " + "Then submit_contract again, extended or unchanged if truly complete." + ) else: result = _run_tool(tools, name, arguments) messages.append( diff --git a/src/fi/alk/generation/traces.py b/src/fi/alk/generation/traces.py index 805389d..4ac3028 100644 --- a/src/fi/alk/generation/traces.py +++ b/src/fi/alk/generation/traces.py @@ -20,6 +20,7 @@ _TRACE_EXTENSIONS = (".json", ".jsonl", ".txt", ".md", ".csv") _MAX_TRACE_CHARS = 7000 _MAX_TRACES_PER_CALL = 4 +_MAX_TRACES_MINED = 40 def load_traces(path: str) -> list[dict[str, str]]: @@ -35,11 +36,17 @@ def load_traces(path: str) -> list[dict[str, str]]: for file_path in paths: try: with open(file_path, encoding="utf-8", errors="ignore") as fh: - text = fh.read(_MAX_TRACE_CHARS) + text = fh.read() except OSError: continue - if text.strip(): - traces.append({"ref": os.path.basename(file_path), "text": text}) + if not text.strip(): + continue + if len(text) > _MAX_TRACE_CHARS: + # Keep the head (intent) AND the tail (resolution): truncating only the end + # of a long call would drop the part that says how it actually ended. + half = _MAX_TRACE_CHARS // 2 + text = text[:half] + "\n... [middle omitted] ...\n" + text[-half:] + traces.append({"ref": os.path.basename(file_path), "text": text}) return traces @@ -89,6 +96,22 @@ def mine_traces( ) -> list[dict[str, Any]]: """Distill raw traces into scenario plans carrying trace provenance.""" brief = contract.brief() + # Large trace sets: drop near-duplicate transcripts deterministically (same token-set + # similarity used for scenario dedup), then cap what is mined. Ten thousand calls are + # mostly repeats of the same few dozen situations; representatives carry the signal. + from .dedup import similarity + + unique: list[dict[str, str]] = [] + for trace in traces: + row = {"situation": trace["text"][:1500]} + if not any( + similarity(row, {"situation": kept["text"][:1500]}) >= 0.75 + for kept in unique + ): + unique.append(trace) + if len(unique) > _MAX_TRACES_MINED: + unique = unique[:_MAX_TRACES_MINED] + traces = unique plans: list[dict[str, Any]] = [] for start in range(0, len(traces), _MAX_TRACES_PER_CALL): batch = traces[start : start + _MAX_TRACES_PER_CALL] diff --git a/tests/test_generation_pipeline.py b/tests/test_generation_pipeline.py index a622b9b..80658b1 100644 --- a/tests/test_generation_pipeline.py +++ b/tests/test_generation_pipeline.py @@ -164,6 +164,15 @@ def agent_repo(tmp_path): def test_full_pipeline_offline(agent_repo, tmp_path): llm = FakeLLMClient( responses=[ + { + "tool_calls": [ + { + "id": "c1", + "name": "submit_contract", + "arguments": {"contract": CONTRACT}, + } + ] + }, { "tool_calls": [ { @@ -461,6 +470,15 @@ def test_exactly_n_scenarios_never_more(agent_repo, tmp_path): two_rows = {"rows": [ROWS["rows"][0], second_row]} llm = FakeLLMClient( responses=[ + { + "tool_calls": [ + { + "id": "c1", + "name": "submit_contract", + "arguments": {"contract": CONTRACT}, + } + ] + }, { "tool_calls": [ { @@ -548,6 +566,15 @@ def test_operator_request_scenarios_come_first_with_provenance(agent_repo, tmp_p } ] }, + { + "tool_calls": [ + { + "id": "c2", + "name": "submit_contract", + "arguments": {"contract": CONTRACT}, + } + ] + }, CATALOG, ROWS, # the dedicated request-planning reply ROWS, # blueprint review echoes the survivor From 5aa15b1091400fdc202ffac2e6b8cefffdfeacc6 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 14 Aug 2026 03:41:04 +0530 Subject: [PATCH 36/55] feat(generation): required environment enum, autonomous trace explorer, failure amplification --- src/fi/alk/generation/cli.py | 14 + src/fi/alk/generation/emit.py | 50 +++- src/fi/alk/generation/environments.py | 114 ++++++++ src/fi/alk/generation/explorer.py | 25 +- src/fi/alk/generation/pipeline.py | 127 +++++++-- src/fi/alk/generation/prompts.py | 46 ++-- src/fi/alk/generation/traces.py | 368 ++++++++++++++++++++++++-- tests/test_generation_pipeline.py | 258 ++++++++++++++++++ 8 files changed, 920 insertions(+), 82 deletions(-) create mode 100644 src/fi/alk/generation/environments.py diff --git a/src/fi/alk/generation/cli.py b/src/fi/alk/generation/cli.py index 2d7626e..4b43acf 100644 --- a/src/fi/alk/generation/cli.py +++ b/src/fi/alk/generation/cli.py @@ -7,6 +7,7 @@ import logging import sys +from . import environments from .llm import DEFAULT_MODEL, LiteLLMClient from .pipeline import GenerationConfig, generate from .sources import resolve_source @@ -23,6 +24,12 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument( "--repo", help="path to the agent's repository folder (repo source)" ) + parser.add_argument( + "--environment", + required=True, + help="runtime the scenarios are staged in: " + + ", ".join(sorted(environments.SUPPORTED)), + ) parser.add_argument("--n", type=int, default=20, help="target number of scenarios") parser.add_argument("--model", default=DEFAULT_MODEL, help="litellm model string") parser.add_argument( @@ -75,9 +82,15 @@ def main(argv: list[str] | None = None) -> int: if guidance.startswith("@"): with open(guidance[1:], encoding="utf-8") as fh: guidance = fh.read() + try: + environment = environments.resolve(args.environment) + except NotImplementedError as exc: + print(str(exc), file=sys.stderr) + return 2 source = resolve_source(args.source, **source_kwargs) llm = LiteLLMClient(model=args.model, budget_usd=args.budget_usd) config = GenerationConfig( + environment=environment, n=args.n, critic_enabled=not args.no_critic, guidance=guidance, @@ -92,6 +105,7 @@ def main(argv: list[str] | None = None) -> int: json.dumps( { "agent": result.contract.agent, + "environment": environment.key, "scenarios": len(result.records), "rejected": len(result.rejected), "out": args.out, diff --git a/src/fi/alk/generation/emit.py b/src/fi/alk/generation/emit.py index d6f5edd..6fc4434 100644 --- a/src/fi/alk/generation/emit.py +++ b/src/fi/alk/generation/emit.py @@ -25,6 +25,7 @@ ) from .contract import AgentContract +from .environments import VOICE, EnvironmentProfile _KIND_TO_GOAL_MACHINE = { "state": "world_success_condition", @@ -189,6 +190,8 @@ def write_outputs( records: list[dict], rejected: list[dict], usage: dict[str, Any], + open_questions: list[str] | None = None, + environment: EnvironmentProfile = VOICE, ) -> None: scenarios_dir = os.path.join(out_dir, "scenarios") alk_dir = os.path.join(out_dir, "alk") @@ -217,7 +220,17 @@ def _dump(path: str, payload: Any) -> None: smoke_manifest(records[0], contract), ) with open(os.path.join(out_dir, "report.md"), "w", encoding="utf-8") as fh: - fh.write(render_report(contract, catalog, records, rejected, usage)) + fh.write( + render_report( + contract, + catalog, + records, + rejected, + usage, + open_questions=open_questions or [], + environment=environment, + ) + ) def render_report( @@ -226,6 +239,8 @@ def render_report( records: list[dict], rejected: list[dict], usage: dict[str, Any], + open_questions: list[str] | None = None, + environment: EnvironmentProfile = VOICE, ) -> str: catalog_names = {str(entry.get("name")) for entry in catalog} reuse: dict[str, int] = {} @@ -243,6 +258,8 @@ def render_report( lines = [ f"# Generated scenarios: {contract.agent}", "", + f"- environment: **{environment.key}** ({environment.label}), " + f"staged by the `{environment.alk_plugin}` runtime", f"- scenarios accepted: **{len(records)}**, rejected by review: {len(rejected)}", f"- checkpoints: {total_checks}, deterministic: {deterministic} " f"({(100 * deterministic // max(total_checks, 1))}%)", @@ -270,6 +287,37 @@ def render_report( f"- **{record.get('id')}**: catches `{record.get('target_failure', '')}`. " f"Matters because: {record.get('why_it_matters', '')}" ) + origins: dict[str, int] = {} + for record in records: + kind = str((record.get("provenance") or {}).get("kind") or "baseline_coverage") + origins[kind] = origins.get(kind, 0) + 1 + # Worth printing whenever anything came from somewhere other than plain coverage planning, + # including a suite built entirely from production traces. + if origins and set(origins) != {"baseline_coverage"}: + lines += ["", "## Where these scenarios came from", ""] + for kind, count in sorted(origins.items(), key=lambda item: -item[1]): + lines.append(f"- `{kind}`: {count}") + pending = environment.pending_kinds() + if pending: + lines += [ + "", + "## Checkpoints this environment cannot grade yet", + "", + f"Scenarios express `{'`, `'.join(pending)}` checkpoints, which the " + f"{environment.label} runtime does not evaluate today. They are recorded in the " + "scenario and graded once that runtime lands; every other checkpoint kind is live.", + ] + if open_questions: + lines += [ + "", + "## Assumptions worth confirming", + "", + "Each of these was decided during planning. Answering any of them in the next run's " + "guidance changes what gets generated.", + "", + ] + for question in open_questions: + lines.append(f"- {question}") if reuse: lines += ["", "## Sub-goal roll-up (appearances across scenarios)", ""] for name, count in sorted(reuse.items(), key=lambda item: -item[1]): diff --git a/src/fi/alk/generation/environments.py b/src/fi/alk/generation/environments.py new file mode 100644 index 0000000..9c0be73 --- /dev/null +++ b/src/fi/alk/generation/environments.py @@ -0,0 +1,114 @@ +"""The environments this harness can generate runnable scenarios for. + +An agent may be reachable several ways at once: the same ordering assistant can take a phone call, +a web chat, or a browser session. Which one a suite targets is therefore a choice the operator +makes, not a property the harness can read off the source, and it is passed in explicitly. + +The set is closed on purpose. A scenario is only worth generating if the runtime can actually stage +it and grade it, so an environment appears here once `fi.simulate` carries a plugin for it. Asking +for anything else raises rather than falling back on a generic shape, because a generic shape yields +scenarios that look correct in a report and cannot be run. +""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class EnvironmentProfile: + """Everything that changes about generation when the target environment changes.""" + + key: str + label: str + alk_plugin: str # the fi.simulate environment plugin a scenario binds to + conversational: bool # a simulated user drives the interaction + input_spec: str # what the scenario's agent_input field must contain + witnessable: tuple[str, ...] # checkpoint kinds this environment can express + gradable_today: tuple[str, ...] # of those, the kinds the runtime already grades + mock_surface: str # how a tool call is intercepted here + compatible_modalities: tuple[str, ...] # advisory cross-check against the contract + + def pending_kinds(self) -> tuple[str, ...]: + return tuple(k for k in self.witnessable if k not in self.gradable_today) + + +VOICE = EnvironmentProfile( + key="voice", + label="voice call", + alk_plugin="voice", + conversational=True, + input_spec=( + "a situation instruction for the simulated caller, written in second person as the " + "caller's own lived circumstance: who they are, what is happening, and what they want. It " + "describes their experience and goal, never instructions about what to say, and never the " + "other side's turns. Facts the agent is expected to ask for live in `facts`, not here. No " + "accent or voice notes" + ), + witnessable=("tool_call_args", "state", "conveyed", "absent", "judge"), + # A live call is graded from the provider's post-call evidence and the transcript. World state + # is expressed by the scenario and asserted once a tool-mocking environment is attached to the + # voice leg, which is the runtime lane's work. + gradable_today=("tool_call_args", "conveyed", "absent", "judge"), + mock_surface=( + "tool calls are answered by the scenario's mock_responses, and the values the agent passed " + "are recovered from the call's recorded tool events" + ), + compatible_modalities=("voice",), +) + +CHAT = EnvironmentProfile( + key="chat", + label="text chat", + alk_plugin="chat", + conversational=True, + input_spec=( + "a situation instruction for the simulated user, second person, their lived circumstance: " + "their goal and what they already know. Facts the agent is meant to elicit are listed " + "separately in `facts`, not here" + ), + witnessable=("tool_call_args", "state", "conveyed", "absent", "judge"), + gradable_today=("tool_call_args", "state", "conveyed", "absent", "judge"), + mock_surface=( + "every tool call is served by the scenario's mock_responses and its state_updates are " + "applied to the world, so both the arguments and the resulting state are observable" + ), + compatible_modalities=("chat", "data_sql", "research", "other"), +) + +SUPPORTED: dict[str, EnvironmentProfile] = {VOICE.key: VOICE, CHAT.key: CHAT} + +# Environments the model may report from the source but that no runtime can stage yet. Named +# separately so the error explains the gap instead of only listing what works. +_NOT_YET_BUILT = { + "browser": "no browser environment plugin exists yet; a scenario would have nothing to drive", + "computer_use": "no desktop environment plugin exists yet", + "code": "no repository or container environment plugin exists yet", +} + + +def resolve(key: str) -> EnvironmentProfile: + """Return the profile for an environment key, or explain why it cannot be generated for.""" + normalized = str(key or "").strip().lower() + if normalized in SUPPORTED: + return SUPPORTED[normalized] + supported = ", ".join(sorted(SUPPORTED)) + if normalized in _NOT_YET_BUILT: + raise NotImplementedError( + f"environment {normalized!r} is not supported: {_NOT_YET_BUILT[normalized]}. " + f"Supported today: {supported}." + ) + raise NotImplementedError( + f"unknown environment {normalized!r}. Supported today: {supported}." + ) + + +def modality_mismatch(profile: EnvironmentProfile, modality: str) -> str: + """Advisory only: the operator's choice always wins, but a mismatch is worth saying out loud.""" + found = str(modality or "").strip().lower() + if not found or found in profile.compatible_modalities: + return "" + return ( + f"the agent's source reads as a {found!r} agent, but scenarios are being generated for the " + f"{profile.key!r} environment; the generated input shape will follow {profile.key!r}" + ) diff --git a/src/fi/alk/generation/explorer.py b/src/fi/alk/generation/explorer.py index df5a4ed..70046bd 100644 --- a/src/fi/alk/generation/explorer.py +++ b/src/fi/alk/generation/explorer.py @@ -73,7 +73,7 @@ If submit_contract returns validation problems, fix them and submit again.""" -_TOOLS: list[dict[str, Any]] = [ +READ_TOOLS: list[dict[str, Any]] = [ { "type": "function", "function": { @@ -122,6 +122,9 @@ }, }, }, +] + +_TOOLS: list[dict[str, Any]] = READ_TOOLS + [ { "type": "function", "function": { @@ -153,20 +156,20 @@ } -class _RepoTools: - """Path-sandboxed read tools over one repository root.""" +class ReadOnlyTree: + """Path-sandboxed read tools over one directory root, shared by every explorer loop.""" def __init__(self, root: str) -> None: self.root = os.path.abspath(root) - def _resolve(self, path: str) -> str: + def resolve(self, path: str) -> str: resolved = os.path.abspath(os.path.join(self.root, str(path or "").lstrip("/"))) if resolved != self.root and not resolved.startswith(self.root + os.sep): raise ValueError(f"path escapes the repository root: {path}") return resolved def list_dir(self, path: str = "") -> str: - target = self._resolve(path) + target = self.resolve(path) if not os.path.isdir(target): return f"not a directory: {path}" entries = [] @@ -181,7 +184,7 @@ def list_dir(self, path: str = "") -> str: return "\n".join(entries) or "(empty)" def read_file(self, path: str, offset: int = 0) -> str: - target = self._resolve(path) + target = self.resolve(path) if not os.path.isfile(target): return f"not a file: {path}" try: @@ -225,7 +228,7 @@ def explore_contract( root: str, llm: LLMClient, *, max_turns: int = _MAX_TURNS ) -> AgentContract: """Run the exploration loop until a valid contract is submitted.""" - tools = _RepoTools(root) + tools = ReadOnlyTree(root) messages: list[dict[str, Any]] = [ {"role": "system", "content": _SYSTEM}, { @@ -263,7 +266,7 @@ def explore_contract( } ) continue - messages.append(_assistant_message(reply)) + messages.append(assistant_message(reply)) for call in calls: name = call.get("name") arguments = call.get("arguments") or {} @@ -283,7 +286,7 @@ def explore_contract( "Then submit_contract again, extended or unchanged if truly complete." ) else: - result = _run_tool(tools, name, arguments) + result = run_read_tool(tools, name, arguments) messages.append( { "role": "tool", @@ -299,7 +302,7 @@ def explore_contract( ) -def _assistant_message(reply: dict[str, Any]) -> dict[str, Any]: +def assistant_message(reply: dict[str, Any]) -> dict[str, Any]: raw = reply.get("raw") if raw is not None: try: @@ -323,7 +326,7 @@ def _assistant_message(reply: dict[str, Any]) -> dict[str, Any]: } -def _run_tool(tools: _RepoTools, name: str, arguments: dict[str, Any]) -> str: +def run_read_tool(tools: ReadOnlyTree, name: str, arguments: dict[str, Any]) -> str: try: if name == "list_dir": return tools.list_dir(arguments.get("path", "")) diff --git a/src/fi/alk/generation/pipeline.py b/src/fi/alk/generation/pipeline.py index 98d71ba..3de9848 100644 --- a/src/fi/alk/generation/pipeline.py +++ b/src/fi/alk/generation/pipeline.py @@ -17,21 +17,23 @@ from __future__ import annotations import logging +import os import re import threading from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from typing import Any -from . import prompts +from . import environments, prompts from .contract import AgentContract, extract_contract from .dedup import near_duplicate from .emit import write_outputs +from .environments import EnvironmentProfile from .oracle import oracle_hint, oracle_problems from .explorer import explore_contract from .llm import LLMClient from .sources import AgentSource -from .traces import load_traces, mine_traces +from .traces import amplify_plans, explore_traces, load_traces, mine_traces from .validators import repair_hint, validate_scenario logger = logging.getLogger(__name__) @@ -43,6 +45,10 @@ @dataclass class GenerationConfig: + # Which runtime the suite is staged in. Chosen by the operator, never inferred: one agent can be + # reachable by several, and only the chosen one determines the input shape and the gradable + # checkpoint kinds. + environment: EnvironmentProfile = environments.VOICE n: int = 20 max_row_rounds: int = 4 max_repairs: int = 3 @@ -62,6 +68,7 @@ class GenerationResult: catalog: list[dict] = field(default_factory=list) records: list[dict] = field(default_factory=list) rejected: list[dict] = field(default_factory=list) + open_questions: list[str] = field(default_factory=list) usage: dict[str, Any] = field(default_factory=dict) @@ -122,9 +129,13 @@ def derive_coverage_plan( config: GenerationConfig, *, total: int | None = None, -) -> list[dict]: - """Partition the target count across use-case nodes. The plan is O(use cases), never O(n), - so planning context stays bounded at any scenario count.""" +) -> tuple[list[dict], list[str]]: + """Partition the target count across use-case nodes, and surface what had to be assumed. + + The plan is O(use cases), never O(n), so planning context stays bounded at any scenario count. + The questions come back with it: this stage makes the largest assumptions in the run, and the + operator can answer them in the next run's guidance instead of discovering them in the output. + """ target = config.n if total is None else total raw = llm.complete_json( prompts.COVERAGE_PLAN_SYSTEM, @@ -135,6 +146,11 @@ def derive_coverage_plan( max_tokens=16_000, ) nodes = raw.get("nodes", raw) if isinstance(raw, dict) else raw + questions = [ + str(q) + for q in (raw.get("open_questions") or [] if isinstance(raw, dict) else []) + if str(q).strip() + ] plan: list[dict] = [] for node in nodes if isinstance(nodes, list) else []: if not isinstance(node, dict) or not node.get("use_case"): @@ -151,7 +167,7 @@ def derive_coverage_plan( scaled[scaled.index(min(scaled))] += 1 for node, count in zip(plan, scaled): node["count"] = count - return plan + return plan, questions def derive_rows( @@ -291,8 +307,7 @@ def materialize_row( row=row, base_environment=contract.base_environment, catalog=catalog, - modality=contract.modality, - conversational=contract.conversational, + environment=config.environment, hint=hint, guidance=config.guidance, ), @@ -311,6 +326,7 @@ def materialize_row( "goal", "target_failure", "unique_end_state", + "provenance", ): record.setdefault(key, row.get(key)) record["id"] = _slugify(record.get("id") or row.get("id", "")) @@ -405,10 +421,17 @@ def generate( logger.info( "contract ready", extra={"agent": contract.agent, "tools": len(contract.tools)} ) + # The operator's environment choice is authoritative; a disagreement with what the source looks + # like is worth saying out loud, because the scenarios will follow the choice either way. + mismatch = environments.modality_mismatch(config.environment, contract.modality) + if mismatch: + logger.warning("environment mismatch: %s", mismatch) + print(f"[generation] NOTE: {mismatch}", flush=True) catalog = derive_catalog(contract, llm) records: list[dict] = [] rejected: list[dict] = [] + open_questions: list[str] = [] feedback = "" def _flush() -> None: @@ -418,6 +441,8 @@ def _flush() -> None: catalog=catalog, records=records, rejected=rejected, + open_questions=open_questions, + environment=config.environment, usage=llm.usage.as_dict(), ) @@ -473,16 +498,36 @@ def _node_rows(node: dict) -> list[dict]: # invented one, so mined plans take their share of N before coverage planning fills # the remainder. Mined plans pass the same gates as everything else. if config.traces_path: - raw_traces = load_traces(config.traces_path) + # A folder of recordings has an unknown shape, so the model navigates it and chooses + # what is worth mining, failing interactions first. A single file needs no exploring. + raw_traces: list[dict] = [] + if os.path.isdir(config.traces_path): + raw_traces = explore_traces(config.traces_path, llm) + if not raw_traces: + raw_traces = load_traces(config.traces_path) if raw_traces: mined = mine_traces(contract, raw_traces, llm, guidance=config.guidance) + # Amplification: a recreation of a real failure protects that one interaction. + # Its neighbours fence the class the failure belongs to, so the suite behaves like + # a regression suite rather than a single pinned data point. + headroom = config.n - len(mined) + neighbours = ( + amplify_plans(contract, mined, llm, limit=headroom) + if headroom > 0 + else [] + ) + mined = mined + neighbours for row in mined: row["id"] = _slugify(row.get("id") or row.get("situation", "")) if config.critic_enabled: mined = review_plan(contract, mined, llm) logger.info( "trace mining", - extra={"traces": len(raw_traces), "plans": len(mined)}, + extra={ + "traces": len(raw_traces), + "plans": len(mined), + "amplified": len(neighbours), + }, ) _materialize_batch(mined[: config.n]) @@ -521,13 +566,27 @@ def _node_rows(node: dict) -> list[dict]: # separately. Planning context is bounded by the node, never by the whole suite; # cross-node overlap is prevented structurally and by the deterministic dedup filter. remaining_target = config.n - len(records) - plan = ( + plan, questions = ( derive_coverage_plan(contract, llm, config, total=remaining_target) if remaining_target > 0 - else [] + else ([], []) ) + open_questions.extend(questions) logger.info("coverage plan", extra={"nodes": len(plan)}) + # Nodes planned in parallel cannot see each other, so overlap is caught here instead. + claimed: list[dict] = [] + + def _claim(rows: list[dict]) -> list[dict]: + with lock: + kept = [] + for row in rows: + if near_duplicate(row, claimed): + continue + claimed.append(row) + kept.append(row) + return kept + def _plan_node(node: dict) -> list[dict]: rows = derive_rows( contract, @@ -539,22 +598,37 @@ def _plan_node(node: dict) -> list[dict]: ) if config.critic_enabled: rows = review_plan(contract, rows, llm) - return rows - - if config.max_workers > 1 and len(plan) > 1: - with ThreadPoolExecutor( + return _claim(rows) + + # Each node flows plan -> review -> materialize on its own, with no barrier between the + # stages: a node whose planning finished early has its scenarios being written while a + # slower node is still planning. Wall-clock becomes the slowest single node rather than + # the sum of the slowest stage in each. + if plan and config.max_workers > 1: + planners = ThreadPoolExecutor( max_workers=min(config.max_workers, len(plan)) - ) as pool: - node_rows = list(pool.map(_plan_node, plan)) + ) + writers = ThreadPoolExecutor(max_workers=config.max_workers) + try: + + def _node_flow(node: dict) -> list: + # Submits and returns; never waits, so a planner thread cannot be blocked + # behind the writer pool it is feeding. + return [ + writers.submit(_materialize_one, row) + for row in _plan_node(node) + ] + + node_futures = [planners.submit(_node_flow, node) for node in plan] + for node_future in node_futures: + for write_future in node_future.result(): + write_future.result() + finally: + planners.shutdown(wait=True) + writers.shutdown(wait=True) else: - node_rows = [_plan_node(node) for node in plan] - # Cross-node near-dup guard after parallel planning (nodes could not see each other). - vetted: list[dict] = [] - for rows in node_rows: - for row in rows: - if not near_duplicate(row, vetted): - vetted.append(row) - _materialize_batch(vetted) + for node in plan: + _materialize_batch(_plan_node(node)) # Replenishment: coverage review names gaps and near-duplicates, then plans the # shortfall suite-wide. Termination is by PROGRESS, not a fixed round count: the loop @@ -616,6 +690,7 @@ def _plan_node(node: dict) -> list[dict]: catalog=catalog, records=records, rejected=rejected, + open_questions=open_questions, usage=llm.usage.as_dict(), ) _flush() diff --git a/src/fi/alk/generation/prompts.py b/src/fi/alk/generation/prompts.py index e7a748b..9185bde 100644 --- a/src/fi/alk/generation/prompts.py +++ b/src/fi/alk/generation/prompts.py @@ -9,6 +9,8 @@ import json +from .environments import EnvironmentProfile + # The scenario model, written as definitions a fresh model can act on. SCENARIO_MODEL = """You help test an AI agent by designing test scenarios. Definitions used throughout: @@ -51,28 +53,6 @@ the contract does not belong in a test. - User personality, accent, or language is NOT varied unless the scenario is specifically about it.""" -AGENT_INPUT_BY_MODALITY = { - "voice": ( - "a situation instruction for the simulated caller, written in second person as the caller's " - "own lived circumstance: who they are, what is happening, and what they want. It describes " - "their experience and goal, never instructions about what to say, and never the other " - "side's turns. Facts the agent is expected to ask for live in `facts`, not here. No accent " - "or voice notes" - ), - "chat": ( - "a situation instruction for the simulated user, second person, lived circumstance: their " - "goal and what they know. Facts the agent should elicit are listed separately in `facts`" - ), - "data_sql": "the plain-English question only: no SQL, no table or column names, no answer", - "code": ( - "the command the agent is invoked with (a real command from the contract) or the issue text " - "handed to it: never the fix, the patch, or the expected review" - ), - "browser": "the natural-language task plus only the starting URL: no selectors, no answer", - "research": "the research question or brief only: no expected findings", - "_default": "exactly what the agent receives at the start, in natural form: never the answer", -} - CHECKPOINT_VOCABULARY = """CHECKPOINT kinds, strongest first. Choose the strongest kind the sub-goal allows; `judge` exists only for sub-goals no state, call, or data value can witness. - tool_call_args (deterministic): passes when the agent called the named tool and every argument @@ -177,8 +157,15 @@ def coverage_plan_prompt(brief: str, *, total: int, guidance: str = "") -> str: that can. - Angles within a node must each produce a DIFFERENT correct outcome, not the same outcome under different wording. +Where the source left something genuinely ambiguous, decide it yourself and keep going, then record +what you decided as a question the person requesting these tests could answer. A question earns its +place only when a different answer would have produced a materially different set of tests: which of +two plausible readings of a rule is the real one, whether an area of the agent is in scope for +testing at all, which of several user populations the suite should assume. Do not record questions +whose answer is already in the contract. {guidance_block(guidance)}Return JSON: {{"nodes": [{{"use_case": "...", "description": "", -"count": , "angles": ["", ...]}}]}}""" +"count": , "angles": ["", ...]}}], "open_questions": ["", ...]}}""" # Contributor stances: benchmark suites get their diversity from many independent contributors @@ -355,16 +342,14 @@ def materialize_prompt( row: dict, base_environment: dict, catalog: list[dict], - modality: str, - conversational: bool, + environment: EnvironmentProfile, hint: str = "", guidance: str = "", ) -> str: - input_spec = AGENT_INPUT_BY_MODALITY.get( - modality, AGENT_INPUT_BY_MODALITY["_default"] - ) + input_spec = environment.input_spec + witnessable = ", ".join(environment.witnessable) conv = "" - if conversational: + if environment.conversational: conv = """- This agent is conversational: `agent_input` is the situation instruction handed to the simulated user, and `facts` lists what that user knows. Every fact the agent is supposed to ask for gets disclosure "on_request"; the simulated user volunteers only "volunteer" facts. @@ -399,6 +384,9 @@ def materialize_prompt( {CHECKPOINT_VOCABULARY} +This test will be staged as a {environment.label}, where {environment.mock_surface}. Only these +checkpoint kinds can be observed there, so every sub-goal must use one of them: {witnessable}. + Write the complete test. Every value must be a real value from the contract's data. Keep the three parts separate: the input never reveals the environment seeding, the checkpoints, or the outcome. diff --git a/src/fi/alk/generation/traces.py b/src/fi/alk/generation/traces.py index 4ac3028..97a05da 100644 --- a/src/fi/alk/generation/traces.py +++ b/src/fi/alk/generation/traces.py @@ -5,26 +5,46 @@ scenario plan in the standard schema, with provenance pinned to the source trace. Mined plans then pass the same gates as invented ones: reality supplies the situation, the contract still supplies every id and value, and the validators still refuse anything ungrounded. + +Two loops sit in front of that. An exploration loop reads a folder whose layout is unknown, works +out how a trace is stored there, and decides which traces are worth mining: where there are more +than a suite can hold, the ones where the interaction went wrong earn their place first. An +amplification loop then takes each failure and asks for the neighbouring situations that share it, +so a suite pins the exact interaction that broke and fences the class it belongs to. """ from __future__ import annotations import json +import logging import os from typing import Any from .contract import AgentContract +from .explorer import READ_TOOLS, ReadOnlyTree, assistant_message, run_read_tool from .llm import LLMClient from .prompts import SCENARIO_MODEL, guidance_block -_TRACE_EXTENSIONS = (".json", ".jsonl", ".txt", ".md", ".csv") +logger = logging.getLogger(__name__) + +_TRACE_EXTENSIONS = (".json", ".jsonl", ".txt", ".md", ".csv", ".log", ".yaml", ".yml") _MAX_TRACE_CHARS = 7000 _MAX_TRACES_PER_CALL = 4 _MAX_TRACES_MINED = 40 +_MAX_EXPLORE_TURNS = 14 +_MAX_RESULT_CHARS = 14_000 + + +def _window(text: str) -> str: + """Keep the head (intent) AND the tail (resolution) of a long interaction.""" + if len(text) <= _MAX_TRACE_CHARS: + return text + half = _MAX_TRACE_CHARS // 2 + return text[:half] + "\n... [middle omitted] ...\n" + text[-half:] def load_traces(path: str) -> list[dict[str, str]]: - """Load raw traces from a file or folder: [{"ref": , "text": }].""" + """Load raw traces from a file or a flat folder, without asking a model anything.""" paths: list[str] = [] if os.path.isfile(path): paths = [path] @@ -41,17 +61,263 @@ def load_traces(path: str) -> list[dict[str, str]]: continue if not text.strip(): continue - if len(text) > _MAX_TRACE_CHARS: - # Keep the head (intent) AND the tail (resolution): truncating only the end - # of a long call would drop the part that says how it actually ended. - half = _MAX_TRACE_CHARS // 2 - text = text[:half] + "\n... [middle omitted] ...\n" + text[-half:] - traces.append({"ref": os.path.basename(file_path), "text": text}) + traces.append({"ref": os.path.basename(file_path), "text": _window(text)}) return traces +# -------------------------------------------------------------------------------------- +# Exploration: an unknown folder of recorded interactions +# -------------------------------------------------------------------------------------- + +_EXPLORER_SYSTEM = """You are a test engineer who has been handed a folder of recorded interactions +between real users and a deployed AI agent. Nobody has told you how the folder is organised or what +format the recordings are in. Your job is to work that out for yourself and then choose which +recordings deserve to become regression tests. + +You have tools to list directories, read files, and search for text. Start by opening enough of the +folder to understand its layout and how a single recording is stored: one file per interaction, many +interactions inside one file, or a folder per interaction. Read whole examples, not fragments, until +you can say what a recording looks like and roughly how many there are. + +Then judge the recordings. A recording is worth turning into a test when the interaction went wrong: +the user did not get what they came for, the agent did something the user had to correct, an error +or failure appears in the exchange, the user repeated themselves or gave up, or the outcome +contradicts what the agent was asked to do. Interactions that simply went well are worth far less, +because a test built from them only confirms what already works. + +How many to select depends on what you find. When the folder holds only a handful of recordings, +take all of them. When it holds more than a suite could reasonably contain, select the ones that +went wrong first, and add successful ones only to cover a common path that no failing recording +touches. Never select more than 25. + +You judge each recording from its own content. There may be a status or outcome field you can trust; +there may not be, in which case you read the exchange and decide. Say which you did. + +When you have chosen, call submit_selection exactly once. If it reports a problem, fix it and submit +again.""" + +_SELECT_TOOL: dict[str, Any] = { + "type": "function", + "function": { + "name": "submit_selection", + "description": "Submit the recordings chosen for regression testing. Call once, after exploring.", + "parameters": { + "type": "object", + "properties": { + "format_notes": { + "type": "string", + "description": "How a recording is stored here and how you judged its outcome.", + }, + "total_seen": { + "type": "integer", + "description": "How many recordings the folder appears to hold in total.", + }, + "selected": { + "type": "array", + "description": "The chosen recordings, most valuable first.", + "items": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Path to the file holding this recording, relative to the folder root.", + }, + "outcome": { + "type": "string", + "enum": ["failed", "succeeded"], + "description": "Whether the interaction went wrong for the user.", + }, + "why": { + "type": "string", + "description": "One line: what went wrong, or why this path is worth keeping.", + }, + }, + "required": ["path", "outcome", "why"], + }, + }, + }, + "required": ["selected", "total_seen", "format_notes"], + }, + }, +} + + +def _folder_overview(root: str) -> str: + """Deterministic census of the folder, so the model starts from facts instead of guesses.""" + by_extension: dict[str, int] = {} + samples: list[str] = [] + total = 0 + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = [d for d in dirnames if not d.startswith(".")] + for filename in sorted(filenames): + if filename.startswith("."): + continue + total += 1 + extension = os.path.splitext(filename)[1] or "(none)" + by_extension[extension] = by_extension.get(extension, 0) + 1 + if len(samples) < 12: + samples.append(os.path.relpath(os.path.join(dirpath, filename), root)) + census = ", ".join( + f"{count} x {extension}" for extension, count in sorted(by_extension.items()) + ) + return ( + f"The folder holds {total} files in total ({census}).\n" + f"A sample of paths:\n" + "\n".join(f"- {s}" for s in samples) + ) + + +def explore_traces( + root: str, llm: LLMClient, *, max_turns: int = _MAX_EXPLORE_TURNS +) -> list[dict[str, str]]: + """Let the model navigate an unknown trace folder and choose what to mine. + + Returns the selected recordings with their text loaded, failing ones first. An exploration that + never submits returns nothing rather than guessing: the caller falls back to a flat load. + """ + tools = ReadOnlyTree(root) + messages: list[dict[str, Any]] = [ + {"role": "system", "content": _EXPLORER_SYSTEM}, + { + "role": "user", + "content": ( + "The recordings folder is mounted for your tools.\n\n" + + _folder_overview(root) + + "\n\nRoot listing:\n" + + tools.list_dir("") + ), + }, + ] + for turn in range(max_turns): + if turn == max_turns - 1: + messages.append( + { + "role": "user", + "content": "Turn budget exhausted. Call submit_selection NOW with your best " + "current selection.", + } + ) + reply = llm.complete_turn( + messages, + tools=READ_TOOLS + [_SELECT_TOOL], + temperature=0.15, + max_tokens=12_000, + ) + calls = reply.get("tool_calls") or [] + if not calls: + messages.append( + {"role": "assistant", "content": reply.get("content") or ""} + ) + messages.append( + { + "role": "user", + "content": "Use the tools. Explore the folder, then call submit_selection.", + } + ) + continue + messages.append(assistant_message(reply)) + for call in calls: + name = call.get("name") + arguments = call.get("arguments") or {} + if name == "submit_selection": + selected, problem = _load_selection(tools, root, arguments) + if selected: + logger.info( + "trace selection", + extra={ + "selected": len(selected), + "total_seen": arguments.get("total_seen"), + }, + ) + return selected + result = problem + else: + result = run_read_tool(tools, name, arguments) + messages.append( + { + "role": "tool", + "tool_call_id": call.get("id") or name, + "content": str(result)[:_MAX_RESULT_CHARS], + } + ) + logger.warning("trace exploration ended without a selection") + return [] + + +def _load_selection( + tools: ReadOnlyTree, root: str, arguments: dict[str, Any] +) -> tuple[list[dict[str, str]], str]: + """Read the chosen files inside the sandbox; report unreadable choices back to the model.""" + raw = arguments.get("selected") + if isinstance(raw, str): + try: + raw = json.loads(raw) + except json.JSONDecodeError: + raw = None + if not isinstance(raw, list) or not raw: + return ( + [], + "submit_selection needs a non-empty 'selected' list; fix and submit again", + ) + loaded: list[dict[str, str]] = [] + missing: list[str] = [] + for entry in raw[:25]: + if not isinstance(entry, dict) or not entry.get("path"): + continue + path = str(entry["path"]) + try: + target = tools.resolve(path) + except ValueError: + missing.append(path) + continue + if not os.path.isfile(target): + missing.append(path) + continue + try: + with open(target, encoding="utf-8", errors="ignore") as fh: + text = fh.read() + except OSError: + missing.append(path) + continue + if not text.strip(): + missing.append(path) + continue + loaded.append( + { + "ref": os.path.relpath(target, os.path.abspath(root)), + "text": _window(text), + "outcome": str(entry.get("outcome", "")).strip().lower(), + "why": str(entry.get("why", "")), + } + ) + if not loaded: + return [], ( + f"none of the selected paths could be read: {missing[:10]}. " + "Use paths relative to the folder root, then submit again" + ) + if missing: + logger.warning("trace selection skipped unreadable paths: %s", missing[:10]) + # Failing interactions carry more test value, so they head the queue for the share of N. + loaded.sort(key=lambda t: 0 if t.get("outcome") == "failed" else 1) + return loaded, "" + + +# -------------------------------------------------------------------------------------- +# Mining: recordings into scenario plans +# -------------------------------------------------------------------------------------- + + def _mining_prompt(brief: str, batch: list[dict[str, str]], guidance: str) -> str: - blob = "\n\n".join(f"--- TRACE {t['ref']} ---\n{t['text']}" for t in batch) + parts = [] + for trace in batch: + header = f"--- TRACE {trace['ref']} ---" + if trace.get("outcome"): + header += ( + f"\n[this interaction was judged to have {trace['outcome']}" + + (f": {trace['why']}" if trace.get("why") else "") + + "]" + ) + parts.append(f"{header}\n{trace['text']}") + blob = "\n\n".join(parts) return f"""{brief} Below are transcripts of REAL interactions this agent (or its production predecessor) had with real @@ -112,6 +378,7 @@ def mine_traces( if len(unique) > _MAX_TRACES_MINED: unique = unique[:_MAX_TRACES_MINED] traces = unique + outcome_by_ref = {t["ref"]: t.get("outcome", "") for t in traces} plans: list[dict[str, Any]] = [] for start in range(0, len(traces), _MAX_TRACES_PER_CALL): batch = traces[start : start + _MAX_TRACES_PER_CALL] @@ -128,13 +395,84 @@ def mine_traces( and row.get("situation") and row.get("target_failure") ): - row["provenance"] = { - "kind": "production_trace", - "trace_ref": str(row.get("trace_ref", "")), - } + ref = str(row.get("trace_ref", "")) + row["provenance"] = {"kind": "production_trace", "trace_ref": ref} + # A recreation of an interaction that went wrong earns a neighbourhood around it. + row["amplify"] = outcome_by_ref.get(ref, "") == "failed" plans.append(row) return plans -def _self_test() -> str: # pragma: no cover - imported for existence checks only - return json.dumps({"module": "traces"}) +# -------------------------------------------------------------------------------------- +# Amplification: fencing the class a real failure belongs to +# -------------------------------------------------------------------------------------- + + +def _amplify_prompt(brief: str, plan: dict[str, Any], want: int) -> str: + return f"""{brief} + +The scenario plan below recreates an interaction this agent had with a real user, and that +interaction went wrong. Recreating it protects against that exact interaction happening again, which +is worth doing, but it protects against nothing else: the same underlying weakness will still show +up the moment a user arrives with a slightly different version of the same situation. + +THE REAL INTERACTION: +{json.dumps(plan)[:4000]} + +Write {want} further scenario plans that surround this one. Each must be able to fail for the SAME +underlying reason as the real interaction, while differing in the circumstances that reach it: the +same rule tested against a different item in the agent's data, the same mistake made at a different +point in the interaction, the same demand arriving with the user's request phrased around a +different need, the same condition met when something else about the world has changed. + +What every plan must satisfy: +- It stands on its own as a test: a competent version of this agent could genuinely fail it. +- It reaches the weakness by a route the real interaction did not already take, so a suite holding + all of them tells you how wide the problem is rather than repeating one data point. +- Every item, value and identifier comes from the contract above. +- The correct end state is a single unambiguous outcome, not a range of acceptable ones. + +Return JSON: {{"rows": [{{"id": "", "use_case": "...", "situation": "", "target_failure": "", +"why_it_matters": "", "unique_end_state": "", "goal": ""}}]}}""" + + +def amplify_plans( + contract: AgentContract, + plans: list[dict[str, Any]], + llm: LLMClient, + *, + per_plan: int = 3, + limit: int = 0, +) -> list[dict[str, Any]]: + """Grow a neighbourhood around each failing recreation, capped by what the suite still needs.""" + brief = contract.brief() + neighbours: list[dict[str, Any]] = [] + for plan in plans: + if not plan.get("amplify"): + continue + if limit and len(neighbours) >= limit: + break + want = per_plan if not limit else max(1, min(per_plan, limit - len(neighbours))) + raw = llm.complete_json( + SCENARIO_MODEL, + _amplify_prompt(brief, plan, want), + temperature=0.45, + max_tokens=16_000, + ) + rows = raw.get("rows", raw) if isinstance(raw, dict) else raw + trace_ref = str((plan.get("provenance") or {}).get("trace_ref", "")) + for row in (rows if isinstance(rows, list) else [])[:want]: + if ( + isinstance(row, dict) + and row.get("situation") + and row.get("target_failure") + ): + row["provenance"] = { + "kind": "trace_amplified", + "trace_ref": trace_ref, + "amplifies": str(plan.get("id", "")), + } + neighbours.append(row) + return neighbours diff --git a/tests/test_generation_pipeline.py b/tests/test_generation_pipeline.py index 80658b1..0685a48 100644 --- a/tests/test_generation_pipeline.py +++ b/tests/test_generation_pipeline.py @@ -593,3 +593,261 @@ def test_operator_request_scenarios_come_first_with_provenance(agent_repo, tmp_p assert len(result.records) == 1 assert result.records[0]["provenance"]["kind"] == "operator_request" assert not llm.responses # request filled N; coverage planning never ran + + +# --------------------------------------------------------------------------- +# Environment selection +# --------------------------------------------------------------------------- + + +def test_unsupported_environment_is_refused_by_name(): + from fi.alk.generation import environments + + for key in ("browser", "computer_use", "code", "telepathy"): + with pytest.raises(NotImplementedError) as excinfo: + environments.resolve(key) + message = str(excinfo.value) + assert "voice" in message and "chat" in message + + +def test_supported_environments_resolve_case_insensitively(): + from fi.alk.generation import environments + + assert environments.resolve("VOICE").key == "voice" + assert environments.resolve(" chat ").alk_plugin == "chat" + + +def test_modality_disagreement_warns_without_overriding_the_choice(): + from fi.alk.generation import environments + + assert not environments.modality_mismatch(environments.CHAT, "data_sql") + warning = environments.modality_mismatch(environments.VOICE, "browser") + assert "browser" in warning and "voice" in warning + + +# --------------------------------------------------------------------------- +# Trace exploration and amplification +# --------------------------------------------------------------------------- + + +def _odd_trace_folder(root, shallow: int = 240): + """A folder nobody documented: nested session dirs, a flat archive, mixed formats.""" + for index in range(shallow): + day = f"2026-08-{(index % 28) + 1:02d}" + session = root / "sessions" / day / f"sess_{index:04d}" + session.mkdir(parents=True, exist_ok=True) + failed = index % 40 == 0 + (session / "transcript.json").write_text( + json.dumps( + { + "id": f"sess_{index:04d}", + "status": "error" if failed else "completed", + "turns": [ + {"role": "user", "text": "one large coffee"}, + { + "role": "agent", + "text": "sorry, I did not catch that" + if failed + else "one large coffee, that is 3.5", + }, + ], + } + ) + ) + archive = root / "archive" + archive.mkdir(parents=True, exist_ok=True) + (archive / "old_call.log").write_text( + "USER: I asked for no onions\nAGENT: added onions\nUSER: that is wrong, again, no onions" + ) + return root + + +def test_trace_explorer_reads_an_unknown_layout_and_puts_failures_first(tmp_path): + from fi.alk.generation.traces import explore_traces + + root = _odd_trace_folder(tmp_path / "traces") + llm = FakeLLMClient( + responses=[ + { + "tool_calls": [ + {"id": "t1", "name": "list_dir", "arguments": {"path": ""}} + ] + }, + { + "tool_calls": [ + { + "id": "t2", + "name": "submit_selection", + "arguments": { + "format_notes": "one json transcript per session folder", + "total_seen": 241, + "selected": [ + { + "path": "sessions/2026-08-02/sess_0001/transcript.json", + "outcome": "succeeded", + "why": "the common happy path", + }, + { + "path": "archive/old_call.log", + "outcome": "failed", + "why": "the agent ignored a stated exclusion", + }, + ], + }, + } + ] + }, + ] + ) + selected = explore_traces(str(root), llm) + assert len(selected) == 2 + # Failing interactions lead, whatever order the model submitted them in. + assert selected[0]["outcome"] == "failed" + assert selected[0]["ref"] == "archive/old_call.log" + assert "no onions" in selected[0]["text"] + + +def test_trace_explorer_refuses_paths_outside_the_folder(tmp_path): + from fi.alk.generation.traces import explore_traces + + root = _odd_trace_folder(tmp_path / "traces", shallow=2) + (tmp_path / "secret.txt").write_text("not a trace") + llm = FakeLLMClient( + responses=[ + { + "tool_calls": [ + { + "id": "t1", + "name": "submit_selection", + "arguments": { + "format_notes": "n/a", + "total_seen": 3, + "selected": [ + { + "path": "../secret.txt", + "outcome": "failed", + "why": "escaping the sandbox", + } + ], + }, + } + ] + }, + { + "tool_calls": [ + { + "id": "t2", + "name": "submit_selection", + "arguments": { + "format_notes": "flat archive", + "total_seen": 3, + "selected": [ + { + "path": "archive/old_call.log", + "outcome": "failed", + "why": "stated exclusion ignored", + } + ], + }, + } + ] + }, + ] + ) + selected = explore_traces(str(root), llm) + assert len(selected) == 1 + assert selected[0]["ref"] == "archive/old_call.log" + + +def test_failing_traces_are_marked_for_amplification_and_fenced(tmp_path): + from fi.alk.generation.traces import amplify_plans, mine_traces + + contract = AgentContract.model_validate(CONTRACT) + traces = [ + { + "ref": "archive/old_call.log", + "text": "USER: no onions\nAGENT: added onions", + "outcome": "failed", + "why": "stated exclusion ignored", + }, + { + "ref": "sessions/ok.json", + "text": "USER: one coffee\nAGENT: one coffee, 3.5", + "outcome": "succeeded", + "why": "happy path", + }, + ] + llm = FakeLLMClient( + responses=[ + { + "rows": [ + { + "id": "recreate-onion", + "trace_ref": "archive/old_call.log", + "use_case": "Order with an exclusion", + "situation": "A caller states an exclusion the agent must honour", + "target_failure": "The agent drops the stated exclusion", + "why_it_matters": "A real customer received the wrong food", + "unique_end_state": "The item is ordered without the excluded ingredient", + "goal": "Order the item as stated", + }, + { + "id": "recreate-coffee", + "trace_ref": "sessions/ok.json", + "use_case": "Order a single item", + "situation": "A caller orders one coffee", + "target_failure": "The agent misprices the order", + "why_it_matters": "This interaction happens constantly", + "unique_end_state": "One coffee ordered at 3.5", + "goal": "Order one coffee", + }, + ] + }, + { + "rows": [ + { + "id": "exclusion-different-item", + "use_case": "Order with an exclusion", + "situation": "The same exclusion is stated against a different item", + "target_failure": "The agent drops the stated exclusion", + "why_it_matters": "The same weakness reaches every item on the menu", + "unique_end_state": "The other item is ordered without the ingredient", + "goal": "Order the other item as stated", + } + ] + }, + ] + ) + plans = mine_traces(contract, traces, llm) + assert [p["amplify"] for p in plans] == [True, False] + + neighbours = amplify_plans(contract, plans, llm, per_plan=1) + assert len(neighbours) == 1 + assert neighbours[0]["provenance"] == { + "kind": "trace_amplified", + "trace_ref": "archive/old_call.log", + "amplifies": "recreate-onion", + } + + +def test_report_names_the_environment_and_the_open_questions(): + from fi.alk.generation import environments + from fi.alk.generation.emit import render_report + + contract = AgentContract.model_validate(CONTRACT) + record = json.loads(json.dumps(SCENARIO)) + record["provenance"] = {"kind": "production_trace", "trace_ref": "call_001.txt"} + report = render_report( + contract, + [], + [record], + [], + {"usd": 0.1}, + open_questions=["Assumed refunds are out of scope for this suite"], + environment=environments.VOICE, + ) + assert "environment: **voice**" in report + assert "Assumed refunds are out of scope" in report + assert "production_trace" in report + # Voice cannot grade world state today, and the report has to say so rather than imply it can. + assert "cannot grade yet" in report and "state" in report From 47259707062049bcf83b43814b56bd207e07731a Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 14 Aug 2026 03:56:45 +0530 Subject: [PATCH 37/55] fix(generation): carry why_it_matters from plan to record so scenarios stop failing their own validator --- src/fi/alk/generation/pipeline.py | 1 + tests/test_generation_pipeline.py | 34 +++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/src/fi/alk/generation/pipeline.py b/src/fi/alk/generation/pipeline.py index 3de9848..51862f0 100644 --- a/src/fi/alk/generation/pipeline.py +++ b/src/fi/alk/generation/pipeline.py @@ -326,6 +326,7 @@ def materialize_row( "goal", "target_failure", "unique_end_state", + "why_it_matters", "provenance", ): record.setdefault(key, row.get(key)) diff --git a/tests/test_generation_pipeline.py b/tests/test_generation_pipeline.py index 0685a48..5266082 100644 --- a/tests/test_generation_pipeline.py +++ b/tests/test_generation_pipeline.py @@ -851,3 +851,37 @@ def test_report_names_the_environment_and_the_open_questions(): assert "production_trace" in report # Voice cannot grade world state today, and the report has to say so rather than imply it can. assert "cannot grade yet" in report and "state" in report + + +def test_plan_fields_survive_materialization_even_when_the_model_omits_them(): + """The plan owns why_it_matters and target_failure; a record must never lose them. + + These fields are the scenario's stated reason to exist, and the validators require them. When + materialization depended on the model echoing them back, an unrelated prompt change silently + turned every scenario into a rejection. + """ + from fi.alk.generation.pipeline import GenerationConfig, materialize_row + + contract = AgentContract.model_validate(CONTRACT) + plan = { + "id": "carried-plan", + "use_case": "Order a single item", + "situation": "A caller orders one item and states a size", + "target_failure": "The agent drops the stated size", + "why_it_matters": "The customer is handed the wrong drink", + "unique_end_state": "One large coffee ordered", + "goal": "Order one large coffee", + "provenance": {"kind": "production_trace", "trace_ref": "call_001.txt"}, + } + stripped = json.loads(json.dumps(SCENARIO)) + for field in ("why_it_matters", "target_failure", "provenance"): + stripped.pop(field, None) + + llm = FakeLLMClient(responses=[stripped, VERDICT]) + record, reason = materialize_row( + contract, plan, [], llm, GenerationConfig(critic_enabled=True) + ) + assert record is not None, reason + assert record["why_it_matters"] == "The customer is handed the wrong drink" + assert record["target_failure"] == "The agent drops the stated size" + assert record["provenance"]["kind"] == "production_trace" From eea39f8c439e89b8c6f49219e2b0d62dddff9002 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 14 Aug 2026 04:25:49 +0530 Subject: [PATCH 38/55] fix(generation): pin an argument only where the user or a rule determines it, else args_present --- src/fi/alk/generation/prompts.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/fi/alk/generation/prompts.py b/src/fi/alk/generation/prompts.py index 9185bde..2783060 100644 --- a/src/fi/alk/generation/prompts.py +++ b/src/fi/alk/generation/prompts.py @@ -59,9 +59,14 @@ listed carried the expected value. definition: {"tool": "", "args_equal": {"": , ...}, "args_present": [""]}. args_equal holds each argument whose correct value the user's request determines; an - argument left out of args_equal is a requirement the test does not protect. A value that only - comes into existence during the run (a generated id, a session handle) cannot be known in advance - and belongs in args_present, never in args_equal. When the same call must happen several times + argument left out of args_equal is a requirement the test does not protect. An argument earns a + pinned value only when this scenario's user actually determines it or a rule the agent enforces + fixes it; where the user says nothing about an argument and no rule settles it, the correct value + is genuinely open, and pinning one there fails an agent that did nothing wrong, which is worse + than not testing the argument at all. Such an argument goes in args_present, so the test still + requires the agent to supply something without inventing a requirement it was never given. A value + that only comes into existence during the run (a generated id, a session handle) cannot be known + in advance and belongs in args_present, never in args_equal. When the same call must happen several times (a quantity of identical items), one checkpoint with "min_count": asserts it; separate identical checkpoints do not. - state (deterministic): passes when the world's final state carries the expected values. @@ -446,6 +451,9 @@ def materialize_prompt( 4. CHECKABLE. Every deterministic checkpoint is computable from the seeded environment plus the expected calls; expected values match what the input implies (an input asking for a large drink must not be checked as medium); conversational checkpoints do not depend on question order. + Every pinned argument value must be traceable to something this scenario's user states or a rule + the agent enforces. Where the scenario asserts a value the user never gave and no rule fixes, a + correct agent fails the test; demand it move to args_present. When more than half the checkpoints are judges, demand deterministic replacements for every one the vocabulary can express deterministically before accepting. 5. SEPARATION. The input reveals nothing the user would not know: no seeded availability, no From 33c53433b9d1f0bde78d35ca36c38af87b77e775 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 14 Aug 2026 04:46:51 +0530 Subject: [PATCH 39/55] fix(generation): an open argument moves to args_present without surrendering the deterministic checkpoint --- src/fi/alk/generation/prompts.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/fi/alk/generation/prompts.py b/src/fi/alk/generation/prompts.py index 2783060..56e3041 100644 --- a/src/fi/alk/generation/prompts.py +++ b/src/fi/alk/generation/prompts.py @@ -63,8 +63,12 @@ pinned value only when this scenario's user actually determines it or a rule the agent enforces fixes it; where the user says nothing about an argument and no rule settles it, the correct value is genuinely open, and pinning one there fails an agent that did nothing wrong, which is worse - than not testing the argument at all. Such an argument goes in args_present, so the test still - requires the agent to supply something without inventing a requirement it was never given. A value + than not testing the argument at all. Such an argument goes in args_present, inside this same + checkpoint, so the test still requires the agent to supply something without inventing a + requirement it was never given. An open argument is a reason to move that one argument into + args_present; it is never a reason to give up the tool_call_args checkpoint or to replace it with + a judge, because the call itself and every argument the user did determine remain exactly + checkable. A value that only comes into existence during the run (a generated id, a session handle) cannot be known in advance and belongs in args_present, never in args_equal. When the same call must happen several times (a quantity of identical items), one checkpoint with "min_count": asserts it; separate From 1b00664fd7f476199f2b02bca770f2d9ad67be5b Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 14 Aug 2026 05:05:17 +0530 Subject: [PATCH 40/55] fix(generation): ground identifier arguments only, and allow parameterless tools to be asserted by the call --- src/fi/alk/generation/validators.py | 32 ++++++++++-- tests/test_generation_pipeline.py | 80 +++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 5 deletions(-) diff --git a/src/fi/alk/generation/validators.py b/src/fi/alk/generation/validators.py index 030c958..17cfc34 100644 --- a/src/fi/alk/generation/validators.py +++ b/src/fi/alk/generation/validators.py @@ -40,6 +40,11 @@ def banned_tokens(contract: AgentContract) -> set[str]: return banned +def _is_identifier_shaped(value: str) -> bool: + """An id or enum token, as opposed to text a scenario composes (a query, a message body).""" + return bool(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.\-]{0,63}", value)) + + def _identifier_values(payload) -> set[str]: """Underscore-shaped string values anywhere in a definition (the id-like ones).""" values: set[str] = set() @@ -62,6 +67,7 @@ def _validate_definition( where: str, legit_vocabulary: set[str], arg_values: dict[str, dict], + argless_tools: frozenset[str] = frozenset(), ) -> list[str]: problems: list[str] = [] unknown_ids = sorted( @@ -77,7 +83,12 @@ def _validate_definition( tool = definition.get("tool") if not isinstance(tool, str) or tool not in tool_names: problems.append(f"{where}:unknown-tool:{tool}") - if not definition.get("args_equal") and not definition.get("args_present"): + # A tool that genuinely declares no parameters is asserted by the call alone. + if ( + not definition.get("args_equal") + and not definition.get("args_present") + and str(tool) not in argless_tools + ): problems.append(f"{where}:tool_call_args-without-args") for arg, value in (definition.get("args_equal") or {}).items(): allowed = arg_values.get(str(tool), {}).get(str(arg)) @@ -86,12 +97,16 @@ def _validate_definition( if value is not None and str(value).lower() not in candidates: problems.append(f"{where}:arg-value-not-allowed:{arg}={value}") continue - # No listed valid values for this argument: a pinned string must still come from - # the contract's own vocabulary. A value found nowhere in the contract is either - # invented or runtime-generated; neither can be pinned in advance. + # No listed valid values for this argument: a pinned identifier must still come from + # the contract's own vocabulary, because an id found nowhere in the contract is either + # invented or runtime-generated and neither can be pinned in advance. This applies to + # identifiers only. Arguments that carry composed text (a query the agent writes, a + # message it sends) are authored per scenario and cannot appear in a contract, so + # requiring them to would make every such agent ungeneratable. if ( isinstance(value, str) and len(value) >= 3 + and _is_identifier_shaped(value) and not value.replace(".", "").replace("-", "").isdigit() and value.lower() not in ("null", "none") and value.lower() not in legit_vocabulary @@ -130,6 +145,7 @@ def validate_scenario(scenario: dict, contract: AgentContract) -> list[str]: tool_names = contract.tool_names() legit_vocabulary = _legit_vocabulary(contract) arg_values = {tool.name: dict(tool.arg_values or {}) for tool in contract.tools} + argless_tools = frozenset(tool.name for tool in contract.tools if not tool.args) for field in ( "id", @@ -184,7 +200,13 @@ def validate_scenario(scenario: dict, contract: AgentContract) -> list[str]: problems.append(f"{where}:no-definition") else: problems += _validate_definition( - kind, definition, tool_names, where, legit_vocabulary, arg_values + kind, + definition, + tool_names, + where, + legit_vocabulary, + arg_values, + argless_tools, ) deterministic = bool(checkpoint.get("deterministic")) if deterministic and kind == "judge": diff --git a/tests/test_generation_pipeline.py b/tests/test_generation_pipeline.py index 5266082..b2d2d52 100644 --- a/tests/test_generation_pipeline.py +++ b/tests/test_generation_pipeline.py @@ -885,3 +885,83 @@ def test_plan_fields_survive_materialization_even_when_the_model_omits_them(): assert record["why_it_matters"] == "The customer is handed the wrong drink" assert record["target_failure"] == "The agent drops the stated size" assert record["provenance"]["kind"] == "production_trace" + + +def test_composed_argument_values_are_not_required_to_exist_in_the_contract(): + """A query the agent writes is authored per scenario; a contract cannot list it. + + The grounding rule exists to catch transposed identifiers. Applied to composed text it made + every query-writing agent ungeneratable: each scenario was rejected for pinning the very SQL + the test exists to check. + """ + from fi.alk.generation.validators import validate_scenario + + contract = AgentContract.model_validate( + { + **CONTRACT, + "conversational": False, + "tools": [ + { + "name": "sql_db_query", + "args": ["query"], + "arg_values": {}, + "description": "Run a SQL query", + }, + { + "name": "sql_db_list_tables", + "args": [], + "arg_values": {}, + "description": "List tables", + }, + ], + } + ) + record = json.loads(json.dumps(SCENARIO)) + record["facts"] = [] + record["sub_goals"] = [ + { + "name": "listed_the_tables", + "milestone": "The agent inspects the schema", + "checkpoint": { + "kind": "tool_call_args", + "deterministic": True, + "detail": "The agent listed the tables", + # A tool with no parameters is asserted by the call alone. + "definition": {"tool": "sql_db_list_tables"}, + }, + }, + { + "name": "ran_the_expected_query", + "milestone": "The agent runs the query", + "checkpoint": { + "kind": "tool_call_args", + "deterministic": True, + "detail": "The agent ran the expected query", + "definition": { + "tool": "sql_db_query", + "args_equal": { + "query": "SELECT BillingCountry, SUM(Total) FROM Invoice GROUP BY BillingCountry" + }, + }, + }, + }, + ] + problems = validate_scenario(record, contract) + assert not [p for p in problems if "pinned-value-not-in-contract" in p], problems + assert not [p for p in problems if "tool_call_args-without-args" in p], problems + + +def test_identifier_shaped_values_are_still_grounded(): + """The original guard must survive: a transposed id is still caught.""" + from fi.alk.generation.validators import validate_scenario + + contract = AgentContract.model_validate(CONTRACT) + record = json.loads(json.dumps(SCENARIO)) + for sub_goal in record["sub_goals"]: + definition = sub_goal["checkpoint"].get("definition") or {} + if definition.get("args_equal"): + key = sorted(definition["args_equal"])[0] + definition["args_equal"][key] = "combo_not_a_real_id" + break + problems = validate_scenario(record, contract) + assert any("combo_not_a_real_id" in p for p in problems), problems From ebc3369a487edde33f6bb5dd0357e845fed029a2 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 14 Aug 2026 05:11:49 +0530 Subject: [PATCH 41/55] docs(generation): README points at the environment profile registry --- src/fi/alk/generation/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fi/alk/generation/README.md b/src/fi/alk/generation/README.md index 78bb208..14e52ab 100644 --- a/src/fi/alk/generation/README.md +++ b/src/fi/alk/generation/README.md @@ -50,7 +50,7 @@ semantics; plain code does structure, dedup, and grounding checks; nothing hardc 4. **Checkpoints assert the right arguments.** Asked for 11 PM, a 10 PM booking must fail. A check is `deterministic: true` only when it carries an executable definition. 5. **Extensible by registry, not by edit.** New agent connections implement `AgentSource` (three - members) and register; new modalities add one entry to `AGENT_INPUT_BY_MODALITY`; the LLM is a + members) and register; a new environment adds one profile to `environments.py`; the LLM is a two-method protocol with the model string as config. ## Extending @@ -58,7 +58,7 @@ semantics; plain code does structure, dedup, and grounding checks; nothing hardc | Want | Do | |---|---| | New agent connection (Vapi, Retell, platform id) | implement `AgentSource`, `@register_source("vapi")` | -| New modality (computer-use, code, ...) | add an `AGENT_INPUT_BY_MODALITY` entry; contract `modality` is open vocabulary | +| New environment (browser, code, computer-use) | add one `EnvironmentProfile` to `environments.py` once `fi.simulate` carries a plugin for it; until then `--environment` refuses it by name | | Different model | `--model vertex_ai/gemini-2.5-pro` or any litellm string; `LLMClient` is a protocol for non-litellm backends | | Different budget | `--budget-usd 5` (hard stop, raises `BudgetExceeded`) | | Stricter or looser QA | critic threshold and retry counts are `GenerationConfig` fields | From 18c575eaf81a47181e62ce68b6464f701c446606 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 14 Aug 2026 10:44:28 +0530 Subject: [PATCH 42/55] perf(generation): stop the repair loop when a rewrite returns the identical failure --- src/fi/alk/generation/pipeline.py | 17 +++++++++++++++++ tests/test_generation_pipeline.py | 25 +++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/src/fi/alk/generation/pipeline.py b/src/fi/alk/generation/pipeline.py index 51862f0..4d788ea 100644 --- a/src/fi/alk/generation/pipeline.py +++ b/src/fi/alk/generation/pipeline.py @@ -72,6 +72,11 @@ class GenerationResult: usage: dict[str, Any] = field(default_factory=dict) +def _signature(problems: list[str]) -> str: + """Identity of a failure, so a repeat can be told from progress.""" + return "|".join(sorted(str(p) for p in problems))[:400] + + def _slugify(value: str) -> str: slug = re.sub(r"[^a-z0-9]+", "-", str(value).lower()).strip("-") return slug[:60] or "scenario" @@ -299,6 +304,7 @@ def materialize_row( hint = "" best: dict | None = None reason = "" + last_signature = "" for _attempt in range(1 + config.max_repairs): raw = llm.complete_json( prompts.SCENARIO_MODEL, @@ -335,11 +341,22 @@ def materialize_row( problems = validate_scenario(record, contract) if problems: best, reason = record, f"validator: {problems[:6]}" + # A rewrite that returns the identical complaint has not understood the instruction, + # and further attempts almost never recover it. Stopping here is most of the cost of + # a rejection: the repair chain is serial, so it is wall-clock as well as spend. + if _signature(problems) == last_signature: + reason = f"unrecoverable, same problem twice: {problems[:6]}" + break + last_signature = _signature(problems) hint = repair_hint(problems) continue inconsistencies = oracle_problems(record) if inconsistencies: best, reason = record, f"oracle: {inconsistencies[:4]}" + if _signature(inconsistencies) == last_signature: + reason = f"unrecoverable, same problem twice: {inconsistencies[:4]}" + break + last_signature = _signature(inconsistencies) hint = oracle_hint(inconsistencies) continue if not config.critic_enabled: diff --git a/tests/test_generation_pipeline.py b/tests/test_generation_pipeline.py index b2d2d52..0938795 100644 --- a/tests/test_generation_pipeline.py +++ b/tests/test_generation_pipeline.py @@ -965,3 +965,28 @@ def test_identifier_shaped_values_are_still_grounded(): break problems = validate_scenario(record, contract) assert any("combo_not_a_real_id" in p for p in problems), problems + + +def test_a_repeated_identical_failure_stops_the_repair_loop(): + """Rewrites that return the same complaint never recover, and the chain is serial. + + Four attempts on a scenario that fails identically each time is the single largest cost in a + run: eight model calls spent to reject one scenario, in a chain that cannot be parallelised. + """ + from fi.alk.generation.pipeline import GenerationConfig, materialize_row + + contract = AgentContract.model_validate(CONTRACT) + broken = json.loads(json.dumps(SCENARIO)) + broken["sub_goals"][0]["checkpoint"]["definition"] = { + "tool": "order_combo_meal", + "args_equal": {"meal_id": "combo_not_a_real_id"}, + } + # Enough responses queued for the full four attempts; the loop must not consume them all. + llm = FakeLLMClient(responses=[json.loads(json.dumps(broken)) for _ in range(4)]) + plan = {"id": "p", "target_failure": "x", "why_it_matters": "y"} + record, reason = materialize_row( + contract, plan, [], llm, GenerationConfig(critic_enabled=True) + ) + assert record is None + assert "same problem twice" in reason + assert llm.usage.calls == 2, f"stopped after {llm.usage.calls} calls, expected 2" From d49d377183a4f74067e60171b6baf3de0443c0e3 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 14 Aug 2026 11:10:01 +0530 Subject: [PATCH 43/55] fix(generation): predict repeated calls for min_count, and ground handles a scenario's own mocks create --- src/fi/alk/generation/oracle.py | 12 +++- src/fi/alk/generation/prompts.py | 14 +++-- src/fi/alk/generation/validators.py | 14 ++++- tests/test_generation_pipeline.py | 91 +++++++++++++++++++++++++++++ 4 files changed, 122 insertions(+), 9 deletions(-) diff --git a/src/fi/alk/generation/oracle.py b/src/fi/alk/generation/oracle.py index 4a32621..d543c75 100644 --- a/src/fi/alk/generation/oracle.py +++ b/src/fi/alk/generation/oracle.py @@ -34,7 +34,17 @@ def predicted_evidence(record: Mapping[str, Any]) -> dict[str, Any]: arguments = dict(definition.get("args_equal") or {}) for arg in definition.get("args_present") or []: arguments.setdefault(str(arg), "") - tool_calls.append({"name": definition.get("tool"), "arguments": arguments}) + # A checkpoint asserting several identical calls predicts several identical calls. + # Emitting one would make every quantity scenario fail the run it itself predicts. + raw_count = definition.get("min_count", definition.get("call_nth", 1)) + try: + repeats = max(1, int(raw_count)) + except (TypeError, ValueError): + repeats = 1 + for _ in range(repeats): + tool_calls.append( + {"name": definition.get("tool"), "arguments": dict(arguments)} + ) # The transcript is predicted ONLY from what the scenario says the agent must communicate. # Seeding it from the conveyed definitions themselves would make those checks self-satisfying; diff --git a/src/fi/alk/generation/prompts.py b/src/fi/alk/generation/prompts.py index 56e3041..925f0f5 100644 --- a/src/fi/alk/generation/prompts.py +++ b/src/fi/alk/generation/prompts.py @@ -68,11 +68,13 @@ requirement it was never given. An open argument is a reason to move that one argument into args_present; it is never a reason to give up the tool_call_args checkpoint or to replace it with a judge, because the call itself and every argument the user did determine remain exactly - checkable. A value - that only comes into existence during the run (a generated id, a session handle) cannot be known - in advance and belongs in args_present, never in args_equal. When the same call must happen several times - (a quantity of identical items), one checkpoint with "min_count": asserts it; separate - identical checkpoints do not. + checkable. Some arguments carry a handle the run itself creates, such as the reference for an item + already added to an order. A handle can still be pinned, but only when this scenario says where it + comes from: the mock that creates it declares that exact value in its state_updates, which is what + makes the value knowable before the run. Where no mock in this scenario produces the handle, the + argument goes in args_present. When the same call must happen several times (a quantity of + identical items), one checkpoint with "min_count": asserts it; separate identical + checkpoints do not. - state (deterministic): passes when the world's final state carries the expected values. definition: {"must": {"": }, "forbidden": {"": }}, evaluated against the seeded environment state after the run. @@ -80,7 +82,7 @@ total, a time, a name) appears in the agent's transcript turns. definition: {"must_include_any": ["", ""]}. The agent's wording is its own; only data values are matchable, because correct phrasing is unbounded. -- absent (deterministic): passes when a named action never occurred. definition: {"no_tool_call": +- absent (deterministic, and it means NEVER, at any point in the interaction, not "not yet"): passes when a named action never occurred. definition: {"no_tool_call": ""} or {"no_tool_call_with": {"tool": "", "args_equal": {...}}}. - judge (not deterministic): definition: {"rubric": ""}. diff --git a/src/fi/alk/generation/validators.py b/src/fi/alk/generation/validators.py index 17cfc34..14bc2a6 100644 --- a/src/fi/alk/generation/validators.py +++ b/src/fi/alk/generation/validators.py @@ -143,7 +143,14 @@ def validate_scenario(scenario: dict, contract: AgentContract) -> list[str]: """Return problems; empty means structurally complete and grounded enough for the critic.""" problems: list[str] = [] tool_names = contract.tool_names() - legit_vocabulary = _legit_vocabulary(contract) + # A scenario may introduce an identifier the contract cannot contain (an order handle, a + # booking reference) PROVIDED it declares where that value comes from: its own environment + # seed or a mock's state_updates. Those are what generate the value during the run, so a + # checkpoint pinning it is checkable, not invented. + legit_vocabulary = _legit_vocabulary(contract) | { + match.lower() + for match in _TOKEN.findall(json.dumps(scenario.get("environment") or {})) + } arg_values = {tool.name: dict(tool.arg_values or {}) for tool in contract.tools} argless_tools = frozenset(tool.name for tool in contract.tools if not tool.args) @@ -300,7 +307,10 @@ def repair_hint(problems: list[str]) -> str: lines.append( f"- A checkpoint pins an argument to a value found nowhere in the contract " f"({problem.split(':')[-1]}). If the value is real, copy it from the contract's " - "data; if it only exists at run time, move the argument to args_present." + "data. If it is a handle the run creates, such as the reference for an item " + "already added, then declare where it comes from: give the mock that creates it a " + "state_updates entry carrying that exact value, which makes it checkable. Move the " + "argument to args_present only when no mock in this scenario produces it." ) elif ":arg-value-not-allowed:" in problem: lines.append( diff --git a/tests/test_generation_pipeline.py b/tests/test_generation_pipeline.py index 0938795..6ca6e83 100644 --- a/tests/test_generation_pipeline.py +++ b/tests/test_generation_pipeline.py @@ -990,3 +990,94 @@ def test_a_repeated_identical_failure_stops_the_repair_loop(): assert record is None assert "same problem twice" in reason assert llm.usage.calls == 2, f"stopped after {llm.usage.calls} calls, expected 2" + + +def test_a_quantity_checkpoint_passes_the_run_it_predicts(): + """min_count asserts several identical calls, so the predicted run must contain several. + + Predicting one call made every quantity scenario contradict itself: three of one run's + fourteen rejections were correct scenarios failing an oracle that under-predicted. + """ + from fi.alk.generation.oracle import oracle_problems, predicted_evidence + + record = json.loads(json.dumps(SCENARIO)) + record["sub_goals"] = [ + { + "name": "two_combos_added", + "milestone": "Two identical combos are ordered", + "checkpoint": { + "kind": "tool_call_args", + "deterministic": True, + "detail": "order_combo_meal called twice", + "definition": { + "tool": "order_combo_meal", + "args_equal": {"meal_id": "combo_big_mac"}, + "min_count": 2, + }, + }, + } + ] + assert len(predicted_evidence(record)["tool_calls"]) == 2 + assert not oracle_problems(record) + + +def test_a_handle_the_scenario_mocks_into_existence_is_groundable(): + """An order handle cannot be in the contract, but the mock that creates it makes it checkable.""" + from fi.alk.generation.validators import validate_scenario + + contract = AgentContract.model_validate(CONTRACT) + tool = contract.tools[0].name + record = json.loads(json.dumps(SCENARIO)) + record["environment"] = { + "seed": {}, + "mock_responses": { + tool: { + "content": "added", + "state_updates": {"order": [{"order_id": "combo_handle_1"}]}, + } + }, + } + record["sub_goals"] = record["sub_goals"][:1] + [ + { + "name": "handle_used", + "milestone": "The agent acts on the item it already added", + "checkpoint": { + "kind": "tool_call_args", + "deterministic": True, + "detail": "acts on the handle its own earlier call created", + "definition": { + "tool": tool, + "args_equal": {"order_id": "combo_handle_1"}, + }, + }, + } + ] + problems = validate_scenario(record, contract) + assert not [p for p in problems if "combo_handle_1" in p], problems + + +def test_a_handle_no_mock_creates_is_still_refused(): + """The relaxation is earned by declaring the source, not by naming something plausible.""" + from fi.alk.generation.validators import validate_scenario + + contract = AgentContract.model_validate(CONTRACT) + tool = contract.tools[0].name + record = json.loads(json.dumps(SCENARIO)) + record["environment"] = {"seed": {}, "mock_responses": {}} + record["sub_goals"] = record["sub_goals"][:1] + [ + { + "name": "handle_used", + "milestone": "The agent acts on an item", + "checkpoint": { + "kind": "tool_call_args", + "deterministic": True, + "detail": "acts on an undeclared handle", + "definition": { + "tool": tool, + "args_equal": {"order_id": "invented_handle_9"}, + }, + }, + } + ] + problems = validate_scenario(record, contract) + assert any("invented_handle_9" in p for p in problems), problems From 5496dcf8c086d6adc082e0595b1c000bf71a2044 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 14 Aug 2026 12:38:17 +0530 Subject: [PATCH 44/55] fix(audit): accept run-created handles a scenario's own mocks declare --- scripts/audit_generated_scenarios.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/scripts/audit_generated_scenarios.py b/scripts/audit_generated_scenarios.py index a3c2e56..9631526 100644 --- a/scripts/audit_generated_scenarios.py +++ b/scripts/audit_generated_scenarios.py @@ -54,11 +54,17 @@ def audit(run_dir: str, agent_repo: str) -> int: subgoal_uses: Counter = Counter() deterministic = total = 0 failures: list[str] = [] + run_declared_handles: list[str] = [] for name in files: record = _load(os.path.join(scenarios_dir, name)) slug = record.get("id", name) fact_values = {str(f.get("value", "")).lower() for f in record.get("facts") or []} + # A scenario may introduce a handle the agent's source cannot contain (an order + # reference) provided its own mock declares that value, which is what creates it during + # the run. Those are grounded; anything else invented is not. + declared = json.dumps(record.get("environment") or {}) + pass for sub_goal in record.get("sub_goals") or []: checkpoint = (sub_goal or {}).get("checkpoint") or {} definition = checkpoint.get("definition") or {} @@ -76,6 +82,11 @@ def audit(run_dir: str, agent_repo: str) -> int: for arg, value in (definition.get("args_equal") or {}).items(): text = str(value) if _IDENTIFIER.match(text) and text not in source: + # Match the bare token: a handle may be declared in state_updates or + # inside the mock's response text, where JSON escaping hides the quotes. + if re.search(rf"\b{re.escape(text)}\b", declared): + run_declared_handles.append(f"{slug}: {arg}={text}") + continue failures.append(f"{slug}: args_equal {arg}={text} not found in agent source") agent_input = str(record.get("agent_input", "")).lower() for token in re.findall(r"[a-z][a-z0-9_]{4,}", agent_input): @@ -87,6 +98,11 @@ def audit(run_dir: str, agent_repo: str) -> int: print(f"checkpoints: {total}, deterministic: {deterministic} ({100 * deterministic // max(total, 1)}%)") print(f"kind mix: {dict(kind_mix)}") print(f"sub-goal names reused in >=2 scenarios: {reused} of {len(subgoal_uses)}") + if run_declared_handles: + print( + f"run-created handles declared by the scenario's own mocks: " + f"{len(run_declared_handles)} (grounded, not failures)" + ) print(f"grounding failures: {len(failures)}") for failure in failures[:30]: print(f" - {failure}") From 3d1887f7a703a614f214e3d5f8ba29794828707a Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 14 Aug 2026 13:17:23 +0530 Subject: [PATCH 45/55] fix(generation): walk nested trace folders and drop plans citing recordings that were never supplied --- src/fi/alk/generation/traces.py | 43 +++++++++++++++---- tests/test_generation_pipeline.py | 69 +++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 8 deletions(-) diff --git a/src/fi/alk/generation/traces.py b/src/fi/alk/generation/traces.py index 97a05da..e342e11 100644 --- a/src/fi/alk/generation/traces.py +++ b/src/fi/alk/generation/traces.py @@ -33,6 +33,7 @@ _MAX_TRACES_MINED = 40 _MAX_EXPLORE_TURNS = 14 _MAX_RESULT_CHARS = 14_000 +_MAX_TRACE_FILES = 400 # the flat fallback reads a bounded slice of a large tree def _window(text: str) -> str: @@ -44,14 +45,26 @@ def _window(text: str) -> str: def load_traces(path: str) -> list[dict[str, str]]: - """Load raw traces from a file or a flat folder, without asking a model anything.""" + """Load raw traces from a file or a folder tree, without asking a model anything. + + Walks recursively. A flat listing silently returned almost nothing for the common case of + recordings filed under dated subdirectories, and because this is the fallback when + exploration does not submit, the grounding disappeared without any error. + """ + root = os.path.abspath(path) paths: list[str] = [] - if os.path.isfile(path): - paths = [path] - elif os.path.isdir(path): - for name in sorted(os.listdir(path)): - if name.endswith(_TRACE_EXTENSIONS) and not name.startswith("."): - paths.append(os.path.join(path, name)) + if os.path.isfile(root): + paths = [root] + elif os.path.isdir(root): + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = sorted(d for d in dirnames if not d.startswith(".")) + for name in sorted(filenames): + if name.endswith(_TRACE_EXTENSIONS) and not name.startswith("."): + paths.append(os.path.join(dirpath, name)) + if len(paths) >= _MAX_TRACE_FILES: + break + if len(paths) >= _MAX_TRACE_FILES: + break traces: list[dict[str, str]] = [] for file_path in paths: try: @@ -61,7 +74,12 @@ def load_traces(path: str) -> list[dict[str, str]]: continue if not text.strip(): continue - traces.append({"ref": os.path.basename(file_path), "text": _window(text)}) + ref = ( + os.path.relpath(file_path, root) + if os.path.isdir(root) + else os.path.basename(file_path) + ) + traces.append({"ref": ref, "text": _window(text)}) return traces @@ -379,6 +397,7 @@ def mine_traces( unique = unique[:_MAX_TRACES_MINED] traces = unique outcome_by_ref = {t["ref"]: t.get("outcome", "") for t in traces} + known_refs = set(outcome_by_ref) plans: list[dict[str, Any]] = [] for start in range(0, len(traces), _MAX_TRACES_PER_CALL): batch = traces[start : start + _MAX_TRACES_PER_CALL] @@ -396,6 +415,14 @@ def mine_traces( and row.get("target_failure") ): ref = str(row.get("trace_ref", "")) + # Provenance has to be verifiable. A plan claiming to recreate a recording + # that was never supplied was invented, and it is worse than a missing plan + # because the report presents it as grounded in production. + if ref not in known_refs: + logger.warning( + "dropping mined plan citing an unknown trace_ref: %r", ref[:80] + ) + continue row["provenance"] = {"kind": "production_trace", "trace_ref": ref} # A recreation of an interaction that went wrong earns a neighbourhood around it. row["amplify"] = outcome_by_ref.get(ref, "") == "failed" diff --git a/tests/test_generation_pipeline.py b/tests/test_generation_pipeline.py index 6ca6e83..1307ecc 100644 --- a/tests/test_generation_pipeline.py +++ b/tests/test_generation_pipeline.py @@ -1081,3 +1081,72 @@ def test_a_handle_no_mock_creates_is_still_refused(): ] problems = validate_scenario(record, contract) assert any("invented_handle_9" in p for p in problems), problems + + +def test_traces_are_loaded_from_nested_folders(tmp_path): + """Recordings are usually filed under dated subdirectories, not at the top level. + + A flat listing returned almost nothing, and because this is the fallback when exploration + does not submit, an entire run silently lost its grounding without erroring. + """ + from fi.alk.generation.traces import load_traces + + nested = tmp_path / "sessions" / "2026-08-17" / "sess_0097" + nested.mkdir(parents=True) + (nested / "transcript.json").write_text( + '{"turns": [{"role": "user", "text": "hi"}]}' + ) + (tmp_path / "README.txt").write_text("call exports") + + traces = load_traces(str(tmp_path)) + refs = {t["ref"] for t in traces} + assert "sessions/2026-08-17/sess_0097/transcript.json" in refs, refs + + +def test_a_plan_citing_an_unsupplied_recording_is_dropped(): + """Provenance must be verifiable, or 'grounded in production' means nothing. + + A run whose trace explorer failed produced three scenarios citing invented recordings named + 'hypothetical_trace_1'. They were reported as recreating real calls. + """ + from fi.alk.generation.traces import mine_traces + + contract = AgentContract.model_validate(CONTRACT) + traces = [ + { + "ref": "archive/real_call.log", + "text": "USER: one coffee", + "outcome": "failed", + } + ] + llm = FakeLLMClient( + responses=[ + { + "rows": [ + { + "id": "real-one", + "trace_ref": "archive/real_call.log", + "use_case": "Order a single item", + "situation": "A caller orders one coffee", + "target_failure": "The agent misprices the order", + "why_it_matters": "It happened to a real customer", + "unique_end_state": "One coffee ordered", + "goal": "Order one coffee", + }, + { + "id": "invented-one", + "trace_ref": "hypothetical_trace_2_change_of_mind", + "use_case": "Order a single item", + "situation": "A caller changes their mind", + "target_failure": "The agent keeps the original item", + "why_it_matters": "Invented provenance", + "unique_end_state": "The new item is ordered", + "goal": "Swap the item", + }, + ] + } + ] + ) + plans = mine_traces(contract, traces, llm) + assert [p["id"] for p in plans] == ["real-one"] + assert plans[0]["provenance"]["trace_ref"] == "archive/real_call.log" From 89fbb4769bf5e5ab72b6de31a5f1d6ff05894a35 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Fri, 14 Aug 2026 13:38:29 +0530 Subject: [PATCH 46/55] fix(generation): every scenario asserts its origin instead of it being inferred from an absent field --- src/fi/alk/generation/emit.py | 2 +- src/fi/alk/generation/pipeline.py | 4 ++++ src/fi/alk/generation/validators.py | 6 ++++++ tests/test_generation_pipeline.py | 18 ++++++++++++++++++ 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/fi/alk/generation/emit.py b/src/fi/alk/generation/emit.py index 6fc4434..20c3770 100644 --- a/src/fi/alk/generation/emit.py +++ b/src/fi/alk/generation/emit.py @@ -289,7 +289,7 @@ def render_report( ) origins: dict[str, int] = {} for record in records: - kind = str((record.get("provenance") or {}).get("kind") or "baseline_coverage") + kind = str((record.get("provenance") or {}).get("kind") or "unlabelled") origins[kind] = origins.get(kind, 0) + 1 # Worth printing whenever anything came from somewhere other than plain coverage planning, # including a suite built entirely from production traces. diff --git a/src/fi/alk/generation/pipeline.py b/src/fi/alk/generation/pipeline.py index 4d788ea..be196dc 100644 --- a/src/fi/alk/generation/pipeline.py +++ b/src/fi/alk/generation/pipeline.py @@ -616,6 +616,8 @@ def _plan_node(node: dict) -> list[dict]: ) if config.critic_enabled: rows = review_plan(contract, rows, llm) + for row in rows: + row.setdefault("provenance", {"kind": "baseline_coverage"}) return _claim(rows) # Each node flows plan -> review -> materialize on its own, with no barrier between the @@ -686,6 +688,8 @@ def _node_flow(node: dict) -> list: break if config.critic_enabled: rows = review_plan(contract, rows, llm) + for row in rows: + row.setdefault("provenance", {"kind": "coverage_gap_fill"}) _materialize_batch(rows) if len(records) == accepted_before: logger.warning( diff --git a/src/fi/alk/generation/validators.py b/src/fi/alk/generation/validators.py index 14bc2a6..711ffa7 100644 --- a/src/fi/alk/generation/validators.py +++ b/src/fi/alk/generation/validators.py @@ -171,6 +171,12 @@ def validate_scenario(scenario: dict, contract: AgentContract) -> list[str]: if isinstance(description, str) and 0 < len(description) < 60: problems.append("description-too-short") + # Every scenario states where it came from. Inferring it from an absent field meant any new + # plan source that forgot to set it would be silently reported as ordinary coverage. + provenance = scenario.get("provenance") + if not isinstance(provenance, dict) or not str(provenance.get("kind", "")).strip(): + problems.append("provenance-missing") + facts = scenario.get("facts") if contract.conversational and not isinstance(facts, list): problems.append("facts-not-a-list") diff --git a/tests/test_generation_pipeline.py b/tests/test_generation_pipeline.py index 1307ecc..23d873d 100644 --- a/tests/test_generation_pipeline.py +++ b/tests/test_generation_pipeline.py @@ -1150,3 +1150,21 @@ def test_a_plan_citing_an_unsupplied_recording_is_dropped(): plans = mine_traces(contract, traces, llm) assert [p["id"] for p in plans] == ["real-one"] assert plans[0]["provenance"]["trace_ref"] == "archive/real_call.log" + + +def test_every_scenario_states_where_it_came_from(): + """Provenance is asserted by the pipeline, never inferred from a missing field. + + Three of one run's twenty scenarios carried no provenance at all and were reported as + ordinary coverage purely because the field was absent. Correct by accident is not correct: + any future plan source that forgot to set it would inherit the same silent label. + """ + from fi.alk.generation.validators import validate_scenario + + contract = AgentContract.model_validate(CONTRACT) + record = json.loads(json.dumps(SCENARIO)) + record.pop("provenance", None) + assert "provenance-missing" in validate_scenario(record, contract) + + record["provenance"] = {"kind": "baseline_coverage"} + assert "provenance-missing" not in validate_scenario(record, contract) From 6a3ffa7ba41d9365df1365c6f5e478b6829dfd37 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 15 Aug 2026 09:31:10 +0530 Subject: [PATCH 47/55] fix(generation): validate the mock layer the runtime executes, and tell the model its real shape --- src/fi/alk/generation/prompts.py | 8 +++- src/fi/alk/generation/validators.py | 59 +++++++++++++++++++++++++++++ tests/test_generation_pipeline.py | 44 +++++++++++++++++++++ 3 files changed, 109 insertions(+), 2 deletions(-) diff --git a/src/fi/alk/generation/prompts.py b/src/fi/alk/generation/prompts.py index 925f0f5..abc0539 100644 --- a/src/fi/alk/generation/prompts.py +++ b/src/fi/alk/generation/prompts.py @@ -429,8 +429,12 @@ def materialize_prompt( specifically about a user attribute - environment: {{"seed": {{}}, "mock_responses": {{"": {{"content": "", - "state_updates": {{}}}}}}}}. Mock only tools this scenario expects - the agent to call. + "state_updates": {{}}}}}}}}. Every tool a checkpoint expects the + agent to call is mocked here, because a tool left unmocked returns nothing to the agent and the + call cannot succeed. Each mocked response is an object, never a list, and uses only these keys: + content, result, success, error, state_updates, artifacts, events. A response written with any + other key is discarded before the agent sees it, and one that is not an object cannot carry + state_updates, so no state checkpoint depending on it can pass. - sub_goals: 3 to 6, per the checkpoint vocabulary above, every definition fully concrete - expected_outcome: {{"world_state": "", "must_convey": [""], "forbidden": [" bool: @@ -171,6 +176,36 @@ def validate_scenario(scenario: dict, contract: AgentContract) -> list[str]: if isinstance(description, str) and 0 < len(description) < 60: problems.append("description-too-short") + # The mock layer is what the runtime actually executes, so it gets checked like everything + # else. An unmocked tool returns nothing to the agent, and a mock the runtime cannot read + # silently drops its response, in both cases producing a scenario that passes every other + # gate and cannot run. + environment = scenario.get("environment") + environment = environment if isinstance(environment, dict) else {} + mocks = environment.get("mock_responses") + mocks = mocks if isinstance(mocks, dict) else {} + expected_calls = { + str(((sg or {}).get("checkpoint") or {}).get("definition", {}).get("tool", "")) + for sg in scenario.get("sub_goals") or [] + if isinstance(sg, dict) + and ((sg.get("checkpoint") or {}).get("kind")) == "tool_call_args" + } + for tool in sorted(t for t in expected_calls if t and t in tool_names): + if tool not in mocks: + problems.append(f"tool-expected-but-not-mocked:{tool}") + for tool, body in mocks.items(): + where = f"mock[{tool}]" + if str(tool) not in tool_names: + problems.append(f"{where}:unknown-tool") + if not isinstance(body, dict): + problems.append(f"{where}:not-an-object") + continue + unknown = sorted(set(body) - _MOCK_KEYS) + if unknown: + problems.append(f"{where}:unreadable-keys:{','.join(unknown)[:60]}") + if "content" not in body and "result" not in body: + problems.append(f"{where}:no-content") + # Every scenario states where it came from. Inferring it from an absent field meant any new # plan source that forgot to set it would be silently reported as ordinary coverage. provenance = scenario.get("provenance") @@ -296,6 +331,30 @@ def repair_hint(problems: list[str]) -> str: "- Every checkpoint is a judge; make the tool-argument and end-state checks " "deterministic per the vocabulary." ) + elif problem.startswith("tool-expected-but-not-mocked:"): + lines.append( + f"- A checkpoint expects a call to {problem.split(':', 1)[1]} but the environment " + "does not mock it, so during the run that call returns nothing to the agent. Add " + "it to mock_responses with the content it should return." + ) + elif ":not-an-object" in problem: + lines.append( + "- A mocked tool response is not an object. Write it as " + '{"content": "", "state_updates": {}}; ' + "any other form cannot carry state_updates, so state checkpoints will not fire." + ) + elif ":unreadable-keys:" in problem: + lines.append( + f"- A mocked tool response uses keys the runtime does not read " + f"({problem.split(':')[-1]}). Only content, result, success, error, " + "state_updates, artifacts and events are read; put the returned text in content " + "and the world changes in state_updates." + ) + elif ":no-content" in problem: + lines.append( + "- A mocked tool response has no content, so the agent receives nothing back " + "from the call. Give it the content the real tool would return." + ) elif ":no-definition" in problem: lines.append( "- A checkpoint has prose but no definition object. Every checkpoint carries the " diff --git a/tests/test_generation_pipeline.py b/tests/test_generation_pipeline.py index 23d873d..2602b18 100644 --- a/tests/test_generation_pipeline.py +++ b/tests/test_generation_pipeline.py @@ -1168,3 +1168,47 @@ def test_every_scenario_states_where_it_came_from(): record["provenance"] = {"kind": "baseline_coverage"} assert "provenance-missing" not in validate_scenario(record, contract) + + +def test_a_tool_a_checkpoint_expects_must_be_mocked(): + """An unmocked tool returns None to the agent, so the call it asserts cannot succeed.""" + from fi.alk.generation.validators import validate_scenario + + contract = AgentContract.model_validate(CONTRACT) + tool = contract.tools[0].name + record = json.loads(json.dumps(SCENARIO)) + record["environment"] = {"seed": {}, "mock_responses": {}} + problems = validate_scenario(record, contract) + assert f"tool-expected-but-not-mocked:{tool}" in problems, problems + + record["environment"]["mock_responses"] = { + tool: {"content": "added", "state_updates": {}} + } + assert not [p for p in validate_scenario(record, contract) if "not-mocked" in p] + + +def test_a_mock_the_runtime_cannot_read_is_refused(): + """Only the runtime's own keys carry a response; anything else is silently dropped. + + One run produced mocks keyed json_response, return_value, call_match and args, plus three + that were lists rather than objects. Every one passed all four gates and would have handed + the agent nothing at run time. + """ + from fi.alk.generation.validators import validate_scenario + + contract = AgentContract.model_validate(CONTRACT) + tool = contract.tools[0].name + record = json.loads(json.dumps(SCENARIO)) + + record["environment"] = {"mock_responses": {tool: {"json_response": {"ok": True}}}} + problems = validate_scenario(record, contract) + assert f"mock[{tool}]:unreadable-keys:json_response" in problems, problems + assert f"mock[{tool}]:no-content" in problems, problems + + record["environment"] = {"mock_responses": {tool: [{"content": "added"}]}} + assert f"mock[{tool}]:not-an-object" in validate_scenario(record, contract) + + record["environment"] = { + "mock_responses": {tool: {"content": "added", "state_updates": {"order": []}}} + } + assert not [p for p in validate_scenario(record, contract) if p.startswith("mock[")] From 2bd2d300185c7d87e048375bf8521da493f50768 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 15 Aug 2026 10:01:38 +0530 Subject: [PATCH 48/55] feat(generation): bridge a generated scenario into the voice simulator prompt with disclosure rules --- src/fi/alk/generation/simulate_bridge.py | 106 +++++++++++++++++++++++ tests/test_generation_pipeline.py | 48 ++++++++++ 2 files changed, 154 insertions(+) create mode 100644 src/fi/alk/generation/simulate_bridge.py diff --git a/src/fi/alk/generation/simulate_bridge.py b/src/fi/alk/generation/simulate_bridge.py new file mode 100644 index 0000000..2bc0311 --- /dev/null +++ b/src/fi/alk/generation/simulate_bridge.py @@ -0,0 +1,106 @@ +"""Turn a generated scenario into the inputs a live voice simulation needs. + +The simulator's system prompt is already built by ``fi.simulate.simulation.voice_prompt``: it +composes identity, situation, objective, personality and the voice execution rules, and leaves one +slot open for extra instruction. Nothing here re-implements that. The generated scenario supplies +what the template was always missing: + +- ``agent_input`` is the situation instruction, and lands in ``Persona.situation``; +- ``expected_outcome.world_state`` is what the caller is trying to reach, and lands in + ``Persona.outcome``; +- ``facts`` carry disclosure rules, which no persona template can express on its own, and are + rendered into the additional-instruction slot so the simulated caller knows what it may volunteer + and what it must wait to be asked for. + +That last part is what makes a generated scenario testable rather than merely playable: a scenario +about eliciting a drink choice only means something if the caller withholds it until asked. +""" + +from __future__ import annotations + +from typing import Any, Mapping + +from fi.simulate.simulation.models import Persona, PersonaFact +from fi.simulate.simulation.voice_prompt import build_voice_simulator_prompt + +_DISCLOSURE_RULES = { + "volunteer": "Say this early, without being asked.", + "on_request": "Do NOT say this until the agent asks for it. If the agent never asks, never say it.", + "withhold": "Never reveal this, whatever the agent asks.", +} + + +def persona_from_record(record: Mapping[str, Any]) -> Persona: + """The generated scenario as the simulator's persona.""" + persona_payload = dict(record.get("persona") or {}) + persona_payload.setdefault("name", "Caller") + persona_payload.setdefault("role", "customer") + outcome = record.get("expected_outcome") or {} + facts = [ + PersonaFact( + key=str(fact["key"]), + value=str(fact.get("value", "")), + disclosure=str(fact.get("disclosure", "on_request")), + ) + for fact in record.get("facts") or [] + if isinstance(fact, Mapping) and fact.get("key") + ] + return Persona( + persona=persona_payload, + situation=str(record.get("agent_input", "")), + outcome=str(outcome.get("world_state") or record.get("goal", "")), + knowledge=facts, + ) + + +def disclosure_instructions(record: Mapping[str, Any]) -> str: + """The caller's facts, grouped by when they are allowed to say them. + + Written as instructions to the person, not as a data structure, because the simulator reads + this as part of its own character rather than as configuration. + """ + facts = [ + f for f in record.get("facts") or [] if isinstance(f, Mapping) and f.get("key") + ] + if not facts: + return "" + lines = [ + "You know the following things. When you may say each one is part of who you are in this " + "call, and getting it wrong changes what is being tested." + ] + for disclosure, rule in _DISCLOSURE_RULES.items(): + group = [ + f for f in facts if str(f.get("disclosure", "on_request")) == disclosure + ] + if not group: + continue + lines.append("") + lines.append(rule) + for fact in group: + label = str(fact["key"]).replace("_", " ") + lines.append(f"- {label}: {fact.get('value', '')}") + turns = record.get("max_reasonable_turns") + if isinstance(turns, int) and turns > 0: + lines.append("") + lines.append( + f"A competent agent can finish this in about {turns} of your turns. If the call runs " + "far past that without progress, wind it up rather than continuing indefinitely." + ) + return "\n".join(lines) + + +def simulator_prompt( + record: Mapping[str, Any], + *, + call_type: str = "inbound", + agent_name: str | None = None, + default_language: str | None = None, +) -> str: + """The finished system prompt for the simulated caller in one generated scenario.""" + return build_voice_simulator_prompt( + persona_from_record(record), + call_type=call_type, # type: ignore[arg-type] + agent_name=agent_name, + additional_instructions=disclosure_instructions(record), + default_language=default_language, + ) diff --git a/tests/test_generation_pipeline.py b/tests/test_generation_pipeline.py index 2602b18..b9ee21d 100644 --- a/tests/test_generation_pipeline.py +++ b/tests/test_generation_pipeline.py @@ -1212,3 +1212,51 @@ def test_a_mock_the_runtime_cannot_read_is_refused(): "mock_responses": {tool: {"content": "added", "state_updates": {"order": []}}} } assert not [p for p in validate_scenario(record, contract) if p.startswith("mock[")] + + +def test_generated_scenario_becomes_a_simulator_prompt(): + """The generated instruction drives the existing voice persona template. + + Nothing about the template is re-implemented here: the scenario supplies the situation, the + objective and, crucially, the disclosure rules, which are what make an elicitation test mean + anything. A caller who volunteers the drink is not testing whether the agent asks for it. + """ + from fi.alk.generation.simulate_bridge import ( + disclosure_instructions, + persona_from_record, + simulator_prompt, + ) + + record = json.loads(json.dumps(SCENARIO)) + record["facts"] = [ + {"key": "drink_choice", "value": "Coca-Cola", "disclosure": "on_request"}, + {"key": "meal_choice", "value": "Big Mac Combo", "disclosure": "volunteer"}, + {"key": "loyalty_number", "value": "99887", "disclosure": "withhold"}, + ] + record["max_reasonable_turns"] = 6 + + persona = persona_from_record(record) + assert persona.situation == record["agent_input"] + assert {f.key for f in persona.knowledge} == { + "drink_choice", + "meal_choice", + "loyalty_number", + } + + instructions = disclosure_instructions(record) + # Each fact sits under the rule that governs it, not in one undifferentiated list. + assert instructions.index("Say this early") < instructions.index("meal choice") + assert instructions.index("until the agent asks") < instructions.index( + "drink choice" + ) + assert instructions.index("Never reveal this") < instructions.index( + "loyalty number" + ) + assert "about 6 of your turns" in instructions + + prompt = simulator_prompt( + record, call_type="inbound", agent_name="drive-thru assistant" + ) + assert record["agent_input"] in prompt + assert "ADDITIONAL SIMULATOR INSTRUCTIONS" in prompt + assert "CONVERSATION EXECUTION RULES" in prompt # the voice rules still apply From 893498c6cb452cc5b27dfe3553491f22446d5af2 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 15 Aug 2026 10:19:30 +0530 Subject: [PATCH 49/55] feat(generation): scenario-driven mock tool server and contract-built vapi assistant --- src/fi/alk/generation/vapi_live.py | 245 +++++++++++++++++++++++++++++ tests/test_generation_pipeline.py | 97 ++++++++++++ 2 files changed, 342 insertions(+) create mode 100644 src/fi/alk/generation/vapi_live.py diff --git a/src/fi/alk/generation/vapi_live.py b/src/fi/alk/generation/vapi_live.py new file mode 100644 index 0000000..a7a16d3 --- /dev/null +++ b/src/fi/alk/generation/vapi_live.py @@ -0,0 +1,245 @@ +"""Run a generated scenario against a real Vapi assistant. + +Three pieces, because a live voice test needs all three and the harness already owns the data for +each: + +1. A **mock tool server**. Vapi executes an assistant's tools by calling a public webhook, so the + scenario's ``mock_responses`` are served over HTTP rather than in-process. The server also + records every call it answers, with arguments, which is what the deterministic checkpoints are + graded against afterwards. The recording is the point: provider evidence can lag or drop, and a + test that cannot be graded is not a test. + +2. An **assistant registry**. Assistants are created once per agent and reused, so the id lives in + a local file rather than being minted fresh on every run. Creating a new assistant per run would + litter the account and lose the tool wiring. + +3. A **scenario binding**. The mock server serves one scenario at a time; binding swaps which + ``mock_responses`` are live without touching the assistant. + +Nothing here decides pass or fail. Grading stays in ``checks.py``, against the calls this server +recorded plus the transcript the run produced. +""" + +from __future__ import annotations + +import json +import logging +import os +import threading +from dataclasses import dataclass, field +from http.server import BaseHTTPRequestHandler, HTTPServer +from typing import Any, Mapping + +from .contract import AgentContract + +logger = logging.getLogger(__name__) + +REGISTRY_PATH = os.environ.get( + "ALK_VAPI_REGISTRY", "artifacts/scenario-gen/vapi_assistants.json" +) +VAPI_API_BASE = os.environ.get("VAPI_API_BASE_URL", "https://api.vapi.ai").rstrip("/") + + +# ---------------------------------------------------------------------------------- +# The mock tool server +# ---------------------------------------------------------------------------------- + + +@dataclass +class ToolCallLog: + """Every tool call the assistant made, in order, with the arguments it passed.""" + + calls: list[dict[str, Any]] = field(default_factory=list) + _lock: threading.Lock = field(default_factory=threading.Lock, repr=False) + + def record(self, name: str, arguments: Mapping[str, Any]) -> None: + with self._lock: + self.calls.append({"name": name, "arguments": dict(arguments)}) + + def snapshot(self) -> list[dict[str, Any]]: + with self._lock: + return [dict(c) for c in self.calls] + + def reset(self) -> None: + with self._lock: + self.calls.clear() + + +class ScenarioMockServer: + """Serves one scenario's mock tool responses over HTTP, and records what was called.""" + + def __init__(self, host: str = "127.0.0.1", port: int = 0) -> None: + self.log = ToolCallLog() + self._scenario: dict[str, Any] = {} + self._state: dict[str, Any] = {} + server = HTTPServer((host, port), _make_handler(self)) + self._server = server + self.port = server.server_address[1] + self._thread = threading.Thread(target=server.serve_forever, daemon=True) + + # -- lifecycle --------------------------------------------------------------- + def start(self) -> "ScenarioMockServer": + self._thread.start() + logger.info("mock tool server listening on port %s", self.port) + return self + + def stop(self) -> None: + self._server.shutdown() + self._server.server_close() + + # -- scenario binding -------------------------------------------------------- + def bind(self, record: Mapping[str, Any]) -> None: + """Make one scenario's mocks live, and clear anything the previous one recorded.""" + environment = record.get("environment") or {} + self._scenario = dict(record) + self._state = json.loads(json.dumps(environment.get("seed") or {})) + self.log.reset() + + @property + def final_state(self) -> dict[str, Any]: + return json.loads(json.dumps(self._state)) + + # -- the actual mock --------------------------------------------------------- + def respond(self, name: str, arguments: Mapping[str, Any]) -> Any: + self.log.record(name, arguments) + mocks = (self._scenario.get("environment") or {}).get("mock_responses") or {} + mock = mocks.get(name) + if not isinstance(mock, Mapping): + # A tool the scenario did not mock still has to answer, or the assistant stalls + # mid-call and the conversation dies for a reason unrelated to what is being tested. + logger.warning( + "no mock declared for %s; answering with an acknowledgement", name + ) + return f"{name} completed." + updates = mock.get("state_updates") + if isinstance(updates, Mapping): + _deep_merge(self._state, updates) + content = mock.get("content", mock.get("result")) + return content if content is not None else f"{name} completed." + + +def _deep_merge(target: dict[str, Any], updates: Mapping[str, Any]) -> None: + for key, value in updates.items(): + if isinstance(value, Mapping) and isinstance(target.get(key), dict): + _deep_merge(target[key], value) + else: + target[key] = json.loads(json.dumps(value)) + + +def _make_handler(owner: "ScenarioMockServer"): + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args: Any) -> None: # silence per-request stderr noise + return + + def do_POST(self) -> None: # noqa: N802 - required name + length = int(self.headers.get("Content-Length") or 0) + raw = self.rfile.read(length) if length else b"{}" + try: + payload = json.loads(raw or b"{}") + except json.JSONDecodeError: + payload = {} + results = [ + {"toolCallId": call_id, "result": owner.respond(name, arguments)} + for call_id, name, arguments in _tool_calls(payload) + ] + body = json.dumps({"results": results}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + return Handler + + +def _tool_calls(payload: Mapping[str, Any]) -> list[tuple[str, str, dict[str, Any]]]: + """Pull (id, name, arguments) out of a Vapi tool-call webhook body.""" + message = payload.get("message") or payload + raw = message.get("toolCalls") or message.get("toolCallList") or [] + calls: list[tuple[str, str, dict[str, Any]]] = [] + for entry in raw if isinstance(raw, list) else []: + if not isinstance(entry, Mapping): + continue + function = entry.get("function") or {} + name = str(function.get("name") or entry.get("name") or "") + arguments = function.get("arguments") or entry.get("arguments") or {} + if isinstance(arguments, str): + try: + arguments = json.loads(arguments) + except json.JSONDecodeError: + arguments = {"_raw": arguments} + if name: + calls.append((str(entry.get("id") or ""), name, dict(arguments))) + return calls + + +# ---------------------------------------------------------------------------------- +# The assistant: built from the contract, registered once, reused +# ---------------------------------------------------------------------------------- + + +def assistant_payload( + contract: AgentContract, *, tool_base_url: str, name: str +) -> dict[str, Any]: + """A Vapi assistant that behaves like the agent the contract describes. + + The system prompt and the tool surface both come from the contract, so the assistant under + test is the agent under test rather than an approximation of it. + """ + rules = "\n".join(f"- {rule}" for rule in contract.hard_constraints) + system = ( + f"{contract.system_prompt_excerpt}\n\n" + f"Rules you always follow:\n{rules}\n\n" + "You are speaking to a customer over a voice channel. Keep replies short and natural. " + "Call the tools you have been given to actually place, change or read the order; never " + "claim an action you have not performed through a tool." + ) + tools = [] + for tool in contract.tools: + properties = {} + for arg in tool.args: + allowed = (tool.arg_values or {}).get(arg) + schema: dict[str, Any] = {"type": "string"} + if isinstance(allowed, list) and allowed: + schema["enum"] = [str(v) for v in allowed][:40] + properties[arg] = schema + tools.append( + { + "type": "function", + "function": { + "name": tool.name, + "description": tool.description or tool.name, + "parameters": { + "type": "object", + "properties": properties, + "required": list(tool.args), + }, + }, + "server": {"url": f"{tool_base_url.rstrip('/')}/tool"}, + } + ) + return { + "name": name, + "firstMessage": "Welcome to the drive thru, what can I get for you?", + "model": { + "provider": "openai", + "model": "gpt-4o", + "messages": [{"role": "system", "content": system}], + "tools": tools, + }, + "transcriber": {"provider": "deepgram", "model": "nova-2"}, + "voice": {"provider": "vapi", "voiceId": "Elliot"}, + } + + +def load_registry() -> dict[str, Any]: + if os.path.exists(REGISTRY_PATH): + with open(REGISTRY_PATH, encoding="utf-8") as fh: + return json.load(fh) + return {} + + +def save_registry(registry: Mapping[str, Any]) -> None: + os.makedirs(os.path.dirname(REGISTRY_PATH) or ".", exist_ok=True) + with open(REGISTRY_PATH, "w", encoding="utf-8") as fh: + json.dump(registry, fh, indent=2, sort_keys=True) diff --git a/tests/test_generation_pipeline.py b/tests/test_generation_pipeline.py index b9ee21d..a13f2fb 100644 --- a/tests/test_generation_pipeline.py +++ b/tests/test_generation_pipeline.py @@ -1260,3 +1260,100 @@ def test_generated_scenario_becomes_a_simulator_prompt(): assert record["agent_input"] in prompt assert "ADDITIONAL SIMULATOR INSTRUCTIONS" in prompt assert "CONVERSATION EXECUTION RULES" in prompt # the voice rules still apply + + +def test_mock_server_answers_from_the_scenario_and_records_the_call(): + """The mock server is both the world the agent acts on and the record used to grade it.""" + import urllib.request + + from fi.alk.generation.vapi_live import ScenarioMockServer + + server = ScenarioMockServer().start() + try: + record = json.loads(json.dumps(SCENARIO)) + tool = AgentContract.model_validate(CONTRACT).tools[0].name + record["environment"] = { + "seed": {"order": {"items": []}}, + "mock_responses": { + tool: { + "content": "added to your order", + "state_updates": {"order": {"items": ["combo_big_mac"]}}, + } + }, + } + server.bind(record) + + body = json.dumps( + { + "message": { + "toolCalls": [ + { + "id": "call_1", + "function": { + "name": tool, + "arguments": '{"meal_id": "combo_big_mac"}', + }, + } + ] + } + } + ).encode() + request = urllib.request.Request( + f"http://127.0.0.1:{server.port}/tool", + data=body, + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=5) as response: + payload = json.loads(response.read()) + + assert payload["results"][0]["toolCallId"] == "call_1" + assert payload["results"][0]["result"] == "added to your order" + # recorded for grading, with the arguments the agent actually passed + assert server.log.snapshot() == [ + {"name": tool, "arguments": {"meal_id": "combo_big_mac"}} + ] + # and the world moved, so state checkpoints have something to assert + assert server.final_state["order"]["items"] == ["combo_big_mac"] + finally: + server.stop() + + +def test_an_unmocked_tool_still_answers_so_the_call_does_not_stall(): + import urllib.request + + from fi.alk.generation.vapi_live import ScenarioMockServer + + server = ScenarioMockServer().start() + try: + server.bind({"environment": {"seed": {}, "mock_responses": {}}}) + body = json.dumps( + {"message": {"toolCalls": [{"id": "c", "function": {"name": "whatever"}}]}} + ).encode() + request = urllib.request.Request( + f"http://127.0.0.1:{server.port}/tool", + data=body, + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=5) as response: + payload = json.loads(response.read()) + assert payload["results"][0]["result"] == "whatever completed." + assert server.log.snapshot()[0]["name"] == "whatever" + finally: + server.stop() + + +def test_assistant_is_built_from_the_contract(): + from fi.alk.generation.vapi_live import assistant_payload + + contract = AgentContract.model_validate(CONTRACT) + payload = assistant_payload( + contract, tool_base_url="https://example.test", name="alk-test" + ) + names = {t["function"]["name"] for t in payload["model"]["tools"]} + assert names == set(contract.tool_names()) + assert all( + t["server"]["url"] == "https://example.test/tool" + for t in payload["model"]["tools"] + ) + system = payload["model"]["messages"][0]["content"] + assert contract.hard_constraints[0] in system From 565aafd1c7f2cedefaa8a109e128c82751bf46fd Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 15 Aug 2026 10:46:28 +0530 Subject: [PATCH 50/55] feat(simulate): drive the acceptance persona from a generated scenario when one is supplied --- oss/simulation-acceptance/voice_cases.py | 42 ++++++++++++++++-------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/oss/simulation-acceptance/voice_cases.py b/oss/simulation-acceptance/voice_cases.py index 9fa2c4d..b91fcec 100644 --- a/oss/simulation-acceptance/voice_cases.py +++ b/oss/simulation-acceptance/voice_cases.py @@ -206,19 +206,35 @@ def build_inputs(case_id: str, run_id: str) -> VoiceInputs: room_mode="managed", room_name_verbatim=bool(room_override), ) - scenario = simulate.Scenario( - name=f"acceptance-{case_id}", - dataset=[ - simulate.Persona( - persona={"name": "Morgan", "role": "customer"}, - situation=( - "A delivery is late. Ask for its current status, expected arrival, " - "and the next action." - ), - outcome="Complete a natural multi-turn conversation and close politely.", - ) - ], - ) + # A generated scenario, when one is supplied, replaces the built-in persona. The scenario + # carries the caller's situation, objective and disclosure rules, which is what turns a + # generic conversation into a specific test. + generated_path = os.environ.get("ALK_SCENARIO", "").strip() + if generated_path: + import json as _json + + from fi.alk.generation.simulate_bridge import persona_from_record + + with open(generated_path, encoding="utf-8") as fh: + record = _json.load(fh) + scenario = simulate.Scenario( + name=str(record.get("id") or f"acceptance-{case_id}"), + dataset=[persona_from_record(record)], + ) + else: + scenario = simulate.Scenario( + name=f"acceptance-{case_id}", + dataset=[ + simulate.Persona( + persona={"name": "Morgan", "role": "customer"}, + situation=( + "A delivery is late. Ask for its current status, expected arrival, " + "and the next action." + ), + outcome="Complete a natural multi-turn conversation and close politely.", + ) + ], + ) llm_provider = os.environ.get("SIMULATOR_LLM_PROVIDER", "google") stt_provider = os.environ.get("SIMULATOR_STT_PROVIDER", "deepgram") tts_provider = os.environ.get("SIMULATOR_TTS_PROVIDER", "deepgram") From d7ee2f91adf32edd11496a86911c9af180fbc467 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 15 Aug 2026 10:47:36 +0530 Subject: [PATCH 51/55] docs(generation): user guide covering generation, auditing and running a scenario live --- src/fi/alk/generation/README.md | 199 ++++++++++++++++++++++++-------- 1 file changed, 149 insertions(+), 50 deletions(-) diff --git a/src/fi/alk/generation/README.md b/src/fi/alk/generation/README.md index 14e52ab..351d9ca 100644 --- a/src/fi/alk/generation/README.md +++ b/src/fi/alk/generation/README.md @@ -1,71 +1,170 @@ # fi.alk.generation -Local-first scenario generation: point at an agent, get a reviewed set of runnable test scenarios. +Point it at an agent and get back test scenarios that can actually be run and graded: what the +caller says, what the tools return, and the exact checks that decide pass or fail. ```bash -python -m fi.alk.generation --repo /path/to/agent --n 20 --out artifacts/scenarios +python -m fi.alk.generation --environment voice --repo /path/to/agent --n 20 --out artifacts/scenarios ``` -## What it produces +## Why this exists -For each scenario, one record with three strictly separated parts: +Writing good tests for a conversational agent is slow, and most generated ones are unusable for the +same two reasons: they reference things the agent does not have, and they are graded by asking a +model whether the conversation "seemed fine". This package fixes both. Every value in a scenario has +to exist in the agent's own code, and most checks are settled by comparing recorded tool calls and +final state in plain Python. -- **(A) agent input** - what the simulated user is told (situation, goal, facts revealed only when - asked). Never contains the answer or hidden state. -- **(B) environment** - seed state plus per-tool mock responses (`static_fixture` tier, the one the - runtime executes today). -- **(C) hidden checks** - sub-goals drawn from a shared per-agent catalog, each with a checkpoint - that asserts the right end state or the right tool call with the right arguments. Deterministic - where possible; judge only where the world is not inspectable. +## Quick start -Emitted artifacts: `scenarios/*.json` (rich records), `alk/*.json` (typed `fi.simulate` `Scenario` -objects with `goal` / `verification` / `constraints` populated), `subgoal_catalog.json`, -`report.md` (coverage + verdicts), `usage.json` (tokens and USD). +```bash +# 1. generate +python -m fi.alk.generation \ + --environment voice \ + --repo /path/to/agent \ + --n 20 \ + --out artifacts/scenarios -## The pipeline +# 2. see what you got +open artifacts/scenarios/report.md +# 3. check nothing was invented +python scripts/audit_generated_scenarios.py artifacts/scenarios /path/to/agent ``` -AgentSource ──▶ evidence blob ──▶ CONTRACT ──▶ sub-goal catalog ──▶ rows (use-case ▸ branch) - (LLM+validate) (LLM+validate) (LLM, round loop, dedup) - │ per row - materialize ─▶ validate ─▶ critic - ▲ │(problems) - └── repair ◀───┘ max 2 - │ accepted - emit + +The audit takes two positional paths. `grounding failures: 0` is the number that matters: anything +higher means a scenario references a tool or value the agent does not have. + +### Useful flags + +| Flag | What it does | +|---|---| +| `--environment` | **Required.** `voice` or `chat`. Anything else is refused by name, because a scenario is only worth generating if a runtime can stage and grade it | +| `--contract` | Reuse a previous run's `contract.json` and skip re-reading the agent. The biggest time and cost saving available | +| `--traces` | A file or folder of real recorded conversations. The harness explores it, picks the ones that went wrong, and builds tests that recreate them | +| `--guidance` | One instruction in your own words. Scenarios answering it are generated before generic coverage | +| `--n` | Target count, delivered exactly, or an explicit statement that fewer distinct ones exist | +| `--budget-usd` | Hard spend ceiling. Partial results are already on disk when it stops | +| `--workers` | Parallel scenario writers, default 8 | +| `--model` | Any litellm model string | + +## What you get + +For each scenario, three strictly separated parts: + +- **The input** - what the simulated user is told: their situation, their goal, and the facts they + hold. Facts carry a disclosure rule, so information the agent is supposed to *ask* for is withheld + until it does. Never contains the answer or anything the user could not know. +- **The environment** - starting state plus a mock response per tool, so the test runs without + touching anything real. +- **The checks** - one per sub-goal, each asserting a tool call with its arguments, a final state, a + value the agent had to say, or an action it must not have taken. A model judges only what none of + those can witness. + +Written to your output directory: + ``` +contract.json what the harness worked out about the agent; pass to --contract next time +scenarios/.json the readable test +alk/.json a typed fi.simulate Scenario for the runtime +report.md what each test catches, where it came from, what had to be assumed +usage.json calls, tokens, spend +``` + +Each scenario records where it came from: a real recorded call, the neighbourhood around a real +failure, your instruction, or baseline coverage. + +## How it works -Generation is a loop over prompts plus deterministic validators, not an agent framework. The LLM does -semantics; plain code does structure, dedup, and grounding checks; nothing hardcodes a domain. +``` +agent source + └── read the code, write down the real tools, values and rules (model, file tools) + └── name the shared milestones a suite reuses (model) + ├── real recordings: explore, pick failures, recreate, (model, file tools) + │ and surround each failure with neighbours + ├── your instruction (model) + └── everything else a complete suite needs (model) + └── per scenario: write it, then four gates + validators every id and tool exists (code) + oracle does it contradict itself (code) + dedup is it a test we already have (code) + reviewer could a good agent fail it (model) + └── suite review: what is the set still missing (model) +``` -## Design rules +The model is called at the named steps and nowhere else. Everything between them is ordinary code, +including every decision about when a loop stops, so behaviour stays stable as the count grows. + +## Running a scenario against a live voice agent + +Generation produces the test. Running it needs a provider and a room. The supported path today is a +Vapi assistant over the web transport, with no phone number involved. + +```python +from fi.alk.generation.simulate_bridge import simulator_prompt, persona_from_record +from fi.alk.generation.vapi_live import ScenarioMockServer, assistant_payload +``` -1. **Grounding is a contract, not a vibe.** Everything the model writes must use interfaces the - extracted `AgentContract` actually lists (exact tool and argument names). Violations are caught - by validators, not by hoping. -2. **Rows are the agent's real use-cases and their branches.** Distinct outcomes are distinct rows. - No happy/edge/adversarial buckets, no infra rows, no forced personas. -3. **Sub-goals are shared.** A per-agent catalog is derived once; scenarios reference catalog names - so results roll up across scenarios (where does payment fail, across all 50 rows). -4. **Checkpoints assert the right arguments.** Asked for 11 PM, a 10 PM booking must fail. A check - is `deterministic: true` only when it carries an executable definition. -5. **Extensible by registry, not by edit.** New agent connections implement `AgentSource` (three - members) and register; a new environment adds one profile to `environments.py`; the LLM is a - two-method protocol with the model string as config. +Three moving parts: + +1. **`ScenarioMockServer`** serves one scenario's mock responses over HTTP, applies its state + updates, and records every tool call with the arguments the agent passed. Providers execute tools + by calling a public webhook, so the server needs a public URL - any HTTP tunnel works. +2. **`assistant_payload`** builds the assistant from the extracted contract: the same system prompt, + the same rules, the same tools with their real argument names, each pointing at that URL. +3. **`persona_from_record`** turns the scenario into the simulated caller, including the disclosure + rules that make an elicitation test mean anything. + +Set `ALK_SCENARIO` to a generated scenario file and the acceptance runner uses it instead of its +built-in persona: + +```bash +export ALK_SCENARIO=artifacts/scenarios/scenarios/.json +python oss/simulation-acceptance/run_voice_case.py 2.1.2 --dry-run # expect dry_run_passed +python oss/simulation-acceptance/run_voice_case.py 2.1.2 +``` + +Afterwards, grade it: + +```python +from fi.alk.generation.checks import evaluate_scenario + +evaluate_scenario( + record, + tool_calls=mock_server.log.snapshot(), + transcript_turns=[m["content"] for m in messages if m["role"] == "assistant"], + final_state=mock_server.final_state, +) +``` + +Tool calls come from the mock server rather than the provider's post-call artifact, because the +server saw every call as it happened while provider evidence can lag or be dropped. + +A run that reports `completed` means a conversation happened. It does not mean the agent behaved +correctly - that is what the checks above are for, and the two can disagree. + +## Configuration + +Credentials are environment variables only and are never written into a scenario. Generation needs +credentials for whichever model provider the `--model` string names. A live voice run additionally +needs the provider API key, an assistant id, a speech provider key, and a LiveKit URL with its key +and secret; the acceptance runner names each one it is missing. ## Extending | Want | Do | |---|---| -| New agent connection (Vapi, Retell, platform id) | implement `AgentSource`, `@register_source("vapi")` | -| New environment (browser, code, computer-use) | add one `EnvironmentProfile` to `environments.py` once `fi.simulate` carries a plugin for it; until then `--environment` refuses it by name | -| Different model | `--model vertex_ai/gemini-2.5-pro` or any litellm string; `LLMClient` is a protocol for non-litellm backends | -| Different budget | `--budget-usd 5` (hard stop, raises `BudgetExceeded`) | -| Stricter or looser QA | critic threshold and retry counts are `GenerationConfig` fields | - -## Boundaries honored - -This package lives on the studio side of the one-way rule: it imports `fi.simulate.simulation.models` -and never the reverse. It emits typed `Scenario` objects; running them is the simulation runtime's -job. Secrets are environment variables only (`GOOGLE_APPLICATION_CREDENTIALS`); nothing is written -into specs. +| A new agent connection | implement `AgentSource` (three members) and `@register_source("name")` | +| A new environment | add one `EnvironmentProfile` to `environments.py`, once the runtime carries a plugin for it | +| A new checkpoint kind | add it in four places: the vocabulary in `prompts.py`, `validators.py`, `checks.py`, and the emit mapping. All four, or the kind gets written and never graded | +| A different similarity measure | replace `similarity` in `dedup.py`; callers are unaffected | +| A different model backend | `LLMClient` is a two-method protocol | + +## Tests + +```bash +python -m pytest tests/test_generation_pipeline.py -q +``` + +Fully offline against a fake model, so they cost nothing. Each one encodes a defect that actually +occurred. From ea3b10123147c62cb7465ea06c8c52dcb8f62109 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 15 Aug 2026 11:13:09 +0530 Subject: [PATCH 52/55] feat(simulate): one command runs the scenario, serves its mocks, grades its checks and writes the trace --- oss/simulation-acceptance/run_voice_case.py | 137 ++++++++-- oss/simulation-acceptance/voice_cases.py | 6 +- src/fi/alk/generation/checks.py | 19 +- src/fi/alk/generation/live_run.py | 262 ++++++++++++++++++++ src/fi/alk/generation/simulate_bridge.py | 9 +- src/fi/alk/generation/vapi_live.py | 8 +- src/fi/simulate/simulation/voice_prompt.py | 3 +- tests/test_generation_pipeline.py | 55 ++++ 8 files changed, 472 insertions(+), 27 deletions(-) create mode 100644 src/fi/alk/generation/live_run.py diff --git a/oss/simulation-acceptance/run_voice_case.py b/oss/simulation-acceptance/run_voice_case.py index 5a4c947..be526dc 100644 --- a/oss/simulation-acceptance/run_voice_case.py +++ b/oss/simulation-acceptance/run_voice_case.py @@ -2,7 +2,9 @@ import argparse import asyncio +import contextlib import json +import os import subprocess import sys from pathlib import Path @@ -19,7 +21,32 @@ def main() -> int: parser.add_argument("case_id", choices=sorted(CASES)) parser.add_argument("--output-root", default="artifacts/simulation-acceptance") parser.add_argument("--dry-run", action="store_true") + # A generated scenario brings its own caller, its own mocked tools and its own checks. + # Supplying one turns this into a graded test rather than a transport check. + parser.add_argument( + "--scenario", + default=os.environ.get("ALK_SCENARIO", ""), + help="path to a generated scenario; drives the caller, the mocks and the checks", + ) + parser.add_argument( + "--agent", + default=os.environ.get("ALK_AGENT", "drive_thru"), + help="registered agent whose assistant serves the scenario's tools", + ) + parser.add_argument( + "--no-mock-tools", + action="store_true", + help="do not serve the scenario's tools; the agent's own tools answer instead", + ) + parser.add_argument( + "--no-grade", action="store_true", help="skip the scenario's checkpoints" + ) + parser.add_argument( + "--no-trace", action="store_true", help="skip writing the run trace" + ) args = parser.parse_args() + if args.scenario: + os.environ["ALK_SCENARIO"] = args.scenario case = CASES[args.case_id] missing = missing_env(case) @@ -81,33 +108,86 @@ def main() -> int: ) return 0 + record = None + if args.scenario: + with open(args.scenario, encoding="utf-8") as fh: + record = json.load(fh) + trigger = _start_livekit_outbound_trigger(case.case_id) + tools = contextlib.nullcontext(None) + if record is not None and not args.no_mock_tools: + from fi.alk.generation.live_run import tool_session + + tools = tool_session(record, agent=args.agent) try: - report = asyncio.run( - simulate.run_voice_simulation( - agent_definition=inputs.agent_definition, - livekit_runtime=inputs.livekit_runtime, - scenario=inputs.scenario, - simulator=inputs.simulator, - simulation_run_id=run_id, - record_audio=True, - recording_root=output_dir / "recordings", - recording_case_directory=output_dir / "recordings", - min_turn_messages=6, - max_seconds=inputs.max_seconds, - connect_timeout=60, - readiness_timeout=120, - cleanup_timeout=30, - conversation_direction=inputs.conversation_direction, - agent_first_silence_timeout_seconds=30, + with tools as tool_state: + report = asyncio.run( + simulate.run_voice_simulation( + agent_definition=inputs.agent_definition, + livekit_runtime=inputs.livekit_runtime, + scenario=inputs.scenario, + simulator=inputs.simulator, + simulation_run_id=run_id, + record_audio=True, + recording_root=output_dir / "recordings", + recording_case_directory=output_dir / "recordings", + min_turn_messages=6, + max_seconds=inputs.max_seconds, + connect_timeout=60, + readiness_timeout=120, + cleanup_timeout=30, + conversation_direction=inputs.conversation_direction, + agent_first_silence_timeout_seconds=30, + ) ) - ) - evaluation = evaluate_agent_report(report, attach=True) + evaluation = evaluate_agent_report(report, attach=True) + recorded_calls = tool_state.calls() if tool_state is not None else [] + final_state = tool_state.final_state if tool_state is not None else {} finally: _finish_livekit_outbound_trigger(trigger) report_path = output_dir / "report.json" report_path.write_text(report.model_dump_json(indent=2), encoding="utf-8") result = report.results[0] + + grading = None + trace_path = None + if record is not None: + messages = [ + m if isinstance(m, dict) else m.model_dump() + for m in (result.messages or []) + ] + # Provider evidence keeps only a count, so the mock server's record is the tool truth. + calls = recorded_calls or _provider_tool_calls(result) + if not args.no_grade: + from fi.alk.generation.live_run import grade + + grading = grade( + record, + messages=messages, + tool_calls=calls, + final_state=final_state, + ) + (output_dir / "checks.json").write_text( + json.dumps(grading, indent=2), encoding="utf-8" + ) + if not args.no_trace: + from fi.alk.generation.live_run import write_trace + + trace_path = write_trace( + str(output_dir), + record=record, + messages=messages, + tool_calls=calls, + final_state=final_state, + grading=grading, + metadata={ + "case_id": case.case_id, + "run_id": run_id, + "stop_reason": result.metadata.get("stop_reason"), + "status": result.metadata.get("status"), + "message_count": len(messages), + }, + ) status = str(result.metadata.get("status") or "unknown") print( json.dumps( @@ -119,18 +199,37 @@ def main() -> int: "failure": result.metadata.get("failure"), "evaluation_passed": evaluation.passed, "evaluation_score": evaluation.score, + **( + { + "scenario": grading.get("scenario_id"), + "checks_passed": grading.get("passed"), + "checks_failed": grading.get("failed"), + "checks_skipped": grading.get("skipped"), + "scenario_verdict": grading.get("verdict"), + } + if grading + else {} + ), + **({"trace": trace_path} if trace_path else {}), "manifest": str(manifest_path), "report": str(report_path), }, indent=2, ) ) + if grading and grading.get("verdict") == "fail": + return 1 return _result_exit_code( status=status, evaluation_passed=evaluation.passed, ) +def _provider_tool_calls(result) -> list[dict]: + raw = getattr(result, "tool_calls", None) or [] + return [r if isinstance(r, dict) else r.model_dump() for r in raw] + + def _result_exit_code(*, status: str, evaluation_passed: bool) -> int: return 0 if status == "completed" and evaluation_passed else 1 diff --git a/oss/simulation-acceptance/voice_cases.py b/oss/simulation-acceptance/voice_cases.py index b91fcec..30b099c 100644 --- a/oss/simulation-acceptance/voice_cases.py +++ b/oss/simulation-acceptance/voice_cases.py @@ -10,11 +10,11 @@ _GOOGLE_PROVIDERS = {"gemini", "google", "vertex"} _MODEL_DEFAULTS = { "llm": { - "gemini": "gemini-2.5-flash-lite", - "google": "gemini-2.5-flash-lite", + "gemini": "gemini-2.5-flash", + "google": "gemini-2.5-flash", "openai": "gpt-4o", "openai_compatible": "gpt-4o", - "vertex": "gemini-2.5-flash-lite", + "vertex": "gemini-2.5-flash", }, "stt": { "cartesia": "ink-2", diff --git a/src/fi/alk/generation/checks.py b/src/fi/alk/generation/checks.py index 4d67433..8df586c 100644 --- a/src/fi/alk/generation/checks.py +++ b/src/fi/alk/generation/checks.py @@ -14,6 +14,7 @@ from __future__ import annotations +import re from dataclasses import dataclass from typing import Any, Mapping, Sequence @@ -92,14 +93,28 @@ def _eval_state( return True, "state matched" +def _spoken(text: str) -> str: + """What a value sounds like, so typography does not decide a test. + + A voice agent says "quarter pounder with cheese combo"; the menu writes it + "Quarter Pounder(R) with Cheese Combo". Only the data value is being asserted, so casing, + symbols and spacing are normalised away before comparing. + """ + return re.sub(r"[^a-z0-9]+", " ", str(text).lower()).strip() + + def _eval_conveyed( definition: Mapping[str, Any], transcript_turns: Sequence[str] ) -> tuple[bool, str]: variants = [str(v) for v in definition.get("must_include_any") or []] - joined = "\n".join(str(turn) for turn in transcript_turns) + joined = _spoken("\n".join(str(turn) for turn in transcript_turns)) for variant in variants: - if variant and variant.lower() in joined.lower(): + if variant and _spoken(variant) in joined: return True, f"value {variant!r} conveyed" + forbidden = [str(v) for v in definition.get("forbidden") or []] + for value in forbidden: + if value and _spoken(value) in joined: + return False, f"the agent named {value!r}, which this scenario forbids" return False, f"none of {variants!r} appeared in the agent's turns" diff --git a/src/fi/alk/generation/live_run.py b/src/fi/alk/generation/live_run.py new file mode 100644 index 0000000..712873b --- /dev/null +++ b/src/fi/alk/generation/live_run.py @@ -0,0 +1,262 @@ +"""Everything a generated scenario needs around a live run, so one command does the whole job. + +A voice run on its own answers "did a conversation happen". A generated scenario also needs its +tools served, its tool calls recorded, its checkpoints graded, and a trace of what occurred. Doing +those as separate steps means they get skipped, so they are wired into the same invocation. + +Three pieces: + +- ``tool_session`` brings up the scenario's mock tools on a public URL and points the provider + assistant at them for the duration of one run, then puts the assistant back as it was. +- ``grade`` runs the scenario's own checkpoints against what the run actually produced. +- ``write_trace`` writes the timeline: every turn, every tool call with its arguments and response, + the world state it left behind, and the verdict per checkpoint. + +Nothing here decides how a scenario is generated or what a checkpoint means. It connects what +already exists. +""" + +from __future__ import annotations + +import contextlib +import json +import logging +import os +import re +import subprocess +import time +from dataclasses import dataclass, field +from typing import Any, Iterator, Mapping, Sequence + +from .checks import evaluate_scenario +from .vapi_live import ScenarioMockServer, assistant_payload, load_registry + +logger = logging.getLogger(__name__) + +_TUNNEL_TIMEOUT_SECONDS = 40 +_TUNNEL_URL = re.compile(r"https://[a-z0-9-]+\.trycloudflare\.com") + + +@dataclass +class ToolSession: + """The live mock-tool surface for one run.""" + + server: ScenarioMockServer + public_url: str + assistant_id: str = "" + started_at: float = field(default_factory=time.time) + + def calls(self) -> list[dict[str, Any]]: + return self.server.log.snapshot() + + @property + def final_state(self) -> dict[str, Any]: + return self.server.final_state + + +def _open_tunnel(port: int) -> tuple[str, subprocess.Popen | None]: + """A public URL for the local mock server. Providers call tools from their own cloud.""" + explicit = os.environ.get("ALK_TOOL_PUBLIC_URL", "").strip() + if explicit: + return explicit.rstrip("/"), None + process = subprocess.Popen( + ["cloudflared", "tunnel", "--url", f"http://localhost:{port}"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + ) + deadline = time.time() + _TUNNEL_TIMEOUT_SECONDS + assert process.stdout is not None + while time.time() < deadline: + line = process.stdout.readline() + if not line: + if process.poll() is not None: + break + continue + found = _TUNNEL_URL.search(line) + if found: + return found.group(0), process + process.terminate() + raise RuntimeError("could not obtain a public URL for the mock tool server") + + +def _patch_assistant_tools(assistant_id: str, contract: Any, public_url: str) -> None: + import httpx + + key = os.environ.get("VAPI_API_KEY", "").strip() + if not key: + raise RuntimeError( + "VAPI_API_KEY is required to point the assistant at the mock tools" + ) + payload = assistant_payload(contract, tool_base_url=public_url, name="") + # Cloudflare blocks some default client signatures on this API; httpx is what works. + response = httpx.patch( + f"https://api.vapi.ai/assistant/{assistant_id}", + headers={"authorization": f"Bearer {key}"}, + json={"model": payload["model"]}, + timeout=60, + ) + if response.status_code != 200: + raise RuntimeError( + f"could not wire the assistant to the mock tools: {response.status_code}" + ) + + +@contextlib.contextmanager +def tool_session( + record: Mapping[str, Any], *, agent: str = "drive_thru" +) -> Iterator[ToolSession]: + """Serve this scenario's mock tools for the length of one run.""" + from .contract import AgentContract + + registry = load_registry().get(agent) or {} + assistant_id = str( + registry.get("assistant_id") or os.environ.get("VAPI_ASSISTANT_ID", "") + ) + contract_path = str(registry.get("contract") or "") + server = ScenarioMockServer( + port=int(os.environ.get("ALK_TOOL_PORT", "8799")) + ).start() + server.bind(record) + tunnel: subprocess.Popen | None = None + try: + public_url, tunnel = _open_tunnel(server.port) + if assistant_id and contract_path and os.path.exists(contract_path): + with open(contract_path, encoding="utf-8") as fh: + contract = AgentContract.model_validate(json.load(fh)) + _patch_assistant_tools(assistant_id, contract, public_url) + else: + logger.warning( + "no registered assistant; tools will not be reachable by the provider" + ) + yield ToolSession( + server=server, public_url=public_url, assistant_id=assistant_id + ) + finally: + if tunnel is not None: + tunnel.terminate() + server.stop() + + +def agent_turns(messages: Sequence[Mapping[str, Any]]) -> list[str]: + return [ + str(m.get("content") or "") + for m in messages + if str(m.get("role")) == "assistant" and m.get("content") + ] + + +def grade( + record: Mapping[str, Any], + *, + messages: Sequence[Mapping[str, Any]], + tool_calls: Sequence[Mapping[str, Any]], + final_state: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """The scenario's own checkpoints against what the run produced.""" + results = evaluate_scenario( + record, + tool_calls=tool_calls, + transcript_turns=agent_turns(messages), + final_state=dict(final_state or {}), + ) + checks = [ + {"name": r.name, "kind": r.kind, "passed": r.passed, "reason": r.reason} + for r in results + ] + graded = [c for c in checks if c["passed"] is not None] + failed = [c for c in graded if c["passed"] is False] + # A conversation that never got going is not a verdict about the agent, so say so rather + # than reporting a confident failure. + user_turns = sum(1 for m in messages if str(m.get("role")) == "user") + return { + "scenario_id": record.get("id"), + "target_failure": record.get("target_failure"), + "checks": checks, + "passed": len(graded) - len(failed), + "failed": len(failed), + "skipped": len(checks) - len(graded), + "verdict": ( + "inconclusive" + if user_turns < 2 or not graded + else ("pass" if not failed else "fail") + ), + } + + +def write_trace( + output_dir: str, + *, + record: Mapping[str, Any], + messages: Sequence[Mapping[str, Any]], + tool_calls: Sequence[Mapping[str, Any]], + final_state: Mapping[str, Any] | None, + grading: Mapping[str, Any] | None, + metadata: Mapping[str, Any] | None = None, +) -> str: + """One file holding what happened, in the order it happened.""" + trace = { + "scenario": { + "id": record.get("id"), + "use_case": record.get("use_case"), + "situation": record.get("situation"), + "target_failure": record.get("target_failure"), + "why_it_matters": record.get("why_it_matters"), + "provenance": record.get("provenance"), + }, + "caller": { + "instruction": record.get("agent_input"), + "facts": record.get("facts"), + "objective": (record.get("expected_outcome") or {}).get("world_state"), + }, + "conversation": [ + {"role": m.get("role"), "content": m.get("content")} for m in messages + ], + "tool_calls": list(tool_calls), + "final_state": dict(final_state or {}), + "checks": (grading or {}).get("checks"), + "verdict": (grading or {}).get("verdict"), + "run": dict(metadata or {}), + } + path = os.path.join(output_dir, "trace.json") + with open(path, "w", encoding="utf-8") as fh: + json.dump(trace, fh, indent=2, ensure_ascii=False, default=str) + _write_readable_trace(output_dir, trace) + return path + + +def _write_readable_trace(output_dir: str, trace: Mapping[str, Any]) -> None: + scenario = trace["scenario"] + lines = [ + f"# {scenario.get('id')}", + "", + f"**Tests for:** {scenario.get('target_failure')}", + f"**Matters because:** {scenario.get('why_it_matters')}", + f"**Origin:** {(scenario.get('provenance') or {}).get('kind')}", + "", + "## What the caller was told", + "", + f"{trace['caller'].get('instruction')}", + "", + "## Conversation", + "", + ] + for turn in trace["conversation"]: + who = "caller" if turn.get("role") == "user" else "agent " + lines.append(f"- **{who}** {turn.get('content')}") + lines += ["", "## Tool calls", ""] + if trace["tool_calls"]: + for call in trace["tool_calls"]: + lines.append(f"- `{call.get('name')}` {json.dumps(call.get('arguments'))}") + else: + lines.append("- none") + lines += ["", "## Checks", ""] + for check in trace.get("checks") or []: + mark = {True: "PASS", False: "FAIL", None: "SKIP"}[check.get("passed")] + lines.append( + f"- **{mark}** `{check.get('name')}` ({check.get('kind')}): {check.get('reason')}" + ) + lines += ["", f"**Verdict: {trace.get('verdict')}**", ""] + with open(os.path.join(output_dir, "trace.md"), "w", encoding="utf-8") as fh: + fh.write("\n".join(lines)) diff --git a/src/fi/alk/generation/simulate_bridge.py b/src/fi/alk/generation/simulate_bridge.py index 2bc0311..8352c32 100644 --- a/src/fi/alk/generation/simulate_bridge.py +++ b/src/fi/alk/generation/simulate_bridge.py @@ -65,8 +65,15 @@ def disclosure_instructions(record: Mapping[str, Any]) -> str: if not facts: return "" lines = [ + "Your objective describes where you end up, not a shortcut for getting there. Play your " + "situation one step at a time, in the order it describes. Where it says you ask for " + "something and then change your mind, you must first ask for that thing and see it " + "confirmed as part of your order, answering any questions needed to complete it, before " + "you mention changing anything. Deciding against it while it is still being set up is the " + "one thing that ruins this call, because the part being tested never happens.", + "", "You know the following things. When you may say each one is part of who you are in this " - "call, and getting it wrong changes what is being tested." + "call, and getting it wrong changes what is being tested.", ] for disclosure, rule in _DISCLOSURE_RULES.items(): group = [ diff --git a/src/fi/alk/generation/vapi_live.py b/src/fi/alk/generation/vapi_live.py index a7a16d3..7eba110 100644 --- a/src/fi/alk/generation/vapi_live.py +++ b/src/fi/alk/generation/vapi_live.py @@ -72,7 +72,13 @@ def __init__(self, host: str = "127.0.0.1", port: int = 0) -> None: self.log = ToolCallLog() self._scenario: dict[str, Any] = {} self._state: dict[str, Any] = {} - server = HTTPServer((host, port), _make_handler(self)) + try: + server = HTTPServer((host, port), _make_handler(self)) + except OSError: + # A leftover server from an earlier run must not block this one; any free port + # works because the public URL is discovered after binding. + logger.warning("port %s busy, binding an ephemeral port instead", port) + server = HTTPServer((host, 0), _make_handler(self)) self._server = server self.port = server.server_address[1] self._thread = threading.Thread(target=server.serve_forever, daemon=True) diff --git a/src/fi/simulate/simulation/voice_prompt.py b/src/fi/simulate/simulation/voice_prompt.py index 0432a2e..86bd100 100644 --- a/src/fi/simulate/simulation/voice_prompt.py +++ b/src/fi/simulate/simulation/voice_prompt.py @@ -279,7 +279,8 @@ def format_voice_persona( rules_section += "10. **Never Break Character:** You are the PERSON described in 'Your Identity' with the situation in 'Your Current Situation.' You are NOT the person on the other end of the line. If you find yourself switching roles - taking on the other person's responsibilities, responding as if you have opposite information or authority, or reversing who called whom - STOP immediately. Stay in your role.\n" rules_section += "11. **Information Sharing:** Only share personal information when it's directly relevant to the conversation or when asked. Don't volunteer unnecessary details about yourself, your background, or your situation unless it naturally fits the context. Real people don't introduce themselves with their entire life story; be selective and purposeful with what you reveal.\n" rules_section += "12. **Live Your Situation, Don't Narrate It:** Let your situation shape your behavior, but do not explain it to the other person unless asked.\n" - rules_section += "13. **Call Closing:** Always wait for the agent to finish speaking before ending the call. Do not cut them off abruptly. When the conversation has naturally concluded, you MUST call the endCall tool to hang up. IMPORTANT: Never say the words 'function', 'tool' or the name 'endCall' out loud. Never say that you are ending the call. Simply say your natural closing sentence once, then silently trigger the endCall tool to terminate the call. Do not leave the call open. CRITICAL: If the agent says goodbye, bye, take care, or any closing phrase, you MUST respond with a brief, natural closing sentence (e.g. 'Alright, thanks, bye!') and then call endCall. Do NOT keep exchanging goodbyes. If you find yourself repeating goodbye phrases, call endCall right away.\n" + rules_section += "13. **Before You Close:** Check your objective first. If what you came for has not happened yet, do not end the call: say plainly what is still missing and give the agent a chance to put it right. If the agent claims something is done that you did not observe, or tells you there is nothing on your order when you asked for items, say so rather than accepting it. Only close once your objective is met, the agent has clearly refused or is unable, or you have raised the gap twice without progress.\n" + rules_section += "14. **Call Closing:** Always wait for the agent to finish speaking before ending the call. Do not cut them off abruptly. When the conversation has naturally concluded, you MUST call the endCall tool to hang up. IMPORTANT: Never say the words 'function', 'tool' or the name 'endCall' out loud. Never say that you are ending the call. Simply say your natural closing sentence once, then silently trigger the endCall tool to terminate the call. Do not leave the call open. CRITICAL: If the agent says goodbye, bye, take care, or any closing phrase, you MUST respond with a brief, natural closing sentence (e.g. 'Alright, thanks, bye!') and then call endCall. Do NOT keep exchanging goodbyes. If you find yourself repeating goodbye phrases, call endCall right away.\n" sections.append(rules_section) return "\n\n".join(sections) diff --git a/tests/test_generation_pipeline.py b/tests/test_generation_pipeline.py index a13f2fb..741e5e5 100644 --- a/tests/test_generation_pipeline.py +++ b/tests/test_generation_pipeline.py @@ -1357,3 +1357,58 @@ def test_assistant_is_built_from_the_contract(): ) system = payload["model"]["messages"][0]["content"] assert contract.hard_constraints[0] in system + + +def test_conveyed_ignores_typography_a_voice_agent_cannot_speak(): + """A menu writes "Big Mac(R) Combo"; an agent says "big mac combo". Only the value is asserted. + + A live run failed this check purely on a registered-trademark glyph, which no speech pipeline + will ever produce. + """ + from fi.alk.generation.checks import evaluate_checkpoint + + passed, _ = evaluate_checkpoint( + "conveyed", + {"must_include_any": ["Quarter Pounder® with Cheese Combo"]}, + transcript_turns=[ + "Got your large quarter pounder with cheese combo, and a large Coke." + ], + ) + assert passed is True + + # a genuinely absent value still fails + passed, _ = evaluate_checkpoint( + "conveyed", + {"must_include_any": ["Hamburger Happy Meal"]}, + transcript_turns=["Got your quarter pounder combo."], + ) + assert passed is False + + +def test_conveyed_fails_when_a_forbidden_value_is_named(): + """Removal scenarios assert what must NOT be said as well as what must.""" + from fi.alk.generation.checks import evaluate_checkpoint + + passed, reason = evaluate_checkpoint( + "conveyed", + { + "must_include_any": ["Quarter Pounder with Cheese Combo"], + "forbidden": ["Hamburger Happy Meal"], + }, + transcript_turns=["You have a hamburger happy meal on the order."], + ) + assert passed is False + assert "forbids" in reason + + +def test_the_caller_is_told_to_play_the_situation_in_order(): + """A caller that decides against an item in advance skips the step under test.""" + from fi.alk.generation.simulate_bridge import disclosure_instructions + + record = json.loads(json.dumps(SCENARIO)) + record["facts"] = [{"key": "drink", "value": "Coke", "disclosure": "on_request"}] + text = disclosure_instructions(record) + assert "one step at a time" in text + assert text.index("in the order it describes") < text.index( + "You know the following" + ) From 75655be36a86068f9f3c5f21de81e40796d8595d Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 15 Aug 2026 11:17:47 +0530 Subject: [PATCH 53/55] docs(generation): document the single command that runs, grades and traces a scenario --- src/fi/alk/generation/README.md | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/fi/alk/generation/README.md b/src/fi/alk/generation/README.md index 351d9ca..9e5314b 100644 --- a/src/fi/alk/generation/README.md +++ b/src/fi/alk/generation/README.md @@ -115,16 +115,24 @@ Three moving parts: 3. **`persona_from_record`** turns the scenario into the simulated caller, including the disclosure rules that make an elicitation test mean anything. -Set `ALK_SCENARIO` to a generated scenario file and the acceptance runner uses it instead of its -built-in persona: +One command runs the whole thing: it serves the scenario's mocks, points the assistant at them, +places the call, grades the checkpoints and writes the trace. ```bash -export ALK_SCENARIO=artifacts/scenarios/scenarios/.json -python oss/simulation-acceptance/run_voice_case.py 2.1.2 --dry-run # expect dry_run_passed -python oss/simulation-acceptance/run_voice_case.py 2.1.2 +python oss/simulation-acceptance/run_voice_case.py 2.1.2 --scenario .json --dry-run +python oss/simulation-acceptance/run_voice_case.py 2.1.2 --scenario .json ``` -Afterwards, grade it: +`2.1.2` is inbound, where the caller speaks first; `2.2.2` is outbound, where the agent does. Both +work with a generated scenario. Without `--scenario` the command behaves exactly as before; +`--no-mock-tools`, `--no-grade` and `--no-trace` turn off each added part. + +Alongside the usual `manifest.json`, `report.json` and `recordings/`, the run writes `checks.json` +(each checkpoint and its verdict) and `trace.json` / `trace.md` (the turns, the tool calls with +their arguments, the resulting state, and the verdict). The exit code is non-zero when the +scenario's own checks fail, even if the conversation completed, so it can gate CI directly. + +To grade a run yourself: ```python from fi.alk.generation.checks import evaluate_scenario From 66fb220527bc2e84c4971aee272978744e8cd582 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 15 Aug 2026 12:54:51 +0530 Subject: [PATCH 54/55] docs(generation): state the outbound path has had one run, not parity with inbound --- src/fi/alk/generation/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/fi/alk/generation/README.md b/src/fi/alk/generation/README.md index 9e5314b..57df691 100644 --- a/src/fi/alk/generation/README.md +++ b/src/fi/alk/generation/README.md @@ -123,9 +123,10 @@ python oss/simulation-acceptance/run_voice_case.py 2.1.2 --scenario .j python oss/simulation-acceptance/run_voice_case.py 2.1.2 --scenario .json ``` -`2.1.2` is inbound, where the caller speaks first; `2.2.2` is outbound, where the agent does. Both -work with a generated scenario. Without `--scenario` the command behaves exactly as before; -`--no-mock-tools`, `--no-grade` and `--no-trace` turn off each added part. +`2.1.2` is inbound, where the caller speaks first; `2.2.2` is outbound, where the agent does. +Inbound is the path that has been exercised; outbound has had a single successful run. Without +`--scenario` the command behaves exactly as before; `--no-mock-tools`, `--no-grade` and +`--no-trace` turn off each added part. Alongside the usual `manifest.json`, `report.json` and `recordings/`, the run writes `checks.json` (each checkpoint and its verdict) and `trace.json` / `trace.md` (the turns, the tool calls with From 9ebd13cfb5287ac8e184ecc5e9761caec8bf5ecf Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 15 Aug 2026 13:31:35 +0530 Subject: [PATCH 55/55] fix(generation): derive the assistant greeting from the contract instead of hardcoding one agent --- oss/simulation-acceptance/run_voice_case.py | 2 +- src/fi/alk/generation/live_run.py | 8 ++++++-- src/fi/alk/generation/vapi_live.py | 11 +++++++++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/oss/simulation-acceptance/run_voice_case.py b/oss/simulation-acceptance/run_voice_case.py index be526dc..35e3c3c 100644 --- a/oss/simulation-acceptance/run_voice_case.py +++ b/oss/simulation-acceptance/run_voice_case.py @@ -30,7 +30,7 @@ def main() -> int: ) parser.add_argument( "--agent", - default=os.environ.get("ALK_AGENT", "drive_thru"), + default=os.environ.get("ALK_AGENT", ""), help="registered agent whose assistant serves the scenario's tools", ) parser.add_argument( diff --git a/src/fi/alk/generation/live_run.py b/src/fi/alk/generation/live_run.py index 712873b..be32763 100644 --- a/src/fi/alk/generation/live_run.py +++ b/src/fi/alk/generation/live_run.py @@ -105,12 +105,16 @@ def _patch_assistant_tools(assistant_id: str, contract: Any, public_url: str) -> @contextlib.contextmanager def tool_session( - record: Mapping[str, Any], *, agent: str = "drive_thru" + record: Mapping[str, Any], *, agent: str = "" ) -> Iterator[ToolSession]: """Serve this scenario's mock tools for the length of one run.""" from .contract import AgentContract - registry = load_registry().get(agent) or {} + all_agents = load_registry() + # With one registered agent there is nothing to choose between, so naming it is optional. + if not agent and len(all_agents) == 1: + agent = next(iter(all_agents)) + registry = all_agents.get(agent) or {} assistant_id = str( registry.get("assistant_id") or os.environ.get("VAPI_ASSISTANT_ID", "") ) diff --git a/src/fi/alk/generation/vapi_live.py b/src/fi/alk/generation/vapi_live.py index 7eba110..348d39a 100644 --- a/src/fi/alk/generation/vapi_live.py +++ b/src/fi/alk/generation/vapi_live.py @@ -185,7 +185,11 @@ def _tool_calls(payload: Mapping[str, Any]) -> list[tuple[str, str, dict[str, An def assistant_payload( - contract: AgentContract, *, tool_base_url: str, name: str + contract: AgentContract, + *, + tool_base_url: str, + name: str, + first_message: str = "", ) -> dict[str, Any]: """A Vapi assistant that behaves like the agent the contract describes. @@ -226,7 +230,10 @@ def assistant_payload( ) return { "name": name, - "firstMessage": "Welcome to the drive thru, what can I get for you?", + # The agent needs an opening line for outbound calls, where it speaks first. Taken from + # the contract so this stays agent-agnostic; override it when the agent has its own. + "firstMessage": first_message + or f"Hello, this is {contract.agent}. How can I help?", "model": { "provider": "openai", "model": "gpt-4o",