From 3e2344445e200d441a0ccc455073c0413fc98b34 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Sat, 4 Jul 2026 21:11:28 +0200 Subject: [PATCH 1/3] Complete hard-deprecation codemod migrations --- apps/cli/src/templates/.agentv/targets.yaml | 58 ++-- .../docs/docs/next/evaluation/eval-cases.mdx | 39 ++- .../docs/docs/next/evaluation/examples.mdx | 25 +- .../docs/docs/next/graders/llm-graders.mdx | 35 ++- .../docs/docs/next/targets/cli-provider.mdx | 12 +- .../docs/docs/next/targets/coding-agents.mdx | 12 +- .../docs/docs/next/targets/configuration.mdx | 14 +- .../docs/next/targets/custom-providers.mdx | 6 +- .../docs/docs/next/targets/llm-providers.mdx | 26 +- .../content/docs/docs/next/targets/retry.mdx | 6 +- examples/contract/.agentv/targets.yaml | 4 +- examples/features/README.md | 2 +- .../execution-metrics/.agentv/targets.yaml | 9 +- examples/features/preprocessors/README.md | 10 +- .../features/preprocessors/evals/suite.yaml | 27 +- .../readme-quickstart/.agentv/tests.yaml | 3 +- .../features/readme-quickstart/targets.yaml | 6 +- .../evals/bug-fixes.eval.yaml | 2 +- .../src/evaluation/loaders/config-loader.ts | 5 + .../src/evaluation/loaders/grader-parser.ts | 5 + .../core/src/evaluation/providers/targets.ts | 89 +++--- .../evaluation/validation/eval-file.schema.ts | 24 +- .../evaluation/validation/eval-validator.ts | 45 ++- .../validation/targets-validator.ts | 48 ++- packages/core/src/evaluation/yaml-parser.ts | 28 +- .../evaluation/loaders/config-loader.test.ts | 4 +- .../evaluation/preprocessors-yaml.test.ts | 48 +-- .../evaluation/providers/cli-schema.test.ts | 8 +- .../providers/fallback-targets.test.ts | 12 +- .../providers/targets-cwd-fallback.test.ts | 8 +- .../test/evaluation/providers/targets.test.ts | 201 +++++++------ .../evaluation/source-traceability.test.ts | 9 - .../validation/eval-file-schema.test.ts | 39 +++ .../validation/eval-validator.test.ts | 72 ++++- .../validation/targets-validator.test.ts | 76 +++-- scripts/migrate-hard-deprecations.test.ts | 127 ++++++++ ...-input.ts => migrate-hard-deprecations.ts} | 284 +++++++++++++++++- skills-data/agentv-eval-writer/SKILL.md | 2 +- .../references/eval.schema.json | 26 +- 39 files changed, 1044 insertions(+), 412 deletions(-) create mode 100644 scripts/migrate-hard-deprecations.test.ts rename scripts/{migrate-direct-input.ts => migrate-hard-deprecations.ts} (56%) diff --git a/apps/cli/src/templates/.agentv/targets.yaml b/apps/cli/src/templates/.agentv/targets.yaml index 0d70548ce..a2796e86b 100644 --- a/apps/cli/src/templates/.agentv/targets.yaml +++ b/apps/cli/src/templates/.agentv/targets.yaml @@ -6,25 +6,25 @@ targets: - id: default provider: azure - endpoint: ${{ AZURE_OPENAI_ENDPOINT }} - api_key: ${{ AZURE_OPENAI_API_KEY }} - model: ${{ AZURE_DEPLOYMENT_NAME }} - # version: ${{ AZURE_OPENAI_API_VERSION }} # Optional: uncomment to override default (v1) + endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}" + api_key: "{{ env.AZURE_OPENAI_API_KEY }}" + model: "{{ env.AZURE_DEPLOYMENT_NAME }}" + # version: "{{ env.AZURE_OPENAI_API_VERSION }}" # Optional: uncomment to override default (v1) - id: codex provider: codex grader_target: azure-llm # Uses the Codex CLI (defaults to `codex` on PATH) - # executable: ${{ CODEX_CLI_PATH }} # Optional: override executable path + # executable: "{{ env.CODEX_CLI_PATH }}" # Optional: override executable path # args: # Optional additional CLI arguments # - --profile - # - ${{ CODEX_PROFILE }} + # - "{{ env.CODEX_PROFILE }}" # - --model - # - ${{ CODEX_MODEL }} + # - "{{ env.CODEX_MODEL }}" # - --ask-for-approval - # - ${{ CODEX_APPROVAL_PRESET }} - cwd: ${{ CODEX_WORKSPACE_DIR }} # Where scratch workspaces are created - log_dir: ${{ CODEX_LOG_DIR }} # Optional: where Codex CLI stream logs are stored (defaults to ./.agentv/logs/codex) + # - "{{ env.CODEX_APPROVAL_PRESET }}" + cwd: "{{ env.CODEX_WORKSPACE_DIR }}" # Where scratch workspaces are created + log_dir: "{{ env.CODEX_LOG_DIR }}" # Optional: where Codex CLI stream logs are stored (defaults to ./.agentv/logs/codex) stream_log: raw # Optional: 'summary' for consolidated logs or 'raw' for per-event logs # Claude - Anthropic's Claude Agent SDK @@ -33,24 +33,24 @@ targets: grader_target: azure-llm # Uses the @anthropic-ai/claude-agent-sdk # model: claude-sonnet-4-20250514 # Optional: override model - # cwd: ${{ CLAUDE_WORKSPACE_DIR }} # Optional: working directory (defaults to process.cwd()) + # cwd: "{{ env.CLAUDE_WORKSPACE_DIR }}" # Optional: working directory (defaults to process.cwd()) # max_turns: 50 # Optional: max conversation turns # max_budget_usd: 5.0 # Optional: max cost budget in USD - # log_dir: ${{ CLAUDE_LOG_DIR }} # Optional: where stream logs are stored (defaults to ./.agentv/logs/claude) + # log_dir: "{{ env.CLAUDE_LOG_DIR }}" # Optional: where stream logs are stored (defaults to ./.agentv/logs/claude) stream_log: raw # Optional: 'summary' for consolidated logs or 'raw' for per-event logs # system_prompt: optional override (default instructs agent to include code in response) - id: azure-llm provider: azure - endpoint: ${{ AZURE_OPENAI_ENDPOINT }} - api_key: ${{ AZURE_OPENAI_API_KEY }} - model: ${{ AZURE_DEPLOYMENT_NAME }} - version: ${{ AZURE_OPENAI_API_VERSION }} + endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}" + api_key: "{{ env.AZURE_OPENAI_API_KEY }}" + model: "{{ env.AZURE_DEPLOYMENT_NAME }}" + version: "{{ env.AZURE_OPENAI_API_VERSION }}" - id: gemini-llm provider: gemini - api_key: ${{ GOOGLE_GENERATIVE_AI_API_KEY }} - model: ${{ GEMINI_MODEL_NAME }} + api_key: "{{ env.GOOGLE_GENERATIVE_AI_API_KEY }}" + model: "{{ env.GEMINI_MODEL_NAME }}" - id: local_cli provider: cli @@ -61,37 +61,37 @@ targets: # Format for each file in {FILES}. {path} and {basename} are automatically shell-escaped, so no quotes needed files_format: --file {path} # Optional working directory resolved from .env - cwd: ${{ CLI_EVALS_DIR }} + cwd: "{{ env.CLI_EVALS_DIR }}" healthcheck: command: uv run ./mock_cli.py --healthcheck - # ── MiMo (Xiaomi) via OpenRouter ─────────────────────────────────── + # -- MiMo (Xiaomi) via OpenRouter ----------------------------------- # All MiMo models are available through OpenRouter with OpenAI-compatible API. # See https://openrouter.ai/xiaomi/mimo-v2.5-pro for pricing and limits. # # Models: - # mimo-v2.5-pro — 1M context, 131K output, flagship - # mimo-v2-pro — 1M context, ~131K output - # mimo-v2.5 — 1M context, ~131K output, multimodal - # mimo-v2-flash — 262K context, 65K output, fast MoE (open-source) - # mimo-v2-omni — 262K context, 65K output, omni-modal + # mimo-v2.5-pro - 1M context, 131K output, flagship + # mimo-v2-pro - 1M context, ~131K output + # mimo-v2.5 - 1M context, ~131K output, multimodal + # mimo-v2-flash - 262K context, 65K output, fast MoE (open-source) + # mimo-v2-omni - 262K context, 65K output, omni-modal - id: mimo provider: openrouter - api_key: ${{ OPENROUTER_API_KEY }} + api_key: "{{ env.OPENROUTER_API_KEY }}" model: xiaomi/mimo-v2.5-pro - id: mimo-flash provider: openrouter - api_key: ${{ OPENROUTER_API_KEY }} + api_key: "{{ env.OPENROUTER_API_KEY }}" model: xiaomi/mimo-v2-flash - # ── Direct provider (not through OpenRouter) ─────────────────────── + # -- Direct provider (not through OpenRouter) ----------------------- # For providers not in pi-ai's model registry, set max_output_tokens # to match your model's actual output limit. Without this, the default # is 16K which may cap output below the model's capability. # - name: mimo-direct # provider: openai # base_url: https://token-plan-sgp.xiaomimimo.com/v1 - # api_key: ${{ XIAOMI_MIMO_API_KEY }} + # api_key: "{{ env.XIAOMI_MIMO_API_KEY }}" # model: xiaomi/mimo-v2.5-pro # max_output_tokens: 131072 diff --git a/apps/web/src/content/docs/docs/next/evaluation/eval-cases.mdx b/apps/web/src/content/docs/docs/next/evaluation/eval-cases.mdx index 7e30b6060..46f881155 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/eval-cases.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/eval-cases.mdx @@ -404,17 +404,29 @@ tests: - Assistant correctly explains the bug and proposes a fix ``` -Suite-level `preprocessors` apply to explicit LLM graders. That matters when the -agent output is a `ContentFile` block rather than plain text: +Suite-level transforms apply before graders see the target output. That matters +when the agent output is a `ContentFile` block rather than plain text: ```yaml -preprocessors: - - type: xlsx - command: ["bun", "run", "scripts/preprocessors/xlsx-to-csv.ts"] +default_test: + options: + transform: >- + return (() => { + const file = Array.isArray(output) + ? output.find((block) => block?.type === "file" && block.path?.endsWith(".xlsx")) + : undefined; + if (!file) return output; + const result = Bun.spawnSync(["bun", "run", "scripts/preprocessors/xlsx-to-csv.ts"], { + stdin: JSON.stringify({ path: file.path, media_type: file.media_type }) + }); + if (result.exitCode !== 0) throw new Error(new TextDecoder().decode(result.stderr).trim()); + return new TextDecoder().decode(result.stdout).trim(); + })() tests: - id: spreadsheet-eval - input: Generate the spreadsheet report + vars: + input: Generate the spreadsheet report assert: - Output includes the revenue rows ``` @@ -466,22 +478,17 @@ tests: value: "fix" ``` -When you need a custom file conversion for only one grader, add `preprocessors` directly to that grader: +When you need a custom output conversion for only one grader, add `transform` directly to that grader: ```yaml -preprocessors: - - type: xlsx - command: ["bun", "run", "scripts/preprocessors/xlsx-to-csv.ts"] - tests: - id: mixed-eval - input: "Debug this function..." + vars: + input: "Debug this function..." assert: - Response is helpful and mentions the fix - - type: llm-rubric # use explicit form for custom preprocessors - preprocessors: - - type: xlsx - command: ["bun", "run", "scripts/preprocessors/xlsx-to-json.ts"] + - type: llm-rubric + transform: file://scripts/transforms/xlsx-to-json.ts - type: contains value: "fix" ``` diff --git a/apps/web/src/content/docs/docs/next/evaluation/examples.mdx b/apps/web/src/content/docs/docs/next/evaluation/examples.mdx index 465e9f2db..e3a050bf4 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/examples.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/examples.mdx @@ -101,22 +101,33 @@ tests: } ``` -## File Output Preprocessing +## File Output Transform Convert a binary file output into text before the `llm-rubric` sees it: ```yaml -description: Grade spreadsheet output via a preprocessor - -preprocessors: - - type: xlsx - command: ["bun", "run", "../scripts/preprocessors/xlsx-to-csv.ts"] +description: Grade spreadsheet output via a transform target: file_output +default_test: + options: + transform: >- + return (() => { + const file = Array.isArray(output) + ? output.find((block) => block?.type === "file" && block.path?.endsWith(".xlsx")) + : undefined; + if (!file) return output; + const result = Bun.spawnSync(["bun", "run", "../scripts/preprocessors/xlsx-to-csv.ts"], { + stdin: JSON.stringify({ path: file.path, media_type: file.media_type }) + }); + if (result.exitCode !== 0) throw new Error(new TextDecoder().decode(result.stderr).trim()); + return new TextDecoder().decode(result.stdout).trim(); + })() tests: - id: spreadsheet-output - input: Generate the spreadsheet report + vars: + input: Generate the spreadsheet report criteria: The extracted spreadsheet content includes the revenue rows assert: - Output contains the transformed spreadsheet text including the revenue rows diff --git a/apps/web/src/content/docs/docs/next/graders/llm-graders.mdx b/apps/web/src/content/docs/docs/next/graders/llm-graders.mdx index 5cdb1f75c..7b870bf2e 100644 --- a/apps/web/src/content/docs/docs/next/graders/llm-graders.mdx +++ b/apps/web/src/content/docs/docs/next/graders/llm-graders.mdx @@ -192,20 +192,32 @@ assert: The `config` object is available as `ctx.config` inside the template function. -## Preprocessing File Outputs +## Transforming File Outputs -If an agent returns a `ContentFile` block instead of plain text, you can preprocess that file into text before `llm-rubric` builds the candidate prompt. +If an agent returns a `ContentFile` block instead of plain text, use `transform` to convert that file into text before `llm-rubric` builds the candidate prompt. -AgentV always tries a default UTF-8 text read first. That is enough for text-based formats such as CSV, JSON, SQL, Markdown, YAML, HTML, XML, and plain text. For binary formats such as `.xlsx`, `.pdf`, or `.docx`, add a preprocessor command: +AgentV always tries a default UTF-8 text read first. That is enough for text-based formats such as CSV, JSON, SQL, Markdown, YAML, HTML, XML, and plain text. For binary formats such as `.xlsx`, `.pdf`, or `.docx`, add a transform: ```yaml -preprocessors: - - type: xlsx - command: ["bun", "run", "scripts/preprocessors/xlsx-to-csv.ts"] +default_test: + options: + transform: >- + return (() => { + const file = Array.isArray(output) + ? output.find((block) => block?.type === "file" && block.path?.endsWith(".xlsx")) + : undefined; + if (!file) return output; + const result = Bun.spawnSync(["bun", "run", "scripts/preprocessors/xlsx-to-csv.ts"], { + stdin: JSON.stringify({ path: file.path, media_type: file.media_type }) + }); + if (result.exitCode !== 0) throw new Error(new TextDecoder().decode(result.stderr).trim()); + return new TextDecoder().decode(result.stdout).trim(); + })() tests: - id: spreadsheet-output - input: Generate the spreadsheet report + vars: + input: Generate the spreadsheet report assert: - Output includes the revenue rows - name: spreadsheet-check @@ -216,15 +228,14 @@ tests: {{ output }} ``` -`type` accepts either a short alias such as `xlsx` or a full MIME type such as `application/vnd.openxmlformats-officedocument.spreadsheetml.sheet`. - Resolution order: -- per-grader `preprocessors` override suite-level entries -- if no preprocessor matches, AgentV falls back to a UTF-8 text read +- `tests[].options.transform` overrides `default_test.options.transform` +- assertion-level `transform` applies to that grader only +- if no transform is configured, AgentV falls back to a UTF-8 text read - if the fallback read looks binary or invalid, the grader receives a warning note instead of failing the test run -See [`examples/features/preprocessors/`](../../../../../examples/features/preprocessors/) for a runnable example with a file-producing target and a custom preprocessor script. +See [`examples/features/preprocessors/`](../../../../../examples/features/preprocessors/) for a runnable example with a file-producing target and a spreadsheet conversion script. ## Available Context Fields diff --git a/apps/web/src/content/docs/docs/next/targets/cli-provider.mdx b/apps/web/src/content/docs/docs/next/targets/cli-provider.mdx index 626d3000e..9f315291e 100644 --- a/apps/web/src/content/docs/docs/next/targets/cli-provider.mdx +++ b/apps/web/src/content/docs/docs/next/targets/cli-provider.mdx @@ -25,9 +25,9 @@ graders: - id: azure-grader provider: azure config: - endpoint: ${{ AZURE_OPENAI_ENDPOINT }} - api_key: ${{ AZURE_OPENAI_API_KEY }} - model: ${{ AZURE_DEPLOYMENT_NAME }} + endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}" + api_key: "{{ env.AZURE_OPENAI_API_KEY }}" + model: "{{ env.AZURE_DEPLOYMENT_NAME }}" defaults: target: my-agent @@ -137,9 +137,9 @@ graders: - id: azure-grader provider: azure config: - endpoint: ${{ AZURE_OPENAI_ENDPOINT }} - api_key: ${{ AZURE_OPENAI_API_KEY }} - model: ${{ AZURE_DEPLOYMENT_NAME }} + endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}" + api_key: "{{ env.AZURE_OPENAI_API_KEY }}" + model: "{{ env.AZURE_DEPLOYMENT_NAME }}" defaults: grader: azure-grader diff --git a/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx b/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx index 971bcfdb3..4b2822ed8 100644 --- a/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx +++ b/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx @@ -90,7 +90,7 @@ targets: target: /results access: rw secrets: - OPENAI_API_KEY: ${{ OPENAI_API_KEY }} + OPENAI_API_KEY: "{{ env.OPENAI_API_KEY }}" config: command: ["codex", "exec", "--json"] model: gpt-5-codex @@ -144,7 +144,7 @@ targets: runtime: host config: command: ["codex-eng", "exec", "--json"] - model: ${{ CODEX_MODEL }} + model: "{{ env.CODEX_MODEL }}" ``` Use `codex-sdk` only when you intentionally want the Codex SDK path: @@ -190,8 +190,8 @@ targets: config: command: ["pi"] subprovider: openrouter - model: ${{ OPENROUTER_MODEL }} - api_key: ${{ OPENROUTER_API_KEY }} + model: "{{ env.OPENROUTER_MODEL }}" + api_key: "{{ env.OPENROUTER_API_KEY }}" ``` Use `pi-sdk` only when you intentionally want the SDK path: @@ -266,8 +266,8 @@ targets: config: command: ["copilot"] subprovider: openai - base_url: ${{ OPENAI_ENDPOINT }} - api_key: ${{ OPENAI_API_KEY }} + base_url: "{{ env.OPENAI_ENDPOINT }}" + api_key: "{{ env.OPENAI_API_KEY }}" api_format: responses ``` diff --git a/apps/web/src/content/docs/docs/next/targets/configuration.mdx b/apps/web/src/content/docs/docs/next/targets/configuration.mdx index 21468005a..1ae98c4c7 100644 --- a/apps/web/src/content/docs/docs/next/targets/configuration.mdx +++ b/apps/web/src/content/docs/docs/next/targets/configuration.mdx @@ -32,9 +32,9 @@ targets: runtime: host config: api_format: chat - base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }} - api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }} - model: ${{ LOCAL_OPENAI_PROXY_MODEL }} + base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" + api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" + model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" - id: codex-local provider: codex-app-server @@ -98,7 +98,7 @@ targets: env: AGENTV_RESULT_DIR: /results secrets: - OPENAI_API_KEY: ${{ OPENAI_API_KEY }} + OPENAI_API_KEY: "{{ env.OPENAI_API_KEY }}" config: command: ["codex", "exec", "--json"] timeout_seconds: 300 @@ -176,7 +176,7 @@ by a separate team; keep fields inline when that is easier to read. ## Environment Variables -Use `${{ VARIABLE_NAME }}` syntax to reference values from your environment. AgentV reads +Use `{{ env.VARIABLE_NAME }}` syntax to reference values from your environment. AgentV reads exported process environment variables directly, and it also loads `.env` files from the eval directory hierarchy when present: @@ -186,8 +186,8 @@ targets: provider: anthropic runtime: host config: - api_key: ${{ ANTHROPIC_API_KEY }} - model: ${{ ANTHROPIC_MODEL }} + api_key: "{{ env.ANTHROPIC_API_KEY }}" + model: "{{ env.ANTHROPIC_MODEL }}" ``` This keeps secrets out of version-controlled files and avoids requiring a CI step that rewrites diff --git a/apps/web/src/content/docs/docs/next/targets/custom-providers.mdx b/apps/web/src/content/docs/docs/next/targets/custom-providers.mdx index 77c5b9d95..1d90fc4a8 100644 --- a/apps/web/src/content/docs/docs/next/targets/custom-providers.mdx +++ b/apps/web/src/content/docs/docs/next/targets/custom-providers.mdx @@ -188,9 +188,9 @@ graders: - id: azure-grader provider: azure config: - endpoint: ${{ AZURE_OPENAI_ENDPOINT }} - api_key: ${{ AZURE_OPENAI_API_KEY }} - model: ${{ AZURE_DEPLOYMENT_NAME }} + endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}" + api_key: "{{ env.AZURE_OPENAI_API_KEY }}" + model: "{{ env.AZURE_DEPLOYMENT_NAME }}" defaults: grader: azure-grader diff --git a/apps/web/src/content/docs/docs/next/targets/llm-providers.mdx b/apps/web/src/content/docs/docs/next/targets/llm-providers.mdx index 5752a96a4..cdd43f262 100644 --- a/apps/web/src/content/docs/docs/next/targets/llm-providers.mdx +++ b/apps/web/src/content/docs/docs/next/targets/llm-providers.mdx @@ -16,7 +16,7 @@ targets: provider: openai runtime: host config: - api_key: ${{ OPENAI_API_KEY }} + api_key: "{{ env.OPENAI_API_KEY }}" model: gpt-4o ``` @@ -47,8 +47,8 @@ targets: config: api_format: chat base_url: https://models.github.ai/inference/v1 - api_key: ${{ GH_MODELS_TOKEN }} - model: ${{ GH_MODELS_MODEL }} + api_key: "{{ env.GH_MODELS_TOKEN }}" + model: "{{ env.GH_MODELS_MODEL }}" # Opt in to Responses API for api.openai.com - id: openai-responses @@ -56,7 +56,7 @@ targets: runtime: host config: api_format: responses - api_key: ${{ OPENAI_API_KEY }} + api_key: "{{ env.OPENAI_API_KEY }}" model: gpt-4o ``` @@ -71,9 +71,9 @@ graders: provider: openai config: api_format: chat - base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }} - api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }} - model: ${{ LOCAL_OPENAI_PROXY_MODEL }} + base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" + api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" + model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" ``` If the local proxy does not require authentication, set @@ -89,9 +89,9 @@ targets: provider: azure runtime: host config: - endpoint: ${{ AZURE_OPENAI_ENDPOINT }} - api_key: ${{ AZURE_OPENAI_API_KEY }} - model: ${{ AZURE_DEPLOYMENT_NAME }} + endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}" + api_key: "{{ env.AZURE_OPENAI_API_KEY }}" + model: "{{ env.AZURE_DEPLOYMENT_NAME }}" ``` | Field | Required | Description | @@ -114,7 +114,7 @@ targets: runtime: host config: base_url: https://.openai.azure.com/openai/deployments/ - api_key: ${{ AZURE_OPENAI_API_KEY }} + api_key: "{{ env.AZURE_OPENAI_API_KEY }}" model: api_format: chat ``` @@ -129,7 +129,7 @@ targets: provider: anthropic runtime: host config: - api_key: ${{ ANTHROPIC_API_KEY }} + api_key: "{{ env.ANTHROPIC_API_KEY }}" model: claude-sonnet-4-20250514 ``` @@ -146,7 +146,7 @@ targets: provider: gemini runtime: host config: - api_key: ${{ GEMINI_API_KEY }} + api_key: "{{ env.GEMINI_API_KEY }}" model: gemini-2.0-flash ``` diff --git a/apps/web/src/content/docs/docs/next/targets/retry.mdx b/apps/web/src/content/docs/docs/next/targets/retry.mdx index 4442cc079..d36f78e93 100644 --- a/apps/web/src/content/docs/docs/next/targets/retry.mdx +++ b/apps/web/src/content/docs/docs/next/targets/retry.mdx @@ -18,9 +18,9 @@ targets: provider: azure runtime: host config: - endpoint: ${{ AZURE_OPENAI_ENDPOINT }} - api_key: ${{ AZURE_OPENAI_API_KEY }} - model: ${{ AZURE_DEPLOYMENT_NAME }} + endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}" + api_key: "{{ env.AZURE_OPENAI_API_KEY }}" + model: "{{ env.AZURE_DEPLOYMENT_NAME }}" max_retries: 5 retry_initial_delay_ms: 2000 retry_max_delay_ms: 120000 diff --git a/examples/contract/.agentv/targets.yaml b/examples/contract/.agentv/targets.yaml index f97d07542..69305e1db 100644 --- a/examples/contract/.agentv/targets.yaml +++ b/examples/contract/.agentv/targets.yaml @@ -3,7 +3,7 @@ targets: provider: openai api_format: chat base_url: https://models.github.ai/inference - api_key: ${{ GH_MODELS_TOKEN }} - model: ${{ CONTRACT_EVAL_MODEL }} + api_key: "{{ env.GH_MODELS_TOKEN }}" + model: "{{ env.CONTRACT_EVAL_MODEL }}" temperature: 0 max_output_tokens: 512 diff --git a/examples/features/README.md b/examples/features/README.md index 6ac10648a..8921a9e48 100644 --- a/examples/features/README.md +++ b/examples/features/README.md @@ -22,7 +22,7 @@ Focused examples for specific AgentV capabilities. Find your use case below, the | [assert-set](assert-set/) | Safety gate and weighted assertion groups | | [threshold-grader](threshold-grader/) | Pass a test if a configurable percentage of sub-graders pass | | [multi-turn-conversation](multi-turn-conversation/) | Grade a multi-turn conversation with per-turn score breakdowns | -| [preprocessors](preprocessors/) | Convert `ContentFile` outputs into grader-readable text before `llm-rubric` runs | +| [preprocessors](preprocessors/) | Convert `ContentFile` outputs with `default_test.options.transform` before `llm-rubric` runs | --- diff --git a/examples/features/execution-metrics/.agentv/targets.yaml b/examples/features/execution-metrics/.agentv/targets.yaml index e9ffc6bc1..ff5d7bee1 100644 --- a/examples/features/execution-metrics/.agentv/targets.yaml +++ b/examples/features/execution-metrics/.agentv/targets.yaml @@ -1,11 +1,10 @@ targets: - id: azure-llm provider: azure - endpoint: ${{ AZURE_OPENAI_ENDPOINT }} - api_key: ${{ AZURE_OPENAI_API_KEY }} - model: ${{ AZURE_DEPLOYMENT_NAME }} - version: ${{ AZURE_OPENAI_API_VERSION }} - + endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}" + api_key: "{{ env.AZURE_OPENAI_API_KEY }}" + model: "{{ env.AZURE_DEPLOYMENT_NAME }}" + version: "{{ env.AZURE_OPENAI_API_VERSION }}" - id: mock_metrics_agent provider: cli grader_target: azure-llm diff --git a/examples/features/preprocessors/README.md b/examples/features/preprocessors/README.md index 67334d602..de63197fa 100644 --- a/examples/features/preprocessors/README.md +++ b/examples/features/preprocessors/README.md @@ -1,10 +1,10 @@ -# Content Preprocessors +# File Output Transforms -Demonstrates how `llm-rubric` preprocessors turn `ContentFile` outputs into text before grading. +Demonstrates how `default_test.options.transform` turns `ContentFile` outputs into text before grading. ## What This Shows -- top-level `preprocessors:` shared by all graders in an eval +- a suite-level output transform shared by all tests in an eval - an agent target returning a `ContentFile` block instead of plain text - an `llm-rubric` receiving transformed spreadsheet text - relative `ContentFile.path` resolution against the target workspace @@ -20,8 +20,8 @@ Expected result: the eval passes because the grader sees the transformed spreads ## Key Files -- `evals/suite.yaml` - eval with top-level `preprocessors` +- `evals/suite.yaml` - eval with `default_test.options.transform` - `.agentv/targets.yaml` - custom file-producing target and custom grader target - `.agentv/providers/file-output.ts` - emits a relative `ContentFile` path - `.agentv/providers/grader-check.ts` - passes only when transformed text reaches the grader prompt -- `scripts/preprocessors/xlsx-to-csv.ts` - example spreadsheet preprocessor script +- `scripts/preprocessors/xlsx-to-csv.ts` - example spreadsheet conversion script diff --git a/examples/features/preprocessors/evals/suite.yaml b/examples/features/preprocessors/evals/suite.yaml index 208541d3d..205e6d0d8 100644 --- a/examples/features/preprocessors/evals/suite.yaml +++ b/examples/features/preprocessors/evals/suite.yaml @@ -1,10 +1,4 @@ description: Convert file outputs to grader-readable text before llm grading -preprocessors: - - type: xlsx - command: - - bun - - run - - ../scripts/preprocessors/xlsx-to-csv.ts prompts: - "{{ input }}" tests: @@ -14,3 +8,24 @@ tests: - The extracted spreadsheet content includes the revenue rows vars: input: Generate the spreadsheet report +default_test: + options: + transform: >- + return (() => { + const content = Array.isArray(output) ? output : []; + const matchers = ["xlsx","application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",".xlsx"]; + const file = content.find((block) => { + if (!block || block.type !== "file") return false; + const mediaType = typeof block.media_type === "string" ? block.media_type : ""; + const filePath = typeof block.path === "string" ? block.path : ""; + return matchers.some((matcher) => mediaType === matcher || filePath.endsWith(matcher)); + }); + if (!file || typeof file.path !== "string") return output; + const result = Bun.spawnSync(["bun","run","../scripts/preprocessors/xlsx-to-csv.ts"], { + stdin: JSON.stringify({ path: file.path, media_type: file.media_type }) + }); + if (result.exitCode !== 0) { + throw new Error(new TextDecoder().decode(result.stderr).trim() || "preprocessor command failed"); + } + return new TextDecoder().decode(result.stdout).trim(); + })() diff --git a/examples/features/readme-quickstart/.agentv/tests.yaml b/examples/features/readme-quickstart/.agentv/tests.yaml index 543bb6028..476c3d3c8 100644 --- a/examples/features/readme-quickstart/.agentv/tests.yaml +++ b/examples/features/readme-quickstart/.agentv/tests.yaml @@ -1,2 +1,3 @@ - id: fizzbuzz - input: Write FizzBuzz in Python. Use lowercase output strings "fizz", "buzz", and "fizzbuzz". + vars: + input: Write FizzBuzz in Python. Use lowercase output strings "fizz", "buzz", and "fizzbuzz". diff --git a/examples/features/readme-quickstart/targets.yaml b/examples/features/readme-quickstart/targets.yaml index 3299f646a..38641bdaa 100644 --- a/examples/features/readme-quickstart/targets.yaml +++ b/examples/features/readme-quickstart/targets.yaml @@ -4,6 +4,6 @@ targets: runtime: host config: api_format: chat - base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }} - api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }} - model: ${{ LOCAL_OPENAI_PROXY_MODEL }} + base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" + api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" + model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" diff --git a/examples/showcase/bug-fix-benchmark/evals/bug-fixes.eval.yaml b/examples/showcase/bug-fix-benchmark/evals/bug-fixes.eval.yaml index f73978120..8f0712f11 100644 --- a/examples/showcase/bug-fix-benchmark/evals/bug-fixes.eval.yaml +++ b/examples/showcase/bug-fix-benchmark/evals/bug-fixes.eval.yaml @@ -13,7 +13,7 @@ workspace: reset: fast target: name: claude-baseline - extends: ${{ AGENT_TARGET }} + extends: "{{ env.AGENT_TARGET }}" hooks: before_each: command: diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts index 28e8cf7d9..5f6f06f73 100644 --- a/packages/core/src/evaluation/loaders/config-loader.ts +++ b/packages/core/src/evaluation/loaders/config-loader.ts @@ -373,6 +373,11 @@ function rejectAuthoredSuiteExecution(rawExecution: JsonValue): void { "Top-level 'execution.max_concurrency' has been removed from eval YAML. Use evaluate_options.max_concurrency for authored suite concurrency.", ); } + if (key === 'workers') { + throw new Error( + "Top-level 'execution.workers' has been removed from eval YAML. Use evaluate_options.max_concurrency for authored suite concurrency.", + ); + } throw new Error( `Top-level 'execution.${key}' is not part of eval YAML. Use supported top-level fields or evaluate_options for authored run controls.`, ); diff --git a/packages/core/src/evaluation/loaders/grader-parser.ts b/packages/core/src/evaluation/loaders/grader-parser.ts index 942d3481f..4e0f7f116 100644 --- a/packages/core/src/evaluation/loaders/grader-parser.ts +++ b/packages/core/src/evaluation/loaders/grader-parser.ts @@ -556,6 +556,11 @@ async function parseGraderList( `Grader '${name}' in '${evalId}': postprocess has been removed. Use transform instead.`, ); } + if (rawEvaluator.preprocessors !== undefined) { + throw new Error( + `Grader '${name}' in '${evalId}': preprocessors has been removed from authored eval YAML. Use transform instead.`, + ); + } const transform = await parseTransformSpec( rawEvaluator.transform as JsonValue | undefined, searchRoots, diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index 18bee5e74..efda8028c 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -260,7 +260,7 @@ export function normalizeCliHealthcheck( * form used by the CLI provider. Resolves environment variables. * * This function resolves environment variable references using - * ${{ VAR_NAME }} syntax and converts external YAML field names to the + * {{ env.VAR_NAME }} syntax and converts external YAML field names to the * internal runtime shape. * * @param input - The loose CLI target input from YAML @@ -913,8 +913,9 @@ export const COMMON_TARGET_SETTINGS = [ 'fallback_targets', ] as const; -const USE_TARGET_ENV_PATTERN = /^\$\{\{\s*([A-Z0-9_]+)\s*\}\}$/i; +const USE_TARGET_ENV_PATTERN = /^\s*\{\{\s*env\.([A-Za-z_][A-Za-z0-9_]*)\s*\}\}\s*$/; const SECRET_ENV_TEMPLATE_PATTERN = /^\s*\{\{\s*env\.([A-Za-z_][A-Za-z0-9_]*)\s*\}\}\s*$/; +const LEGACY_ENV_TEMPLATE_PATTERN = /^\s*\$\{\{\s*([A-Z0-9_]+)\s*\}\}\s*$/i; const BASE_TARGET_SCHEMA = z .object({ @@ -1035,6 +1036,14 @@ export function resolveDelegatedTargetDefinition( return definition; } + const legacyEnvMatch = rawUseTarget.match(LEGACY_ENV_TEMPLATE_PATTERN); + if (legacyEnvMatch) { + const envVarName = legacyEnvMatch[1] ?? 'VARIABLE_NAME'; + throw new Error( + `Target "${definition.name}" uses removed legacy use_target syntax \${{ ${envVarName} }}. Use {{ env.${envVarName} }} instead.`, + ); + } + const envMatch = rawUseTarget.match(USE_TARGET_ENV_PATTERN); const envVarName = envMatch?.[1]; const resolvedName = envVarName ? (env[envVarName]?.trim() ?? '') : rawUseTarget; @@ -1042,7 +1051,7 @@ export function resolveDelegatedTargetDefinition( if (resolvedName.length === 0) { if (envVarName) { throw new Error( - `Target "${definition.name}" uses use_target: \${{ ${envVarName} }}, but ${envVarName} is not set. Set ${envVarName} to the name of a concrete target (for example, "azure") before running the eval.`, + `Target "${definition.name}" uses use_target: {{ env.${envVarName} }}, but ${envVarName} is not set. Set ${envVarName} to the name of a concrete target (for example, "azure") before running the eval.`, ); } @@ -1055,7 +1064,7 @@ export function resolveDelegatedTargetDefinition( if (!next) { if (envVarName) { throw new Error( - `Target "${definition.name}" uses use_target: \${{ ${envVarName} }}, which resolved to "${resolvedName}", but no target named "${resolvedName}" exists.`, + `Target "${definition.name}" uses use_target: {{ env.${envVarName} }}, which resolved to "${resolvedName}", but no target named "${resolvedName}" exists.`, ); } @@ -2578,7 +2587,7 @@ const cliErrorMap: z.ZodErrorMap = (issue, ctx) => { * 3. Validates CLI placeholders in the command * * @param target - The raw target definition from YAML - * @param env - Environment variable lookup for ${{ VAR }} resolution + * @param env - Environment variable lookup for {{ env.VAR }} resolution * @param evalFilePath - Optional path to eval file for relative path resolution * @returns Normalized CLI configuration matching CliResolvedConfig */ @@ -2754,9 +2763,9 @@ function resolveCopilotLogConfig( } /** - * Resolve a string value from targets.yaml, supporting `${{ VARIABLE }}` env var syntax. + * Resolve a string value from targets.yaml, supporting `{{ env.VARIABLE }}` env var syntax. * - * Security: By default (`allowLiteral: false`), values MUST use the `${{ VARIABLE_NAME }}` + * Security: By default (`allowLiteral: false`), values MUST use the `{{ env.VARIABLE_NAME }}` * syntax to reference environment variables. Literal strings are rejected to prevent * secrets (API keys, tokens) from being committed in plaintext to targets.yaml. * Only non-sensitive fields like `cwd` or `model` use `allowLiteral: true`. @@ -2778,37 +2787,32 @@ function resolveOptionalString( return undefined; } - // Check for ${{ variable }} syntax - const envVarMatch = trimmed.match(/^\$\{\{\s*([A-Z0-9_]+)\s*\}\}$/i); - if (envVarMatch) { - const varName = envVarMatch[1]; - const envValue = env[varName]; - const optionalEnv = options?.optionalEnv ?? false; - - // Treat empty or undefined env vars the same way - if (envValue === undefined || envValue.trim().length === 0) { - if (optionalEnv) { - return undefined; - } - const status = envValue === undefined ? 'is not set' : 'is empty'; - throw new Error(`Environment variable '${varName}' required for ${description} ${status}`); - } - return envValue; + const legacyEnvVarMatch = trimmed.match(LEGACY_ENV_TEMPLATE_PATTERN); + if (legacyEnvVarMatch) { + const varName = legacyEnvVarMatch[1]; + throw new Error( + `${description} uses removed legacy environment syntax \${{ ${varName} }}. Use {{ env.${varName} }} instead.`, + ); } if (trimmed.includes('{{') && trimmed.includes('env.')) { const allowLiteral = options?.allowLiteral ?? false; const isSecretField = /\b(api key|bearer token|github token|token|secret)\b/i.test(description); + const wholeEnvMatch = trimmed.match(SECRET_ENV_TEMPLATE_PATTERN); if (!allowLiteral && isSecretField && !SECRET_ENV_TEMPLATE_PATTERN.test(trimmed)) { - throw new Error( - `${description} must use a whole \${{ VARIABLE_NAME }} or {{ env.VARIABLE_NAME }} reference`, - ); + throw new Error(`${description} must use a whole {{ env.VARIABLE_NAME }} reference`); } const rendered = renderEnvTemplateString(trimmed, env).trim(); if (rendered.length === 0) { if (options?.optionalEnv ?? false) { return undefined; } + if (wholeEnvMatch) { + const varName = wholeEnvMatch[1] ?? 'VARIABLE_NAME'; + throw new Error( + `${description} env template {{ env.${varName} }} resolved to an empty value; ${varName} is not set`, + ); + } throw new Error(`${description} env template resolved to an empty value`); } return rendered; @@ -2818,7 +2822,7 @@ function resolveOptionalString( const allowLiteral = options?.allowLiteral ?? false; if (!allowLiteral) { throw new Error( - `${description} must use \${{ VARIABLE_NAME }} syntax for environment variables or be marked as allowing literals`, + `${description} must use {{ env.VARIABLE_NAME }} syntax for environment variables or be marked as allowing literals`, ); } return trimmed; @@ -2895,19 +2899,28 @@ function resolveOptionalStringArray( throw new Error(`${description}[${i}] cannot be empty`); } - // Check for ${{ variable }} syntax - const envVarMatch = trimmed.match(/^\$\{\{\s*([A-Z0-9_]+)\s*\}\}$/i); - if (envVarMatch) { - const varName = envVarMatch[1]; - const envValue = env[varName]; - if (envValue !== undefined) { - if (envValue.trim().length === 0) { - throw new Error(`Environment variable '${varName}' for ${description}[${i}] is empty`); + const legacyEnvVarMatch = trimmed.match(LEGACY_ENV_TEMPLATE_PATTERN); + if (legacyEnvVarMatch) { + const varName = legacyEnvVarMatch[1]; + throw new Error( + `${description}[${i}] uses removed legacy environment syntax \${{ ${varName} }}. Use {{ env.${varName} }} instead.`, + ); + } + + if (trimmed.includes('{{') && trimmed.includes('env.')) { + const wholeEnvMatch = trimmed.match(SECRET_ENV_TEMPLATE_PATTERN); + const rendered = renderEnvTemplateString(trimmed, env).trim(); + if (rendered.length === 0) { + if (wholeEnvMatch) { + const varName = wholeEnvMatch[1] ?? 'VARIABLE_NAME'; + throw new Error( + `${description}[${i}] env template {{ env.${varName} }} resolved to an empty value; ${varName} is not set`, + ); } - resolved.push(envValue); - continue; + throw new Error(`${description}[${i}] env template resolved to an empty value`); } - throw new Error(`Environment variable '${varName}' for ${description}[${i}] is not set`); + resolved.push(rendered); + continue; } // Treat as literal value diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index 60d2a81f1..5b4b375fe 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -91,11 +91,6 @@ const PromptSchema = z.union([ const PromptsSchema = z.union([PromptSchema, z.array(PromptSchema).min(1)]); -const PreprocessorSchema = z.object({ - type: z.string().min(1), - command: z.union([z.string(), z.array(z.string())]), -}); - /** Score range for analytic rubrics */ const ScoreRangeSchema = z.object({ score_range: z.tuple([z.number().int().min(0).max(10), z.number().int().min(0).max(10)]), @@ -122,7 +117,6 @@ const ScriptGraderSchema = EvaluatorCommonSchema.extend({ cwd: z.string().optional(), target: z.union([z.boolean(), z.object({ max_calls: z.number().optional() })]).optional(), config: z.record(z.unknown()).optional(), - preprocessors: z.array(PreprocessorSchema).optional(), }); const LlmGraderSchema = EvaluatorCommonSchema.extend({ @@ -134,7 +128,6 @@ const LlmGraderSchema = EvaluatorCommonSchema.extend({ config: z.record(z.unknown()).optional(), max_steps: z.number().int().min(1).max(50).optional(), temperature: z.number().min(0).max(2).optional(), - preprocessors: z.array(PreprocessorSchema).optional(), }); const IncludeSchema = z @@ -276,6 +269,20 @@ const EvaluatorSchema = z.union([ ]); const AssertionObjectSchema = JsonObjectSchema.superRefine((value, ctx) => { + if (value.preprocessors !== undefined) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['preprocessors'], + message: 'preprocessors has been removed from authored eval YAML. Use transform instead.', + }); + } + if (value.postprocess !== undefined) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['postprocess'], + message: 'postprocess has been removed. Use transform instead.', + }); + } const rawType = value.type; if (typeof rawType !== 'string') { return; @@ -771,8 +778,7 @@ export const EvalFileSchema: z.ZodType = z execution: z.never().optional(), // Suite-level assert entries assert: z.array(AssertionItemSchema).optional(), - // Suite-level content preprocessors shared by evaluators - preprocessors: z.array(PreprocessorSchema).optional(), + preprocessors: z.never().optional(), // Workspace (inline object or path to external workspace YAML file) workspace: z.union([WorkspaceSchema, z.string()]).optional(), }) diff --git a/packages/core/src/evaluation/validation/eval-validator.ts b/packages/core/src/evaluation/validation/eval-validator.ts index cbf742638..c07056271 100644 --- a/packages/core/src/evaluation/validation/eval-validator.ts +++ b/packages/core/src/evaluation/validation/eval-validator.ts @@ -170,7 +170,6 @@ const KNOWN_TOP_LEVEL_FIELDS = new Set([ 'nunjucks_filters', 'extensions', 'on_run_complete', - 'preprocessors', 'workspace', 'metadata', 'governance', @@ -245,6 +244,14 @@ const REMOVED_TOP_LEVEL_FIELDS = new Map([ "Top-level 'early_exit' has been removed. Use evaluate_options.repeat.early_exit instead.", ], ['budget_usd', "Top-level 'budget_usd' has been removed. Use evaluate_options.budget_usd."], + [ + 'on_run_complete', + "Top-level 'on_run_complete' has been removed. Use extensions with an afterAll hook instead.", + ], + [ + 'preprocessors', + "Top-level 'preprocessors' has been removed from authored eval YAML. Use default_test.options.transform or assertion-level transform instead.", + ], ]); /** Deprecated top-level fields with migration hints. */ @@ -798,6 +805,16 @@ function validateExecutionPolicy( }); continue; } + if (key === 'workers') { + errors.push({ + severity: 'error', + filePath, + location: `${location}.${key}`, + message: + "Top-level 'execution.workers' has been removed from eval YAML. Use evaluate_options.max_concurrency for authored suite concurrency.", + }); + continue; + } errors.push({ severity: 'error', filePath, @@ -1269,6 +1286,14 @@ function validateTestOptions( }); return; } + if (options.postprocess !== undefined) { + errors.push({ + severity: 'error', + filePath, + location: `${location}.postprocess`, + message: `${location}.postprocess has been removed. Use ${location}.transform instead.`, + }); + } validateEvaluateOptionsRepeat(options.repeat, `${location}.repeat`, filePath, errors); } @@ -1324,6 +1349,7 @@ function validateDefaultTest( errors, customAssertionTypes, ); + validateTestOptions(defaultTest.options, 'default_test.options', filePath, errors); const threshold = defaultTest.threshold; if ( threshold !== undefined && @@ -2227,6 +2253,23 @@ function validateAssertArray( for (const { item, index } of objectItems) { const itemLocation = `${location}[${index}]`; + if (item.postprocess !== undefined) { + errors.push({ + severity: 'error', + filePath, + location: `${itemLocation}.postprocess`, + message: `${itemLocation}.postprocess has been removed. Use ${itemLocation}.transform instead.`, + }); + } + if (item.preprocessors !== undefined) { + errors.push({ + severity: 'error', + filePath, + location: `${itemLocation}.preprocessors`, + message: `${itemLocation}.preprocessors has been removed from authored eval YAML. Use ${itemLocation}.transform instead.`, + }); + } + validateAssertArray( item.assert, `${itemLocation}.assert`, diff --git a/packages/core/src/evaluation/validation/targets-validator.ts b/packages/core/src/evaluation/validation/targets-validator.ts index 1b62084ba..802ab066e 100644 --- a/packages/core/src/evaluation/validation/targets-validator.ts +++ b/packages/core/src/evaluation/validation/targets-validator.ts @@ -1,7 +1,6 @@ import { readFile } from 'node:fs/promises'; import path from 'node:path'; -import { interpolateEnv } from '../interpolation.js'; import { CLI_PLACEHOLDERS, COMMON_TARGET_SETTINGS, @@ -15,6 +14,7 @@ import type { ValidationError, ValidationResult } from './types.js'; type JsonValue = string | number | boolean | null | JsonObject | JsonArray; type JsonObject = { readonly [key: string]: JsonValue }; type JsonArray = readonly JsonValue[]; +const LEGACY_ENV_PATTERN = /\$\{\{\s*([A-Z_][A-Z0-9_]*)\s*\}\}/g; function isObject(value: unknown): value is JsonObject { return typeof value === 'object' && value !== null && !Array.isArray(value); @@ -24,6 +24,45 @@ function isNonEmptyString(value: unknown): value is string { return typeof value === 'string' && value.trim().length > 0; } +function isEnvTemplated(value: unknown): value is string { + return ( + typeof value === 'string' && + (/^\s*\{\{\s*env\.[\s\S]+?\}\}\s*$/.test(value) || + /^\s*\$\{\{\s*[A-Z_][A-Z0-9_]*\s*\}\}\s*$/.test(value)) + ); +} + +function validateLegacyEnvTemplates( + value: unknown, + filePath: string, + location: string, + errors: ValidationError[], +): void { + if (typeof value === 'string') { + for (const match of value.matchAll(LEGACY_ENV_PATTERN)) { + const name = match[1]; + errors.push({ + severity: 'error', + filePath, + location, + message: `Legacy environment syntax \${{ ${name} }} has been removed. Use {{ env.${name} }} instead.`, + }); + } + return; + } + if (Array.isArray(value)) { + value.forEach((entry, index) => + validateLegacyEnvTemplates(entry, filePath, `${location}[${index}]`, errors), + ); + return; + } + if (isObject(value)) { + for (const [key, nested] of Object.entries(value)) { + validateLegacyEnvTemplates(nested, filePath, `${location}.${key}`, errors); + } + } +} + // Cross-provider settings derived from the schema source of truth in targets.ts. // Adding a field to COMMON_TARGET_SETTINGS automatically makes it valid here. const COMMON_SETTINGS = new Set(COMMON_TARGET_SETTINGS); @@ -388,7 +427,7 @@ export async function validateTargetsFile(filePath: string): Promise rejectPostprocess(entry, `assert[${index}]`)); + suite.assert.forEach((entry, index) => { + rejectPostprocess(entry, `assert[${index}]`); + rejectPreprocessors(entry, `assert[${index}]`); + }); + } + if (suite.preprocessors !== undefined) { + throw new Error( + 'preprocessors has been removed from authored eval YAML. Use default_test.options.transform or assertion-level transform instead.', + ); } if (!Array.isArray(suite.tests)) { return; @@ -608,9 +627,10 @@ function rejectAuthoredPostprocess(suite: RawTestSuite): void { } rejectPostprocess(entry.options, `tests[${index}].options`); if (Array.isArray(entry.assert)) { - entry.assert.forEach((assertion, assertionIndex) => - rejectPostprocess(assertion, `tests[${index}].assert[${assertionIndex}]`), - ); + entry.assert.forEach((assertion, assertionIndex) => { + rejectPostprocess(assertion, `tests[${index}].assert[${assertionIndex}]`); + rejectPreprocessors(assertion, `tests[${index}].assert[${assertionIndex}]`); + }); } }); } diff --git a/packages/core/test/evaluation/loaders/config-loader.test.ts b/packages/core/test/evaluation/loaders/config-loader.test.ts index e5ac5cdda..351954bfb 100644 --- a/packages/core/test/evaluation/loaders/config-loader.test.ts +++ b/packages/core/test/evaluation/loaders/config-loader.test.ts @@ -202,7 +202,7 @@ describe('loadConfig', () => { ' env:', ' AGENTV_RESULT_DIR: /results', ' secrets:', - ' OPENAI_API_KEY: ${{ OPENAI_API_KEY }}', + ' OPENAI_API_KEY: "{{ env.OPENAI_API_KEY }}"', ' config:', ' command: "agent-cli run {PROMPT_FILE} {OUTPUT_FILE}"', '', @@ -225,7 +225,7 @@ describe('loadConfig', () => { { source: './.agentv/results', target: '/results', access: 'rw' }, ], env: { AGENTV_RESULT_DIR: '/results' }, - secrets: { OPENAI_API_KEY: '${{ OPENAI_API_KEY }}' }, + secrets: { OPENAI_API_KEY: '{{ env.OPENAI_API_KEY }}' }, }, config: { command: 'agent-cli run {PROMPT_FILE} {OUTPUT_FILE}' }, }); diff --git a/packages/core/test/evaluation/preprocessors-yaml.test.ts b/packages/core/test/evaluation/preprocessors-yaml.test.ts index 222bec034..81d7f9030 100644 --- a/packages/core/test/evaluation/preprocessors-yaml.test.ts +++ b/packages/core/test/evaluation/preprocessors-yaml.test.ts @@ -13,12 +13,10 @@ describe('eval YAML preprocessors', () => { tempDirs.length = 0; }); - it('merges suite-level preprocessors into llm-graders and resolves command paths', async () => { + it('rejects removed suite-level preprocessors with transform guidance', async () => { const dir = await mkdtemp(path.join(tmpdir(), 'agentv-yaml-preprocessors-')); tempDirs.push(dir); - await writeFile(path.join(dir, 'xlsx-default.js'), 'console.log("default")', 'utf8'); - await writeFile(path.join(dir, 'xlsx-override.js'), 'console.log("override")', 'utf8'); await writeFile( path.join(dir, 'suite.eval.yaml'), `preprocessors: @@ -31,49 +29,24 @@ prompts: tests: - id: report criteria: works - assert: - - name: grade - type: llm-grader - prompt: Evaluate {{ output }} - preprocessors: - - type: xlsx - command: - - node - - xlsx-override.js vars: input: grade this `, 'utf8', ); - const tests = await loadTests(path.join(dir, 'suite.eval.yaml'), dir); - const evaluator = tests[0]?.assertions?.[0]; - - expect(evaluator?.type).toBe('llm-grader'); - if (!evaluator || evaluator.type !== 'llm-grader') { - throw new Error('expected llm-grader evaluator'); - } - - expect(evaluator.preprocessors).toHaveLength(1); - expect(evaluator.preprocessors?.[0]?.resolvedCommand?.[1]).toBe( - path.join(dir, 'xlsx-override.js'), + await expect(loadTests(path.join(dir, 'suite.eval.yaml'), dir)).rejects.toThrow( + 'preprocessors has been removed from authored eval YAML. Use default_test.options.transform or assertion-level transform instead.', ); }); - it('lets alias-based evaluator overrides replace MIME-typed suite defaults', async () => { + it('rejects removed assertion preprocessors with transform guidance', async () => { const dir = await mkdtemp(path.join(tmpdir(), 'agentv-yaml-preprocessors-')); tempDirs.push(dir); - await writeFile(path.join(dir, 'xlsx-default.js'), 'console.log("default")', 'utf8'); - await writeFile(path.join(dir, 'xlsx-override.js'), 'console.log("override")', 'utf8'); await writeFile( path.join(dir, 'suite.eval.yaml'), - `preprocessors: - - type: application/vnd.openxmlformats-officedocument.spreadsheetml.sheet - command: - - node - - xlsx-default.js -prompts: + `prompts: - "{{ input }}" tests: - id: report @@ -93,15 +66,8 @@ tests: 'utf8', ); - const tests = await loadTests(path.join(dir, 'suite.eval.yaml'), dir); - const evaluator = tests[0]?.assertions?.[0]; - if (!evaluator || evaluator.type !== 'llm-grader') { - throw new Error('expected llm-grader evaluator'); - } - - expect(evaluator.preprocessors).toHaveLength(1); - expect(evaluator.preprocessors?.[0]?.resolvedCommand?.[1]).toBe( - path.join(dir, 'xlsx-override.js'), + await expect(loadTests(path.join(dir, 'suite.eval.yaml'), dir)).rejects.toThrow( + 'tests[0].assert[0].preprocessors has been removed from authored eval YAML. Use tests[0].assert[0].transform instead.', ); }); }); diff --git a/packages/core/test/evaluation/providers/cli-schema.test.ts b/packages/core/test/evaluation/providers/cli-schema.test.ts index a2646b7d4..77d604812 100644 --- a/packages/core/test/evaluation/providers/cli-schema.test.ts +++ b/packages/core/test/evaluation/providers/cli-schema.test.ts @@ -177,7 +177,7 @@ describe('normalizeCliHealthcheck', () => { it('normalizes HTTP and command healthchecks with timeout conversion', () => { // HTTP with snake_case timeout const httpInput = { - url: '${{ HEALTH_URL }}', + url: '{{ env.HEALTH_URL }}', timeout_seconds: 30, }; @@ -241,8 +241,8 @@ describe('normalizeCliTargetInput', () => { const input = { name: 'test-target', provider: 'cli', - command: '${{ CLI_CMD }}', - cwd: '${{ WORK_DIR }}', + command: '{{ env.CLI_CMD }}', + cwd: '{{ env.WORK_DIR }}', }; const result = normalizeCliTargetInput(input, mockEnv); @@ -287,7 +287,7 @@ describe('normalizeCliTargetInput', () => { provider: 'cli', command: 'agent {PROMPT}', healthcheck: { - url: '${{ HEALTH_ENDPOINT }}', + url: '{{ env.HEALTH_ENDPOINT }}', timeout_seconds: 10, }, }; diff --git a/packages/core/test/evaluation/providers/fallback-targets.test.ts b/packages/core/test/evaluation/providers/fallback-targets.test.ts index 0f3b7a51c..0279acbb2 100644 --- a/packages/core/test/evaluation/providers/fallback-targets.test.ts +++ b/packages/core/test/evaluation/providers/fallback-targets.test.ts @@ -135,8 +135,8 @@ describe('resolveTargetDefinition - fallback_targets', () => { const definition = { name: 'test-openai', provider: 'openai', - api_key: '${{ TEST_KEY }}', - model: '${{ TEST_MODEL }}', + api_key: '{{ env.TEST_KEY }}', + model: '{{ env.TEST_MODEL }}', fallback_targets: ['azure-llm', 'gemini-flash'], }; @@ -148,8 +148,8 @@ describe('resolveTargetDefinition - fallback_targets', () => { const definition = { name: 'test-openai', provider: 'openai', - api_key: '${{ TEST_KEY }}', - model: '${{ TEST_MODEL }}', + api_key: '{{ env.TEST_KEY }}', + model: '{{ env.TEST_MODEL }}', fallbackTargets: ['backup-1'], }; @@ -162,8 +162,8 @@ describe('resolveTargetDefinition - fallback_targets', () => { const definition = { name: 'test-openai', provider: 'openai', - api_key: '${{ TEST_KEY }}', - model: '${{ TEST_MODEL }}', + api_key: '{{ env.TEST_KEY }}', + model: '{{ env.TEST_MODEL }}', }; const resolved = resolveTargetDefinition(definition, env); diff --git a/packages/core/test/evaluation/providers/targets-cwd-fallback.test.ts b/packages/core/test/evaluation/providers/targets-cwd-fallback.test.ts index 24a0a1a11..e8e6e741a 100644 --- a/packages/core/test/evaluation/providers/targets-cwd-fallback.test.ts +++ b/packages/core/test/evaluation/providers/targets-cwd-fallback.test.ts @@ -8,7 +8,7 @@ describe('CLI cwd fallback to eval directory', () => { name: 'test-cli', provider: 'cli', command: 'echo {PROMPT}', - cwd: '${{ EMPTY_CWD_VAR }}', + cwd: '{{ env.EMPTY_CWD_VAR }}', }; const env = { @@ -29,7 +29,7 @@ describe('CLI cwd fallback to eval directory', () => { name: 'test-cli', provider: 'cli', command: 'echo {PROMPT}', - cwd: '${{ UNDEFINED_CWD_VAR }}', + cwd: '{{ env.UNDEFINED_CWD_VAR }}', }; const env = {}; // Env var not defined @@ -48,7 +48,7 @@ describe('CLI cwd fallback to eval directory', () => { name: 'test-cli', provider: 'cli', command: 'echo {PROMPT}', - cwd: '${{ EXPLICIT_CWD }}', + cwd: '{{ env.EXPLICIT_CWD }}', }; const env = { @@ -69,7 +69,7 @@ describe('CLI cwd fallback to eval directory', () => { name: 'test-cli', provider: 'cli', command: 'echo {PROMPT}', - cwd: '${{ MISSING_VAR }}', + cwd: '{{ env.MISSING_VAR }}', }; const env = {}; diff --git a/packages/core/test/evaluation/providers/targets.test.ts b/packages/core/test/evaluation/providers/targets.test.ts index 0864a5136..4ab98c297 100644 --- a/packages/core/test/evaluation/providers/targets.test.ts +++ b/packages/core/test/evaluation/providers/targets.test.ts @@ -45,7 +45,7 @@ const { extractLastAssistantContent } = await import('../../../src/evaluation/pr describe('resolveDelegatedTargetDefinition', () => { it('throws a helpful error when an env-backed use_target variable is missing', () => { const definitions = new Map([ - ['grader', { name: 'grader', use_target: '${{ GRADER_TARGET }}' }], + ['grader', { name: 'grader', use_target: '{{ env.GRADER_TARGET }}' }], ['azure', { name: 'azure', provider: 'azure' }], ]); @@ -56,7 +56,7 @@ describe('resolveDelegatedTargetDefinition', () => { it('throws a helpful error when an env-backed use_target resolves to a missing target', () => { const definitions = new Map([ - ['grader', { name: 'grader', use_target: '${{ GRADER_TARGET }}' }], + ['grader', { name: 'grader', use_target: '{{ env.GRADER_TARGET }}' }], ]); expect(() => @@ -66,9 +66,20 @@ describe('resolveDelegatedTargetDefinition', () => { ).toThrow(/resolved to "azure".*no target named "azure" exists/i); }); - it('resolves a delegated target chain to a concrete definition', () => { + it('rejects legacy env syntax in use_target', () => { const definitions = new Map([ ['grader', { name: 'grader', use_target: '${{ GRADER_TARGET }}' }], + ['azure', { name: 'azure', provider: 'azure' }], + ]); + + expect(() => resolveDelegatedTargetDefinition('grader', definitions, {})).toThrow( + /removed legacy use_target syntax.*Use {{ env\.GRADER_TARGET }}/, + ); + }); + + it('resolves a delegated target chain to a concrete definition', () => { + const definitions = new Map([ + ['grader', { name: 'grader', use_target: '{{ env.GRADER_TARGET }}' }], ['llm', { name: 'llm', use_target: 'azure' }], ['azure', { name: 'azure', provider: 'azure' }], ]); @@ -131,8 +142,8 @@ describe('resolveTargetDefinition', () => { id: 'candidate-agent', provider: 'openai', config: { - api_key: '${{ OPENAI_API_KEY }}', - model: '${{ OPENAI_MODEL }}', + api_key: '{{ env.OPENAI_API_KEY }}', + model: '{{ env.OPENAI_MODEL }}', }, } as never, { OPENAI_API_KEY: 'secret', OPENAI_MODEL: 'gpt-5-mini' }, @@ -146,7 +157,7 @@ describe('resolveTargetDefinition', () => { expect(target.config.model).toBe('gpt-5-mini'); }); - it("throws when settings don't use ${{ }} syntax", () => { + it("throws when settings don't use {{ env.* }} syntax", () => { const env = { AZURE_OPENAI_ENDPOINT: 'https://example.openai.azure.com', AZURE_OPENAI_API_KEY: 'secret', @@ -167,7 +178,7 @@ describe('resolveTargetDefinition', () => { ).toThrow(/must use.*VARIABLE_NAME.*syntax/i); }); - it('resolves azure settings using ${{ variable }} syntax', () => { + it('resolves azure settings using {{ env.VAR }} syntax', () => { const env = { AZURE_OPENAI_ENDPOINT: 'https://example.openai.azure.com', AZURE_OPENAI_API_KEY: 'secret', @@ -178,9 +189,9 @@ describe('resolveTargetDefinition', () => { { name: 'default', provider: 'azure', - endpoint: '${{ AZURE_OPENAI_ENDPOINT }}', - api_key: '${{ AZURE_OPENAI_API_KEY }}', - model: '${{ AZURE_DEPLOYMENT_NAME }}', + endpoint: '{{ env.AZURE_OPENAI_ENDPOINT }}', + api_key: '{{ env.AZURE_OPENAI_API_KEY }}', + model: '{{ env.AZURE_DEPLOYMENT_NAME }}', }, env, ); @@ -194,7 +205,7 @@ describe('resolveTargetDefinition', () => { }); }); - it('resolves with ${{ }} syntax with extra whitespace', () => { + it('resolves with {{ env.* }} syntax with extra whitespace', () => { const env = { MY_VAR: 'test-value', MY_API_KEY: 'literal-key', @@ -205,9 +216,9 @@ describe('resolveTargetDefinition', () => { { name: 'test', provider: 'azure', - endpoint: '${{ MY_VAR }}', - api_key: '${{ MY_API_KEY }}', - model: '${{ MY_MODEL }}', + endpoint: '{{ env.MY_VAR }}', + api_key: '{{ env.MY_API_KEY }}', + model: '{{ env.MY_MODEL }}', }, env, ); @@ -219,7 +230,7 @@ describe('resolveTargetDefinition', () => { expect(target.config.resourceName).toBe('test-value'); }); - it('resolves with ${{ }} syntax without spaces', () => { + it('resolves with {{ env.* }} syntax without spaces', () => { const env = { MY_ENDPOINT: 'https://no-spaces.example.com', MY_KEY: 'key123', @@ -230,9 +241,9 @@ describe('resolveTargetDefinition', () => { { name: 'no-spaces', provider: 'azure', - endpoint: '${{MY_ENDPOINT}}', - api_key: '${{MY_KEY}}', - model: '${{MY_MODEL}}', + endpoint: '{{ env.MY_ENDPOINT }}', + api_key: '{{ env.MY_KEY }}', + model: '{{ env.MY_MODEL }}', }, env, ); @@ -245,7 +256,7 @@ describe('resolveTargetDefinition', () => { expect(target.config.apiKey).toBe('key123'); }); - it('throws when ${{ variable }} reference is missing from env', () => { + it('throws when {{ env.VAR }} reference is missing from env', () => { const env = {} satisfies Record; expect(() => @@ -253,7 +264,7 @@ describe('resolveTargetDefinition', () => { { name: 'broken', provider: 'azure', - endpoint: '${{ MISSING_VAR }}', + endpoint: '{{ env.MISSING_VAR }}', api_key: 'key', model: 'model', }, @@ -274,10 +285,10 @@ describe('resolveTargetDefinition', () => { { name: 'azure-version', provider: 'azure', - endpoint: '${{ AZURE_OPENAI_ENDPOINT }}', - api_key: '${{ AZURE_OPENAI_API_KEY }}', - model: '${{ AZURE_DEPLOYMENT_NAME }}', - version: '${{ CUSTOM_VERSION }}', + endpoint: '{{ env.AZURE_OPENAI_ENDPOINT }}', + api_key: '{{ env.AZURE_OPENAI_API_KEY }}', + model: '{{ env.AZURE_DEPLOYMENT_NAME }}', + version: '{{ env.CUSTOM_VERSION }}', }, env, ); @@ -302,9 +313,9 @@ describe('resolveTargetDefinition', () => { { name: 'azure-with-api-format', provider: 'azure', - endpoint: '${{ AZURE_OPENAI_ENDPOINT }}', - api_key: '${{ AZURE_OPENAI_API_KEY }}', - model: '${{ AZURE_DEPLOYMENT_NAME }}', + endpoint: '{{ env.AZURE_OPENAI_ENDPOINT }}', + api_key: '{{ env.AZURE_OPENAI_API_KEY }}', + model: '{{ env.AZURE_DEPLOYMENT_NAME }}', api_format: 'responses', }, env, @@ -322,7 +333,7 @@ describe('resolveTargetDefinition', () => { { name: 'openai-with-judge-target', provider: 'openai', - api_key: '${{ OPENAI_API_KEY }}', + api_key: '{{ env.OPENAI_API_KEY }}', model: 'gpt-5-mini', judge_target: 'grader', } as never, @@ -413,9 +424,9 @@ describe('resolveTargetDefinition', () => { { name: 'azure-default-version', provider: 'azure', - endpoint: '${{ AZURE_OPENAI_ENDPOINT }}', - api_key: '${{ AZURE_OPENAI_API_KEY }}', - model: '${{ AZURE_DEPLOYMENT_NAME }}', + endpoint: '{{ env.AZURE_OPENAI_ENDPOINT }}', + api_key: '{{ env.AZURE_OPENAI_API_KEY }}', + model: '{{ env.AZURE_DEPLOYMENT_NAME }}', }, env, ); @@ -438,9 +449,9 @@ describe('resolveTargetDefinition', () => { { name: 'broken', provider: 'azure', - endpoint: '${{ AZURE_OPENAI_ENDPOINT }}', - api_key: '${{ AZURE_OPENAI_API_KEY }}', - model: '${{ AZURE_DEPLOYMENT_NAME }}', + endpoint: '{{ env.AZURE_OPENAI_ENDPOINT }}', + api_key: '{{ env.AZURE_OPENAI_API_KEY }}', + model: '{{ env.AZURE_DEPLOYMENT_NAME }}', }, env, ), @@ -478,7 +489,7 @@ describe('resolveTargetDefinition', () => { { name: 'editor', provider: 'vscode', - executable: '${{ VSCODE_CMD }}', + executable: '{{ env.VSCODE_CMD }}', }, env, ); @@ -539,7 +550,7 @@ describe('resolveTargetDefinition', () => { { name: 'gemini-target', provider: 'gemini', - api_key: '${{ GOOGLE_API_KEY }}', + api_key: '{{ env.GOOGLE_API_KEY }}', }, env, ); @@ -565,8 +576,8 @@ describe('resolveTargetDefinition', () => { { name: 'gemini-pro', provider: 'gemini', - api_key: '${{ GOOGLE_API_KEY }}', - model: '${{ GOOGLE_GEMINI_MODEL }}', + api_key: '{{ env.GOOGLE_API_KEY }}', + model: '{{ env.GOOGLE_GEMINI_MODEL }}', }, env, ); @@ -591,7 +602,7 @@ describe('resolveTargetDefinition', () => { { name: 'gemini-flash', provider: 'gemini', - api_key: '${{ GOOGLE_API_KEY }}', + api_key: '{{ env.GOOGLE_API_KEY }}', model: 'gemini-1.5-flash', }, env, @@ -619,9 +630,9 @@ describe('resolveTargetDefinition', () => { { name: 'openai-target', provider: 'openai', - endpoint: '${{ OPENAI_ENDPOINT }}', - api_key: '${{ OPENAI_API_KEY }}', - model: '${{ OPENAI_MODEL }}', + endpoint: '{{ env.OPENAI_ENDPOINT }}', + api_key: '{{ env.OPENAI_API_KEY }}', + model: '{{ env.OPENAI_MODEL }}', }, env, ); @@ -735,8 +746,8 @@ describe('resolveTargetDefinition', () => { { name: 'openrouter-target', provider: 'openrouter', - api_key: '${{ OPENROUTER_API_KEY }}', - model: '${{ OPENROUTER_MODEL }}', + api_key: '{{ env.OPENROUTER_API_KEY }}', + model: '{{ env.OPENROUTER_MODEL }}', }, env, ); @@ -758,7 +769,7 @@ describe('resolveTargetDefinition', () => { { name: 'broken-gemini', provider: 'gemini', - api_key: '${{ GOOGLE_API_KEY }}', + api_key: '{{ env.GOOGLE_API_KEY }}', }, {}, ), @@ -803,7 +814,7 @@ describe('resolveTargetDefinition', () => { name: 'shell-cli', provider: 'cli', command: 'code chat {PROMPT} {FILES}', - cwd: '${{ WORKDIR }}', + cwd: '{{ env.WORKDIR }}', timeout_seconds: 3, files_format: '--file {path}', }, @@ -852,7 +863,7 @@ describe('resolveTargetDefinition', () => { ).toThrow(/unsupported placeholder/i); }); - it('resolves codex-cli command argv using ${{ }} syntax', () => { + it('resolves codex-cli command argv using {{ env.* }} syntax', () => { const env = { CODEX_PROFILE: 'default', CODEX_MODEL: 'gpt-4', @@ -865,9 +876,9 @@ describe('resolveTargetDefinition', () => { command: [ 'codex-personal', '--profile', - '${{ CODEX_PROFILE }}', + '{{ env.CODEX_PROFILE }}', '--model', - '${{ CODEX_MODEL }}', + '{{ env.CODEX_MODEL }}', ], }, env, @@ -893,8 +904,8 @@ describe('resolveTargetDefinition', () => { name: 'codex-cli', provider: 'codex-cli', command: ['codex'], - model: '${{ CODEX_MODEL }}', - reasoning_effort: '${{ CODEX_REASONING_EFFORT }}', + model: '{{ env.CODEX_MODEL }}', + reasoning_effort: '{{ env.CODEX_REASONING_EFFORT }}', }, { CODEX_MODEL: 'gpt-5.5', @@ -917,11 +928,11 @@ describe('resolveTargetDefinition', () => { name: 'codex-local-openai', provider: 'codex-cli', command: ['codex-eng'], - model: '${{ CODEX_MODEL }}', + model: '{{ env.CODEX_MODEL }}', reasoning_effort: 'medium', model_verbosity: 'medium', - base_url: '${{ OPENAI_BASE_URL }}', - api_key: '${{ OPENAI_API_KEY }}', + base_url: '{{ env.OPENAI_BASE_URL }}', + api_key: '{{ env.OPENAI_API_KEY }}', api_format: 'responses', sandbox_mode: 'danger-full-access', approval_policy: 'never', @@ -1116,9 +1127,9 @@ describe('resolveTargetDefinition', () => { name: 'copilot-cli-openai-flat', provider: 'copilot-cli', subprovider: 'openai', - base_url: '${{ OPENAI_ENDPOINT }}', - api_key: '${{ OPENAI_API_KEY }}', - bearer_token: '${{ OPTIONAL_BEARER_TOKEN }}', + base_url: '{{ env.OPENAI_ENDPOINT }}', + api_key: '{{ env.OPENAI_API_KEY }}', + bearer_token: '{{ env.OPTIONAL_BEARER_TOKEN }}', api_format: 'responses', api_version: '2024-10-21', }, @@ -1152,8 +1163,8 @@ describe('resolveTargetDefinition', () => { provider: 'copilot-sdk', model: 'gpt-5', subprovider: 'openai', - base_url: '${{ OPENAI_ENDPOINT }}', - api_key: '${{ OPENAI_API_KEY }}', + base_url: '{{ env.OPENAI_ENDPOINT }}', + api_key: '{{ env.OPENAI_API_KEY }}', api_format: 'responses', }, env, @@ -1185,11 +1196,11 @@ describe('resolveTargetDefinition', () => { provider: 'copilot-sdk', model: 'gpt-5', subprovider: 'openai', - base_url: '${{ OPENAI_ENDPOINT }}', - api_key: '${{ OPENAI_API_KEY }}', + base_url: '{{ env.OPENAI_ENDPOINT }}', + api_key: '{{ env.OPENAI_API_KEY }}', api_format: 'responses', model_id: 'gpt-5', - wire_model: '${{ WIRE_MODEL }}', + wire_model: '{{ env.WIRE_MODEL }}', }, env, ); @@ -1214,7 +1225,7 @@ describe('resolveTargetDefinition', () => { { name: 'copilot-sdk-with-args', provider: 'copilot-sdk', - args: ['--plugin-dir', '${{ COPILOT_PLUGIN_DIR }}'], + args: ['--plugin-dir', '{{ env.COPILOT_PLUGIN_DIR }}'], }, { COPILOT_PLUGIN_DIR: './plugins' }, ); @@ -1265,7 +1276,7 @@ describe('resolveTargetDefinition', () => { name: 'copilot-sdk-bearer', provider: 'copilot-sdk', base_url: 'https://custom-endpoint.example.com/v1', - bearer_token: '${{ MY_TOKEN }}', + bearer_token: '{{ env.MY_TOKEN }}', }, env, ); @@ -1291,7 +1302,7 @@ describe('resolveTargetDefinition', () => { model: 'gpt-5', subprovider: 'openai', base_url: 'https://resource.openai.azure.com/openai/v1/', - api_key: '${{ FOUNDRY_KEY }}', + api_key: '{{ env.FOUNDRY_KEY }}', api_format: 'responses', }, env, @@ -1329,9 +1340,9 @@ describe('resolveTargetDefinition', () => { { name: 'deprecated-camel-case', provider: 'openai', - baseUrl: '${{ OPENAI_BASE_URL }}', - apiKey: '${{ OPENAI_API_KEY }}', - model: '${{ OPENAI_MODEL }}', + baseUrl: '{{ env.OPENAI_BASE_URL }}', + apiKey: '{{ env.OPENAI_API_KEY }}', + model: '{{ env.OPENAI_MODEL }}', maxTokens: 100, }, { @@ -1349,8 +1360,8 @@ describe('resolveTargetDefinition', () => { name: 'azure-chat', provider: 'openai', base_url: 'https://resource.openai.azure.com/openai/deployments/gpt-4o', - api_key: '${{ AZURE_OPENAI_API_KEY }}', - model: '${{ AZURE_DEPLOYMENT_NAME }}', + api_key: '{{ env.AZURE_OPENAI_API_KEY }}', + model: '{{ env.AZURE_DEPLOYMENT_NAME }}', api_format: 'chat', }, { @@ -1485,9 +1496,9 @@ describe('createProvider', () => { { name: 'openai-target', provider: 'openai', - endpoint: '${{ OPENAI_ENDPOINT }}', - api_key: '${{ OPENAI_API_KEY }}', - model: '${{ OPENAI_MODEL }}', + endpoint: '{{ env.OPENAI_ENDPOINT }}', + api_key: '{{ env.OPENAI_API_KEY }}', + model: '{{ env.OPENAI_MODEL }}', }, env, ); @@ -1512,9 +1523,9 @@ describe('createProvider', () => { { name: 'openai-resp', provider: 'openai', - endpoint: '${{ OPENAI_ENDPOINT }}', - api_key: '${{ OPENAI_API_KEY }}', - model: '${{ OPENAI_MODEL }}', + endpoint: '{{ env.OPENAI_ENDPOINT }}', + api_key: '{{ env.OPENAI_API_KEY }}', + model: '{{ env.OPENAI_MODEL }}', api_format: 'responses', }, env, @@ -1535,8 +1546,8 @@ describe('createProvider', () => { { name: 'openrouter-target', provider: 'openrouter', - api_key: '${{ OPENROUTER_API_KEY }}', - model: '${{ OPENROUTER_MODEL }}', + api_key: '{{ env.OPENROUTER_API_KEY }}', + model: '{{ env.OPENROUTER_MODEL }}', }, env, ); @@ -1558,8 +1569,8 @@ describe('createProvider', () => { { name: 'anthropic-target', provider: 'anthropic', - api_key: '${{ ANTHROPIC_API_KEY }}', - model: '${{ ANTHROPIC_MODEL }}', + api_key: '{{ env.ANTHROPIC_API_KEY }}', + model: '{{ env.ANTHROPIC_MODEL }}', thinking_budget: 4096, }, env, @@ -1579,7 +1590,7 @@ describe('createProvider', () => { it('routes gemini targets through pi-ai google-generative-ai', async () => { const env = { GOOGLE_API_KEY: 'gemini-key' } satisfies Record; const resolved = resolveTargetDefinition( - { name: 'gemini-target', provider: 'gemini', api_key: '${{ GOOGLE_API_KEY }}' }, + { name: 'gemini-target', provider: 'gemini', api_key: '{{ env.GOOGLE_API_KEY }}' }, env, ); const provider = createProvider(resolved); @@ -1598,9 +1609,9 @@ describe('createProvider', () => { { name: 'azure-target', provider: 'azure', - endpoint: '${{ AZURE_OPENAI_ENDPOINT }}', - api_key: '${{ AZURE_OPENAI_API_KEY }}', - model: '${{ AZURE_DEPLOYMENT_NAME }}', + endpoint: '{{ env.AZURE_OPENAI_ENDPOINT }}', + api_key: '{{ env.AZURE_OPENAI_API_KEY }}', + model: '{{ env.AZURE_DEPLOYMENT_NAME }}', }, env, ); @@ -1626,9 +1637,9 @@ describe('createProvider', () => { name: 'pi-azure', provider: 'pi-coding-agent', subprovider: 'azure', - base_url: '${{ AZURE_OPENAI_ENDPOINT }}', - model: '${{ AZURE_DEPLOYMENT_NAME }}', - api_key: '${{ AZURE_OPENAI_API_KEY }}', + base_url: '{{ env.AZURE_OPENAI_ENDPOINT }}', + model: '{{ env.AZURE_DEPLOYMENT_NAME }}', + api_key: '{{ env.AZURE_OPENAI_API_KEY }}', tools: 'read,bash,edit,write', }, env, @@ -1688,9 +1699,9 @@ describe('createProvider', () => { name: 'pi-cli-azure', provider: 'pi-cli', subprovider: 'azure', - base_url: '${{ AZURE_OPENAI_ENDPOINT }}', - model: '${{ AZURE_DEPLOYMENT_NAME }}', - api_key: '${{ AZURE_OPENAI_API_KEY }}', + base_url: '{{ env.AZURE_OPENAI_ENDPOINT }}', + model: '{{ env.AZURE_DEPLOYMENT_NAME }}', + api_key: '{{ env.AZURE_OPENAI_API_KEY }}', }, env, ); @@ -1710,7 +1721,7 @@ describe('createProvider', () => { provider: 'pi-cli', subprovider: 'openai', base_url: 'http://127.0.0.1:10531/v1', - api_key: '${{ OPENAI_API_KEY }}', + api_key: '{{ env.OPENAI_API_KEY }}', model: 'gpt-5.3-codex-spark', }, { OPENAI_API_KEY: 'local-key' }, @@ -1728,8 +1739,8 @@ describe('createProvider', () => { { name: 'pi-cli-local-endpoint', provider: 'pi-cli', - base_url: '${{ AGENTV_OPENAI_BASE_URL }}', - api_key: '${{ AGENTV_OPENAI_API_KEY }}', + base_url: '{{ env.AGENTV_OPENAI_BASE_URL }}', + api_key: '{{ env.AGENTV_OPENAI_API_KEY }}', model: 'gpt-5.3-codex-spark', }, { @@ -1750,7 +1761,7 @@ describe('createProvider', () => { name: 'pi-cli-openai', provider: 'pi-cli', subprovider: 'openai', - api_key: '${{ OPENAI_API_KEY }}', + api_key: '{{ env.OPENAI_API_KEY }}', model: 'gpt-4o', }, { OPENAI_API_KEY: 'real-openai-key' }, @@ -1769,7 +1780,7 @@ describe('createProvider', () => { provider: 'pi-cli', subprovider: 'openai-codex', model: 'gpt-5.5', - reasoning_effort: '${{ PI_THINKING }}', + reasoning_effort: '{{ env.PI_THINKING }}', }, { PI_THINKING: 'medium' }, ); diff --git a/packages/core/test/evaluation/source-traceability.test.ts b/packages/core/test/evaluation/source-traceability.test.ts index 7e617887c..e53358923 100644 --- a/packages/core/test/evaluation/source-traceability.test.ts +++ b/packages/core/test/evaluation/source-traceability.test.ts @@ -64,14 +64,6 @@ tests: - bun - graders/code.ts cwd: graders - - metric: preprocessed - type: llm-grader - prompt: Inline prompt - preprocessors: - - type: text - command: - - bun - - graders/pre.ts vars: input: - role: user @@ -96,7 +88,6 @@ tests: 'assertion_template', 'input_file', 'llm_grader_prompt', - 'preprocessor_command', 'prompt_script', 'script_grader_command', 'script_grader_cwd', diff --git a/packages/core/test/evaluation/validation/eval-file-schema.test.ts b/packages/core/test/evaluation/validation/eval-file-schema.test.ts index 9473804ff..94c61bd31 100644 --- a/packages/core/test/evaluation/validation/eval-file-schema.test.ts +++ b/packages/core/test/evaluation/validation/eval-file-schema.test.ts @@ -83,6 +83,45 @@ describe('EvalFileSchema input shorthand', () => { expect(result.success).toBe(false); }); + it('rejects removed authored preprocessors and postprocess fields', () => { + const result = EvalFileSchema.safeParse({ + preprocessors: [{ type: 'xlsx', command: ['node', 'xlsx.js'] }], + default_test: { + options: { + postprocess: 'output.trim()', + }, + }, + tests: [ + { + ...baseTest, + options: { + postprocess: 'output.trim()', + }, + assert: [ + { + type: 'llm-rubric', + value: 'Good', + preprocessors: [{ type: 'xlsx', command: ['node', 'xlsx.js'] }], + }, + { + type: 'llm-rubric', + value: 'Also good', + postprocess: 'output.trim()', + }, + ], + }, + ], + }); + + expect(result.success).toBe(false); + if (result.success) throw new Error('Expected removed fields to be rejected'); + const messages = collectIssueMessages(result.error.issues); + expect(messages.some((message) => message.includes('preprocessors has been removed'))).toBe( + true, + ); + expect(messages.some((message) => message.includes('postprocess has been removed'))).toBe(true); + }); + it('accepts shared composable graph fields in eval YAML', () => { const result = EvalFileSchema.safeParse({ targets: [ diff --git a/packages/core/test/evaluation/validation/eval-validator.test.ts b/packages/core/test/evaluation/validation/eval-validator.test.ts index 15f6c6a0c..db4ea01bb 100644 --- a/packages/core/test/evaluation/validation/eval-validator.test.ts +++ b/packages/core/test/evaluation/validation/eval-validator.test.ts @@ -233,7 +233,77 @@ tests: expect.objectContaining({ severity: 'error', location: 'execution.workers', - message: expect.stringContaining('authored run controls'), + message: expect.stringContaining('evaluate_options.max_concurrency'), + }), + ); + }); + + it('rejects removed preprocessor and postprocess authored fields', async () => { + const filePath = path.join(tempDir, 'removed-transform-fields.yaml'); + await writeFile( + filePath, + `preprocessors: + - type: xlsx + command: ["node", "xlsx.js"] +prompts: + - "{{ input }}" +default_test: + options: + postprocess: output.trim() +tests: + - id: local-case + vars: + input: "Hello" + options: + postprocess: output.trim() + assert: + - type: llm-rubric + value: "Good" + postprocess: output.trim() + - type: llm-rubric + value: "Also good" + preprocessors: + - type: xlsx + command: ["node", "xlsx.js"] +`, + ); + + const result = await validateEvalFile(filePath); + + expect(result.valid).toBe(false); + expect(result.errors).toContainEqual( + expect.objectContaining({ + severity: 'error', + location: 'preprocessors', + message: expect.stringContaining('default_test.options.transform'), + }), + ); + expect(result.errors).toContainEqual( + expect.objectContaining({ + severity: 'error', + location: 'default_test.options.postprocess', + message: expect.stringContaining('default_test.options.transform'), + }), + ); + expect(result.errors).toContainEqual( + expect.objectContaining({ + severity: 'error', + location: 'tests[0].options.postprocess', + message: expect.stringContaining('tests[0].options.transform'), + }), + ); + expect(result.errors).toContainEqual( + expect.objectContaining({ + severity: 'error', + location: 'tests[0].assert[0].postprocess', + message: expect.stringContaining('tests[0].assert[0].transform'), + }), + ); + expect(result.errors).toContainEqual( + expect.objectContaining({ + severity: 'error', + location: 'tests[0].assert[1].preprocessors', + message: expect.stringContaining('tests[0].assert[1].transform'), }), ); }); diff --git a/packages/core/test/evaluation/validation/targets-validator.test.ts b/packages/core/test/evaluation/validation/targets-validator.test.ts index dfcfb7b3a..6bb6e99c1 100644 --- a/packages/core/test/evaluation/validation/targets-validator.test.ts +++ b/packages/core/test/evaluation/validation/targets-validator.test.ts @@ -24,7 +24,7 @@ describe('validateTargetsFile', () => { `targets: - id: openrouter-target provider: openrouter - api_key: \${{ OPENROUTER_API_KEY }} + api_key: "{{ env.OPENROUTER_API_KEY }}" model: openai/gpt-5-mini `, ); @@ -49,10 +49,10 @@ describe('validateTargetsFile', () => { provider: codex-cli config: command: ["codex"] - model: \${{ CODEX_MODEL }} + model: "{{ env.CODEX_MODEL }}" reasoning_effort: low - base_url: \${{ OPENAI_BASE_URL }} - api_key: \${{ OPENAI_API_KEY }} + base_url: "{{ env.OPENAI_BASE_URL }}" + api_key: "{{ env.OPENAI_API_KEY }}" api_format: responses grader_target: grader fallback_targets: [backup-agent] @@ -60,7 +60,7 @@ describe('validateTargetsFile', () => { - id: grader provider: openai config: - api_key: \${{ OPENAI_API_KEY }} + api_key: "{{ env.OPENAI_API_KEY }}" model: gpt-5-mini - id: backup-agent provider: mock @@ -302,8 +302,8 @@ targets: - id: codex-target provider: codex-cli command: ["codex"] - model: \${{ CODEX_MODEL }} - reasoning_effort: \${{ CODEX_REASONING_EFFORT }} + model: "{{ env.CODEX_MODEL }}" + reasoning_effort: "{{ env.CODEX_REASONING_EFFORT }}" `, ); @@ -321,16 +321,16 @@ targets: provider: copilot-sdk model: gpt-5 subprovider: openai - base_url: \${{ OPENAI_ENDPOINT }} - api_key: \${{ OPENAI_API_KEY }} + base_url: "{{ env.OPENAI_ENDPOINT }}" + api_key: "{{ env.OPENAI_API_KEY }}" api_format: responses model_id: gpt-5 - wire_model: \${{ OPENAI_MODEL }} + wire_model: "{{ env.OPENAI_MODEL }}" - id: copilot-cli-custom-provider provider: copilot-cli subprovider: openai - base_url: \${{ OPENAI_ENDPOINT }} - api_key: \${{ OPENAI_API_KEY }} + base_url: "{{ env.OPENAI_ENDPOINT }}" + api_key: "{{ env.OPENAI_API_KEY }}" api_format: responses `, ); @@ -349,11 +349,11 @@ targets: - id: codex-local-openai provider: codex-cli command: ["codex"] - model: \${{ CODEX_MODEL }} + model: "{{ env.CODEX_MODEL }}" reasoning_effort: medium model_verbosity: medium - base_url: \${{ OPENAI_ENDPOINT }} - api_key: \${{ OPENAI_API_KEY }} + base_url: "{{ env.OPENAI_ENDPOINT }}" + api_key: "{{ env.OPENAI_API_KEY }}" api_format: responses sandbox_mode: danger-full-access approval_policy: never @@ -375,20 +375,20 @@ targets: provider: copilot-sdk custom_provider: type: openai - base_url: \${{ OPENAI_ENDPOINT }} - api_key: \${{ OPENAI_API_KEY }} + base_url: "{{ env.OPENAI_ENDPOINT }}" + api_key: "{{ env.OPENAI_API_KEY }}" - label: copilot-sdk-byok provider: copilot-sdk byok: type: openai - base_url: \${{ OPENAI_ENDPOINT }} - api_key: \${{ OPENAI_API_KEY }} + base_url: "{{ env.OPENAI_ENDPOINT }}" + api_key: "{{ env.OPENAI_API_KEY }}" - label: copilot-cli-custom provider: copilot-cli custom_provider: type: openai - base_url: \${{ OPENAI_ENDPOINT }} - api_key: \${{ OPENAI_API_KEY }} + base_url: "{{ env.OPENAI_ENDPOINT }}" + api_key: "{{ env.OPENAI_API_KEY }}" `, ); @@ -414,9 +414,9 @@ targets: filePath, `targets: - id: default - use_target: \${{ AGENT_TARGET }} + use_target: "{{ env.AGENT_TARGET }}" - id: grader - use_target: \${{ GRADER_TARGET }} + use_target: "{{ env.GRADER_TARGET }}" - id: codex-agent provider: codex-cli command: ["codex"] @@ -454,6 +454,30 @@ targets: } }); + it('rejects legacy env interpolation in target YAML', async () => { + const filePath = path.join(tempDir, 'legacy-env-target.yaml'); + await writeFile( + filePath, + `targets: + - id: openai-target + provider: openai + api_key: \${{ OPENAI_API_KEY }} + model: gpt-5-mini +`, + ); + + const result = await validateTargetsFile(filePath); + + expect(result.valid).toBe(false); + expect(result.errors).toContainEqual( + expect.objectContaining({ + severity: 'error', + location: 'targets[0].api_key', + message: expect.stringContaining('Use {{ env.OPENAI_API_KEY }} instead'), + }), + ); + }); + it('rejects removed judge_target alias', async () => { const filePath = path.join(tempDir, 'judge-target-alias.yaml'); await writeFile( @@ -525,9 +549,9 @@ targets: `targets: - label: azure-responses provider: azure - endpoint: \${{ AZURE_OPENAI_ENDPOINT }} - api_key: \${{ AZURE_OPENAI_API_KEY }} - model: \${{ AZURE_DEPLOYMENT_NAME }} + endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}" + api_key: "{{ env.AZURE_OPENAI_API_KEY }}" + model: "{{ env.AZURE_DEPLOYMENT_NAME }}" api_format: responses `, ); diff --git a/scripts/migrate-hard-deprecations.test.ts b/scripts/migrate-hard-deprecations.test.ts new file mode 100644 index 000000000..0166d6518 --- /dev/null +++ b/scripts/migrate-hard-deprecations.test.ts @@ -0,0 +1,127 @@ +import { describe, expect, it } from 'bun:test'; +import { parse } from 'yaml'; + +import { _internal } from './migrate-hard-deprecations.ts'; + +function migrateSnippet(source: string): string { + const migrated = _internal.migrateYamlSnippet(source, '/tmp/suite.eval.yaml'); + if (migrated === undefined) { + throw new Error('expected snippet to migrate'); + } + return migrated; +} + +function asRecord(value: unknown): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('expected object'); + } + return value as Record; +} + +describe('migrate-hard-deprecations', () => { + it('migrates suite execution, artifact, and workspace legacy keys', () => { + const migrated = migrateSnippet(`description: legacy +execution: + workers: 4 +workspace: + isolation: per_case + repos: + - path: ./repo + repo: https://github.com/org/repo.git +results: + timing_path: .agentv/results/run/timing.json + manifest_path: .agentv/results/run/manifest.json +tests: + - id: one + vars: + input: hi +`); + const parsed = parse(migrated) as Record; + + expect(parsed.execution).toBeUndefined(); + expect(parsed.evaluate_options).toEqual({ max_concurrency: 4 }); + expect(parsed.workspace).toEqual({ + scope: 'attempt', + repos: [{ path: './repo', repo: 'https://github.com/org/repo.git' }], + }); + expect(parsed.results).toEqual({ + metrics_path: '.agentv/results/run/metrics.json', + index_path: '.agentv/results/run/manifest.json', + }); + }); + + it('migrates legacy env templates while preserving shell-time command variables', () => { + const parsed = parse(`targets: + - id: cli + provider: cli + command: echo $FOO \${BAR} + cwd: \${{ WORKSPACE_DIR }} + args: + - \${{ CODEX_MODEL }} +`) as unknown; + + expect(_internal.migrateYamlValue(parsed, '/tmp/targets.yaml')).toBe(true); + expect(parsed).toEqual({ + targets: [ + { + id: 'cli', + provider: 'cli', + command: 'echo $FOO ${BAR}', + cwd: '{{ env.WORKSPACE_DIR }}', + args: ['{{ env.CODEX_MODEL }}'], + }, + ], + }); + }); + + it('migrates postprocess and preprocessors to Promptfoo-compatible transform fields', () => { + const migrated = migrateSnippet(`preprocessors: + - type: xlsx + command: + - bun + - scripts/xlsx.ts +default_test: + vars: + topic: revenue +prompts: + - "{{ input }}" +tests: + - id: one + options: + postprocess: output.toUpperCase() + assert: + - type: llm-rubric + value: ok + postprocess: output.trim() + - type: llm-rubric + value: also ok + preprocessors: + - type: xlsx + command: + - node + - xlsx.js + vars: + input: hi +`); + const parsed = asRecord(parse(migrated)); + const defaultTest = asRecord(parsed.default_test); + const defaultOptions = asRecord(defaultTest.options); + const tests = parsed.tests as Array>; + const firstTest = asRecord(tests[0]); + const firstTestOptions = asRecord(firstTest.options); + const assertions = firstTest.assert as Array>; + const firstAssertion = asRecord(assertions[0]); + const secondAssertion = asRecord(assertions[1]); + + expect(parsed.preprocessors).toBeUndefined(); + expect(defaultOptions.postprocess).toBeUndefined(); + expect(defaultOptions.transform).toContain('return (() =>'); + expect(defaultOptions.transform).toContain('Bun.spawnSync(["bun","scripts/xlsx.ts"]'); + expect(firstTestOptions).toEqual({ transform: 'output.toUpperCase()' }); + expect(firstAssertion).toMatchObject({ transform: 'output.trim()' }); + expect(firstAssertion.postprocess).toBeUndefined(); + expect(secondAssertion.preprocessors).toBeUndefined(); + expect(secondAssertion.transform).toContain('return (() =>'); + expect(secondAssertion.transform).toContain('Bun.spawnSync(["node","xlsx.js"]'); + }); +}); diff --git a/scripts/migrate-direct-input.ts b/scripts/migrate-hard-deprecations.ts similarity index 56% rename from scripts/migrate-direct-input.ts rename to scripts/migrate-hard-deprecations.ts index edb7ebb69..81319238e 100644 --- a/scripts/migrate-direct-input.ts +++ b/scripts/migrate-hard-deprecations.ts @@ -4,8 +4,17 @@ import { parse, stringify } from 'yaml'; type JsonObject = Record; -const ROOTS = ['examples', 'evals', 'apps/cli/test/commands/eval/pipeline/fixtures']; +const ROOTS = [ + 'examples', + 'evals', + 'apps/cli/src/templates', + 'apps/cli/test/commands/eval/pipeline/fixtures', +]; const INPUT_PROMPT = '{{ input }}'; +const LEGACY_ENV_PATTERN = /\$\{\{\s*([A-Z_][A-Z0-9_]*)\s*\}\}/g; +const PREPROCESSOR_MEDIA_TYPES: Readonly> = { + xlsx: ['xlsx', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', '.xlsx'], +}; const TEST_TS_FILES = [ 'apps/cli/test/commands/eval/artifact-writer.test.ts', 'apps/cli/test/commands/eval/bundle.test.ts', @@ -20,11 +29,9 @@ const TEST_TS_FILES = [ 'packages/core/test/evaluation/criteria-optional.test.ts', 'packages/core/test/evaluation/extensions.test.ts', 'packages/core/test/evaluation/interpolation-integration.test.ts', - 'packages/core/test/evaluation/preprocessors-yaml.test.ts', 'packages/core/test/evaluation/repo-schema-validation.test.ts', 'packages/core/test/evaluation/rubric-operators-yaml.test.ts', 'packages/core/test/evaluation/source-traceability.test.ts', - 'packages/core/test/evaluation/workspace-config-parsing.test.ts', 'packages/core/test/evaluation/yaml-parser-tags-map.test.ts', 'packages/core/test/evaluation/loaders/ts-eval-loader.test.ts', 'packages/core/test/evaluation/loaders/case-file-loader.test.ts', @@ -42,7 +49,7 @@ function walk(dir: string, files: string[] = []): string[] { const fullPath = path.join(dir, entry); const stat = statSync(fullPath); if (stat.isDirectory()) { - if (!['node_modules', 'dist', '.agentv', '.beads'].includes(entry)) { + if (!['node_modules', 'dist', '.beads'].includes(entry)) { walk(fullPath, files); } } else if (/\.(ya?ml)$/i.test(entry)) { @@ -56,6 +63,214 @@ function clone(value: T): T { return value === undefined ? value : (JSON.parse(JSON.stringify(value)) as T); } +function migrateLegacyEnvReference(value: string): string { + return value.replace(LEGACY_ENV_PATTERN, (_match, name: string) => `{{ env.${name} }}`); +} + +function setNestedObjectValue(parent: JsonObject, key: string, value: unknown): void { + const child = isObject(parent[key]) ? parent[key] : {}; + parent[key] = child; + Object.assign(child, value); +} + +function migrateExecutionConcurrency(suite: JsonObject): boolean { + if (!isObject(suite.execution)) return false; + + const execution = suite.execution; + const concurrency = execution.max_concurrency ?? execution.workers; + let changed = false; + if (concurrency !== undefined) { + const evaluateOptions = isObject(suite.evaluate_options) ? suite.evaluate_options : {}; + if (evaluateOptions.max_concurrency === undefined) { + evaluateOptions.max_concurrency = clone(concurrency); + suite.evaluate_options = evaluateOptions; + } + Reflect.deleteProperty(execution, 'max_concurrency'); + Reflect.deleteProperty(execution, 'workers'); + changed = true; + } + + if (Object.keys(execution).length === 0) { + Reflect.deleteProperty(suite, 'execution'); + changed = true; + } + return changed; +} + +function transformWrapperForPreprocessors(preprocessors: unknown): string | undefined { + if (!Array.isArray(preprocessors) || preprocessors.length !== 1) return undefined; + const preprocessor = preprocessors[0]; + if (!isObject(preprocessor)) return undefined; + const rawType = typeof preprocessor.type === 'string' ? preprocessor.type : undefined; + const rawCommand = preprocessor.command; + const command = + typeof rawCommand === 'string' + ? [rawCommand] + : Array.isArray(rawCommand) && rawCommand.every((entry) => typeof entry === 'string') + ? rawCommand + : undefined; + if (!rawType || !command || command.length === 0) return undefined; + + const matchers = PREPROCESSOR_MEDIA_TYPES[rawType] ?? [rawType]; + const commandLiteral = JSON.stringify(command); + const matcherLiteral = JSON.stringify(matchers); + return `return (() => { + const content = Array.isArray(output) ? output : []; + const matchers = ${matcherLiteral}; + const file = content.find((block) => { + if (!block || block.type !== "file") return false; + const mediaType = typeof block.media_type === "string" ? block.media_type : ""; + const filePath = typeof block.path === "string" ? block.path : ""; + return matchers.some((matcher) => mediaType === matcher || filePath.endsWith(matcher)); + }); + if (!file || typeof file.path !== "string") return output; + const result = Bun.spawnSync(${commandLiteral}, { + stdin: JSON.stringify({ path: file.path, media_type: file.media_type }) + }); + if (result.exitCode !== 0) { + throw new Error(new TextDecoder().decode(result.stderr).trim() || "preprocessor command failed"); + } + return new TextDecoder().decode(result.stdout).trim(); +})()`; +} + +function migrateOptionsPostprocess(options: unknown): boolean { + if (!isObject(options) || options.postprocess === undefined) return false; + if (options.transform === undefined) { + options.transform = clone(options.postprocess); + } + Reflect.deleteProperty(options, 'postprocess'); + return true; +} + +function migrateAssertionTransform(assertion: JsonObject): boolean { + let changed = false; + if (assertion.postprocess !== undefined) { + if (assertion.transform === undefined) { + assertion.transform = clone(assertion.postprocess); + } + Reflect.deleteProperty(assertion, 'postprocess'); + changed = true; + } + if (assertion.preprocessors !== undefined) { + const transform = transformWrapperForPreprocessors(assertion.preprocessors); + if (transform && assertion.transform === undefined) { + assertion.transform = transform; + } + Reflect.deleteProperty(assertion, 'preprocessors'); + changed = true; + } + return changed; +} + +function migrateAssertions(assertions: unknown): boolean { + if (!Array.isArray(assertions)) return false; + return assertions + .filter(isObject) + .map((assertion) => { + const nested = migrateAssertions(assertion.assert); + return migrateAssertionTransform(assertion) || nested; + }) + .some(Boolean); +} + +function migrateSuitePreprocessors(suite: JsonObject): boolean { + if (suite.preprocessors === undefined) return false; + const transform = transformWrapperForPreprocessors(suite.preprocessors); + if (transform) { + const defaultTest = isObject(suite.default_test) ? suite.default_test : {}; + suite.default_test = defaultTest; + setNestedObjectValue(defaultTest, 'options', { + ...(isObject(defaultTest.options) ? defaultTest.options : {}), + ...(isObject(defaultTest.options) && defaultTest.options.transform !== undefined + ? {} + : { transform }), + }); + } + Reflect.deleteProperty(suite, 'preprocessors'); + return true; +} + +function migrateWorkspace(workspace: unknown): boolean { + if (!isObject(workspace)) return false; + if (workspace.isolation === undefined && workspace.mode === undefined) return false; + + const raw = workspace.isolation ?? workspace.mode; + if (workspace.scope === undefined) { + if (raw === 'shared' || raw === 'suite') { + workspace.scope = 'suite'; + } else if (raw === 'per_case' || raw === 'per_test' || raw === 'attempt' || raw === 'fresh') { + workspace.scope = 'attempt'; + } + } + Reflect.deleteProperty(workspace, 'isolation'); + Reflect.deleteProperty(workspace, 'mode'); + return true; +} + +function migrateDeprecatedArtifactKeys(value: unknown): boolean { + if (Array.isArray(value)) { + return value.map(migrateDeprecatedArtifactKeys).some(Boolean); + } + if (!isObject(value)) return false; + + let changed = false; + for (const nested of Object.values(value)) { + changed = migrateDeprecatedArtifactKeys(nested) || changed; + } + if (Object.hasOwn(value, 'timing_path')) { + if (value.metrics_path === undefined) { + value.metrics_path = + typeof value.timing_path === 'string' + ? value.timing_path.replace(/timing\.json/g, 'metrics.json') + : clone(value.timing_path); + } + Reflect.deleteProperty(value, 'timing_path'); + changed = true; + } + if (Object.hasOwn(value, 'manifest_path')) { + if (value.index_path === undefined) { + value.index_path = clone(value.manifest_path); + } + Reflect.deleteProperty(value, 'manifest_path'); + changed = true; + } + return changed; +} + +function migrateLegacyEnvReferences(value: unknown): boolean { + if (Array.isArray(value)) { + return value + .map((entry, index) => { + if (typeof entry === 'string') { + const next = migrateLegacyEnvReference(entry); + if (next !== entry) { + value[index] = next; + return true; + } + return false; + } + return migrateLegacyEnvReferences(entry); + }) + .some(Boolean); + } + if (!isObject(value)) return false; + + let changed = false; + for (const [key, nested] of Object.entries(value)) { + if (typeof nested === 'string') { + const next = migrateLegacyEnvReference(nested); + if (next !== nested) { + value[key] = next; + changed = true; + } + } else { + changed = migrateLegacyEnvReferences(nested) || changed; + } + } + return changed; +} + function inputFilesToMessage( inputFiles: unknown, input: unknown, @@ -245,10 +460,22 @@ function migrateYamlValue( if (!isObject(value)) return false; + let changed = false; + changed = migrateLegacyEnvReferences(value) || changed; + changed = migrateDeprecatedArtifactKeys(value) || changed; + changed = migrateExecutionConcurrency(value) || changed; + changed = migrateWorkspace(value.workspace) || changed; + changed = + migrateOptionsPostprocess( + isObject(value.default_test) ? value.default_test.options : undefined, + ) || changed; + changed = migrateSuitePreprocessors(value) || changed; + changed = migrateAssertions(value.assert) || changed; + const isSuiteLike = value.tests !== undefined || value.eval_cases !== undefined || value.imports !== undefined; if (!isSuiteLike && inheritedSuiteInput === undefined) { - return false; + return changed; } const hasSuiteInput = Object.hasOwn(value, 'input') || Object.hasOwn(value, 'input_files'); @@ -256,11 +483,13 @@ function migrateYamlValue( ? inputFilesToMessage(value.input_files, value.input, fileDir) : inheritedSuiteInput; - let changed = false; const rawTests = value.tests ?? value.eval_cases; if (Array.isArray(rawTests)) { for (const entry of rawTests) { if (isObject(entry) && !Object.hasOwn(entry, 'include')) { + changed = migrateWorkspace(entry.workspace) || changed; + changed = migrateOptionsPostprocess(entry.options) || changed; + changed = migrateAssertions(entry.assert) || changed; changed = migrateCase(entry, fileDir, suiteInput) || changed; } } @@ -295,8 +524,20 @@ function migrateYamlFile(filePath: string, inheritedSuiteInput?: unknown): boole } function migrateYamlSnippet(source: string, filePath: string): string | undefined { - if (!source.includes('input:') && !source.includes('input_files:')) return undefined; - if (source.includes('${')) return undefined; + if ( + !source.includes('input:') && + !source.includes('input_files:') && + !source.includes('${{') && + !source.includes('execution:') && + !source.includes('preprocessors:') && + !source.includes('postprocess:') && + !source.includes('timing_path:') && + !source.includes('manifest_path:') && + !source.includes('isolation:') + ) { + return undefined; + } + if (source.replace(LEGACY_ENV_PATTERN, '').includes('${')) return undefined; try { const parsed = parse(source, { uniqueKeys: false }) as unknown; if (Array.isArray(parsed)) { @@ -352,14 +593,25 @@ function migrateTestTemplateLiterals(filePath: string): boolean { return changed; } -let changedCount = 0; -for (const root of ROOTS) { - for (const filePath of walk(root)) { - if (migrateYamlFile(filePath)) changedCount += 1; +export function migrateRepository(): number { + let changedCount = 0; + for (const root of ROOTS) { + for (const filePath of walk(root)) { + if (migrateYamlFile(filePath)) changedCount += 1; + } } -} -for (const filePath of TEST_TS_FILES) { - if (migrateTestTemplateLiterals(filePath)) changedCount += 1; + for (const filePath of TEST_TS_FILES) { + if (migrateTestTemplateLiterals(filePath)) changedCount += 1; + } + return changedCount; } -console.log(`Migrated ${changedCount} YAML roots`); +export const _internal = { + migrateYamlSnippet, + migrateYamlValue, + transformWrapperForPreprocessors, +}; + +if (import.meta.main) { + console.log(`Migrated ${migrateRepository()} YAML roots`); +} diff --git a/skills-data/agentv-eval-writer/SKILL.md b/skills-data/agentv-eval-writer/SKILL.md index b96d51450..1a1db5a91 100644 --- a/skills-data/agentv-eval-writer/SKILL.md +++ b/skills-data/agentv-eval-writer/SKILL.md @@ -215,7 +215,7 @@ then render those vars from the prompt template next to the input. **JSONL format:** One test per line as JSON. Optional `.yaml` sidecar for shared defaults. See `examples/features/basic-jsonl/`. -**Environment variables:** All string fields support `${{ VAR }}` interpolation. Missing vars resolve to empty string. Works in eval files, external case files, and workspace configs. `.env` files are loaded automatically. +**Environment variables:** Use `{{ env.VAR }}` templates in authored config. Missing vars resolve to empty string. Works in eval files, external case files, and workspace configs. `.env` files are loaded automatically. ## Metadata diff --git a/skills-data/agentv-eval-writer/references/eval.schema.json b/skills-data/agentv-eval-writer/references/eval.schema.json index 7dbc2a45a..ecc32f844 100644 --- a/skills-data/agentv-eval-writer/references/eval.schema.json +++ b/skills-data/agentv-eval-writer/references/eval.schema.json @@ -10415,31 +10415,7 @@ } }, "preprocessors": { - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "type": "string", - "minLength": 1 - }, - "command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ] - } - }, - "required": ["type", "command"], - "additionalProperties": false - } + "not": {} }, "workspace": { "anyOf": [ From 9d26fdadd539d296624ef54d5ab5d4f26d80ca55 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Sat, 4 Jul 2026 21:19:22 +0200 Subject: [PATCH 2/3] Fix manifest path codemod target --- scripts/migrate-hard-deprecations.test.ts | 7 ++++++- scripts/migrate-hard-deprecations.ts | 9 ++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/scripts/migrate-hard-deprecations.test.ts b/scripts/migrate-hard-deprecations.test.ts index 0166d6518..8ff749a97 100644 --- a/scripts/migrate-hard-deprecations.test.ts +++ b/scripts/migrate-hard-deprecations.test.ts @@ -31,6 +31,8 @@ workspace: results: timing_path: .agentv/results/run/timing.json manifest_path: .agentv/results/run/manifest.json +results_jsonl: + manifest_path: .agentv/results/run/manifest.jsonl tests: - id: one vars: @@ -46,7 +48,10 @@ tests: }); expect(parsed.results).toEqual({ metrics_path: '.agentv/results/run/metrics.json', - index_path: '.agentv/results/run/manifest.json', + index_path: '.agentv/results/run/index.jsonl', + }); + expect(parsed.results_jsonl).toEqual({ + index_path: '.agentv/results/run/index.jsonl', }); }); diff --git a/scripts/migrate-hard-deprecations.ts b/scripts/migrate-hard-deprecations.ts index 81319238e..a9da4777e 100644 --- a/scripts/migrate-hard-deprecations.ts +++ b/scripts/migrate-hard-deprecations.ts @@ -230,7 +230,7 @@ function migrateDeprecatedArtifactKeys(value: unknown): boolean { } if (Object.hasOwn(value, 'manifest_path')) { if (value.index_path === undefined) { - value.index_path = clone(value.manifest_path); + value.index_path = migrateManifestPathToIndexPath(value.manifest_path); } Reflect.deleteProperty(value, 'manifest_path'); changed = true; @@ -238,6 +238,13 @@ function migrateDeprecatedArtifactKeys(value: unknown): boolean { return changed; } +function migrateManifestPathToIndexPath(value: unknown): unknown { + if (typeof value !== 'string') { + return clone(value); + } + return value.replace(/manifest\.jsonl?$/i, 'index.jsonl'); +} + function migrateLegacyEnvReferences(value: unknown): boolean { if (Array.isArray(value)) { return value From 17720bbb768da193ce77a5d4fa2d1210745e21ab Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Sat, 4 Jul 2026 21:27:16 +0200 Subject: [PATCH 3/3] Fix rerun env fixture syntax --- apps/cli/test/commands/runs/rerun.test.ts | 4 ++-- scripts/migrate-hard-deprecations.ts | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/cli/test/commands/runs/rerun.test.ts b/apps/cli/test/commands/runs/rerun.test.ts index cda0f2043..c13bbdeeb 100644 --- a/apps/cli/test/commands/runs/rerun.test.ts +++ b/apps/cli/test/commands/runs/rerun.test.ts @@ -280,7 +280,7 @@ describe('agentv runs rerun', () => { const created = await fixture(`targets: - id: captured provider: cli - command: \${{ LOCAL_AGENT_COMMAND }} + command: "{{ env.LOCAL_AGENT_COMMAND }}" `); const missing = await runCli(created, [ @@ -292,7 +292,7 @@ describe('agentv runs rerun', () => { '--dry-run', ]); expect(missing.exitCode).toBe(1); - expect(missing.stderr).toContain('Missing environment variable(s)'); + expect(missing.stderr).toContain("Failed to resolve target 'captured'"); expect(missing.stderr).toContain('LOCAL_AGENT_COMMAND'); const withAmbientEnv = await runCli( diff --git a/scripts/migrate-hard-deprecations.ts b/scripts/migrate-hard-deprecations.ts index a9da4777e..6f4d8bdb0 100644 --- a/scripts/migrate-hard-deprecations.ts +++ b/scripts/migrate-hard-deprecations.ts @@ -23,6 +23,7 @@ const TEST_TS_FILES = [ 'apps/cli/test/commands/eval/task-bundle.test.ts', 'apps/cli/test/commands/grade/grade-prepared.test.ts', 'apps/cli/test/commands/prepare/prepare.test.ts', + 'apps/cli/test/commands/runs/rerun.test.ts', 'apps/cli/test/commands/workspace/deps.test.ts', 'apps/cli/test/eval.integration.test.ts', 'packages/core/test/evaluation/conversation-mode.test.ts',