From 90e5cdc7e1277be4a8d25a020b942be2c5020289 Mon Sep 17 00:00:00 2001 From: Christopher Date: Mon, 6 Jul 2026 21:32:25 +1000 Subject: [PATCH] fix(eval): repair broken targets.yaml and two eval-run bugs found during dogfood .agentv/targets.yaml failed validation on main: a recent hard-deprecation pass rejected the legacy ${{ VAR }} interpolation syntax and removed the use_target field with no replacement, but never migrated this repo-root file. Migrate all scalar fields to {{ env.VAR }} and turn default/agent/llm/grader from use_target aliases into concrete targets mirroring today's AGENT_TARGET/LLM_TARGET/GRADER_TARGET values, so `target: llm` etc. across examples/ keep resolving without editing 65+ example eval files. Also fixes two bugs surfaced while dogfooding: - `--results-repo current/.` (the exact syntax documented in --help) fell through to being parsed as an owner/repo GitHub shorthand instead of the source repo, producing a bogus `git clone https://github.com/current/..git`. - `metadata.eval_file` in summary.json could be an absolute host path instead of repo-relative, since the two call sites building it never relativized against cwd (unlike the third call site nearby, which already did). Co-Authored-By: Claude Sonnet 5 Entire-Checkpoint: bfb9174c1739 --- .agentv/targets.yaml | 141 ++++++++++++++----------- apps/cli/src/commands/eval/run-eval.ts | 7 +- 2 files changed, 84 insertions(+), 64 deletions(-) diff --git a/.agentv/targets.yaml b/.agentv/targets.yaml index 6d1176e6d..1c7a4cd34 100644 --- a/.agentv/targets.yaml +++ b/.agentv/targets.yaml @@ -4,48 +4,67 @@ # # "grader" is the LLM used for scoring; agent targets reference it via # grader_target so eval execution and grading use separate models. +# +# default/agent/llm/grader are concrete targets, not env-var-driven aliases: +# AgentV removed the `use_target` indirection field with no replacement, so a +# target id can no longer redirect to "whichever target another target names". +# These four mirror today's AGENT_TARGET/LLM_TARGET/GRADER_TARGET selection +# (copilot-cli / azure / azure) so `target: llm` etc. across examples/ keep +# resolving. To switch the active provider, edit the definitions below +# directly (or point examples at a different concrete target id) instead of +# changing an env var. targets: - # ── Default target (use) ─────────────────────────────────────────── - # Evals without an explicit target resolve to "default". The use - # redirects to a named target, controlled via AGENT_TARGET env var. - # One env var switches the entire provider config (auth, model, etc.). - # Example: AGENT_TARGET=copilot-cli or AGENT_TARGET=claude + # ── Default target ────────────────────────────────────────────────── + # Evals without an explicit target resolve to "default" by convention. - id: default - use_target: ${{ AGENT_TARGET }} + provider: copilot-cli + model: "{{ env.COPILOT_MODEL }}" + grader_target: grader + stream_log: raw - id: agent - use_target: ${{ AGENT_TARGET }} + provider: copilot-cli + model: "{{ env.COPILOT_MODEL }}" + grader_target: grader + stream_log: raw # ── LLM target (text generation, no agent binary needed) ──────────── - # Delegates to LLM_TARGET — same provider used for grading and LLM evals. - id: llm - use_target: ${{ LLM_TARGET }} + provider: azure + endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}" + api_key: "{{ env.AZURE_OPENAI_API_KEY }}" + model: "{{ env.AZURE_DEPLOYMENT_NAME }}" + version: "{{ env.AZURE_OPENAI_API_VERSION }}" # ── Grader (LLM-as-judge) ────────────────────────────────────────── - # Used by agent targets via grader_target. Switch provider via GRADER_TARGET. + # Used by agent targets via grader_target. - id: grader - use_target: ${{ GRADER_TARGET }} + provider: azure + endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}" + api_key: "{{ env.AZURE_OPENAI_API_KEY }}" + model: "{{ env.AZURE_DEPLOYMENT_NAME }}" + version: "{{ env.AZURE_OPENAI_API_VERSION }}" # ── Named agent targets ─────────────────────────────────────────── - id: copilot provider: copilot-cli - model: ${{ COPILOT_MODEL }} + model: "{{ env.COPILOT_MODEL }}" grader_target: grader stream_log: raw - id: copilot-sdk provider: copilot-sdk - model: ${{ COPILOT_MODEL }} + model: "{{ env.COPILOT_MODEL }}" grader_target: grader stream_log: raw - id: copilot-sdk-azure provider: copilot-sdk - model: ${{ AZURE_DEPLOYMENT_NAME }} + model: "{{ env.AZURE_DEPLOYMENT_NAME }}" subprovider: azure - base_url: ${{ AZURE_OPENAI_ENDPOINT }} - api_key: ${{ AZURE_OPENAI_API_KEY }} + base_url: "{{ env.AZURE_OPENAI_ENDPOINT }}" + api_key: "{{ env.AZURE_OPENAI_API_KEY }}" grader_target: grader stream_log: raw @@ -60,15 +79,15 @@ targets: - id: pi provider: pi-cli subprovider: openrouter - model: ${{ OPENROUTER_MODEL }} - api_key: ${{ OPENROUTER_API_KEY }} + model: "{{ env.OPENROUTER_MODEL }}" + api_key: "{{ env.OPENROUTER_API_KEY }}" grader_target: grader - id: pi-sdk provider: pi-coding-agent subprovider: openai - base_url: ${{ OPENAI_ENDPOINT }} - api_key: ${{ OPENAI_API_KEY }} + base_url: "{{ env.OPENAI_ENDPOINT }}" + api_key: "{{ env.OPENAI_API_KEY }}" model: gpt-5.5 grader_target: openai thinking: low @@ -78,37 +97,37 @@ targets: - id: pi-azure provider: pi-cli subprovider: azure - base_url: ${{ AZURE_OPENAI_ENDPOINT }} - model: ${{ AZURE_DEPLOYMENT_NAME }} - api_key: ${{ AZURE_OPENAI_API_KEY }} + base_url: "{{ env.AZURE_OPENAI_ENDPOINT }}" + model: "{{ env.AZURE_DEPLOYMENT_NAME }}" + api_key: "{{ env.AZURE_OPENAI_API_KEY }}" grader_target: grader - id: pi-sdk-azure provider: pi-coding-agent subprovider: azure - base_url: ${{ AZURE_OPENAI_ENDPOINT }} + base_url: "{{ env.AZURE_OPENAI_ENDPOINT }}" model: gpt-5.5 - api_key: ${{ AZURE_OPENAI_API_KEY }} + api_key: "{{ env.AZURE_OPENAI_API_KEY }}" grader_target: grader thinking: low stream_log: raw - id: codex provider: codex - executable: ${{ CODEX_EXECUTABLE }} - model: ${{ CODEX_MODEL }} - model_reasoning_effort: ${{ CODEX_REASONING_EFFORT }} + executable: "{{ env.CODEX_EXECUTABLE }}" + model: "{{ env.CODEX_MODEL }}" + model_reasoning_effort: "{{ env.CODEX_REASONING_EFFORT }}" grader_target: grader - cwd: ${{ CODEX_WORKSPACE_DIR }} - log_dir: ${{ CODEX_LOG_DIR }} + cwd: "{{ env.CODEX_WORKSPACE_DIR }}" + log_dir: "{{ env.CODEX_LOG_DIR }}" stream_log: raw # ── LLM targets (direct model access) ───────────────────────────── - id: gh-models provider: openai base_url: https://models.github.ai/inference - api_key: ${{ GH_MODELS_TOKEN }} - model: ${{ GH_MODELS_MODEL }} + api_key: "{{ env.GH_MODELS_TOKEN }}" + model: "{{ env.GH_MODELS_MODEL }}" # Single Azure target. Always uses Azure's Responses API # (`/openai/v1/responses`); the api version defaults to `v1` and can be @@ -117,52 +136,52 @@ targets: # `base_url` instead. - id: azure provider: azure - endpoint: ${{ AZURE_OPENAI_ENDPOINT }} - api_key: ${{ AZURE_OPENAI_API_KEY }} - model: ${{ AZURE_DEPLOYMENT_NAME }} - version: ${{ AZURE_OPENAI_API_VERSION }} + endpoint: "{{ env.AZURE_OPENAI_ENDPOINT }}" + api_key: "{{ env.AZURE_OPENAI_API_KEY }}" + model: "{{ env.AZURE_DEPLOYMENT_NAME }}" + version: "{{ env.AZURE_OPENAI_API_VERSION }}" - id: gemini provider: gemini - api_key: ${{ GOOGLE_GENERATIVE_AI_API_KEY }} - model: ${{ GEMINI_MODEL_NAME }} + api_key: "{{ env.GOOGLE_GENERATIVE_AI_API_KEY }}" + model: "{{ env.GEMINI_MODEL_NAME }}" - id: openai provider: openai - endpoint: ${{ OPENAI_ENDPOINT }} - api_key: ${{ OPENAI_API_KEY }} - model: ${{ OPENAI_MODEL }} + endpoint: "{{ env.OPENAI_ENDPOINT }}" + api_key: "{{ env.OPENAI_API_KEY }}" + model: "{{ env.OPENAI_MODEL }}" # Local OpenAI-compatible endpoint. Useful for dogfood against a local proxy # without changing provider-specific target labels. - id: local-openai provider: openai - base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }} - api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }} - model: ${{ LOCAL_OPENAI_PROXY_MODEL }} + base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" + api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" + model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" grader_target: local-openai-grader - id: local-openai-grader provider: openai - base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }} - api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }} - model: ${{ LOCAL_OPENAI_PROXY_MODEL }} + base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" + api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" + model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" - id: pi-cli-openai provider: pi-cli subprovider: openai - base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }} - api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }} - model: ${{ LOCAL_OPENAI_PROXY_MODEL }} + base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" + api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" + model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" grader_target: local-openai-grader thinking: low stream_log: raw - id: codex-sdk-openai provider: codex - base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }} - api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }} - model: ${{ LOCAL_OPENAI_PROXY_MODEL }} + base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" + api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" + model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" api_format: responses model_reasoning_effort: low grader_target: local-openai-grader @@ -171,34 +190,34 @@ targets: - id: copilot-sdk-openai provider: copilot-sdk subprovider: openai - base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }} - api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }} - model: ${{ LOCAL_OPENAI_PROXY_MODEL }} + base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}" + api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}" + model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" grader_target: local-openai-grader stream_log: raw - id: openrouter provider: openrouter - api_key: ${{ OPENROUTER_API_KEY }} - model: ${{ OPENROUTER_MODEL }} + api_key: "{{ env.OPENROUTER_API_KEY }}" + model: "{{ env.OPENROUTER_MODEL }}" # ── MiMo (Xiaomi) via OpenRouter ─────────────────────────────────── - id: mimo provider: openrouter - api_key: ${{ OPENROUTER_API_KEY }} + api_key: "{{ env.OPENROUTER_API_KEY }}" model: xiaomi/mimo-v2.5-pro grader_target: grader - id: mimo-flash provider: openrouter - api_key: ${{ OPENROUTER_API_KEY }} + api_key: "{{ env.OPENROUTER_API_KEY }}" model: xiaomi/mimo-v2-flash grader_target: grader - id: mimo-direct provider: openai base_url: https://token-plan-sgp.xiaomimimo.com/v1 - api_key: ${{ XIAOMI_MIMO_API_KEY }} + api_key: "{{ env.XIAOMI_MIMO_API_KEY }}" model: xiaomi/mimo-v2.5-pro max_output_tokens: 131072 grader_target: grader diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index b93d8819e..6643fff71 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -342,6 +342,7 @@ function resultsRepoOverride( } if ( value === 'current' || + value === 'current/.' || value === '.' || value.startsWith('./') || value.startsWith('../') || @@ -350,7 +351,7 @@ function resultsRepoOverride( value.startsWith('~\\') || /^[A-Za-z]:[/\\]/.test(value) ) { - return { repo_path: value === 'current' ? '.' : value }; + return { repo_path: value === 'current' || value === 'current/.' ? '.' : value }; } return { repo: value }; } @@ -2325,7 +2326,7 @@ export async function runEvalCommand( // writeArtifactsFromResults. // Skip on resume — we want to preserve the *original* planned count. if (!isResumeAppend && totalEvalCount > 0) { - const evalFile = activeTestFiles.length === 1 ? activeTestFiles[0] : ''; + const evalFile = activeTestFiles.length === 1 ? path.relative(cwd, activeTestFiles[0]) : ''; await writeInitialRunSummaryArtifact(runDir, { evalFile, plannedTestCount: totalEvalCount, @@ -2621,7 +2622,7 @@ export async function runEvalCommand( // Per-result artifact directories are allocated from row identity and // exposed through index.jsonl fields. if (allResults.length > 0) { - const evalFile = activeTestFiles.length === 1 ? activeTestFiles[0] : ''; + const evalFile = activeTestFiles.length === 1 ? path.relative(cwd, activeTestFiles[0]) : ''; const sourceTests = activeSourceTests; const taskBundleTargets = buildTaskBundleTargetSelections(activeTestFiles, fileMetadata); if (isResumeAppend) {