Migrate example eval suites to suite.yaml

.agents/verification.md

@@ -138,12 +138,13 @@ bun apps/cli/src/cli.ts eval examples/features/rubric/evals/dataset.eval.yaml --
 Use live dogfood before marking PRs ready when they affect eval execution, experiments, repeat runs, targets, providers, graders, or artifact provenance.
 
 - Live means both sides are real: a live agent/provider target and a live grader target. Do not count `mock`, replay/frozen transcript runs, or deterministic-only assertions as dogfood for these changes.
+- Dogfood must use a threshold high enough to prove correctness for the eval under test. A threshold of `0` is only an execution smoke check and does not count as dogfood evidence.
 - Prefer the smallest realistic eval: one or two cases, bounded timeouts, and `workers: 1` for heavyweight agent providers.
 - For artifact/result contract changes, prefer letting AgentV choose the canonical run directory and capture the printed `Artifact workspace written to:` and `Results written to:` paths for evidence. Do not precompute `--output` unless the test specifically needs a fixed path.
 - For native experiment changes, run through `agentv eval run ... --experiment <experiment.yaml|ts>` so resolution, setup, scripts, target selection, run knobs, and artifact metadata are exercised together.
 - For repeat-run changes, use `evaluate_options.repeat.count >= 2` when validating repeated attempts. Inspect root `index.jsonl`, root `summary.json`, and the repeated case folder. Use `repeat` for authored configuration and `attempts[]` for produced executions. The repeated case folder should carry aggregate `summary.json` with flattened snake_case timing fields; attempt-specific outputs, transcripts, and metrics live under `attempt-N/`. Each `attempt-N/` folder should contain `result.json`, `grading.json`, `metrics.json`, `transcript.json`, `transcript-raw.jsonl`, and `outputs/answer.md` when answer output is available. `result.json` should point at `./grading.json`, `./metrics.json`, `./transcript.json`, and `./transcript-raw.jsonl` through the corresponding path fields.
 - For local OpenAI-compatible grading through the OAuth proxy, use `endpoint: http://127.0.0.1:10531/v1`, but still route `api_key` and `model` through environment references such as `${{ LOCAL_OPENAI_PROXY_API_KEY }}` and `${{ LOCAL_OPENAI_PROXY_MODEL }}`. Literal secrets and literal model values are intentionally rejected by target validation unless a resolver explicitly allows them.
-- For `codex`/Codex SDK live dogfood through the same local proxy, configure the agent target with `provider: codex`, `base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }}`, `api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }}`, `model: ${{ LOCAL_OPENAI_PROXY_MODEL }}`, `api_format: responses`, `grader_target: <local-openai-grader>`, `workers: 1`, and a bounded `timeout_seconds`. Configure the grader target as `provider: openai`, `api_format: chat`, and the same local proxy env references. A minimal run should use `bun apps/cli/src/cli.ts eval run <eval.yaml> --targets <targets.yaml> --target <codex-target> --workers 1`.
+- For `codex`/Codex SDK live dogfood through the same local proxy, configure the agent target with `provider: codex`, `base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }}`, `api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }}`, `model: ${{ LOCAL_OPENAI_PROXY_MODEL }}`, `api_format: responses`, `grader_target: <local-openai-grader>`, `workers: 1`, and a bounded `timeout_seconds`. Configure the grader target as `provider: openai` with the same local proxy env references; OpenAI-compatible targets default to chat completions unless `api_format: responses` is explicitly required. A minimal run should use `bun apps/cli/src/cli.ts eval run <eval.yaml> --targets <targets.yaml> --target <codex-target> --workers 1`.
 - If the local proxy returns `401 token_expired`, the blocker is stale Codex OAuth, not AgentV target configuration. Refresh from a trusted local terminal with `codex logout`, `codex login --device-auth`, then restart `openai-oauth` and rerun the same eval command.
 - Preserve review evidence in `agentv-private` on an orphan `evidence/<bead-or-feature-slug>` branch. Include the run bundle, source eval/experiment/targets files, a short README, an artifact tree, contract checks, and screenshots when folder structure or UI behavior is under review.
 - If comparing against an external convention such as Vercel `agent-eval`, verify both semantic provenance and the physical `attempt-N` artifact layout for repeat runs.

.agentv/config.yaml

@@ -1,21 +1,23 @@
 $schema: agentv-config-v2
 
-# Example .agentv/config.yaml Configuration
+# Repository AgentV configuration.
 
-# Customize which YAML files are discovered as evals during interactive mode
-# (`agentv eval` with no args). Defaults to dataset*.yaml and eval.yaml under evals/.
+# Discover the current example suite names during interactive eval selection.
+# Legacy *.eval.yaml fixtures remain discoverable for compatibility coverage.
 eval_patterns:
-  - "**/evals/**/dataset*.yaml"
-  - "**/evals/**/eval.yaml"
+  - "**/evals/**/suite.yaml"
+  - "**/evals/**/suite.yml"
+  - "**/evals/**/*.eval.yaml"
+  - "**/evals/**/*.eval.yml"
+  - "**/evals/**/*.eval.ts"
+
+# Publish completed run bundles to the git-backed results branch.
+results:
+  path: .
+  branch: agentv/results/v1
+  auto_push: true
 
 # Execution defaults (overridden by CLI flags)
 execution:
   verbose: false
-  trace_file: .agentv/results/trace-{timestamp}.jsonl
   keep_workspaces: false
-  otel_file: .agentv/results/otel-{timestamp}.json
-  # OTel live export (set credentials in .env)
-  # export_otel: true
-  # otel_backend: langfuse
-  # otel_capture_content: false
-  # otel_group_turns: true

.agentv/targets.yaml

@@ -11,36 +11,36 @@ targets:
   # redirects to a named target, controlled via AGENT_TARGET env var.
   # One env var switches the entire provider config (auth, model, etc.).
   # Example: AGENT_TARGET=copilot-cli  or  AGENT_TARGET=claude
-  - name: default
+  - label: default
     use_target: ${{ AGENT_TARGET }}
 
-  - name: agent
+  - label: agent
     use_target: ${{ AGENT_TARGET }}
 
   # ── LLM target (text generation, no agent binary needed) ────────────
   # Delegates to LLM_TARGET — same provider used for grading and LLM evals.
-  - name: llm
+  - label: llm
     use_target: ${{ LLM_TARGET }}
 
   # ── Grader (LLM-as-judge) ──────────────────────────────────────────
   # Used by agent targets via grader_target. Switch provider via GRADER_TARGET.
-  - name: grader
+  - label: grader
     use_target: ${{ GRADER_TARGET }}
 
   # ── Named agent targets ───────────────────────────────────────────
-  - name: copilot
+  - label: copilot
     provider: copilot-cli
     model: ${{ COPILOT_MODEL }}
     grader_target: grader
     stream_log: raw
 
-  - name: copilot-sdk
+  - label: copilot-sdk
     provider: copilot-sdk
     model: ${{ COPILOT_MODEL }}
     grader_target: grader
     stream_log: raw
 
-  - name: copilot-sdk-azure
+  - label: copilot-sdk-azure
     provider: copilot-sdk
     model: ${{ AZURE_DEPLOYMENT_NAME }}
     subprovider: azure
@@ -49,40 +49,22 @@ targets:
     grader_target: grader
     stream_log: raw
 
-  - name: claude
+  - label: claude
     provider: claude-cli
     grader_target: grader
 
-  # Claude via Z.ai provider (GLM models). Requires cc-mirror:
-  #   npx cc-mirror quick --provider zai --api-key "$Z_AI_API_KEY"
-  # Alternative: set Z_AI_API_KEY in ~/.cc-mirror/claude-zai/config/settings.json
-  # See https://github.com/numman-ali/cc-mirror
-  - name: claude-zai
-    provider: cc-mirror
-    executable: claude-zai
-    grader_target: grader
-
-  # Generic cc-mirror target. Set CC_MIRROR_VARIANT to the installed variant
-  # name (the directory under ~/.cc-mirror/, e.g. claude-zai, my-kimi).
-  # Setup: npx cc-mirror quick --provider <provider> --name <variant-name> --api-key "$KEY"
-  # See https://github.com/numman-ali/cc-mirror
-  - name: cc-mirror
-    provider: cc-mirror
-    variant: ${{ CC_MIRROR_VARIANT }}
-    grader_target: grader
-
-  - name: claude-sdk
+  - label: claude-sdk
     provider: claude-sdk
     grader_target: grader
 
-  - name: pi
+  - label: pi
     provider: pi-cli
     subprovider: openrouter
     model: ${{ OPENROUTER_MODEL }}
     api_key: ${{ OPENROUTER_API_KEY }}
     grader_target: grader
 
-  - name: pi-sdk
+  - label: pi-sdk
     provider: pi-coding-agent
     subprovider: openai
     base_url: ${{ OPENAI_ENDPOINT }}
@@ -93,15 +75,15 @@ targets:
     stream_log: raw
 
 
-  - name: pi-azure
+  - label: pi-azure
     provider: pi-cli
     subprovider: azure
     base_url: ${{ AZURE_OPENAI_ENDPOINT }}
     model: ${{ AZURE_DEPLOYMENT_NAME }}
     api_key: ${{ AZURE_OPENAI_API_KEY }}
     grader_target: grader
 
-  - name: pi-sdk-azure
+  - label: pi-sdk-azure
     provider: pi-coding-agent
     subprovider: azure
     base_url: ${{ AZURE_OPENAI_ENDPOINT }}
@@ -111,7 +93,7 @@ targets:
     thinking: low
     stream_log: raw
 
-  - name: codex
+  - label: codex
     provider: codex
     executable: ${{ CODEX_EXECUTABLE }}
     model: ${{ CODEX_MODEL }}
@@ -122,7 +104,7 @@ targets:
     stream_log: raw
 
   # ── LLM targets (direct model access) ─────────────────────────────
-  - name: gh-models
+  - label: gh-models
     provider: openai
     base_url: https://models.github.ai/inference
     api_key: ${{ GH_MODELS_TOKEN }}
@@ -133,43 +115,87 @@ targets:
   # overridden via AZURE_OPENAI_API_VERSION. Chat-completions-only Azure
   # deployments must use `provider: openai` with a deployment-scoped
   # `base_url` instead.
-  - name: azure
+  - label: azure
     provider: azure
     endpoint: ${{ AZURE_OPENAI_ENDPOINT }}
     api_key: ${{ AZURE_OPENAI_API_KEY }}
     model: ${{ AZURE_DEPLOYMENT_NAME }}
     version: ${{ AZURE_OPENAI_API_VERSION }}
 
-  - name: gemini
+  - label: gemini
     provider: gemini
     api_key: ${{ GOOGLE_GENERATIVE_AI_API_KEY }}
     model: ${{ GEMINI_MODEL_NAME }}
 
-  - name: openai
+  - label: openai
     provider: openai
     endpoint: ${{ OPENAI_ENDPOINT }}
     api_key: ${{ OPENAI_API_KEY }}
     model: ${{ OPENAI_MODEL }}
 
-  - name: openrouter
+  # Local OpenAI-compatible endpoint. Useful for dogfood against a local proxy
+  # without changing provider-specific target labels.
+  - label: local-openai
+    provider: openai
+    base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }}
+    api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }}
+    model: ${{ LOCAL_OPENAI_PROXY_MODEL }}
+    grader_target: local-openai-grader
+
+  - label: local-openai-grader
+    provider: openai
+    base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }}
+    api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }}
+    model: ${{ LOCAL_OPENAI_PROXY_MODEL }}
+
+  - label: pi-cli-openai
+    provider: pi-cli
+    subprovider: openai
+    base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }}
+    api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }}
+    model: ${{ LOCAL_OPENAI_PROXY_MODEL }}
+    grader_target: local-openai-grader
+    thinking: low
+    stream_log: raw
+
+  - label: codex-sdk-openai
+    provider: codex
+    base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }}
+    api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }}
+    model: ${{ LOCAL_OPENAI_PROXY_MODEL }}
+    api_format: responses
+    model_reasoning_effort: low
+    grader_target: local-openai-grader
+    stream_log: raw
+
+  - label: copilot-sdk-openai
+    provider: copilot-sdk
+    subprovider: openai
+    base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }}
+    api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }}
+    model: ${{ LOCAL_OPENAI_PROXY_MODEL }}
+    grader_target: local-openai-grader
+    stream_log: raw
+
+  - label: openrouter
     provider: openrouter
     api_key: ${{ OPENROUTER_API_KEY }}
     model: ${{ OPENROUTER_MODEL }}
 
   # ── MiMo (Xiaomi) via OpenRouter ───────────────────────────────────
-  - name: mimo
+  - label: mimo
     provider: openrouter
     api_key: ${{ OPENROUTER_API_KEY }}
     model: xiaomi/mimo-v2.5-pro
     grader_target: grader
 
-  - name: mimo-flash
+  - label: mimo-flash
     provider: openrouter
     api_key: ${{ OPENROUTER_API_KEY }}
     model: xiaomi/mimo-v2-flash
     grader_target: grader
 
-  - name: mimo-direct
+  - label: mimo-direct
     provider: openai
     base_url: https://token-plan-sgp.xiaomimimo.com/v1
     api_key: ${{ XIAOMI_MIMO_API_KEY }}

.github/workflows/ci.yml

@@ -303,8 +303,5 @@ jobs:
             }
           '
 
-      - name: Check evals directories have eval files
-        run: bun scripts/validate-eval-dirs.ts
-
       - name: Validate eval schemas
-        run: bun apps/cli/dist/cli.js validate 'examples/features/**/evals/**/*.eval.yaml' 'examples/features/**/*.EVAL.yaml'
+        run: bun apps/cli/dist/cli.js validate 'examples/features/**/evals/**/suite.yaml' 'examples/features/**/evals/**/*.eval.yaml' 'examples/features/**/*.EVAL.yaml'

apps/cli/src/commands/eval/discover.ts

@@ -17,8 +17,8 @@ export interface DiscoveredEvalFile {
  * Discover eval files by glob pattern matching.
  *
  * Uses `eval_patterns` from `.agentv/config.yaml` if configured,
- * otherwise falls back to default patterns that match `suite*.yaml`,
- * `eval.yaml`, and `dataset*.yaml` files under `evals/` directories.
+ * otherwise falls back to default patterns that match `suite.yaml`,
+ * `eval.yaml`, and `*.eval.yaml` files under `evals/` directories.
  */
 export async function discoverEvalFiles(cwd: string): Promise<readonly DiscoveredEvalFile[]> {
   const repoRoot = await findRepoRoot(cwd);

apps/cli/src/commands/eval/shared.ts

@@ -79,8 +79,8 @@ export async function resolveEvalPaths(
       if (candidateStats.isDirectory()) {
         // Auto-expand directory to recursive eval file glob
         const filePattern = options.allowReadAdapters
-          ? '{*.eval.yaml,*.eval.yml,eval.yaml,eval.yml,*.eval.ts,*.eval.mts,evals.json,*.evals.json}'
-          : '{*.eval.yaml,*.eval.yml,eval.yaml,eval.yml,*.eval.ts,*.eval.mts}';
+          ? '{suite.yaml,suite.yml,*.eval.yaml,*.eval.yml,eval.yaml,eval.yml,*.eval.ts,*.eval.mts,evals.json,*.evals.json}'
+          : '{suite.yaml,suite.yml,*.eval.yaml,*.eval.yml,eval.yaml,eval.yml,*.eval.ts,*.eval.mts}';
         const dirGlob = path.posix.join(candidatePath.replace(/\\/g, '/'), `**/${filePattern}`);
         const dirMatches = await fg(dirGlob, {
           absolute: true,
@@ -143,7 +143,7 @@ export async function resolveEvalPaths(
     throw new Error(
       `No eval files matched any provided paths or globs: ${includePatterns.join(
         ', ',
-      )}. Provide YAML, JSONL, TypeScript, or supported read-adapter paths/globs (e.g., "evals/**/eval.yaml", "evals/**/*.eval.ts", "skills/**/evals.json").`,
+      )}. Provide YAML, JSONL, TypeScript, or supported read-adapter paths/globs (e.g., "evals/**/suite.yaml", "evals/**/*.eval.ts", "skills/**/evals.json").`,
     );
   }
 

apps/cli/src/commands/results/remote.ts

@@ -187,7 +187,6 @@ export async function loadNormalizedResultsConfig(
       : (getProjectForPath(repoRoot) ?? getProjectForPath(cwd));
   const projectResults = project?.results
     ? ({
-        mode: 'github' as const,
         ...(project.results.repo !== undefined && { repo: project.results.repo }),
         ...(project.results.path !== undefined && { path: project.results.path }),
         ...(project.results.branch !== undefined && { branch: project.results.branch }),

apps/cli/src/commands/validate/validate-files.ts

@@ -76,7 +76,7 @@ async function validateSingleFile(filePath: string): Promise<ValidationResult> {
           severity: 'warning',
           filePath: absolutePath,
           message:
-            'File type not recognized. Eval files must end in .eval.yaml. Skipping validation.',
+            'File type not recognized. Eval files should be named suite.yaml or end in .eval.yaml. Skipping validation.',
         },
       ],
     };
@@ -164,8 +164,13 @@ function isYamlFile(filePath: string): boolean {
   return ext === '.yaml' || ext === '.yml';
 }
 
-/** Returns true only for *.eval.yaml / *.eval.yml files (used for directory scanning). */
+/** Returns true for native eval YAML suite files used during directory scanning. */
 function isEvalYamlFile(filePath: string): boolean {
   const lower = path.basename(filePath).toLowerCase();
-  return lower.endsWith('.eval.yaml') || lower.endsWith('.eval.yml');
+  return (
+    lower === 'suite.yaml' ||
+    lower === 'suite.yml' ||
+    lower.endsWith('.eval.yaml') ||
+    lower.endsWith('.eval.yml')
+  );
 }