From 2024330af8a1cdedc672f0d3d3f82af05029bfbb Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Sat, 4 Jul 2026 05:34:36 +0200 Subject: [PATCH] docs(examples): migrate authored eval contracts --- .../contract/evals/release-gate.eval.yaml | 1 + .../evals/repo-materialization.eval.yaml | 1 + .../agent-skills-evals/csv-analyzer.EVAL.yaml | 4 ++ .../multi-provider-skill-trigger.EVAL.yaml | 9 ++- .../scripts/win-rate-summary.ts | 12 ++-- examples/features/compare/evals/suite.yaml | 2 +- .../evals/skill-trigger.EVAL.yaml | 15 ++--- .../scripts/copilot-log-workspace.mjs | 61 +++++++++++++++++++ examples/features/docker-workspace/README.md | 2 + .../evals/docker-example.EVAL.yaml | 1 + .../evals/field-accuracy.eval.yaml | 2 +- .../file-changes-graders/evals/suite.yaml | 1 + .../file-changes-with-repos/evals/suite.yaml | 14 ++--- .../scripts/setup-nested-repo.mjs | 31 ++++++++++ .../features/file-changes/evals/suite.yaml | 1 + .../functional-grading/evals/suite.yaml | 1 + .../features/repo-lifecycle/evals/suite.yaml | 1 + .../tool-calls-template/evals/suite.yaml | 16 ++--- .../vitest-workspace-grader/evals/suite.yaml | 1 + .../workspace-artifact/evals/suite.yaml | 1 + .../workspace-multi-repo/evals/suite.yaml | 1 + .../features/workspace-setup-script/README.md | 1 + .../evals/dataset-vscode.eval.yaml | 1 + .../workspace-setup-script/evals/suite.yaml | 1 + .../workspace-shared-config/workspace.yaml | 1 + .../evals/bug-fixes.eval.yaml | 1 + .../showcase/cross-repo-sync/evals/suite.yaml | 14 ++--- .../showcase/cross-repo-sync/scripts/reset.ts | 18 ++++-- .../showcase/cross-repo-sync/scripts/setup.ts | 24 +++++--- scripts/validate-example-evals.ts | 19 +++--- 30 files changed, 185 insertions(+), 73 deletions(-) create mode 100644 examples/features/copilot-log-eval/scripts/copilot-log-workspace.mjs create mode 100644 examples/features/file-changes-with-repos/scripts/setup-nested-repo.mjs diff --git a/examples/contract/evals/release-gate.eval.yaml b/examples/contract/evals/release-gate.eval.yaml index 29ca05dc5..2dba4f7ce 100644 --- a/examples/contract/evals/release-gate.eval.yaml +++ b/examples/contract/evals/release-gate.eval.yaml @@ -2,6 +2,7 @@ name: release-contract description: Lightweight release gate for npm latest promotion. workspace: + scope: suite template: ../workspace-template target: github-models-contract diff --git a/examples/contract/evals/repo-materialization.eval.yaml b/examples/contract/evals/repo-materialization.eval.yaml index 32e164466..ebbd5a30c 100644 --- a/examples/contract/evals/repo-materialization.eval.yaml +++ b/examples/contract/evals/repo-materialization.eval.yaml @@ -3,6 +3,7 @@ description: Release gate for public repo materialization, previous-commit checkout, and file input substitution. workspace: + scope: suite repos: - path: ./fixture repo: EntityProcess/agentv-contract-fixture diff --git a/examples/features/agent-skills-evals/csv-analyzer.EVAL.yaml b/examples/features/agent-skills-evals/csv-analyzer.EVAL.yaml index 1723c30ad..7869c63fa 100644 --- a/examples/features/agent-skills-evals/csv-analyzer.EVAL.yaml +++ b/examples/features/agent-skills-evals/csv-analyzer.EVAL.yaml @@ -1,6 +1,10 @@ tags: [ agent, skill-trigger ] +extensions: + - agentv:agent-rules + workspace: + scope: suite template: workspace/ tests: diff --git a/examples/features/agent-skills-evals/multi-provider-skill-trigger.EVAL.yaml b/examples/features/agent-skills-evals/multi-provider-skill-trigger.EVAL.yaml index 0aa85d7a4..a34d4c8b7 100644 --- a/examples/features/agent-skills-evals/multi-provider-skill-trigger.EVAL.yaml +++ b/examples/features/agent-skills-evals/multi-provider-skill-trigger.EVAL.yaml @@ -1,7 +1,8 @@ # Multi-provider skill-trigger evaluation example. # -# Uses an isolated workspace template (workspace/) so no global skill installation -# is needed. The workspace contains: +# Uses an isolated workspace template (workspace/) and the built-in +# agentv:agent-rules extension so no global skill installation is needed. The +# workspace contains: # .claude/skills/acme-deploy/ — for claude-cli/copilot-cli providers # .agents/skills/acme-deploy/ — for codex/pi providers # .codex/skills/acme-deploy/ — for codex provider (fallback) @@ -19,7 +20,11 @@ # The grader automatically resolves the correct tool names for each # provider. No provider-specific config needed in test cases. +extensions: + - agentv:agent-rules + workspace: + scope: suite template: workspace/ tests: diff --git a/examples/features/benchmark-tooling/scripts/win-rate-summary.ts b/examples/features/benchmark-tooling/scripts/win-rate-summary.ts index cac7eba9f..5d4221ae5 100644 --- a/examples/features/benchmark-tooling/scripts/win-rate-summary.ts +++ b/examples/features/benchmark-tooling/scripts/win-rate-summary.ts @@ -1,18 +1,18 @@ #!/usr/bin/env bun /** - * win-rate-summary — Aggregate win/loss/tie rates from agentv compare output. + * win-rate-summary — Aggregate win/loss/tie rates from agentv results compare output. * * Usage: * bun win-rate-summary.ts [--tolerance ] [--json] * bun win-rate-summary.ts [--tolerance ] [--json] * - * Input: JSON output from `agentv compare --json`, saved to a file. + * Input: JSON output from `agentv results compare --json`, saved to a file. * When a directory is given, all .json files are read and each is * treated as a separate metric (filename becomes the metric label). * * Tie Policy: * A result is classified as a "tie" when |delta| < tolerance. - * Default tolerance: 0.1 (same as agentv compare default threshold). + * Default tolerance: 0.1 (same as agentv results compare default threshold). * Set --tolerance 0 for strict comparison (no ties unless delta == 0). */ @@ -211,7 +211,7 @@ function main(): void { console.log( `Usage: bun win-rate-summary.ts [--tolerance ] [--json] -Reads JSON output from \`agentv compare --json\` and computes aggregate win/loss/tie rates. +Reads JSON output from \`agentv results compare --json\` and computes aggregate win/loss/tie rates. Options: --tolerance Delta tolerance for tie classification (default: 0.1) @@ -226,7 +226,7 @@ Per-Metric Breakdown: Each file is treated as a separate metric; the filename becomes the label. Examples: - agentv compare baseline.jsonl candidate.jsonl --json > comparison.json + agentv results compare baseline.jsonl candidate.jsonl --json > comparison.json bun win-rate-summary.ts comparison.json bun win-rate-summary.ts comparison.json --tolerance 0.05 --json bun win-rate-summary.ts ./comparisons/ # per-metric from directory`, @@ -267,7 +267,7 @@ Examples: for (const { label, data } of inputs) { if (!data.matched || !Array.isArray(data.matched)) { console.error( - `Error: "${label}" has no matched array. Is this agentv compare --json output?`, + `Error: "${label}" has no matched array. Is this agentv results compare --json output?`, ); process.exit(1); } diff --git a/examples/features/compare/evals/suite.yaml b/examples/features/compare/evals/suite.yaml index df73997dd..6325fb0ff 100644 --- a/examples/features/compare/evals/suite.yaml +++ b/examples/features/compare/evals/suite.yaml @@ -3,7 +3,7 @@ # agentv eval evals/suite.yaml --target baseline # agentv eval evals/suite.yaml --target candidate # Then compare: -# agentv compare .agentv/results/default//index.jsonl .agentv/results/default//index.jsonl +# agentv results compare .agentv/results/default//index.jsonl .agentv/results/default//index.jsonl name: compare-demo description: Demo eval for generating baseline and candidate results to compare diff --git a/examples/features/copilot-log-eval/evals/skill-trigger.EVAL.yaml b/examples/features/copilot-log-eval/evals/skill-trigger.EVAL.yaml index 51272fa38..33c5e7545 100644 --- a/examples/features/copilot-log-eval/evals/skill-trigger.EVAL.yaml +++ b/examples/features/copilot-log-eval/evals/skill-trigger.EVAL.yaml @@ -18,19 +18,12 @@ tags: [ agent ] target: copilot-log +extensions: + - file://../scripts/copilot-log-workspace.mjs:beforeAll + workspace: + scope: suite template: ../workspace/ - hooks: - before_all: - command: - - node - - ../../workspace-setup-script/scripts/workspace-setup.mjs - - --from - - ../workspace/.allagents/workspace.yaml - - --marketplace-source - - ../../../../.claude-plugin - - --require - - .github/skills/agentv-bench/SKILL.md tests: # Negative case: csv-analyzer skill should NOT be triggered diff --git a/examples/features/copilot-log-eval/scripts/copilot-log-workspace.mjs b/examples/features/copilot-log-eval/scripts/copilot-log-workspace.mjs new file mode 100644 index 000000000..e1cbf745d --- /dev/null +++ b/examples/features/copilot-log-eval/scripts/copilot-log-workspace.mjs @@ -0,0 +1,61 @@ +// @ts-check + +import { spawnSync } from 'node:child_process'; +import { existsSync, rmSync } from 'node:fs'; +import { join, resolve } from 'node:path'; + +const REQUIRED_FILES = ['.github/skills/agentv-bench/SKILL.md']; + +/** + * @param {{ workspace_path?: string; eval_dir: string }} context + */ +export function beforeAll(context) { + const workspacePath = context.workspace_path; + if (!workspacePath) { + throw new Error('workspace_path not provided to copilot-log setup extension'); + } + + rmSync(join(workspacePath, '.allagents'), { recursive: true, force: true }); + + const npx = process.platform === 'win32' ? 'npx.cmd' : 'npx'; + run(npx, [ + '--yes', + 'allagents', + 'workspace', + 'init', + workspacePath, + '--from', + resolve(context.eval_dir, '../workspace/.allagents/workspace.yaml'), + ]); + run( + npx, + [ + '--yes', + 'allagents', + 'plugin', + 'marketplace', + 'add', + resolve(context.eval_dir, '../../../../.claude-plugin'), + '--scope', + 'project', + ], + workspacePath, + ); + run(npx, ['--yes', 'allagents', 'workspace', 'sync'], workspacePath); + + const missing = REQUIRED_FILES.filter((file) => !existsSync(join(workspacePath, file))); + if (missing.length > 0) { + throw new Error(`Required artifacts not found in workspace: ${missing.join(', ')}`); + } +} + +function run(command, args, cwd = undefined) { + const result = spawnSync(command, args, { + stdio: ['ignore', 'inherit', 'inherit'], + shell: process.platform === 'win32', + ...(cwd ? { cwd } : {}), + }); + if (result.status !== 0) { + throw new Error(`${command} ${args.join(' ')} failed with exit ${result.status ?? 1}`); + } +} diff --git a/examples/features/docker-workspace/README.md b/examples/features/docker-workspace/README.md index 79bcdaf77..9096dc89b 100644 --- a/examples/features/docker-workspace/README.md +++ b/examples/features/docker-workspace/README.md @@ -30,6 +30,7 @@ image that has the repository, dependencies, and test infrastructure ready. ```yaml workspace: + scope: suite docker: image: swebench/sweb.eval.x86_64.django__django-15180 timeout: 1800 # seconds (default: 1800) @@ -41,6 +42,7 @@ For evals that need a repo pinned to a dataset snapshot, use `workspace.repos[]. ```yaml workspace: + scope: suite docker: image: swebench/sweb.eval.x86_64.django__django-15180 repos: diff --git a/examples/features/docker-workspace/evals/docker-example.EVAL.yaml b/examples/features/docker-workspace/evals/docker-example.EVAL.yaml index 0a59bb125..310c4ea06 100644 --- a/examples/features/docker-workspace/evals/docker-example.EVAL.yaml +++ b/examples/features/docker-workspace/evals/docker-example.EVAL.yaml @@ -13,6 +13,7 @@ description: Example eval using Docker workspace for grading target: mock_agent workspace: + scope: suite docker: image: python:3.11-slim timeout: 300 diff --git a/examples/features/document-extraction/evals/field-accuracy.eval.yaml b/examples/features/document-extraction/evals/field-accuracy.eval.yaml index d060e3a34..6a1dba9d2 100644 --- a/examples/features/document-extraction/evals/field-accuracy.eval.yaml +++ b/examples/features/document-extraction/evals/field-accuracy.eval.yaml @@ -249,7 +249,7 @@ tests: EXPECTED GROUND TRUTH: supplier.name = "Acme Shipping" (normalized) This simulates OCR output that preserves document formatting. - Uses code-grader with config pass-through for multi-field fuzzy matching. + Uses a script grader with config pass-through for multi-field fuzzy matching. expected_output: - role: assistant content: diff --git a/examples/features/file-changes-graders/evals/suite.yaml b/examples/features/file-changes-graders/evals/suite.yaml index 5b7b7e05b..f1f146c99 100644 --- a/examples/features/file-changes-graders/evals/suite.yaml +++ b/examples/features/file-changes-graders/evals/suite.yaml @@ -12,6 +12,7 @@ description: Verify file_changes diffs are accessible to LLM grader (llm-rubric, built-in, and copilot-cli) workspace: + scope: suite template: ../workspace-template target: mock_agent diff --git a/examples/features/file-changes-with-repos/evals/suite.yaml b/examples/features/file-changes-with-repos/evals/suite.yaml index d03b981f1..f1ec2ccc5 100644 --- a/examples/features/file-changes-with-repos/evals/suite.yaml +++ b/examples/features/file-changes-with-repos/evals/suite.yaml @@ -23,17 +23,11 @@ description: Verify file_changes captures workspace-root files AND changes inside nested repos workspace: + scope: suite template: ../workspace-template - hooks: - before_all: - command: - - bash - - -c - - >- - cd "{{workspace_path}}/my-lib" && git -c init.defaultBranch=main init - && git -c user.email=test@agentv.dev -c user.name="AgentV Test" add . - && git -c user.email=test@agentv.dev -c user.name="AgentV Test" commit - -m "init" + +extensions: + - file://../scripts/setup-nested-repo.mjs:beforeAll target: mock_agent diff --git a/examples/features/file-changes-with-repos/scripts/setup-nested-repo.mjs b/examples/features/file-changes-with-repos/scripts/setup-nested-repo.mjs new file mode 100644 index 000000000..e63ed395d --- /dev/null +++ b/examples/features/file-changes-with-repos/scripts/setup-nested-repo.mjs @@ -0,0 +1,31 @@ +// @ts-check + +import { execFileSync } from 'node:child_process'; +import { join } from 'node:path'; + +/** + * @param {{ workspace_path?: string }} context + */ +export function beforeAll(context) { + const workspacePath = context.workspace_path; + if (!workspacePath) { + throw new Error('workspace_path not provided to nested repo setup extension'); + } + + const repoPath = join(workspacePath, 'my-lib'); + run('git', ['-c', 'init.defaultBranch=main', 'init'], repoPath); + run( + 'git', + ['-c', 'user.email=test@agentv.dev', '-c', 'user.name=AgentV Test', 'add', '.'], + repoPath, + ); + run( + 'git', + ['-c', 'user.email=test@agentv.dev', '-c', 'user.name=AgentV Test', 'commit', '-m', 'init'], + repoPath, + ); +} + +function run(command, args, cwd) { + execFileSync(command, args, { cwd, stdio: 'inherit' }); +} diff --git a/examples/features/file-changes/evals/suite.yaml b/examples/features/file-changes/evals/suite.yaml index 947cc4230..33376a20e 100644 --- a/examples/features/file-changes/evals/suite.yaml +++ b/examples/features/file-changes/evals/suite.yaml @@ -13,6 +13,7 @@ name: file-changes description: Verify file_changes captures edits, creates, and deletes across multiple tests workspace: + scope: suite template: ../workspace-template target: mock_agent diff --git a/examples/features/functional-grading/evals/suite.yaml b/examples/features/functional-grading/evals/suite.yaml index af1410347..c72b07570 100644 --- a/examples/features/functional-grading/evals/suite.yaml +++ b/examples/features/functional-grading/evals/suite.yaml @@ -14,6 +14,7 @@ name: functional-grading description: Functional grading with workspace_path — deploy-and-test pattern workspace: + scope: suite template: ../workspace-template target: mock_agent diff --git a/examples/features/repo-lifecycle/evals/suite.yaml b/examples/features/repo-lifecycle/evals/suite.yaml index 3ff56aee5..d413497b8 100644 --- a/examples/features/repo-lifecycle/evals/suite.yaml +++ b/examples/features/repo-lifecycle/evals/suite.yaml @@ -3,6 +3,7 @@ description: >- check out a specific commit, and have the agent work on it. workspace: + scope: suite repos: - path: ./repo repo: https://github.com/EntityProcess/agentv.git diff --git a/examples/features/tool-calls-template/evals/suite.yaml b/examples/features/tool-calls-template/evals/suite.yaml index 07f6182f4..45eb85b5e 100644 --- a/examples/features/tool-calls-template/evals/suite.yaml +++ b/examples/features/tool-calls-template/evals/suite.yaml @@ -4,8 +4,8 @@ # whether an agent invoked the right skills — without needing the # skill-trigger evaluator. # -# Skills live in workspace/.agents/skills/. The setup hook copies -# them to .claude/skills/ so copilot and other providers can discover them. +# Skills live in workspace/.agents/skills/. The built-in agent-rules extension +# stages them so supported providers can discover them. # # Run: # bun agentv eval examples/features/tool-calls-template/evals/suite.yaml --target copilot @@ -13,16 +13,12 @@ name: tool-calls-template description: Rubric assertions with {{ tool_calls }} for skill verification +extensions: + - agentv:agent-rules + workspace: + scope: suite template: ../workspace/ - hooks: - before_all: - command: - - bash - - -c - - 'WS=$(python3 -c "import - json,sys;print(json.load(sys.stdin)[\"workspace_path\"])") && mkdir -p - "$WS/.claude" && cp -r "$WS/.agents/skills" "$WS/.claude/skills"' tests: - id: deploy-skill-triggered diff --git a/examples/features/vitest-workspace-grader/evals/suite.yaml b/examples/features/vitest-workspace-grader/evals/suite.yaml index dd3a24aff..2e2e02150 100644 --- a/examples/features/vitest-workspace-grader/evals/suite.yaml +++ b/examples/features/vitest-workspace-grader/evals/suite.yaml @@ -2,6 +2,7 @@ name: vitest-workspace-grader description: Deterministic workspace grading with a Vitest verifier file. workspace: + scope: suite template: ../workspace-template target: mock_agent diff --git a/examples/features/workspace-artifact/evals/suite.yaml b/examples/features/workspace-artifact/evals/suite.yaml index 1dac56e0d..e7cf02438 100644 --- a/examples/features/workspace-artifact/evals/suite.yaml +++ b/examples/features/workspace-artifact/evals/suite.yaml @@ -23,6 +23,7 @@ name: workspace-artifact description: Verify file_changes captures generated artifacts (CSV) under workspace_path workspace: + scope: suite template: ../workspace-template target: mock_csv_agent diff --git a/examples/features/workspace-multi-repo/evals/suite.yaml b/examples/features/workspace-multi-repo/evals/suite.yaml index cc682cab9..ebb81267a 100644 --- a/examples/features/workspace-multi-repo/evals/suite.yaml +++ b/examples/features/workspace-multi-repo/evals/suite.yaml @@ -3,6 +3,7 @@ description: >- cloned into the workspace. workspace: + scope: suite template: ../workspace-template hooks: after_each: diff --git a/examples/features/workspace-setup-script/README.md b/examples/features/workspace-setup-script/README.md index 2400ee644..2b65b4519 100644 --- a/examples/features/workspace-setup-script/README.md +++ b/examples/features/workspace-setup-script/README.md @@ -39,6 +39,7 @@ extensions: - file://../scripts/workspace-setup.mjs:beforeAll workspace: + scope: suite template: ../workspace-template repos: - path: ./my-repo diff --git a/examples/features/workspace-setup-script/evals/dataset-vscode.eval.yaml b/examples/features/workspace-setup-script/evals/dataset-vscode.eval.yaml index 4637ca1c0..28478bd97 100644 --- a/examples/features/workspace-setup-script/evals/dataset-vscode.eval.yaml +++ b/examples/features/workspace-setup-script/evals/dataset-vscode.eval.yaml @@ -6,6 +6,7 @@ extensions: - file://../scripts/workspace-setup.mjs:beforeAll workspace: + scope: suite template: ../workspace-template hooks: after_each: diff --git a/examples/features/workspace-setup-script/evals/suite.yaml b/examples/features/workspace-setup-script/evals/suite.yaml index 0e8e1e59c..758e48dd8 100644 --- a/examples/features/workspace-setup-script/evals/suite.yaml +++ b/examples/features/workspace-setup-script/evals/suite.yaml @@ -6,6 +6,7 @@ extensions: - file://../scripts/workspace-setup.mjs:beforeAll workspace: + scope: suite template: ../workspace-template repos: - path: ./my-repo diff --git a/examples/features/workspace-shared-config/workspace.yaml b/examples/features/workspace-shared-config/workspace.yaml index 55a8b443d..a74989a77 100644 --- a/examples/features/workspace-shared-config/workspace.yaml +++ b/examples/features/workspace-shared-config/workspace.yaml @@ -1,3 +1,4 @@ +scope: suite template: ./workspace-template hooks: after_each: diff --git a/examples/showcase/bug-fix-benchmark/evals/bug-fixes.eval.yaml b/examples/showcase/bug-fix-benchmark/evals/bug-fixes.eval.yaml index 61135038c..20e287948 100644 --- a/examples/showcase/bug-fix-benchmark/evals/bug-fixes.eval.yaml +++ b/examples/showcase/bug-fix-benchmark/evals/bug-fixes.eval.yaml @@ -14,6 +14,7 @@ description: | superpowers (obra), compound-engineering (Every Inc), agent-skills (Addy Osmani). workspace: + scope: suite repos: - path: ./repo repo: https://github.com/EntityProcess/agentv diff --git a/examples/showcase/cross-repo-sync/evals/suite.yaml b/examples/showcase/cross-repo-sync/evals/suite.yaml index 509738b0d..4fff42e31 100644 --- a/examples/showcase/cross-repo-sync/evals/suite.yaml +++ b/examples/showcase/cross-repo-sync/evals/suite.yaml @@ -3,17 +3,13 @@ description: Evaluate agent ability to sync AgentV implementation with agenteval version: "1.0" tags: [ showcase, workspace, cross-repo ] +extensions: + - file://../scripts/setup.ts:beforeEach + - file://../scripts/reset.ts:afterEach + workspace: + scope: suite template: ../workspace-template - hooks: - before_each: - command: [ "bash", "../scripts/run-ts.sh", "../scripts/setup.ts" ] - timeout_ms: 900000 - cwd: . - after_each: - command: [ "bash", "../scripts/run-ts.sh", "../scripts/reset.ts" ] - timeout_ms: 5000 - cwd: . target: mock_agent diff --git a/examples/showcase/cross-repo-sync/scripts/reset.ts b/examples/showcase/cross-repo-sync/scripts/reset.ts index b7ddb968e..6cd790514 100644 --- a/examples/showcase/cross-repo-sync/scripts/reset.ts +++ b/examples/showcase/cross-repo-sync/scripts/reset.ts @@ -9,6 +9,8 @@ */ import { existsSync, readFileSync, rmSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; interface StdinPayload { workspace_path: string; @@ -16,11 +18,8 @@ interface StdinPayload { eval_run_id: string; } -function main(): void { - const input = readFileSync(0, 'utf-8'); - const payload: StdinPayload = JSON.parse(input); - - const agentevalsDir = `${payload.workspace_path}/agentevals`; +export function afterEach(context: StdinPayload): void { + const agentevalsDir = `${context.workspace_path}/agentevals`; if (existsSync(agentevalsDir)) { rmSync(agentevalsDir, { recursive: true, force: true }); @@ -28,4 +27,11 @@ function main(): void { } } -main(); +function main(): void { + const input = readFileSync(0, 'utf-8'); + afterEach(JSON.parse(input)); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) { + main(); +} diff --git a/examples/showcase/cross-repo-sync/scripts/setup.ts b/examples/showcase/cross-repo-sync/scripts/setup.ts index 80c4210fa..0e4295b30 100644 --- a/examples/showcase/cross-repo-sync/scripts/setup.ts +++ b/examples/showcase/cross-repo-sync/scripts/setup.ts @@ -14,6 +14,8 @@ import { execFile } from 'node:child_process'; import { existsSync, readFileSync, rmSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; import { promisify } from 'node:util'; const execFileAsync = promisify(execFile); @@ -42,11 +44,8 @@ async function run(cmd: string, args: string[], cwd?: string): Promise { return stdout.trim(); } -async function main(): Promise { - const input = readFileSync(0, 'utf-8'); - const payload: StdinPayload = JSON.parse(input); - - const { workspace_path, case_metadata } = payload; +export async function beforeEach(context: StdinPayload): Promise { + const { workspace_path, case_metadata } = context; if (!case_metadata?.agentevals_before) { throw new Error('case_metadata must include agentevals_before commit SHA'); } @@ -74,7 +73,14 @@ async function main(): Promise { console.log(`Cloned agentevals at ${commit} → ${agentevalsDir} (.git preserved)`); } -main().catch((err) => { - console.error(err); - process.exit(1); -}); +async function main(): Promise { + const input = readFileSync(0, 'utf-8'); + await beforeEach(JSON.parse(input)); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) { + main().catch((err) => { + console.error(err); + process.exit(1); + }); +} diff --git a/scripts/validate-example-evals.ts b/scripts/validate-example-evals.ts index 8bb47e18a..f0606d3a4 100644 --- a/scripts/validate-example-evals.ts +++ b/scripts/validate-example-evals.ts @@ -1,8 +1,10 @@ /** - * Validate all eval YAML files under examples/features and examples/contract. + * Validate public eval YAML files under examples/. * - * Finds files matching the public examples/contract eval patterns and runs - * AgentV schema validation on each one. + * Finds files matching public eval patterns and runs AgentV schema validation + * on each one. Sidecar files such as default-test.yaml, cases.yaml, and + * *.grader-scores.yaml are intentionally excluded because they are referenced + * inputs, not runnable eval suites. * * Used by CI and local verification to catch invalid eval files before merge. * @@ -23,16 +25,17 @@ const ROOT = path.resolve(import.meta.dir, '..'); function findEvalFiles(): string[] { const patterns = [ - 'examples/features/**/evals/*.eval.yaml', - 'examples/features/**/evals/*.EVAL.yaml', - 'examples/features/**/*.EVAL.yaml', - 'examples/contract/**/evals/*.eval.yaml', - 'examples/contract/**/evals/*.EVAL.yaml', + 'examples/**/*.eval.yaml', + 'examples/**/*.EVAL.yaml', + 'examples/**/EVAL.yaml', + 'examples/**/evals/suite.yaml', + 'examples/**/suites/*.eval.yaml', ]; const files = new Set(); for (const pattern of patterns) { for (const match of globSync(pattern, { cwd: ROOT })) { + if (match.endsWith('.grader-scores.yaml')) continue; files.add(path.resolve(ROOT, match)); } }