docs(examples): migrate authored eval contracts

examples/contract/evals/release-gate.eval.yaml

@@ -2,6 +2,7 @@ name: release-contract
 description: Lightweight release gate for npm latest promotion.
 
 workspace:
+  scope: suite
   template: ../workspace-template
 
 target: github-models-contract

examples/contract/evals/repo-materialization.eval.yaml

@@ -3,6 +3,7 @@ description: Release gate for public repo materialization, previous-commit
   checkout, and file input substitution.
 
 workspace:
+  scope: suite
   repos:
     - path: ./fixture
       repo: EntityProcess/agentv-contract-fixture

examples/features/agent-skills-evals/csv-analyzer.EVAL.yaml

@@ -1,6 +1,10 @@
 tags: [ agent, skill-trigger ]
 
+extensions:
+  - agentv:agent-rules
+
 workspace:
+  scope: suite
   template: workspace/
 
 tests:

examples/features/agent-skills-evals/multi-provider-skill-trigger.EVAL.yaml

@@ -1,7 +1,8 @@
 # Multi-provider skill-trigger evaluation example.
 #
-# Uses an isolated workspace template (workspace/) so no global skill installation
-# is needed. The workspace contains:
+# Uses an isolated workspace template (workspace/) and the built-in
+# agentv:agent-rules extension so no global skill installation is needed. The
+# workspace contains:
 #   .claude/skills/acme-deploy/  — for claude-cli/copilot-cli providers
 #   .agents/skills/acme-deploy/  — for codex/pi providers
 #   .codex/skills/acme-deploy/   — for codex provider (fallback)
@@ -19,7 +20,11 @@
 # The grader automatically resolves the correct tool names for each
 # provider. No provider-specific config needed in test cases.
 
+extensions:
+  - agentv:agent-rules
+
 workspace:
+  scope: suite
   template: workspace/
 
 tests:

examples/features/benchmark-tooling/scripts/win-rate-summary.ts

@@ -1,18 +1,18 @@
 #!/usr/bin/env bun
 /**
- * win-rate-summary — Aggregate win/loss/tie rates from agentv compare output.
+ * win-rate-summary — Aggregate win/loss/tie rates from agentv results compare output.
  *
  * Usage:
  *   bun win-rate-summary.ts <comparison.json> [--tolerance <n>] [--json]
  *   bun win-rate-summary.ts <dir-of-comparisons/> [--tolerance <n>] [--json]
  *
- * Input: JSON output from `agentv compare --json`, saved to a file.
+ * Input: JSON output from `agentv results compare --json`, saved to a file.
  *        When a directory is given, all .json files are read and each is
  *        treated as a separate metric (filename becomes the metric label).
  *
  * Tie Policy:
  *   A result is classified as a "tie" when |delta| < tolerance.
- *   Default tolerance: 0.1 (same as agentv compare default threshold).
+ *   Default tolerance: 0.1 (same as agentv results compare default threshold).
  *   Set --tolerance 0 for strict comparison (no ties unless delta == 0).
  */
 
@@ -211,7 +211,7 @@ function main(): void {
     console.log(
       `Usage: bun win-rate-summary.ts <comparison.json | dir/> [--tolerance <n>] [--json]
 
-Reads JSON output from \`agentv compare --json\` and computes aggregate win/loss/tie rates.
+Reads JSON output from \`agentv results compare --json\` and computes aggregate win/loss/tie rates.
 
 Options:
   --tolerance <n>  Delta tolerance for tie classification (default: 0.1)
@@ -226,7 +226,7 @@ Per-Metric Breakdown:
   Each file is treated as a separate metric; the filename becomes the label.
 
 Examples:
-  agentv compare baseline.jsonl candidate.jsonl --json > comparison.json
+  agentv results compare baseline.jsonl candidate.jsonl --json > comparison.json
   bun win-rate-summary.ts comparison.json
   bun win-rate-summary.ts comparison.json --tolerance 0.05 --json
   bun win-rate-summary.ts ./comparisons/              # per-metric from directory`,
@@ -267,7 +267,7 @@ Examples:
   for (const { label, data } of inputs) {
     if (!data.matched || !Array.isArray(data.matched)) {
       console.error(
-        `Error: "${label}" has no matched array. Is this agentv compare --json output?`,
+        `Error: "${label}" has no matched array. Is this agentv results compare --json output?`,
       );
       process.exit(1);
     }

examples/features/compare/evals/suite.yaml

@@ -3,7 +3,7 @@
 #   agentv eval evals/suite.yaml --target baseline
 #   agentv eval evals/suite.yaml --target candidate
 # Then compare:
-#   agentv compare .agentv/results/default/<baseline-timestamp>/index.jsonl .agentv/results/default/<candidate-timestamp>/index.jsonl
+#   agentv results compare .agentv/results/default/<baseline-timestamp>/index.jsonl .agentv/results/default/<candidate-timestamp>/index.jsonl
 
 name: compare-demo
 description: Demo eval for generating baseline and candidate results to compare

examples/features/copilot-log-eval/evals/skill-trigger.EVAL.yaml

@@ -18,19 +18,12 @@ tags: [ agent ]
 
 target: copilot-log
 
+extensions:
+  - file://../scripts/copilot-log-workspace.mjs:beforeAll
+
 workspace:
+  scope: suite
   template: ../workspace/
-  hooks:
-    before_all:
-      command:
-        - node
-        - ../../workspace-setup-script/scripts/workspace-setup.mjs
-        - --from
-        - ../workspace/.allagents/workspace.yaml
-        - --marketplace-source
-        - ../../../../.claude-plugin
-        - --require
-        - .github/skills/agentv-bench/SKILL.md
 
 tests:
   # Negative case: csv-analyzer skill should NOT be triggered

examples/features/copilot-log-eval/scripts/copilot-log-workspace.mjs

@@ -0,0 +1,61 @@
+// @ts-check
+
+import { spawnSync } from 'node:child_process';
+import { existsSync, rmSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+
+const REQUIRED_FILES = ['.github/skills/agentv-bench/SKILL.md'];
+
+/**
+ * @param {{ workspace_path?: string; eval_dir: string }} context
+ */
+export function beforeAll(context) {
+  const workspacePath = context.workspace_path;
+  if (!workspacePath) {
+    throw new Error('workspace_path not provided to copilot-log setup extension');
+  }
+
+  rmSync(join(workspacePath, '.allagents'), { recursive: true, force: true });
+
+  const npx = process.platform === 'win32' ? 'npx.cmd' : 'npx';
+  run(npx, [
+    '--yes',
+    'allagents',
+    'workspace',
+    'init',
+    workspacePath,
+    '--from',
+    resolve(context.eval_dir, '../workspace/.allagents/workspace.yaml'),
+  ]);
+  run(
+    npx,
+    [
+      '--yes',
+      'allagents',
+      'plugin',
+      'marketplace',
+      'add',
+      resolve(context.eval_dir, '../../../../.claude-plugin'),
+      '--scope',
+      'project',
+    ],
+    workspacePath,
+  );
+  run(npx, ['--yes', 'allagents', 'workspace', 'sync'], workspacePath);
+
+  const missing = REQUIRED_FILES.filter((file) => !existsSync(join(workspacePath, file)));
+  if (missing.length > 0) {
+    throw new Error(`Required artifacts not found in workspace: ${missing.join(', ')}`);
+  }
+}
+
+function run(command, args, cwd = undefined) {
+  const result = spawnSync(command, args, {
+    stdio: ['ignore', 'inherit', 'inherit'],
+    shell: process.platform === 'win32',
+    ...(cwd ? { cwd } : {}),
+  });
+  if (result.status !== 0) {
+    throw new Error(`${command} ${args.join(' ')} failed with exit ${result.status ?? 1}`);
+  }
+}

examples/features/docker-workspace/README.md

@@ -30,6 +30,7 @@ image that has the repository, dependencies, and test infrastructure ready.
 
 ```yaml
 workspace:
+  scope: suite
   docker:
     image: swebench/sweb.eval.x86_64.django__django-15180
     timeout: 1800    # seconds (default: 1800)
@@ -41,6 +42,7 @@ For evals that need a repo pinned to a dataset snapshot, use `workspace.repos[].
 
 ```yaml
 workspace:
+  scope: suite
   docker:
     image: swebench/sweb.eval.x86_64.django__django-15180
   repos:

examples/features/docker-workspace/evals/docker-example.EVAL.yaml

@@ -13,6 +13,7 @@ description: Example eval using Docker workspace for grading
 target: mock_agent
 
 workspace:
+  scope: suite
   docker:
     image: python:3.11-slim
     timeout: 300

examples/features/document-extraction/evals/field-accuracy.eval.yaml

@@ -249,7 +249,7 @@ tests:
         EXPECTED GROUND TRUTH: supplier.name = "Acme Shipping" (normalized)
 
         This simulates OCR output that preserves document formatting.
-        Uses code-grader with config pass-through for multi-field fuzzy matching.
+        Uses a script grader with config pass-through for multi-field fuzzy matching.
     expected_output:
       - role: assistant
         content:

examples/features/file-changes-graders/evals/suite.yaml

@@ -12,6 +12,7 @@ description: Verify file_changes diffs are accessible to LLM grader (llm-rubric,
   built-in, and copilot-cli)
 
 workspace:
+  scope: suite
   template: ../workspace-template
 
 target: mock_agent

examples/features/file-changes-with-repos/evals/suite.yaml

@@ -23,17 +23,11 @@ description: Verify file_changes captures workspace-root files AND changes
   inside nested repos
 
 workspace:
+  scope: suite
   template: ../workspace-template
-  hooks:
-    before_all:
-      command:
-        - bash
-        - -c
-        - >-
-          cd "{{workspace_path}}/my-lib" && git -c init.defaultBranch=main init
-          && git -c user.email=test@agentv.dev -c user.name="AgentV Test" add .
-          && git -c user.email=test@agentv.dev -c user.name="AgentV Test" commit
-          -m "init"
+
+extensions:
+  - file://../scripts/setup-nested-repo.mjs:beforeAll
 
 target: mock_agent
 

examples/features/file-changes-with-repos/scripts/setup-nested-repo.mjs

@@ -0,0 +1,31 @@
+// @ts-check
+
+import { execFileSync } from 'node:child_process';
+import { join } from 'node:path';
+
+/**
+ * @param {{ workspace_path?: string }} context
+ */
+export function beforeAll(context) {
+  const workspacePath = context.workspace_path;
+  if (!workspacePath) {
+    throw new Error('workspace_path not provided to nested repo setup extension');
+  }
+
+  const repoPath = join(workspacePath, 'my-lib');
+  run('git', ['-c', 'init.defaultBranch=main', 'init'], repoPath);
+  run(
+    'git',
+    ['-c', 'user.email=test@agentv.dev', '-c', 'user.name=AgentV Test', 'add', '.'],
+    repoPath,
+  );
+  run(
+    'git',
+    ['-c', 'user.email=test@agentv.dev', '-c', 'user.name=AgentV Test', 'commit', '-m', 'init'],
+    repoPath,
+  );
+}
+
+function run(command, args, cwd) {
+  execFileSync(command, args, { cwd, stdio: 'inherit' });
+}

examples/features/file-changes/evals/suite.yaml

@@ -13,6 +13,7 @@ name: file-changes
 description: Verify file_changes captures edits, creates, and deletes across multiple tests
 
 workspace:
+  scope: suite
   template: ../workspace-template
 
 target: mock_agent

examples/features/functional-grading/evals/suite.yaml

@@ -14,6 +14,7 @@ name: functional-grading
 description: Functional grading with workspace_path — deploy-and-test pattern
 
 workspace:
+  scope: suite
   template: ../workspace-template
 
 target: mock_agent

examples/features/repo-lifecycle/evals/suite.yaml

@@ -3,6 +3,7 @@ description: >-
   check out a specific commit, and have the agent work on it.
 
 workspace:
+  scope: suite
   repos:
     - path: ./repo
       repo: https://github.com/EntityProcess/agentv.git

examples/features/tool-calls-template/evals/suite.yaml

@@ -4,25 +4,21 @@
 # whether an agent invoked the right skills — without needing the
 # skill-trigger evaluator.
 #
-# Skills live in workspace/.agents/skills/. The setup hook copies
-# them to .claude/skills/ so copilot and other providers can discover them.
+# Skills live in workspace/.agents/skills/. The built-in agent-rules extension
+# stages them so supported providers can discover them.
 #
 # Run:
 #   bun agentv eval examples/features/tool-calls-template/evals/suite.yaml --target copilot
 
 name: tool-calls-template
 description: Rubric assertions with {{ tool_calls }} for skill verification
 
+extensions:
+  - agentv:agent-rules
+
 workspace:
+  scope: suite
   template: ../workspace/
-  hooks:
-    before_all:
-      command:
-        - bash
-        - -c
-        - 'WS=$(python3 -c "import
-          json,sys;print(json.load(sys.stdin)[\"workspace_path\"])") && mkdir -p
-          "$WS/.claude" && cp -r "$WS/.agents/skills" "$WS/.claude/skills"'
 
 tests:
   - id: deploy-skill-triggered

examples/features/vitest-workspace-grader/evals/suite.yaml

@@ -2,6 +2,7 @@ name: vitest-workspace-grader
 description: Deterministic workspace grading with a Vitest verifier file.
 
 workspace:
+  scope: suite
   template: ../workspace-template
 
 target: mock_agent

examples/features/workspace-artifact/evals/suite.yaml

@@ -23,6 +23,7 @@ name: workspace-artifact
 description: Verify file_changes captures generated artifacts (CSV) under workspace_path
 
 workspace:
+  scope: suite
   template: ../workspace-template
 
 target: mock_csv_agent

examples/features/workspace-multi-repo/evals/suite.yaml

@@ -3,6 +3,7 @@ description: >-
   cloned into the workspace.
 
 workspace:
+  scope: suite
   template: ../workspace-template
   hooks:
     after_each:

examples/features/workspace-setup-script/README.md

@@ -39,6 +39,7 @@ extensions:
   - file://../scripts/workspace-setup.mjs:beforeAll
 
 workspace:
+  scope: suite
   template: ../workspace-template
   repos:
     - path: ./my-repo

examples/features/workspace-setup-script/evals/dataset-vscode.eval.yaml

@@ -6,6 +6,7 @@ extensions:
   - file://../scripts/workspace-setup.mjs:beforeAll
 
 workspace:
+  scope: suite
   template: ../workspace-template
   hooks:
     after_each:

examples/features/workspace-setup-script/evals/suite.yaml

@@ -6,6 +6,7 @@ extensions:
   - file://../scripts/workspace-setup.mjs:beforeAll
 
 workspace:
+  scope: suite
   template: ../workspace-template
   repos:
     - path: ./my-repo

examples/features/workspace-shared-config/workspace.yaml

@@ -1,3 +1,4 @@
+scope: suite
 template: ./workspace-template
 hooks:
   after_each: